Skip to content

Commit

Permalink
Fix/blue green deploy (backport #4414) (#4420)
Browse files Browse the repository at this point in the history 
* Fix/blue green deploy (#4414)

* fix(blue-green-deploy): removing old failed manifests

* fix(blue-green-deploy): refactoring

* fix(blue-green-deploy): refactoring

* fix(blue-green-deploy): added docs

* fix(blue-green-deploy): added test to validate new behaviour

(cherry picked from commit 5156eec)

# Conflicts:
#	orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStage.java
#	orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStageTest.java

* chore(conflict): resolved conflicts (#4423)

---------

Co-authored-by: Krzysztof Kotula <[email protected]>
Co-authored-by: ovidiupopa07 <[email protected]>
  • Loading branch information
@mergify @kkotula @ovidiupopa07
3 people authored Mar 28, 2023
1 parent bb38bc6 commit ccdbea7
Show file tree
Hide file tree
Showing 2 changed files with 355 additions and 102 deletions.
311 changes: 233 additions & 78 deletions ...in/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,31 @@
import com.netflix.spinnaker.orca.clouddriver.tasks.artifacts.CleanupArtifactsTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.*;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.DeployManifestContext.TrafficManagement;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ResolveDeploySourceManifestTask;
import com.netflix.spinnaker.orca.pipeline.ExpressionAwareStageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.tasks.artifacts.BindProducedArtifactsTask;
import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;

@Component
@RequiredArgsConstructor
public class DeployManifestStage extends ExpressionAwareStageDefinitionBuilder {

public static final String PIPELINE_CONFIG_TYPE = "deployManifest";

private final OortService oortService;
private final OldManifestActionAppender oldManifestActionAppender;

@Autowired
public DeployManifestStage(OortService oortService) {
this.oortService = oortService;
private static boolean shouldRemoveStageOutputs(@NotNull StageExecution stage) {
return stage.getContext().getOrDefault("noOutput", "false").toString().equals("true");
}

@Override
Expand All @@ -68,102 +72,253 @@ public void taskGraph(@Nonnull StageExecution stage, @Nonnull TaskNode.Builder b
.withTask(BindProducedArtifactsTask.TASK_NAME, BindProducedArtifactsTask.class);
}

@Override
public boolean processExpressions(
@Nonnull StageExecution stage,
@Nonnull ContextParameterProcessor contextParameterProcessor,
@Nonnull ExpressionEvaluationSummary summary) {
DeployManifestContext context = stage.mapTo(DeployManifestContext.class);
if (context.isSkipExpressionEvaluation()) {
processDefaultEntries(
stage, contextParameterProcessor, summary, Collections.singletonList("manifests"));
return false;
}
return true;
}

@Override
public void afterStages(@Nonnull StageExecution stage, @Nonnull StageGraphBuilder graph) {
TrafficManagement trafficManagement =
stage.mapTo(DeployManifestContext.class).getTrafficManagement();
if (trafficManagement.isEnabled()) {
switch (trafficManagement.getOptions().getStrategy()) {
case RED_BLACK:
disableOldManifests(stage.getContext(), graph);
oldManifestActionAppender.deleteOrDisableOldManifest(stage.getContext(), graph);
break;
case HIGHLANDER:
disableOldManifests(stage.getContext(), graph);
deleteOldManifests(stage.getContext(), graph);
oldManifestActionAppender.disableOldManifest(stage.getContext(), graph);
oldManifestActionAppender.deleteOldManifest(stage.getContext(), graph);
break;
case NONE:
// do nothing
}
}
if (stage.getContext().getOrDefault("noOutput", "false").toString().equals("true")) {
if (shouldRemoveStageOutputs(stage)) {
stage.setOutputs(emptyMap());
}
}

private void disableOldManifests(Map<String, Object> parentContext, StageGraphBuilder graph) {
addStagesForOldManifests(parentContext, graph, DisableManifestStage.PIPELINE_CONFIG_TYPE);
}
/** {@code OldManifestActionAppender} appends new stages to old manifests */
@Component
@RequiredArgsConstructor
static class OldManifestActionAppender {

private void deleteOldManifests(Map<String, Object> parentContext, StageGraphBuilder graph) {
addStagesForOldManifests(parentContext, graph, DeleteManifestStage.PIPELINE_CONFIG_TYPE);
}
private final GetDeployedManifests deployedManifests;
private final ManifestOperationsHelper manifestOperationsHelper;

private void addStagesForOldManifests(
Map<String, Object> parentContext, StageGraphBuilder graph, String stageType) {
List<Map<String, ?>> deployedManifests = getNewManifests(parentContext);
String account = (String) parentContext.get("account");
Map manifestMoniker = (Map) parentContext.get("moniker");
String application = (String) manifestMoniker.get("app");

deployedManifests.forEach(
manifest -> {
Map manifestMetadata = (Map) manifest.get("metadata");
String manifestName =
String.format("replicaSet %s", (String) manifestMetadata.get("name"));
String namespace = (String) manifestMetadata.get("namespace");
Map annotations = (Map) manifestMetadata.get("annotations");
String clusterName = (String) annotations.get("moniker.spinnaker.io/cluster");
String cloudProvider = "kubernetes";

ImmutableList<String> previousManifestNames =
getOldManifestNames(application, account, clusterName, namespace, manifestName);
previousManifestNames.forEach(
name -> {
graph.append(
(stage) -> {
stage.setType(stageType);
Map<String, Object> context = stage.getContext();
context.put("account", account);
context.put("app", application);
context.put("cloudProvider", cloudProvider);
context.put("manifestName", name);
context.put("location", namespace);
});
});
});
}
/**
* Appends delete stages to already deployed manifests that preceded the current stage manifest
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void deleteOldManifest(Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) ->
appendStage(graph, manifest, name, DeleteManifestStage.PIPELINE_CONFIG_TYPE));
}

/**
* Appends disable stages to already deployed manifests that preceded the current stage manifest
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void disableOldManifest(Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) ->
appendStage(graph, manifest, name, DisableManifestStage.PIPELINE_CONFIG_TYPE));
}

/**
* Appends disable or delete stages to already deployed manifests that preceded the current
* stage manifest. The specific stage that will be appended depends on the status of the
* previous deployment.
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void deleteOrDisableOldManifest(
Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) -> {
var oldManifestIsUnstable =
this.manifestOperationsHelper.previousDeploymentNeitherStableNorFailed(
manifest.getAccount(), name);
var nextStageType =
oldManifestIsUnstable
? DeleteManifestStage.PIPELINE_CONFIG_TYPE
: DisableManifestStage.PIPELINE_CONFIG_TYPE;
appendStage(graph, manifest, name, nextStageType);
});
}

private List<Map<String, ?>> getNewManifests(Map<String, Object> parentContext) {
List<Map<String, ?>> manifests = (List<Map<String, ?>>) parentContext.get("outputs.manifests");
return manifests.stream()
.filter(manifest -> manifest.get("kind").equals("ReplicaSet"))
.collect(Collectors.toList());
private void applyAction(
Map<String, Object> parentContext, BiConsumer<String, DeployedManifest> action) {
this.deployedManifests
.get(parentContext)
.forEach(
currentlyDeployedManifest ->
manifestOperationsHelper
.getOldManifestNames(currentlyDeployedManifest)
.forEach(
oldManifestName ->
action.accept(oldManifestName, currentlyDeployedManifest)));
}

private void appendStage(
StageGraphBuilder graph, DeployedManifest manifest, String name, String stageType) {
graph.append(
(stage) -> {
stage.setType(stageType);
Map<String, Object> context = stage.getContext();
context.put("account", manifest.getAccount());
context.put("app", manifest.getApplication());
context.put("cloudProvider", manifest.getCloudProvider());
context.put("manifestName", name);
context.put("location", manifest.getNamespace());
});
}
}

private ImmutableList<String> getOldManifestNames(
String application,
String account,
String clusterName,
String namespace,
String newManifestName) {
return oortService
.getClusterManifests(account, namespace, "replicaSet", application, clusterName)
.stream()
.filter(m -> !m.getFullResourceName().equals(newManifestName))
.map(ManifestCoordinates::getFullResourceName)
.collect(toImmutableList());
/**
* Delegate class to handle all manifest-related actions in this file such as fetching manifest
* from external service or extracting manifest params from parentContext
*/
@Component
@RequiredArgsConstructor
static class ManifestOperationsHelper {

private static final String REPLICA_SET = "replicaSet";
private static final String KIND = "kind";
private static final String OUTPUTS_MANIFEST = "outputs.manifests";

private final OortService oortService;

/**
* This returns all replicaSet manifests from the cluster. The search is performed in an
* external service, and search parameters match manifests deployed in the current stage.
*
* @param dm - deployment manifest of current stage
* @return list of all manifest already deployed to the cluster
*/
ImmutableList<String> getOldManifestNames(DeployedManifest dm) {
return oortService
.getClusterManifests(
dm.getAccount(),
dm.getNamespace(),
REPLICA_SET,
dm.getApplication(),
dm.getClusterName())
.stream()
.filter(m -> !m.getFullResourceName().equals(dm.getManifestName()))
.map(ManifestCoordinates::getFullResourceName)
.collect(toImmutableList());
}

/**
* Returns replicaSet manifests from the {@code parentContext}
*
* @param parentContext of currently processed stage
* @return list of replicaSet manifests deployed in the cluster - obtained directly from the
* {@code parentContext}
*/
List<Map<String, ?>> getNewManifests(Map<String, Object> parentContext) {
var manifestsFromParentContext = (List<Map<String, ?>>) parentContext.get(OUTPUTS_MANIFEST);
return manifestsFromParentContext.stream()
.filter(manifest -> REPLICA_SET.equalsIgnoreCase((String) manifest.get(KIND)))
.collect(Collectors.toList());
}

/**
* During a B/G deployment, if the blue deployment fails to create pods (due to issues such as
* an incorrect image name), the deployment will not fail, but will wait indefinitely to achieve
* stability. This is indicated by status.failed=false and status.stable=false. This method
* checks for such a situation.
*
* @param account used to run deployment
* @param name of the manifest
* @return true, if manifest was not deployed correctly and waits to get stable, false otherwise
*/
boolean previousDeploymentNeitherStableNorFailed(String account, String name) {
var oldManifest = this.oortService.getManifest(account, name, false);

var status = oldManifest.getStatus();
var notStable = !status.getStable().isState();
var notFailed = !status.getFailed().isState();

return notFailed && notStable;
}
}

@Override
public boolean processExpressions(
@Nonnull StageExecution stage,
@Nonnull ContextParameterProcessor contextParameterProcessor,
@Nonnull ExpressionEvaluationSummary summary) {
DeployManifestContext context = stage.mapTo(DeployManifestContext.class);
if (context.isSkipExpressionEvaluation()) {
processDefaultEntries(
stage, contextParameterProcessor, summary, Collections.singletonList("manifests"));
return false;
/** Delegate class for fetching and mapping manifests deployed in the cluster */
@Component
@RequiredArgsConstructor
static class GetDeployedManifests {

private final ManifestOperationsHelper manifestOperationsHelper;

/**
* This method encapsulates fetching deployed manifests and mapping them to a new designated
* type, {@code DeployedManifest}
*
* @param parentContext is the context of currently processed stage
* @return list of replicaSet manifests deployed in currently processed stage
*/
List<DeployedManifest> get(Map<String, Object> parentContext) {

var deployedManifests = new ArrayList<DeployedManifest>();
var account = (String) parentContext.get("account");
var manifestMoniker = (Map) parentContext.get("moniker");
var application = (String) manifestMoniker.get("app");

this.manifestOperationsHelper
.getNewManifests(parentContext)
.forEach(
manifest -> {
var manifestMetadata = (Map) manifest.get("metadata");
var annotations = (Map) manifestMetadata.get("annotations");

deployedManifests.add(
new DeployedManifest(
account,
manifestMoniker,
application,
(Map) manifest.get("metadata"),
String.format("replicaSet %s", manifestMetadata.get("name")),
(String) manifestMetadata.get("namespace"),
(Map) manifestMetadata.get("annotations"),
(String) annotations.get("moniker.spinnaker.io/cluster"),
"kubernetes"));
});
return deployedManifests;
}
return true;
}

@Getter
@RequiredArgsConstructor
static class DeployedManifest {
private final String account;
private final Map manifestMoniker;
private final String application;
private final Map manifestMetadata;
private final String manifestName;
private final String namespace;
private final Map annotations;
private final String clusterName;
private final String cloudProvider;
}
}
Loading

0 comments on commit ccdbea7

Please sign in to comment.