feat: add merge support for casc defined system credentials

Enables support for merging casc defined credentials with existing credentials
(i.e. manually created).

fixes JENKINS-64079

src/main/java/com/cloudbees/plugins/credentials/SystemCredentialsProvider.java

@@ -186,6 +186,26 @@ public synchronized void setDomainCredentialsMap(Map<Domain, List<Credentials>>
         this.domainCredentialsMap = DomainCredentials.toCopyOnWriteMap(domainCredentialsMap);
     }
 
+    /**
+     * Merge the given credentials with the current set. Replace existing domain credentials or add new credentials.
+     * Existing credentials not in the given set will not be removed.
+     *
+     * @param domainCredentialsMap credentials to add or update
+     */
+    public synchronized void mergeDomainCredentialsMap(Map<Domain, List<Credentials>> domainCredentialsMap) {
+        for (Map.Entry<Domain, List<Credentials>> entry : DomainCredentials.toCopyOnWriteMap(domainCredentialsMap).entrySet()) {
+            List<Credentials> target = this.domainCredentialsMap.get(entry.getKey());
+            if (target == null) {
+                this.domainCredentialsMap.put(entry.getKey(), entry.getValue());
+            } else {
+                target.removeAll(entry.getValue());
+                target.addAll(entry.getValue());
+                this.domainCredentialsMap.remove(entry.getKey());
+                this.domainCredentialsMap.put(entry.getKey(), target);
+            }
+        }
+    }
+
     /**
      * Short-cut method for {@link Jenkins#checkPermission(hudson.security.Permission)}
      *

src/main/java/com/cloudbees/plugins/credentials/casc/SystemCredentialsProviderConfigurator.java

@@ -30,6 +30,7 @@
 import edu.umd.cs.findbugs.annotations.CheckForNull;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import hudson.Extension;
+import hudson.Util;
 import io.jenkins.plugins.casc.Attribute;
 import io.jenkins.plugins.casc.BaseConfigurator;
 import io.jenkins.plugins.casc.ConfigurationContext;
@@ -39,11 +40,21 @@
 import org.kohsuke.accmod.Restricted;
 import org.kohsuke.accmod.restrictions.NoExternalUse;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
- * A configurator for system credentials provider located beneath the {@link CredentialsRootConfigurator}
+ * A configurator for system credentials provider located beneath the {@link CredentialsRootConfigurator}. The default
+ * merge strategy will replace all existing credentials. To merge CasC credentials with existing credentials use
+ * the env var {@code CASC_CREDENTIALS_MERGE_STRATEGY} or system property {@code casc.credentials.merge.strategy}
+ * to set the strategy to "{@code merge}". The "{@code merge}" strategy will not remove credentials don't exist in
+ * CasC configuration.
+ * 
+ * @see SystemCredentialsProvider#mergeDomainCredentialsMap(Map) 
+ * @see SystemCredentialsProvider#setDomainCredentialsMap(Map)
  */
 @Extension(optional = true, ordinal = 2)
 @Restricted(NoExternalUse.class)
@@ -64,17 +75,29 @@ protected SystemCredentialsProvider instance(Mapping mapping, ConfigurationConte
     public Set<Attribute<SystemCredentialsProvider, ?>> describe() {
         return Collections.singleton(
             new MultivaluedAttribute<SystemCredentialsProvider, DomainCredentials>("domainCredentials", DomainCredentials.class)
-                .setter( (target, value) -> target.setDomainCredentialsMap(DomainCredentials.asMap(value)))
+                .setter((target, value) -> {
+                    String strategy = Util.fixEmptyAndTrim(
+                            System.getProperty("casc.credentials.merge.strategy",
+                            System.getenv("CASC_CREDENTIALS_MERGE_STRATEGY")
+                    ));
+
+                    if ("merge".equalsIgnoreCase(strategy)) {
+                        target.mergeDomainCredentialsMap(DomainCredentials.asMap(value));
+                    } else {
+                        target.setDomainCredentialsMap(DomainCredentials.asMap(value));
+                    }
+                })
         );
     }
 
     @CheckForNull
     @Override
     public CNode describe(SystemCredentialsProvider instance, ConfigurationContext context) throws Exception {
         Mapping mapping = new Mapping();
-        for (Attribute attribute : describe()) {
+        for (Attribute<SystemCredentialsProvider, ?> attribute : describe()) {
             mapping.put(attribute.getName(), attribute.describe(instance, context));
         }
         return mapping;
     }
+
 }

src/test/java/com/cloudbees/plugins/credentials/SystemCredentialsProviderTest.java

@@ -24,6 +24,8 @@
 package com.cloudbees.plugins.credentials;
 
 import com.cloudbees.plugins.credentials.common.IdCredentials;
+import com.cloudbees.plugins.credentials.domains.Domain;
+import com.cloudbees.plugins.credentials.domains.DomainCredentials;
 import com.cloudbees.plugins.credentials.impl.DummyCredentials;
 import com.cloudbees.plugins.credentials.impl.DummyIdCredentials;
 import edu.umd.cs.findbugs.annotations.NonNull;
@@ -39,10 +41,20 @@
 import hudson.tasks.BuildStepDescriptor;
 import hudson.tasks.Builder;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
 import jenkins.security.QueueItemAuthenticatorConfiguration;
 import org.acegisecurity.Authentication;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.hudson.test.JenkinsRule;
@@ -51,6 +63,7 @@
 import org.jvnet.hudson.test.TestExtension;
 import org.kohsuke.stapler.DataBoundConstructor;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -164,6 +177,72 @@ public void given_globalScopeCredential_when_builtAsUserWithoutUseItem_then_cred
         r.assertBuildStatus(Result.FAILURE, prj.scheduleBuild2(0).get());
     }
 
+    @Test
+    public void mergeDomainCredentialsMap() {
+        SystemCredentialsProvider provider = SystemCredentialsProvider.getInstance();
+
+        // initial creds
+        Map<Domain, List<Credentials>> creds = new HashMap<>();
+        creds.put(null, Arrays.asList(
+                new DummyIdCredentials("foo-manchu", CredentialsScope.GLOBAL, "foo", "manchu", "Dr. Fu Manchu"),
+                new DummyIdCredentials("bar-manchu", CredentialsScope.GLOBAL, "bar", "manchu", "Dr. Bar Manchu")
+        ));
+        Domain catsDotCom = new Domain("cats.com", "cats dot com", Collections.emptyList());
+        creds.put(catsDotCom, Arrays.asList(
+                new DummyIdCredentials("kitty-cat", CredentialsScope.GLOBAL, "kitty", "manchu", "Mrs. Kitty"),
+                new DummyIdCredentials("garfield-cat", CredentialsScope.GLOBAL, "garfield", "manchu", "Garfield")
+        ));
+        provider.setDomainCredentialsMap(creds);
+
+        // merge creds
+        Map<Domain, List<Credentials>> update = new HashMap<>();
+        update.put(null, Arrays.asList(
+                new DummyIdCredentials("foo-manchu", CredentialsScope.GLOBAL, "foo", "Man-chu", "Dr. Fu Manchu Phd"),
+                new DummyIdCredentials("strange", CredentialsScope.GLOBAL, "strange", "manchu", "Dr. Strange")
+        ));
+        Domain catsDotCom2 = new Domain("cats.com", "cats.com domain for cats", Collections.emptyList());
+        update.put(catsDotCom2, Arrays.asList(
+                new DummyIdCredentials("garfield-cat", CredentialsScope.GLOBAL, "garfield", "manchu", "Garfield the Cat"),
+        new DummyIdCredentials("eek-cat", CredentialsScope.GLOBAL, "eek", "manchu", "Eek the Cat")
+        ));
+        Domain dogsDotCom = new Domain("dogs.com", "dogs.com domain for dogs", Collections.emptyList());
+        update.put(dogsDotCom, Arrays.asList(
+                new DummyIdCredentials("snoopy", CredentialsScope.GLOBAL, "snoopy", "manchu", "Snoop-a-Loop")
+        ));
+
+        // do merge
+        provider.mergeDomainCredentialsMap(update);
+
+        // verify
+        List<DomainCredentials> domainCreds = provider.getDomainCredentials();
+        assertEquals(3, domainCreds.size());
+        for (DomainCredentials dc : domainCreds) {
+            if (dc.getDomain().isGlobal()) {
+                assertEquals(3, dc.getCredentials().size());
+                assertDummyCreds(dc.getCredentials(), DummyIdCredentials::getUsername, "bar", "foo", "strange");
+                assertDummyCreds(dc.getCredentials(), DummyIdCredentials::getDescription, "Dr. Bar Manchu", "Dr. Fu Manchu Phd", "Dr. Strange");
+            } else if (StringUtils.equals(dc.getDomain().getName(), "cats.com")) {
+                assertEquals("cats.com domain for cats", dc.getDomain().getDescription());
+                assertEquals(3, dc.getCredentials().size());
+                assertDummyCreds(dc.getCredentials(), DummyIdCredentials::getUsername, "eek", "garfield", "kitty");
+                assertDummyCreds(dc.getCredentials(), DummyIdCredentials::getDescription, "Eek the Cat", "Garfield the Cat", "Mrs. Kitty");
+            } else if (StringUtils.equals(dc.getDomain().getName(), "dogs.com")) {
+                assertEquals("dogs.com domain for dogs", dc.getDomain().getDescription());
+                assertEquals(1, dc.getCredentials().size());
+                assertDummyCreds(dc.getCredentials(), DummyIdCredentials::getUsername, "snoopy");
+            }
+        }
+    }
+
+    private <T> void assertDummyCreds(List<Credentials> creds, Function<DummyIdCredentials, T> valSupplier, T... expected) {
+        List<T> vals = creds.stream()
+                .filter(c -> c instanceof DummyIdCredentials)
+                .map(c -> valSupplier.apply((DummyIdCredentials) c))
+                .sorted()
+                .collect(Collectors.toList());
+        assertEquals(Arrays.asList(expected), vals);
+    }
+
     public static class HasCredentialBuilder extends Builder {
 
         private final String id;

src/test/java/com/cloudbees/plugins/credentials/casc/MergeSystemCredentialsTest.java

@@ -0,0 +1,100 @@
+/*
+ * The MIT License
+ *
+ *  Copyright (c) 2018, CloudBees, Inc.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ *
+ */
+
+package com.cloudbees.plugins.credentials.casc;
+
+import com.cloudbees.plugins.credentials.CredentialsMatchers;
+import com.cloudbees.plugins.credentials.CredentialsProvider;
+import com.cloudbees.plugins.credentials.CredentialsScope;
+import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
+import com.cloudbees.plugins.credentials.common.UsernamePasswordCredentials;
+import com.cloudbees.plugins.credentials.domains.Domain;
+import com.cloudbees.plugins.credentials.domains.HostnameRequirement;
+import com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl;
+import hudson.security.ACL;
+import hudson.util.Secret;
+import io.jenkins.plugins.casc.ConfigurationAsCode;
+import io.jenkins.plugins.casc.ConfigurationContext;
+import io.jenkins.plugins.casc.ConfiguratorException;
+import io.jenkins.plugins.casc.ConfiguratorRegistry;
+import io.jenkins.plugins.casc.misc.ConfiguredWithCode;
+import io.jenkins.plugins.casc.misc.JenkinsConfiguredWithCodeRule;
+import io.jenkins.plugins.casc.model.Mapping;
+import io.jenkins.plugins.casc.model.Sequence;
+import jenkins.model.Jenkins;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.jvnet.hudson.test.JenkinsRule;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertNotNull;
+
+public class MergeSystemCredentialsTest {
+
+    @ClassRule
+    public static JenkinsConfiguredWithCodeRule j = new JenkinsConfiguredWithCodeRule();
+
+    @Before
+    public void setup() {
+        System.setProperty("casc.credentials.merge.strategy", "merge");
+    }
+
+    @After
+    public void teardown() {
+        System.clearProperty("casc.credentials.merge.strategy");
+    }
+
+    @Test
+    public void merge_system_credentials() throws ConfiguratorException {
+        UsernamePasswordCredentials foo = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, "foo", "", "Foo", "Bar");
+        UsernamePasswordCredentials bar = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, "bar", "", "Bar", "Foo");
+        Domain testCom = new Domain("test.com", "test dot com", Collections.emptyList());
+        SystemCredentialsProvider.getInstance().getCredentials().add(foo);
+        SystemCredentialsProvider.getInstance().getDomainCredentialsMap().put(testCom, new CopyOnWriteArrayList<>(Collections.singletonList(bar)));
+        ConfigurationAsCode.get().configure(getClass().getResource("MergeSystemCredentialsTest.yaml").toExternalForm());
+        System.out.println(SystemCredentialsProvider.getInstance().getDomainCredentialsMap());
+        List<UsernamePasswordCredentials> ups = CredentialsProvider.lookupCredentials(
+            UsernamePasswordCredentials.class, j.jenkins, ACL.SYSTEM,
+            Collections.singletonList(new HostnameRequirement("api.test.com"))
+        );
+        assertThat(ups, hasSize(3));
+        bar = CredentialsMatchers.firstOrNull(ups, CredentialsMatchers.withId("bar"));
+        assertThat(bar, not(nullValue()));
+        assertThat(bar.getUsername(), equalTo("bar_usr"));
+    }
+
+}

src/test/resources/com/cloudbees/plugins/credentials/casc/MergeSystemCredentialsTest.yaml

@@ -0,0 +1,20 @@
+credentials:
+  system:
+    domainCredentials:
+      - domain:
+          name: "test.com"
+          description: "test.com domain"
+          specifications:
+            - hostnameSpecification:
+                includes: "*.test.com"
+        credentials:
+          - usernamePassword:
+                scope: SYSTEM
+                id: bar
+                username: bar_usr
+                password: "pwd"
+          - usernamePassword:
+              scope:    SYSTEM
+              id:       sudo_password
+              username: root
+              password: "password"