From b9060fc00df89a4c73d5b98947688b200b79901f Mon Sep 17 00:00:00 2001 From: K0K0V0K <109747532+K0K0V0K@users.noreply.github.com> Date: Sat, 23 Nov 2024 17:15:34 +0100 Subject: [PATCH] YARN-11738 Modernize SecretManager config (#7144) --- .../fs/CommonConfigurationKeysPublic.java | 9 ++++ .../hadoop/security/token/SecretManager.java | 46 ++++++++++++------- .../src/main/resources/core-default.xml | 22 +++++++++ .../TestNMTokenSecretManagerInNM.java | 2 + 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 9036f34542d88..86cad28d3ee33 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1011,6 +1011,15 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY = "hadoop.security.credstore.java-keystore-provider.password-file"; + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY = + "hadoop.security.secret-manager.key-generator.algorithm"; + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT = + "HmacSHA1"; + + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY = + "hadoop.security.secret-manager.key-length"; + public static final int HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT = 64; + /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java index 514806ddec804..002917e6d15b5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java @@ -27,8 +27,13 @@ import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.StandbyException; @@ -40,6 +45,8 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public abstract class SecretManager { + + public static final Logger LOG = LoggerFactory.getLogger(SecretManager.class); /** * The token was invalid and the message explains why. */ @@ -107,16 +114,23 @@ public byte[] retriableRetrievePassword(T identifier) public void checkAvailableForRead() throws StandbyException { // Default to being available for read. } - - /** - * The name of the hashing algorithm. - */ - private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; - /** - * The length of the random keys to use. - */ - private static final int KEY_LENGTH = 64; + private static final String SELECTED_ALGORITHM; + private static final int SELECTED_LENGTH; + + static { + Configuration conf = new Configuration(); + String algorithm = conf.get( + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT); + LOG.info("Selected hash algorithm: {}", algorithm); + SELECTED_ALGORITHM = algorithm; + int length = conf.getInt( + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT); + LOG.info("Selected hash key length:{}", length); + SELECTED_LENGTH = length; + } /** * A thread local store for the Macs. @@ -126,10 +140,9 @@ public void checkAvailableForRead() throws StandbyException { @Override protected Mac initialValue() { try { - return Mac.getInstance(DEFAULT_HMAC_ALGORITHM); + return Mac.getInstance(SELECTED_ALGORITHM); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + - " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa); } } }; @@ -140,11 +153,10 @@ protected Mac initialValue() { private final KeyGenerator keyGen; { try { - keyGen = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM); - keyGen.init(KEY_LENGTH); + keyGen = KeyGenerator.getInstance(SELECTED_ALGORITHM); + keyGen.init(SELECTED_LENGTH); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + - " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa); } } @@ -185,6 +197,6 @@ public static byte[] createPassword(byte[] identifier, * @return the secret key */ protected static SecretKey createSecretKey(byte[] key) { - return new SecretKeySpec(key, DEFAULT_HMAC_ALGORITHM); + return new SecretKeySpec(key, SELECTED_ALGORITHM); } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8e62550cd239b..dd3504527ecd9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1063,6 +1063,28 @@ + + hadoop.security.secret-manager.key-generator.algorithm + HmacSHA1 + + The configuration key specifying the KeyGenerator algorithm used in SecretManager + for generating secret keys. The algorithm must be a KeyGenerator algorithm supported by + the Java Cryptography Architecture (JCA). Common examples include "HmacSHA1", + "HmacSHA256", and "HmacSHA512". + + + + + hadoop.security.secret-manager.key-length + 64 + + The configuration key specifying the key length of the generated secret keys + in SecretManager. The key length must be appropriate for the algorithm. + For example, longer keys are generally more secure but may not be supported + by all algorithms. + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java index 5c1f3a1c31064..153b650c20485 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java @@ -60,6 +60,8 @@ public void testRecovery() throws IOException { secretMgr.setNodeId(nodeId); MasterKey currentKey = keygen.generateKey(); secretMgr.setMasterKey(currentKey); + // check key is 64 bit long (8 byte) + assertEquals(8, currentKey.getBytes().array().length); NMTokenIdentifier attemptToken1 = getNMTokenId(secretMgr.createNMToken(attempt1, nodeId, "user1")); NMTokenIdentifier attemptToken2 =