Skip to content

Commit

Permalink
HADOOP-18919. Zookeeper SSL/TLS support in HDFS ZKFC (#6194)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dombizita
dombizita authored Oct 23, 2023
1 parent 5eeab5e commit 4c04818
Show file tree
Hide file tree
Showing 13 changed files with 263 additions and 112 deletions.
32 changes: 27 additions & 5 deletions ...common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.client.ZKClientConfig;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
Expand All @@ -48,6 +50,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.naming.ConfigurationException;

import org.apache.hadoop.security.SecurityUtil.TruststoreKeystore;

/**
*
* This class implements a simple library to perform leader election on top of
Expand Down Expand Up @@ -170,6 +176,7 @@ enum State {
private final int zkSessionTimeout;
private final List<ACL> zkAcl;
private final List<ZKAuthInfo> zkAuthInfo;
private TruststoreKeystore truststoreKeystore;
private byte[] appData;
private final String zkLockFilePath;
private final String zkBreadCrumbPath;
Expand Down Expand Up @@ -209,6 +216,7 @@ enum State {
* @param app
* reference to callback interface object
* @param maxRetryNum maxRetryNum.
* @param truststoreKeystore truststore keystore, that we will use for ZK if SSL/TLS is enabled
* @throws IOException raised on errors performing I/O.
* @throws HadoopIllegalArgumentException
* if valid data is not supplied.
Expand All @@ -218,10 +226,10 @@ enum State {
public ActiveStandbyElector(String zookeeperHostPorts,
int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
List<ZKAuthInfo> authInfo, ActiveStandbyElectorCallback app,
int maxRetryNum) throws IOException, HadoopIllegalArgumentException,
KeeperException {
int maxRetryNum, TruststoreKeystore truststoreKeystore)
throws IOException, HadoopIllegalArgumentException, KeeperException {
this(zookeeperHostPorts, zookeeperSessionTimeout, parentZnodeName, acl,
authInfo, app, maxRetryNum, true);
authInfo, app, maxRetryNum, true, truststoreKeystore);
}

/**
Expand Down Expand Up @@ -254,6 +262,7 @@ public ActiveStandbyElector(String zookeeperHostPorts,
* @param failFast
* whether need to add the retry when establishing ZK connection.
* @param maxRetryNum max Retry Num
* @param truststoreKeystore truststore keystore, that we will use for ZK if SSL/TLS is enabled
* @throws IOException
* raised on errors performing I/O.
* @throws HadoopIllegalArgumentException
Expand All @@ -264,7 +273,7 @@ public ActiveStandbyElector(String zookeeperHostPorts,
public ActiveStandbyElector(String zookeeperHostPorts,
int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
List<ZKAuthInfo> authInfo, ActiveStandbyElectorCallback app,
int maxRetryNum, boolean failFast) throws IOException,
int maxRetryNum, boolean failFast, TruststoreKeystore truststoreKeystore) throws IOException,
HadoopIllegalArgumentException, KeeperException {
if (app == null || acl == null || parentZnodeName == null
|| zookeeperHostPorts == null || zookeeperSessionTimeout <= 0) {
Expand All @@ -279,6 +288,7 @@ public ActiveStandbyElector(String zookeeperHostPorts,
zkLockFilePath = znodeWorkingDir + "/" + LOCK_FILENAME;
zkBreadCrumbPath = znodeWorkingDir + "/" + BREADCRUMB_FILENAME;
this.maxRetryNum = maxRetryNum;
this.truststoreKeystore = truststoreKeystore;

// establish the ZK Connection for future API calls
if (failFast) {
Expand Down Expand Up @@ -740,7 +750,19 @@ protected synchronized ZooKeeper connectToZooKeeper() throws IOException,
* @throws IOException raised on errors performing I/O.
*/
protected ZooKeeper createZooKeeper() throws IOException {
return new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
ZKClientConfig zkClientConfig = new ZKClientConfig();
if (truststoreKeystore != null) {
try {
SecurityUtil.setSslConfiguration(zkClientConfig, truststoreKeystore);
} catch (ConfigurationException ce) {
throw new IOException(ce);
}
}
return initiateZookeeper(zkClientConfig);
}

protected ZooKeeper initiateZookeeper(ZKClientConfig zkClientConfig) throws IOException {
return new ZooKeeper(zkHostPort, zkSessionTimeout, watcher, zkClientConfig);
}

private void fatalError(String errorMessage) {
Expand Down
6 changes: 5 additions & 1 deletion ...common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hadoop.security.SecurityUtil.TruststoreKeystore;

@InterfaceAudience.LimitedPrivate("HDFS")
public abstract class ZKFailoverController {

Expand Down Expand Up @@ -147,6 +149,7 @@ protected abstract void checkRpcAdminAccess()
protected abstract InetSocketAddress getRpcAddressToBindTo();
protected abstract PolicyProvider getPolicyProvider();
protected abstract List<HAServiceTarget> getAllOtherNodes();
protected abstract boolean isSSLEnabled();

/**
* Return the name of a znode inside the configured parent znode in which
Expand Down Expand Up @@ -372,9 +375,10 @@ private void initZK() throws HadoopIllegalArgumentException, IOException,
int maxRetryNum = conf.getInt(
CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_KEY,
CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_DEFAULT);
TruststoreKeystore truststoreKeystore = isSSLEnabled() ? new TruststoreKeystore(conf) : null;
elector = new ActiveStandbyElector(zkQuorum,
zkTimeout, getParentZnode(), zkAcls, zkAuths,
new ElectorCallbacks(), maxRetryNum);
new ElectorCallbacks(), maxRetryNum, truststoreKeystore);
}

private String getParentZnode() {
Expand Down
102 changes: 102 additions & 0 deletions ...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.concurrent.TimeUnit;

import javax.annotation.Nullable;
import javax.naming.ConfigurationException;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.kerberos.KerberosTicket;

Expand All @@ -53,6 +54,8 @@
import org.apache.hadoop.util.StopWatch;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ZKUtil;
import org.apache.zookeeper.client.ZKClientConfig;
import org.apache.zookeeper.common.ClientX509Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xbill.DNS.Name;
Expand Down Expand Up @@ -786,4 +789,103 @@ public static List<ZKUtil.ZKAuthInfo> getZKAuthInfos(Configuration conf,
throw e;
}
}

public static void validateSslConfiguration(TruststoreKeystore truststoreKeystore)
throws ConfigurationException {
if (org.apache.commons.lang3.StringUtils.isEmpty(truststoreKeystore.keystoreLocation)) {
throw new ConfigurationException(
"The keystore location parameter is empty for the ZooKeeper client connection.");
}
if (org.apache.commons.lang3.StringUtils.isEmpty(truststoreKeystore.keystorePassword)) {
throw new ConfigurationException(
"The keystore password parameter is empty for the ZooKeeper client connection.");
}
if (org.apache.commons.lang3.StringUtils.isEmpty(truststoreKeystore.truststoreLocation)) {
throw new ConfigurationException(
"The truststore location parameter is empty for the ZooKeeper client connection.");
}
if (org.apache.commons.lang3.StringUtils.isEmpty(truststoreKeystore.truststorePassword)) {
throw new ConfigurationException(
"The truststore password parameter is empty for the ZooKeeper client connection.");
}
}

/**
* Configure ZooKeeper Client with SSL/TLS connection.
* @param zkClientConfig ZooKeeper Client configuration
* @param truststoreKeystore truststore keystore, that we use to set the SSL configurations
* @throws ConfigurationException if the SSL configs are empty
*/
public static void setSslConfiguration(ZKClientConfig zkClientConfig,
TruststoreKeystore truststoreKeystore)
throws ConfigurationException {
setSslConfiguration(zkClientConfig, truststoreKeystore, new ClientX509Util());
}

public static void setSslConfiguration(ZKClientConfig zkClientConfig,
TruststoreKeystore truststoreKeystore,
ClientX509Util x509Util)
throws ConfigurationException {
validateSslConfiguration(truststoreKeystore);
LOG.info("Configuring the ZooKeeper client to use SSL/TLS encryption for connecting to the "
+ "ZooKeeper server.");
LOG.debug("Configuring the ZooKeeper client with {} location: {}.",
truststoreKeystore.keystoreLocation,
CommonConfigurationKeys.ZK_SSL_KEYSTORE_LOCATION);
LOG.debug("Configuring the ZooKeeper client with {} location: {}.",
truststoreKeystore.truststoreLocation,
CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_LOCATION);

zkClientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, "true");
zkClientConfig.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET,
"org.apache.zookeeper.ClientCnxnSocketNetty");
zkClientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(),
truststoreKeystore.keystoreLocation);
zkClientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(),
truststoreKeystore.keystorePassword);
zkClientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(),
truststoreKeystore.truststoreLocation);
zkClientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(),
truststoreKeystore.truststorePassword);
}

/**
* Helper class to contain the Truststore/Keystore paths for the ZK client connection over
* SSL/TLS.
*/
public static class TruststoreKeystore {
private final String keystoreLocation;
private final String keystorePassword;
private final String truststoreLocation;
private final String truststorePassword;

/**
* Configuration for the ZooKeeper connection when SSL/TLS is enabled.
* When a value is not configured, ensure that empty string is set instead of null.
*
* @param conf ZooKeeper Client configuration
*/
public TruststoreKeystore(Configuration conf) {
keystoreLocation = conf.get(CommonConfigurationKeys.ZK_SSL_KEYSTORE_LOCATION, "");
keystorePassword = conf.get(CommonConfigurationKeys.ZK_SSL_KEYSTORE_PASSWORD, "");
truststoreLocation = conf.get(CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_LOCATION, "");
truststorePassword = conf.get(CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_PASSWORD, "");
}

public String getKeystoreLocation() {
return keystoreLocation;
}

public String getKeystorePassword() {
return keystorePassword;
}

public String getTruststoreLocation() {
return truststoreLocation;
}

public String getTruststorePassword() {
return truststorePassword;
}
}
}
97 changes: 2 additions & 95 deletions ...-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ZKClientConfig;
import org.apache.zookeeper.common.ClientX509Util;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;

Expand All @@ -49,7 +48,7 @@

import org.apache.hadoop.util.Preconditions;

import javax.naming.ConfigurationException;
import org.apache.hadoop.security.SecurityUtil.TruststoreKeystore;

/**
* Helper class that provides utility methods specific to ZK operations.
Expand Down Expand Up @@ -570,64 +569,12 @@ public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
setJaasConfiguration(zkClientConfig);
}
if (sslEnabled) {
setSslConfiguration(zkClientConfig);
SecurityUtil.setSslConfiguration(zkClientConfig, truststoreKeystore);
}
return new ZooKeeper(connectString, sessionTimeout, watcher,
canBeReadOnly, zkClientConfig);
}

/**
* Configure ZooKeeper Client with SSL/TLS connection.
* @param zkClientConfig ZooKeeper Client configuration
*/
private void setSslConfiguration(ZKClientConfig zkClientConfig) throws ConfigurationException {
this.setSslConfiguration(zkClientConfig, new ClientX509Util());
}

private void setSslConfiguration(ZKClientConfig zkClientConfig, ClientX509Util x509Util)
throws ConfigurationException {
validateSslConfiguration();
LOG.info("Configuring the ZooKeeper client to use SSL/TLS encryption for connecting to the "
+ "ZooKeeper server.");
LOG.debug("Configuring the ZooKeeper client with {} location: {}.",
this.truststoreKeystore.keystoreLocation,
CommonConfigurationKeys.ZK_SSL_KEYSTORE_LOCATION);
LOG.debug("Configuring the ZooKeeper client with {} location: {}.",
this.truststoreKeystore.truststoreLocation,
CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_LOCATION);

zkClientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, "true");
zkClientConfig.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET,
"org.apache.zookeeper.ClientCnxnSocketNetty");
zkClientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(),
this.truststoreKeystore.keystoreLocation);
zkClientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(),
this.truststoreKeystore.keystorePassword);
zkClientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(),
this.truststoreKeystore.truststoreLocation);
zkClientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(),
this.truststoreKeystore.truststorePassword);
}

private void validateSslConfiguration() throws ConfigurationException {
if (StringUtils.isEmpty(this.truststoreKeystore.keystoreLocation)) {
throw new ConfigurationException(
"The keystore location parameter is empty for the ZooKeeper client connection.");
}
if (StringUtils.isEmpty(this.truststoreKeystore.keystorePassword)) {
throw new ConfigurationException(
"The keystore password parameter is empty for the ZooKeeper client connection.");
}
if (StringUtils.isEmpty(this.truststoreKeystore.truststoreLocation)) {
throw new ConfigurationException(
"The truststore location parameter is empty for the ZooKeeper client connection.");
}
if (StringUtils.isEmpty(this.truststoreKeystore.truststorePassword)) {
throw new ConfigurationException(
"The truststore password parameter is empty for the ZooKeeper client connection.");
}
}

private boolean isJaasConfigurationSet(ZKClientConfig zkClientConfig) {
String clientConfig = zkClientConfig.getProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY,
ZKClientConfig.LOGIN_CONTEXT_NAME_KEY_DEFAULT);
Expand All @@ -649,44 +596,4 @@ private void setJaasConfiguration(ZKClientConfig zkClientConfig) throws IOExcept
zkClientConfig.setProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY, JAAS_CLIENT_ENTRY);
}
}

/**
* Helper class to contain the Truststore/Keystore paths for the ZK client connection over
* SSL/TLS.
*/
public static class TruststoreKeystore {
private final String keystoreLocation;
private final String keystorePassword;
private final String truststoreLocation;
private final String truststorePassword;

/**
* Configuration for the ZooKeeper connection when SSL/TLS is enabled.
* When a value is not configured, ensure that empty string is set instead of null.
*
* @param conf ZooKeeper Client configuration
*/
public TruststoreKeystore(Configuration conf) {
keystoreLocation = conf.get(CommonConfigurationKeys.ZK_SSL_KEYSTORE_LOCATION, "");
keystorePassword = conf.get(CommonConfigurationKeys.ZK_SSL_KEYSTORE_PASSWORD, "");
truststoreLocation = conf.get(CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_LOCATION, "");
truststorePassword = conf.get(CommonConfigurationKeys.ZK_SSL_TRUSTSTORE_PASSWORD, "");
}

public String getKeystoreLocation() {
return keystoreLocation;
}

public String getKeystorePassword() {
return keystorePassword;
}

public String getTruststoreLocation() {
return truststoreLocation;
}

public String getTruststorePassword() {
return truststorePassword;
}
}
}
5 changes: 5 additions & 0 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,5 +370,10 @@ protected List<HAServiceTarget> getAllOtherNodes() {
}
return services;
}

@Override
protected boolean isSSLEnabled() {
return false;
}
}
}
Loading

0 comments on commit 4c04818

Please sign in to comment.