diff --git a/build.gradle b/build.gradle index 7d6b4dc299..3d6b2b2340 100644 --- a/build.gradle +++ b/build.gradle @@ -293,6 +293,13 @@ task integrationTest(type: Test) { check.dependsOn integrationTest dependencies { + api "org.apache.httpcomponents:httpclient:${versions.httpclient}" + api "org.apache.httpcomponents:httpcore:${versions.httpcore}" + api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}" + api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}" + implementation "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}" + testImplementation "org.apache.httpcomponents:fluent-hc:${versions.httpclient}" + implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5' implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" @@ -301,7 +308,6 @@ dependencies { implementation 'commons-cli:commons-cli:1.3.1' implementation "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' - implementation 'org.apache.httpcomponents:httpclient-cache:4.5.13' implementation 'io.jsonwebtoken:jjwt-api:0.10.8' implementation('org.apache.cxf:cxf-rt-rs-security-jose:3.4.5') { exclude(group: 'jakarta.activation', module: 'jakarta.activation-api') @@ -348,8 +354,6 @@ dependencies { implementation 'commons-lang:commons-lang:2.4' implementation 'commons-collections:commons-collections:3.2.2' implementation 'com.jayway.jsonpath:json-path:2.4.0' - implementation 'org.apache.httpcomponents:httpclient:4.5.13' - implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'net.minidev:json-smart:2.4.7' runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.10.8' runtimeOnly 'io.jsonwebtoken:jjwt-jackson:0.10.8' @@ -386,7 +390,6 @@ dependencies { testImplementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1' testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9' testImplementation 'javax.servlet:servlet-api:2.5' - testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.13' testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}" testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}:test" testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test" diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index 54e4894a78..68ced71b7b 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -43,15 +43,17 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.message.BasicHeader; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; @@ -94,17 +96,26 @@ default TestRestClient getRestClient(UserCredentialsHolder user, Header... heade default RestHighLevelClient getRestHighLevelClient(UserCredentialsHolder user) { InetSocketAddress httpAddress = getHttpAddress(); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user.getName(), user.getPassword())); - - RestClientBuilder.HttpClientConfigCallback configCallback = httpClientBuilder -> { - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy( - new SSLIOSessionStrategy(getSSLContext(), null, null, NoopHostnameVerifier.INSTANCE)); - - return httpClientBuilder; - }; - - RestClientBuilder builder = RestClient.builder(new HttpHost(httpAddress.getHostString(), httpAddress.getPort(), "https")) + HttpHost httpHost = new HttpHost("https", httpAddress.getHostString(), httpAddress.getPort()); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(user.getName(), user.getPassword().toCharArray())); + + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder + .create() + .setSslContext(getSSLContext()) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + + RestClientBuilder.HttpClientConfigCallback configCallback = httpClientBuilder -> + httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager(connectionManager); + + RestClientBuilder builder = RestClient.builder(httpHost) .setHttpClientConfigCallback(configCallback); diff --git a/src/main/java/org/opensearch/security/httpclient/HttpClient.java b/src/main/java/org/opensearch/security/httpclient/HttpClient.java index 281235f5e0..c4340a7a57 100644 --- a/src/main/java/org/opensearch/security/httpclient/HttpClient.java +++ b/src/main/java/org/opensearch/security/httpclient/HttpClient.java @@ -13,7 +13,6 @@ import java.io.Closeable; import java.io.IOException; -import java.net.Socket; import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.KeyStore; @@ -28,21 +27,25 @@ import java.util.stream.Collectors; import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; import com.google.common.collect.Lists; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpHost; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.apache.http.message.BasicHeader; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; -import org.apache.http.ssl.PrivateKeyDetails; -import org.apache.http.ssl.PrivateKeyStrategy; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.PrivateKeyDetails; +import org.apache.hc.core5.ssl.PrivateKeyStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.ssl.SSLContexts; +import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -153,7 +156,7 @@ private HttpClient(final KeyStore trustStore, final String basicCredentials, fin HttpHost[] hosts = Arrays.stream(servers) .map(s->s.split(":")) - .map(s->new HttpHost(s[0], Integer.parseInt(s[1]),ssl?"https":"http")) + .map(s->new HttpHost(ssl?"https":"http", s[0], Integer.parseInt(s[1]))) .collect(Collectors.toList()).toArray(new HttpHost[0]); @@ -223,7 +226,7 @@ private final HttpAsyncClientBuilder asyncClientBuilder(HttpAsyncClientBuilder h sslContextBuilder.loadKeyMaterial(keystore, keyPassword, new PrivateKeyStrategy() { @Override - public String chooseAlias(Map aliases, Socket socket) { + public String chooseAlias(Map aliases, SSLParameters sslParameters) { if(aliases == null || aliases.isEmpty()) { return keystoreAlias; } @@ -238,13 +241,19 @@ public String chooseAlias(Map aliases, Socket socket) final HostnameVerifier hnv = verifyHostnames?new DefaultHostnameVerifier():NoopHostnameVerifier.INSTANCE; - final SSLContext sslContext = sslContextBuilder.build(); - httpClientBuilder.setSSLStrategy(new SSLIOSessionStrategy( - sslContext, - supportedProtocols, - supportedCipherSuites, - hnv - )); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder + .create() + .setSslContext(sslContextBuilder.build()) + .setCiphers(supportedCipherSuites) + .setTlsVersions(supportedProtocols) + .setHostnameVerifier(hnv) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + + httpClientBuilder.setConnectionManager(connectionManager); } if (basicCredentials != null) { @@ -255,9 +264,9 @@ public String chooseAlias(Map aliases, Socket socket) int timeout = 5; RequestConfig config = RequestConfig.custom() - .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout(timeout * 1000) - .setSocketTimeout(timeout * 1000).build(); + .setConnectTimeout(Timeout.ofSeconds(timeout)) + .setConnectionRequestTimeout(Timeout.ofSeconds(timeout)) + .setResponseTimeout(Timeout.ofSeconds(timeout)).build(); httpClientBuilder.setDefaultRequestConfig(config); diff --git a/src/main/java/org/opensearch/security/tools/SecurityAdmin.java b/src/main/java/org/opensearch/security/tools/SecurityAdmin.java index 2553a13677..5277eb227f 100644 --- a/src/main/java/org/opensearch/security/tools/SecurityAdmin.java +++ b/src/main/java/org/opensearch/security/tools/SecurityAdmin.java @@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -70,12 +71,15 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.http.HttpHost; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.ssl.SSLContexts; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; @@ -1387,26 +1391,29 @@ private static RestHighLevelClient getRestHighLevelClient(SSLContext sslContext, String[] enabledProtocols, String[] enabledCiphers, String hostname, - int port) { + int port) throws NoSuchAlgorithmException { final HostnameVerifier hnv = !nhnv ? new DefaultHostnameVerifier() : NoopHostnameVerifier.INSTANCE; String[] supportedProtocols = enabledProtocols.length > 0 ? enabledProtocols : null; String[] supportedCipherSuites = enabledCiphers.length > 0 ? enabledCiphers : null; - HttpHost httpHost = new HttpHost(hostname, port, "https"); + final HttpHost httpHost = new HttpHost("https", hostname, port); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder + .create() + .setSslContext(SSLContext.getDefault()) + .setCiphers(supportedCipherSuites) + .setTlsVersions(supportedProtocols) + .setHostnameVerifier(hnv) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); RestClientBuilder restClientBuilder = RestClient.builder(httpHost) - .setHttpClientConfigCallback( - builder -> builder.setSSLStrategy( - new SSLIOSessionStrategy( - sslContext, - supportedProtocols, - supportedCipherSuites, - hnv - ) - ) - ); + .setHttpClientConfigCallback(builder -> builder.setConnectionManager(connectionManager)); + return new RestHighLevelClient(restClientBuilder); } diff --git a/src/test/java/org/opensearch/security/sanity/tests/SecurityRestTestCase.java b/src/test/java/org/opensearch/security/sanity/tests/SecurityRestTestCase.java index 2418bd2194..81665e2231 100644 --- a/src/test/java/org/opensearch/security/sanity/tests/SecurityRestTestCase.java +++ b/src/test/java/org/opensearch/security/sanity/tests/SecurityRestTestCase.java @@ -14,9 +14,10 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.Arrays; import java.util.Map; -import org.apache.http.HttpHost; +import org.apache.hc.core5.http.HttpHost; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -82,7 +83,9 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE // create client with passed user String userName = System.getProperty("user"); String password = System.getProperty("password"); - return new SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(60000).build(); + // TODO: see please https://github.com/opensearch-project/common-utils/issues/285 + return new SecureRestClientBuilder(Arrays.stream(hosts).map(h -> new org.apache.http.HttpHost(h.getHostName(), h.getPort(), h.getSchemeName())).toArray(org.apache.http.HttpHost[]::new), + isHttps(), userName, password).setSocketTimeout(60000).build(); } else { RestClientBuilder builder = RestClient.builder(hosts); diff --git a/src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java b/src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java index b95104dd9f..9df4530721 100644 --- a/src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java +++ b/src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java @@ -45,11 +45,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.netty.handler.ssl.OpenSsl; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.http.Header; -import org.apache.http.HttpHost; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.message.BasicHeader; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.logging.log4j.LogManager; @@ -155,15 +158,22 @@ protected RestHighLevelClient getRestClient(ClusterInfo info, String keyStoreNam sslContextBuilder.loadTrustMaterial(trustStore, null); SSLContext sslContext = sslContextBuilder.build(); - HttpHost httpHost = new HttpHost(info.httpHost, info.httpPort, "https"); + HttpHost httpHost = new HttpHost("https", info.httpHost, info.httpPort); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder + .create() + .setSslContext(sslContext) + .setTlsVersions("TLSv1", "TLSv1.1", "TLSv1.2", "SSLv3") + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + RestClientBuilder restClientBuilder = RestClient.builder(httpHost) .setHttpClientConfigCallback( - builder -> builder.setSSLStrategy( - new SSLIOSessionStrategy(sslContext, - new String[] { "TLSv1", "TLSv1.1", "TLSv1.2", "SSLv3"}, - null, - NoopHostnameVerifier.INSTANCE))); + builder -> builder.setConnectionManager(connectionManager)); return new RestHighLevelClient(restClientBuilder); } catch (Exception e) { log.error("Cannot create client", e);