-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Changed allowlist config to denylist ip config for datasource uri hos…
…ts (#2042) Signed-off-by: Vamsi Manohar <[email protected]> (cherry picked from commit 9082a46) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
- Loading branch information
1 parent
beaa750
commit 5c2bd03
Showing
18 changed files
with
294 additions
and
125 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
common/src/main/java/org/opensearch/sql/common/interceptors/URIValidatorInterceptor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * | ||
| * * Copyright OpenSearch Contributors | ||
| * * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.sql.common.interceptors; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import lombok.NonNull; | ||
| import okhttp3.Interceptor; | ||
| import okhttp3.Request; | ||
| import okhttp3.Response; | ||
| import org.jetbrains.annotations.NotNull; | ||
| import org.opensearch.sql.common.setting.Settings; | ||
| import org.opensearch.sql.common.utils.URIValidationUtils; | ||
|
|
||
| public class URIValidatorInterceptor implements Interceptor { | ||
|
|
||
| private final List<String> denyHostList; | ||
|
|
||
| public URIValidatorInterceptor(@NonNull List<String> denyHostList) { | ||
| this.denyHostList = denyHostList; | ||
| } | ||
|
|
||
| @NotNull | ||
| @Override | ||
| public Response intercept(Interceptor.Chain chain) throws IOException { | ||
| Request request = chain.request(); | ||
| String host = request.url().host(); | ||
| boolean isValidHost = URIValidationUtils.validateURIHost(host, denyHostList); | ||
| if (isValidHost) { | ||
| return chain.proceed(request); | ||
| } else { | ||
| throw new IllegalArgumentException( | ||
| String.format( | ||
| "Disallowed hostname in the uri. Validate with %s config", | ||
| Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST.getKeyValue())); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
common/src/main/java/org/opensearch/sql/common/utils/URIValidationUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package org.opensearch.sql.common.utils; | ||
|
|
||
| import inet.ipaddr.IPAddressString; | ||
| import java.net.InetAddress; | ||
| import java.net.UnknownHostException; | ||
| import java.util.List; | ||
|
|
||
| /** Utility Class for URI host validation. */ | ||
| public class URIValidationUtils { | ||
|
|
||
| public static boolean validateURIHost(String host, List<String> denyHostList) | ||
| throws UnknownHostException { | ||
| IPAddressString ipStr = new IPAddressString(InetAddress.getByName(host).getHostAddress()); | ||
| for (String denyHost : denyHostList) { | ||
| IPAddressString denyHostStr = new IPAddressString(denyHost); | ||
| if (denyHostStr.contains(ipStr)) { | ||
| return false; | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
...sources/src/main/java/org/opensearch/sql/datasources/utils/DatasourceValidationUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package org.opensearch.sql.datasources.utils; | ||
|
|
||
| import java.net.URI; | ||
| import java.net.URISyntaxException; | ||
| import java.net.UnknownHostException; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import lombok.experimental.UtilityClass; | ||
| import org.apache.commons.validator.routines.DomainValidator; | ||
| import org.opensearch.sql.common.utils.URIValidationUtils; | ||
|
|
||
| /** Common Validation methods for all datasource connectors. */ | ||
| @UtilityClass | ||
| public class DatasourceValidationUtils { | ||
|
|
||
| public static final int MAX_LENGTH_FOR_CONFIG_PROPERTY = 1000; | ||
|
|
||
| public static void validateHost(String uriString, List<String> denyHostList) | ||
| throws URISyntaxException, UnknownHostException { | ||
| validateDomain(uriString); | ||
| if (!URIValidationUtils.validateURIHost(new URI(uriString).getHost(), denyHostList)) { | ||
| throw new IllegalArgumentException( | ||
| "Disallowed hostname in the uri. " | ||
| + "Validate with plugins.query.datasources.uri.hosts.denylist config"); | ||
| } | ||
| ; | ||
| } | ||
|
|
||
| public static void validateLengthAndRequiredFields( | ||
| Map<String, String> config, Set<String> fields) { | ||
| Set<String> missingFields = new HashSet<>(); | ||
| Set<String> invalidLengthFields = new HashSet<>(); | ||
| for (String field : fields) { | ||
| if (!config.containsKey(field)) { | ||
| missingFields.add(field); | ||
| } else if (config.get(field).length() > MAX_LENGTH_FOR_CONFIG_PROPERTY) { | ||
| invalidLengthFields.add(field); | ||
| } | ||
| } | ||
| StringBuilder errorStringBuilder = new StringBuilder(); | ||
| if (missingFields.size() > 0) { | ||
| errorStringBuilder.append( | ||
| String.format( | ||
| "Missing %s fields in the Prometheus connector properties.", missingFields)); | ||
| } | ||
|
|
||
| if (invalidLengthFields.size() > 0) { | ||
| errorStringBuilder.append( | ||
| String.format("Fields %s exceeds more than 1000 characters.", invalidLengthFields)); | ||
| } | ||
| if (errorStringBuilder.length() > 0) { | ||
| throw new IllegalArgumentException(errorStringBuilder.toString()); | ||
| } | ||
| } | ||
|
|
||
| private static void validateDomain(String uriString) throws URISyntaxException { | ||
| URI uri = new URI(uriString); | ||
| String host = uri.getHost(); | ||
| if (host == null | ||
| || (!(DomainValidator.getInstance().isValid(host) | ||
| || DomainValidator.getInstance().isValidLocalTld(host)))) { | ||
| throw new IllegalArgumentException( | ||
| String.format("Invalid hostname in the uri: %s", uriString)); | ||
| } | ||
| } | ||
| } |
94 changes: 94 additions & 0 deletions
94
...ces/src/test/java/org/opensearch/sql/datasources/utils/DatasourceValidationUtilsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| package org.opensearch.sql.datasources.utils; | ||
|
|
||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.Set; | ||
| import lombok.SneakyThrows; | ||
| import org.apache.commons.lang3.RandomStringUtils; | ||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.junit.jupiter.api.extension.ExtendWith; | ||
| import org.mockito.junit.jupiter.MockitoExtension; | ||
|
|
||
| @ExtendWith(MockitoExtension.class) | ||
| public class DatasourceValidationUtilsTest { | ||
|
|
||
| @SneakyThrows | ||
| @Test | ||
| public void testValidateHost() { | ||
| IllegalArgumentException illegalArgumentException = | ||
| Assertions.assertThrows( | ||
| IllegalArgumentException.class, | ||
| () -> | ||
| DatasourceValidationUtils.validateHost( | ||
| "http://localhost:9090", Collections.singletonList("127.0.0.0/8"))); | ||
| Assertions.assertEquals( | ||
| "Disallowed hostname in the uri. Validate with plugins.query.datasources.uri.hosts.denylist" | ||
| + " config", | ||
| illegalArgumentException.getMessage()); | ||
| } | ||
|
|
||
| @SneakyThrows | ||
| @Test | ||
| public void testValidateHostWithSuccess() { | ||
| Assertions.assertDoesNotThrow( | ||
| () -> | ||
| DatasourceValidationUtils.validateHost( | ||
| "http://localhost:9090", Collections.singletonList("192.168.0.0/8"))); | ||
| } | ||
|
|
||
| @SneakyThrows | ||
| @Test | ||
| public void testValidateHostWithInvalidDomain() { | ||
| IllegalArgumentException illegalArgumentException = | ||
| Assertions.assertThrows( | ||
| IllegalArgumentException.class, | ||
| () -> | ||
| DatasourceValidationUtils.validateHost( | ||
| "http:://prometheus:9090", Collections.singletonList("127.0.0.0/8"))); | ||
| Assertions.assertEquals( | ||
| "Invalid hostname in the uri: http:://prometheus:9090", | ||
| illegalArgumentException.getMessage()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testValidateLengthAndRequiredFieldsWithAbsentField() { | ||
| HashMap<String, String> config = new HashMap<>(); | ||
| config.put("s3.uri", "test"); | ||
| IllegalArgumentException illegalArgumentException = | ||
| Assertions.assertThrows( | ||
| IllegalArgumentException.class, | ||
| () -> | ||
| DatasourceValidationUtils.validateLengthAndRequiredFields( | ||
| config, Set.of("s3.uri", "s3.auth.type"))); | ||
| Assertions.assertEquals( | ||
| "Missing [s3.auth.type] fields in the Prometheus connector properties.", | ||
| illegalArgumentException.getMessage()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testValidateLengthAndRequiredFieldsWithInvalidLength() { | ||
| HashMap<String, String> config = new HashMap<>(); | ||
| config.put("s3.uri", RandomStringUtils.random(1001)); | ||
| IllegalArgumentException illegalArgumentException = | ||
| Assertions.assertThrows( | ||
| IllegalArgumentException.class, | ||
| () -> | ||
| DatasourceValidationUtils.validateLengthAndRequiredFields( | ||
| config, Set.of("s3.uri", "s3.auth.type"))); | ||
| Assertions.assertEquals( | ||
| "Missing [s3.auth.type] fields in the Prometheus connector properties.Fields " | ||
| + "[s3.uri] exceeds more than 1000 characters.", | ||
| illegalArgumentException.getMessage()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testValidateLengthAndRequiredFieldsWithSuccess() { | ||
| HashMap<String, String> config = new HashMap<>(); | ||
| config.put("s3.uri", "test"); | ||
| Assertions.assertDoesNotThrow( | ||
| () -> | ||
| DatasourceValidationUtils.validateLengthAndRequiredFields( | ||
| config, Collections.emptySet())); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.