From 1302e366c0f7012e2bc72d3ab9fc9eab3543dcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Bobowski?= <145468486+sfc-gh-mbobowski@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:41:07 +0200 Subject: [PATCH] SNOW-1728000 CloudPlatform enum and unused code removal in e2e tests (#967) --- .github/workflows/End2EndTestApache.yml | 4 +-- .github/workflows/End2EndTestConfluent.yml | 4 +-- .github/workflows/IntegrationTest.yml | 2 +- .github/workflows/StressTest.yml | 2 +- test/cloud_platform.py | 8 ++++++ test/test_verify.py | 32 ++++++++++------------ 6 files changed, 27 insertions(+), 25 deletions(-) create mode 100644 test/cloud_platform.py diff --git a/.github/workflows/End2EndTestApache.yml b/.github/workflows/End2EndTestApache.yml index 3888baa1a..30c17cb78 100644 --- a/.github/workflows/End2EndTestApache.yml +++ b/.github/workflows/End2EndTestApache.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast matrix: apache_kafka_version: [ '2.5.1', '2.8.1', '3.2.1' ] - snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ] + snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] java_test_version: [ '8', '11', '17' ] steps: - name: Checkout Code @@ -84,8 +84,6 @@ jobs: env: SNOWFLAKE_CREDENTIAL_FILE: "../profile.json" SF_CLOUD_PLATFORM: ${{ matrix.snowflake_cloud }} - # Will not run delivery guarantee tests in GCS (Check runDeliveryGuaranteeTests in test_verify.py) - ENABLE_DELIVERY_GUARANTEE_TESTS: "True" run: | cd test ./run_test_apache.sh ${{ matrix.apache_kafka_version }} ./apache_properties \ No newline at end of file diff --git a/.github/workflows/End2EndTestConfluent.yml b/.github/workflows/End2EndTestConfluent.yml index 01e25a852..30fc47773 100644 --- a/.github/workflows/End2EndTestConfluent.yml +++ b/.github/workflows/End2EndTestConfluent.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast matrix: confluent_version: [ '6.2.6', '7.6.0' ] - snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ] + snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] java_test_version: [ '8', '11', '17' ] steps: - name: Checkout Code @@ -84,8 +84,6 @@ jobs: env: SNOWFLAKE_CREDENTIAL_FILE: "../profile.json" SF_CLOUD_PLATFORM: ${{ matrix.snowflake_cloud }} - # Will not run delivery guarantee tests in GCS (Check runDeliveryGuaranteeTests in test_verify.py) - ENABLE_DELIVERY_GUARANTEE_TESTS: "True" run: | cd test ./run_test_confluent.sh ${{ matrix.confluent_version }} ./apache_properties \ No newline at end of file diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml index 3f347250b..fd88201e9 100644 --- a/.github/workflows/IntegrationTest.yml +++ b/.github/workflows/IntegrationTest.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast matrix: - snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ] + snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] steps: - name: Checkout Code uses: actions/checkout@v2 diff --git a/.github/workflows/StressTest.yml b/.github/workflows/StressTest.yml index 81935f380..091822f5d 100644 --- a/.github/workflows/StressTest.yml +++ b/.github/workflows/StressTest.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast matrix: confluent_version: [ '5.5.11', '6.2.6', '7.6.0' ] - snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ] + snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] java_test_version: [ '8', '11', '17' ] steps: - name: Checkout Code diff --git a/test/cloud_platform.py b/test/cloud_platform.py new file mode 100644 index 000000000..360695450 --- /dev/null +++ b/test/cloud_platform.py @@ -0,0 +1,8 @@ +from enum import Enum + +# Enum for cloud platforms supported by Snowflake (AWS/AZURE/GCP) +class CloudPlatform(Enum): + AWS = 1 + AZURE = 2 + GCP = 3 + ALL = 4 diff --git a/test/test_verify.py b/test/test_verify.py index 0756cb0cd..bd896a02c 100755 --- a/test/test_verify.py +++ b/test/test_verify.py @@ -21,6 +21,8 @@ import test_suit from test_suit.test_utils import parsePrivateKey, RetryableError +from cloud_platform import CloudPlatform + @dataclass class ConnectorParameters: @@ -45,14 +47,10 @@ def errorExit(message): class KafkaTest: def __init__(self, kafkaAddress, schemaRegistryAddress, kafkaConnectAddress, credentialPath, - connectorParameters: ConnectorParameters, testVersion, enableSSL, snowflakeCloudPlatform, - enableDeliveryGuaranteeTests=False): + connectorParameters: ConnectorParameters, testVersion, enableSSL, snowflakeCloudPlatform): self.testVersion = testVersion self.credentialPath = credentialPath - # can be None or one of AWS, AZURE, GCS self.snowflakeCloudPlatform = snowflakeCloudPlatform - # default is false or set to true as env variable - self.enableDeliveryGuaranteeTests = enableDeliveryGuaranteeTests with open(self.credentialPath) as f: credentialJson = json.load(f) testHost = credentialJson["host"] @@ -519,6 +517,15 @@ def run_test_set_with_parameters(kafka_test: KafkaTest, testSet, nameSalt, press runTestSet(kafka_test, testSet, nameSalt, pressure, skipProxy, allowedTestsCsv) +def __parseCloudPlatform() -> CloudPlatform: + if "SF_CLOUD_PLATFORM" in os.environ: + rawCloudPlatform = os.environ['SF_CLOUD_PLATFORM'] + return CloudPlatform[rawCloudPlatform] + else: + print("No SF_CLOUD_PLATFORM defined. Fallback to ALL.") + return CloudPlatform.ALL + + if __name__ == "__main__": if len(sys.argv) < 10: errorExit( @@ -547,16 +554,8 @@ def run_test_set_with_parameters(kafka_test: KafkaTest, testSet, nameSalt, press errorExit("\n=== Provided SNOWFLAKE_CREDENTIAL_FILE {} does not exist. Aborting. ===".format( credentialPath)) - # This will either be AWS, AZURE or GCS - snowflakeCloudPlatform = None - - # If it is not set, we will not run delivery guarantee tests - enableDeliveryGuaranteeTests = False - if "SF_CLOUD_PLATFORM" in os.environ: - snowflakeCloudPlatform = os.environ['SF_CLOUD_PLATFORM'] - - if "ENABLE_DELIVERY_GUARANTEE_TESTS" in os.environ: - enableDeliveryGuaranteeTests = (os.environ['ENABLE_DELIVERY_GUARANTEE_TESTS'] == 'True') + snowflakeCloudPlatform: CloudPlatform = __parseCloudPlatform() + print("Running tests for platform {} and distribution {}".format(snowflakeCloudPlatform, testSet)) parametersList = ConnectorParametersList([ ConnectorParameters(snowflake_streaming_enable_single_buffer='false'), @@ -572,8 +571,7 @@ def run_test_set_with_parameters(kafka_test: KafkaTest, testSet, nameSalt, press parameters, testVersion, enableSSL, - snowflakeCloudPlatform, - False), + snowflakeCloudPlatform), testSet, nameSalt + str(idx), pressure,