From 204e31199bc0a5d5ae90bca2e2cfccb476aaf91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Bobowski?= <145468486+sfc-gh-mbobowski@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:48:19 +0200 Subject: [PATCH] SNOW-1728000 Cloud platform tests filtering (#968) --- test/test_selector.py | 86 ++++- test/test_suites.py | 792 ++++++++++++++++++++++++++++++------------ test/test_verify.py | 17 +- 3 files changed, 648 insertions(+), 247 deletions(-) diff --git a/test/test_selector.py b/test/test_selector.py index 271c7e735..45def964c 100644 --- a/test/test_selector.py +++ b/test/test_selector.py @@ -1,18 +1,86 @@ from test_suites import create_end_to_end_test_suites +from collections import OrderedDict +from test_suites import EndToEndTestSuite +from cloud_platform import CloudPlatform import test_suit + # TestSelector is responsible for selecting a subset of tests to be run # It is meant to filter tests by platform, cloud vendor or any other predicate needed class TestSelector: - def selectTestsToBeRun(self, driver, nameSalt, schemaRegistryAddress, testPlatform, allowedTestsCsv): - test_suites = create_end_to_end_test_suites(driver, nameSalt, schemaRegistryAddress, testPlatform, allowedTestsCsv) + def select_tests_to_be_run( + self, + driver, + name_salt: str, + schema_registry_address: str, + test_platform: str, + cloud_platform: CloudPlatform, + allowed_tests_csv: str, + ): + test_suites = create_end_to_end_test_suites( + driver, name_salt, schema_registry_address, test_platform + ) + test_suites = self.__filter_by_allow_list(allowed_tests_csv, test_suites) + test_suites = self.__filter_by_test_platform(test_platform, test_suites) + test_suites = self.__filter_by_cloud_platform(cloud_platform, test_suites) + + print("Running", len(test_suites), "tests") + + return [ + single_end_to_end_test.test_instance + for single_end_to_end_test in test_suites.values() + ] + + def __filter_by_allow_list( + self, + allowed_tests_csv: str, + tests_to_be_filtered: OrderedDict[str, EndToEndTestSuite], + ) -> OrderedDict[str, EndToEndTestSuite]: + if allowed_tests_csv is None or allowed_tests_csv == "": + return tests_to_be_filtered + else: + testsToRun = dict( + (k, v) + for k, v in tests_to_be_filtered.items() + if k in allowed_tests_csv.split(",") + ) + return testsToRun + + def __filter_by_test_platform( + self, + test_platform: str, + tests_to_be_filtered: OrderedDict[str, EndToEndTestSuite], + ) -> OrderedDict[str, EndToEndTestSuite]: + if test_platform == "apache": + return dict( + (k, v) + for k, v in tests_to_be_filtered.items() + if v.run_in_apache == True + ) + elif test_platform == "confluent": + return dict( + (k, v) + for k, v in tests_to_be_filtered.items() + if v.run_in_confluent == True + ) + elif test_platform == "clean": + return tests_to_be_filtered + else: + raise test_suit.test_utils.NonRetryableError( + "unknown test_platform={}".format(test_platform) + ) - if testPlatform == "apache": - return [single_end_to_end_test.test_instance for single_end_to_end_test in test_suites.values() if single_end_to_end_test.run_in_apache == True] - elif testPlatform == "confluent": - return [single_end_to_end_test.test_instance for single_end_to_end_test in test_suites.values() if single_end_to_end_test.run_in_confluent == True] - elif testPlatform == "clean": - return [single_end_to_end_test.test_instance for single_end_to_end_test in test_suites.values()] + def __filter_by_cloud_platform( + self, + cloud_platform: CloudPlatform, + tests_to_be_filtered: OrderedDict[str, EndToEndTestSuite], + ) -> OrderedDict[str, EndToEndTestSuite]: + if cloud_platform == CloudPlatform.ALL: + return tests_to_be_filtered else: - raise test_suit.test_utils.NonRetryableError("unknown testPlatform={}".format(testPlatform)) + return dict( + (k, v) + for k, v in tests_to_be_filtered.items() + if v.cloud_platform in (cloud_platform, CloudPlatform.ALL) + ) diff --git a/test/test_suites.py b/test/test_suites.py index 9a34ba56b..9e2bb16c9 100644 --- a/test/test_suites.py +++ b/test/test_suites.py @@ -1,9 +1,13 @@ from collections import OrderedDict from test_suit.resilience_tests.test_kc_delete_create import TestKcDeleteCreate -from test_suit.resilience_tests.test_kc_delete_create_chaos import TestKcDeleteCreateChaos +from test_suit.resilience_tests.test_kc_delete_create_chaos import ( + TestKcDeleteCreateChaos, +) from test_suit.resilience_tests.test_kc_delete_resume import TestKcDeleteResume -from test_suit.resilience_tests.test_kc_delete_resume_chaos import TestKcDeleteResumeChaos +from test_suit.resilience_tests.test_kc_delete_resume_chaos import ( + TestKcDeleteResumeChaos, +) from test_suit.resilience_tests.test_kc_pause_create import TestKcPauseCreate from test_suit.resilience_tests.test_kc_pause_create_chaos import TestKcPauseCreateChaos from test_suit.resilience_tests.test_kc_pause_resume import TestKcPauseResume @@ -12,62 +16,106 @@ from test_suit.resilience_tests.test_kc_recreate_chaos import TestKcRecreateChaos from test_suit.resilience_tests.test_kc_restart import TestKcRestart from test_suit.test_auto_table_creation import TestAutoTableCreation -from test_suit.test_auto_table_creation_topic2table import TestAutoTableCreationTopic2Table +from test_suit.test_auto_table_creation_topic2table import ( + TestAutoTableCreationTopic2Table, +) from test_suit.test_avro_avro import TestAvroAvro from test_suit.test_avrosr_avrosr import TestAvrosrAvrosr from test_suit.test_confluent_protobuf_protobuf import TestConfluentProtobufProtobuf from test_suit.test_json_json import TestJsonJson -from test_suit.test_multiple_topic_to_one_table_snowpipe import TestMultipleTopicToOneTableSnowpipe -from test_suit.test_multiple_topic_to_one_table_snowpipe_streaming import \ - TestMultipleTopicToOneTableSnowpipeStreaming +from test_suit.test_multiple_topic_to_one_table_snowpipe import ( + TestMultipleTopicToOneTableSnowpipe, +) +from test_suit.test_multiple_topic_to_one_table_snowpipe_streaming import ( + TestMultipleTopicToOneTableSnowpipeStreaming, +) from test_suit.test_native_complex_smt import TestNativeComplexSmt from test_suit.test_native_string_avrosr import TestNativeStringAvrosr -from test_suit.test_native_string_json_without_schema import TestNativeStringJsonWithoutSchema -from test_suit.test_native_string_protobuf import TestNativeStringProtobuf +from test_suit.test_native_string_json_without_schema import ( + TestNativeStringJsonWithoutSchema, +) from test_suit.test_nullable_values_after_smt import TestNullableValuesAfterSmt from test_suit.test_schema_evolution_avro_sr import TestSchemaEvolutionAvroSR -from test_suit.test_schema_evolution_avro_sr_logical_types import TestSchemaEvolutionAvroSRLogicalTypes +from test_suit.test_schema_evolution_avro_sr_logical_types import ( + TestSchemaEvolutionAvroSRLogicalTypes, +) from test_suit.test_schema_evolution_drop_table import TestSchemaEvolutionDropTable from test_suit.test_schema_evolution_json import TestSchemaEvolutionJson -from test_suit.test_schema_evolution_json_ignore_tombstone import TestSchemaEvolutionJsonIgnoreTombstone -from test_suit.test_schema_evolution_multi_topic_drop_table import TestSchemaEvolutionMultiTopicDropTable -from test_suit.test_schema_evolution_nullable_values_after_smt import TestSchemaEvolutionNullableValuesAfterSmt -from test_suit.test_schema_evolution_nonnullable_json import TestSchemaEvolutionNonNullableJson -from test_suit.test_schema_evolution_w_auto_table_creation_avro_sr import \ - TestSchemaEvolutionWithAutoTableCreationAvroSR -from test_suit.test_schema_evolution_w_auto_table_creation_json import \ - TestSchemaEvolutionWithAutoTableCreationJson -from test_suit.test_schema_evolution_w_random_row_count import \ - TestSchemaEvolutionWithRandomRowCount +from test_suit.test_schema_evolution_json_ignore_tombstone import ( + TestSchemaEvolutionJsonIgnoreTombstone, +) +from test_suit.test_schema_evolution_multi_topic_drop_table import ( + TestSchemaEvolutionMultiTopicDropTable, +) +from test_suit.test_schema_evolution_nullable_values_after_smt import ( + TestSchemaEvolutionNullableValuesAfterSmt, +) +from test_suit.test_schema_evolution_nonnullable_json import ( + TestSchemaEvolutionNonNullableJson, +) +from test_suit.test_schema_evolution_w_auto_table_creation_avro_sr import ( + TestSchemaEvolutionWithAutoTableCreationAvroSR, +) +from test_suit.test_schema_evolution_w_auto_table_creation_json import ( + TestSchemaEvolutionWithAutoTableCreationJson, +) +from test_suit.test_schema_evolution_w_random_row_count import ( + TestSchemaEvolutionWithRandomRowCount, +) from test_suit.test_schema_mapping import TestSchemaMapping -from test_suit.test_schema_not_supported_converter import TestSchemaNotSupportedConverter -from test_suit.test_snowpipe_streaming_channel_migration_disabled import \ - TestSnowpipeStreamingStringJsonChannelMigrationDisabled -from test_suit.test_snowpipe_streaming_schema_mapping_dlq import TestSnowpipeStreamingSchemaMappingDLQ -from test_suit.test_snowpipe_streaming_nullable_values_after_smt import TestSnowpipeStreamingNullableValuesAfterSmt -from test_suit.test_snowpipe_streaming_string_avro_sr import TestSnowpipeStreamingStringAvroSR -from test_suit.test_snowpipe_streaming_string_json import TestSnowpipeStreamingStringJson -from test_suit.test_snowpipe_streaming_string_json_dlq import TestSnowpipeStreamingStringJsonDLQ -from test_suit.test_snowpipe_streaming_string_json_ignore_tombstone import \ - TestSnowpipeStreamingStringJsonIgnoreTombstone +from test_suit.test_schema_not_supported_converter import ( + TestSchemaNotSupportedConverter, +) +from test_suit.test_snowpipe_streaming_channel_migration_disabled import ( + TestSnowpipeStreamingStringJsonChannelMigrationDisabled, +) +from test_suit.test_snowpipe_streaming_schema_mapping_dlq import ( + TestSnowpipeStreamingSchemaMappingDLQ, +) +from test_suit.test_snowpipe_streaming_nullable_values_after_smt import ( + TestSnowpipeStreamingNullableValuesAfterSmt, +) +from test_suit.test_snowpipe_streaming_string_avro_sr import ( + TestSnowpipeStreamingStringAvroSR, +) +from test_suit.test_snowpipe_streaming_string_json import ( + TestSnowpipeStreamingStringJson, +) +from test_suit.test_snowpipe_streaming_string_json_dlq import ( + TestSnowpipeStreamingStringJsonDLQ, +) +from test_suit.test_snowpipe_streaming_string_json_ignore_tombstone import ( + TestSnowpipeStreamingStringJsonIgnoreTombstone, +) +from test_suit.test_native_string_protobuf import TestNativeStringProtobuf from test_suit.test_string_avro import TestStringAvro from test_suit.test_string_avrosr import TestStringAvrosr from test_suit.test_string_json import TestStringJson from test_suit.test_string_json_ignore_tombstone import TestStringJsonIgnoreTombstone -from test_suit.test_streaming_client_parameter_override import TestStreamingClientParameterOverride +from test_suit.test_streaming_client_parameter_override import ( + TestStreamingClientParameterOverride, +) +from cloud_platform import CloudPlatform class EndToEndTestSuite: - ''' + """ Placeholder class for defining what a single end to end test looks like. Just modify the caller constructor of this class to disable, enable in confluent or Apache Kafka. In future can add whether it runs in snowpipe or snowpipe streaming mode. - ''' + """ - def __init__(self, test_instance, run_in_confluent, run_in_apache): + def __init__( + self, + test_instance, + run_in_confluent, + run_in_apache, + cloud_platform: CloudPlatform, + ): self._test_instance = test_instance self._run_in_confluent = run_in_confluent self._run_in_apache = run_in_apache + self._cloud_platform = cloud_platform @property def test_instance(self): @@ -81,206 +129,492 @@ def run_in_confluent(self): def run_in_apache(self): return self._run_in_apache + @property + def cloud_platform(self): + return self._cloud_platform -def create_end_to_end_test_suites(driver, nameSalt, schemaRegistryAddress, testSet, allowedTestsCsv): - ''' + +def create_end_to_end_test_suites(driver, nameSalt, schemaRegistryAddress, testSet): + """ Creates all End to End tests which needs to run against Confluent Kafka or Apache Kafka. :param driver: Driver holds all helper function for tests - Create topic, create connector, send data are few functions amongst many present in Class KafkaTest. :param nameSalt: random string appended for uniqueness of Connector Name :param schemaRegistryAddress: Schema registry For confluent runs :param testSet: confluent Kafka or apache Kafka (OSS) - :param allowedTestsCsv: comma separated list of tests to be run. Run all tests when no value given. :return: - ''' - test_suites = OrderedDict([ - ("TestStringJson", EndToEndTestSuite( - test_instance=TestStringJson(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestStringJsonIgnoreTombstone", EndToEndTestSuite( - test_instance=TestStringJsonIgnoreTombstone(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestJsonJson", EndToEndTestSuite( - test_instance=TestJsonJson(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestStringAvro", EndToEndTestSuite( - test_instance=TestStringAvro(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestAvroAvro", EndToEndTestSuite( - test_instance=TestAvroAvro(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestStringAvrosr", EndToEndTestSuite( - test_instance=TestStringAvrosr(driver, nameSalt), run_in_confluent=True, run_in_apache=False - )), - ("TestAvrosrAvrosr", EndToEndTestSuite( - test_instance=TestAvrosrAvrosr(driver, nameSalt), run_in_confluent=True, run_in_apache=False - )), - ("TestNativeStringAvrosr", EndToEndTestSuite( - test_instance=TestNativeStringAvrosr(driver, nameSalt), run_in_confluent=True, - run_in_apache=False - )), - ("TestNativeStringJsonWithoutSchema", EndToEndTestSuite( - test_instance=TestNativeStringJsonWithoutSchema(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestNativeComplexSmt", EndToEndTestSuite( - test_instance=TestNativeComplexSmt(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestNativeStringProtobuf", EndToEndTestSuite( - test_instance=TestNativeStringProtobuf(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestNullableValuesAfterSmt", EndToEndTestSuite( - test_instance=TestNullableValuesAfterSmt(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestConfluentProtobufProtobuf", EndToEndTestSuite( - test_instance=TestConfluentProtobufProtobuf(driver, nameSalt), run_in_confluent=False, - run_in_apache=False - )), - ("TestSnowpipeStreamingNullableValuesAfterSmt", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingNullableValuesAfterSmt(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSnowpipeStreamingStringJson", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingStringJson(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSnowpipeStreamingStringJsonChannelMigrationDisabled", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingStringJsonChannelMigrationDisabled(driver, nameSalt), - run_in_confluent=True, run_in_apache=True - )), - ("TestSnowpipeStreamingStringJsonIgnoreTombstone", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingStringJsonIgnoreTombstone(driver, nameSalt), - run_in_confluent=True, - run_in_apache=True - )), - ("TestSnowpipeStreamingStringJsonDLQ", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingStringJsonDLQ(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSnowpipeStreamingStringAvroSR", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingStringAvroSR(driver, nameSalt), run_in_confluent=True, - run_in_apache=False - )), - ("TestMultipleTopicToOneTableSnowpipeStreaming", EndToEndTestSuite( - test_instance=TestMultipleTopicToOneTableSnowpipeStreaming(driver, nameSalt), - run_in_confluent=True, run_in_apache=True - )), - ("TestMultipleTopicToOneTableSnowpipe", EndToEndTestSuite( - test_instance=TestMultipleTopicToOneTableSnowpipe(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaMapping", EndToEndTestSuite( - test_instance=TestSchemaMapping(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestSnowpipeStreamingSchemaMappingDLQ", EndToEndTestSuite( - test_instance=TestSnowpipeStreamingSchemaMappingDLQ(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestAutoTableCreation", EndToEndTestSuite( - test_instance=TestAutoTableCreation(driver, nameSalt, schemaRegistryAddress, testSet), - run_in_confluent=True, run_in_apache=False - )), - ("TestAutoTableCreationTopic2Table", EndToEndTestSuite( - test_instance=TestAutoTableCreationTopic2Table(driver, nameSalt, schemaRegistryAddress, testSet), - run_in_confluent=True, run_in_apache=False - )), - ("TestSchemaEvolutionJson", EndToEndTestSuite( - test_instance=TestSchemaEvolutionJson(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaEvolutionJsonIgnoreTombstone", EndToEndTestSuite( - test_instance=TestSchemaEvolutionJsonIgnoreTombstone(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaEvolutionAvroSR", EndToEndTestSuite( - test_instance=TestSchemaEvolutionAvroSR(driver, nameSalt), run_in_confluent=True, - run_in_apache=False - )), - ("TestSchemaEvolutionAvroSRLogicalTypes", EndToEndTestSuite( - test_instance=TestSchemaEvolutionAvroSRLogicalTypes(driver, nameSalt), run_in_confluent=False, - run_in_apache=False - )), - ("TestSchemaEvolutionWithAutoTableCreationJson", EndToEndTestSuite( - test_instance=TestSchemaEvolutionWithAutoTableCreationJson(driver, nameSalt), - run_in_confluent=True, run_in_apache=True - )), - ("TestSchemaEvolutionWithAutoTableCreationAvroSR", EndToEndTestSuite( - test_instance=TestSchemaEvolutionWithAutoTableCreationAvroSR(driver, nameSalt), - run_in_confluent=True, run_in_apache=False - )), - ("TestSchemaEvolutionWithRandomRowCount", EndToEndTestSuite( - test_instance=TestSchemaEvolutionWithRandomRowCount(driver, nameSalt), - run_in_confluent=True, run_in_apache=True - )), - ("TestSchemaEvolutionNonNullableJson", EndToEndTestSuite( - test_instance=TestSchemaEvolutionNonNullableJson(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaEvolutionNullableValuesAfterSmt", EndToEndTestSuite( - test_instance=TestSchemaEvolutionNullableValuesAfterSmt(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaNotSupportedConverter", EndToEndTestSuite( - test_instance=TestSchemaNotSupportedConverter(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestKcDeleteCreate", EndToEndTestSuite( - test_instance=TestKcDeleteCreate(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcDeleteCreateChaos", EndToEndTestSuite( - test_instance=TestKcDeleteCreateChaos(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestKcDeleteResume", EndToEndTestSuite( - test_instance=TestKcDeleteResume(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcDeleteResumeChaos", EndToEndTestSuite( - test_instance=TestKcDeleteResumeChaos(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestKcPauseCreate", EndToEndTestSuite( - test_instance=TestKcPauseCreate(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcPauseCreateChaos", EndToEndTestSuite( - test_instance=TestKcPauseCreateChaos(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestKcPauseResume", EndToEndTestSuite( - test_instance=TestKcPauseResume(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcPauseResumeChaos", EndToEndTestSuite( - test_instance=TestKcPauseResumeChaos(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestKcRecreate", EndToEndTestSuite( - test_instance=TestKcRecreate(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcRecreateChaos", EndToEndTestSuite( - test_instance=TestKcRecreateChaos(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestKcRestart", EndToEndTestSuite( - test_instance=TestKcRestart(driver, nameSalt), run_in_confluent=True, run_in_apache=True - )), - ("TestSchemaEvolutionDropTable", EndToEndTestSuite( - test_instance=TestSchemaEvolutionDropTable(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestSchemaEvolutionMultiTopicDropTable", EndToEndTestSuite( - test_instance=TestSchemaEvolutionMultiTopicDropTable(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ("TestStreamingClientParameterOverride", EndToEndTestSuite( - test_instance=TestStreamingClientParameterOverride(driver, nameSalt), run_in_confluent=True, - run_in_apache=True - )), - ]) - - # Return all suites or only selected subset - if allowedTestsCsv is None or allowedTestsCsv == "": - return test_suites - else: - testsToRun = dict((k, v) for k, v in test_suites.items() if k in allowedTestsCsv.split(',')) - print("Running", len(testsToRun), "tests") - return testsToRun + """ + return OrderedDict( + [ + ( + "TestStringJson", + EndToEndTestSuite( + test_instance=TestStringJson(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestStringJsonIgnoreTombstone", + EndToEndTestSuite( + test_instance=TestStringJsonIgnoreTombstone(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestJsonJson", + EndToEndTestSuite( + test_instance=TestJsonJson(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestStringAvro", + EndToEndTestSuite( + test_instance=TestStringAvro(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestAvroAvro", + EndToEndTestSuite( + test_instance=TestAvroAvro(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestStringAvrosr", + EndToEndTestSuite( + test_instance=TestStringAvrosr(driver, nameSalt), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestAvrosrAvrosr", + EndToEndTestSuite( + test_instance=TestAvrosrAvrosr(driver, nameSalt), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestNativeStringAvrosr", + EndToEndTestSuite( + test_instance=TestNativeStringAvrosr(driver, nameSalt), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestNativeStringJsonWithoutSchema", + EndToEndTestSuite( + test_instance=TestNativeStringJsonWithoutSchema(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestNativeComplexSmt", + EndToEndTestSuite( + test_instance=TestNativeComplexSmt(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestNativeStringProtobuf", + EndToEndTestSuite( + test_instance=TestNativeStringProtobuf(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestNullableValuesAfterSmt", + EndToEndTestSuite( + test_instance=TestNullableValuesAfterSmt(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestConfluentProtobufProtobuf", + EndToEndTestSuite( + test_instance=TestConfluentProtobufProtobuf(driver, nameSalt), + run_in_confluent=False, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingNullableValuesAfterSmt", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingNullableValuesAfterSmt( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingStringJson", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingStringJson(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingStringJsonChannelMigrationDisabled", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingStringJsonChannelMigrationDisabled( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingStringJsonIgnoreTombstone", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingStringJsonIgnoreTombstone( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingStringJsonDLQ", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingStringJsonDLQ(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingStringAvroSR", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingStringAvroSR(driver, nameSalt), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestMultipleTopicToOneTableSnowpipeStreaming", + EndToEndTestSuite( + test_instance=TestMultipleTopicToOneTableSnowpipeStreaming( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestMultipleTopicToOneTableSnowpipe", + EndToEndTestSuite( + test_instance=TestMultipleTopicToOneTableSnowpipe(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaMapping", + EndToEndTestSuite( + test_instance=TestSchemaMapping(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSnowpipeStreamingSchemaMappingDLQ", + EndToEndTestSuite( + test_instance=TestSnowpipeStreamingSchemaMappingDLQ( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestAutoTableCreation", + EndToEndTestSuite( + test_instance=TestAutoTableCreation( + driver, nameSalt, schemaRegistryAddress, testSet + ), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestAutoTableCreationTopic2Table", + EndToEndTestSuite( + test_instance=TestAutoTableCreationTopic2Table( + driver, nameSalt, schemaRegistryAddress, testSet + ), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionJson", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionJson(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionJsonIgnoreTombstone", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionJsonIgnoreTombstone( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionAvroSR", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionAvroSR(driver, nameSalt), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionAvroSRLogicalTypes", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionAvroSRLogicalTypes( + driver, nameSalt + ), + run_in_confluent=False, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionWithAutoTableCreationJson", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionWithAutoTableCreationJson( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionWithAutoTableCreationAvroSR", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionWithAutoTableCreationAvroSR( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=False, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionWithRandomRowCount", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionWithRandomRowCount( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionNonNullableJson", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionNonNullableJson(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionNullableValuesAfterSmt", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionNullableValuesAfterSmt( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaNotSupportedConverter", + EndToEndTestSuite( + test_instance=TestSchemaNotSupportedConverter(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcDeleteCreate", + EndToEndTestSuite( + test_instance=TestKcDeleteCreate(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcDeleteCreateChaos", + EndToEndTestSuite( + test_instance=TestKcDeleteCreateChaos(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcDeleteResume", + EndToEndTestSuite( + test_instance=TestKcDeleteResume(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcDeleteResumeChaos", + EndToEndTestSuite( + test_instance=TestKcDeleteResumeChaos(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcPauseCreate", + EndToEndTestSuite( + test_instance=TestKcPauseCreate(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcPauseCreateChaos", + EndToEndTestSuite( + test_instance=TestKcPauseCreateChaos(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcPauseResume", + EndToEndTestSuite( + test_instance=TestKcPauseResume(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcPauseResumeChaos", + EndToEndTestSuite( + test_instance=TestKcPauseResumeChaos(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcRecreate", + EndToEndTestSuite( + test_instance=TestKcRecreate(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcRecreateChaos", + EndToEndTestSuite( + test_instance=TestKcRecreateChaos(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestKcRestart", + EndToEndTestSuite( + test_instance=TestKcRestart(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionDropTable", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionDropTable(driver, nameSalt), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestSchemaEvolutionMultiTopicDropTable", + EndToEndTestSuite( + test_instance=TestSchemaEvolutionMultiTopicDropTable( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ( + "TestStreamingClientParameterOverride", + EndToEndTestSuite( + test_instance=TestStreamingClientParameterOverride( + driver, nameSalt + ), + run_in_confluent=True, + run_in_apache=True, + cloud_platform=CloudPlatform.ALL, + ), + ), + ] + ) diff --git a/test/test_verify.py b/test/test_verify.py index bd896a02c..91ee8250f 100755 --- a/test/test_verify.py +++ b/test/test_verify.py @@ -47,10 +47,9 @@ def errorExit(message): class KafkaTest: def __init__(self, kafkaAddress, schemaRegistryAddress, kafkaConnectAddress, credentialPath, - connectorParameters: ConnectorParameters, testVersion, enableSSL, snowflakeCloudPlatform): + connectorParameters: ConnectorParameters, testVersion, enableSSL): self.testVersion = testVersion self.credentialPath = credentialPath - self.snowflakeCloudPlatform = snowflakeCloudPlatform with open(self.credentialPath) as f: credentialJson = json.load(f) testHost = credentialJson["host"] @@ -466,7 +465,7 @@ def runStressTests(driver, testSet, nameSalt): ############################ Stress Tests Round 2 ############################ -def runTestSet(driver, testSet, nameSalt, enable_stress_test, skipProxy, allowedTestsCsv): +def runTestSet(driver, testSet, nameSalt, enable_stress_test, skipProxy, cloud_platform, allowedTestsCsv): if enable_stress_test: runStressTests(driver, testSet, nameSalt) else: @@ -474,7 +473,7 @@ def runTestSet(driver, testSet, nameSalt, enable_stress_test, skipProxy, allowed print(datetime.now().strftime("\n%H:%M:%S "), "=== Round 1 ===") testSelector = TestSelector() - end_to_end_tests_suite = testSelector.selectTestsToBeRun(driver, nameSalt, schemaRegistryAddress, testSet, allowedTestsCsv) + end_to_end_tests_suite = testSelector.select_tests_to_be_run(driver, nameSalt, schemaRegistryAddress, testSet, cloud_platform, allowedTestsCsv) execution(testSet, end_to_end_tests_suite, driver, nameSalt) @@ -494,7 +493,7 @@ def runTestSet(driver, testSet, nameSalt, enable_stress_test, skipProxy, allowed print("Proxy Test should be the last test, since it modifies the JVM values") proxy_tests_suite = [EndToEndTestSuite( - test_instance=TestStringJsonProxy(driver, nameSalt), run_in_confluent=True, run_in_apache=True + test_instance=TestStringJsonProxy(driver, nameSalt), run_in_confluent=True, run_in_apache=True, cloud_platform = CloudPlatform.ALL )] end_to_end_proxy_tests_suite = [single_end_to_end_test.test_instance for single_end_to_end_test in proxy_tests_suite] @@ -513,8 +512,8 @@ def execution(testSet, testSuitList, driver, nameSalt, round=1): testExecutor.execute(testSuitList, driver, nameSalt, round) -def run_test_set_with_parameters(kafka_test: KafkaTest, testSet, nameSalt, pressure, skipProxy, allowedTestsCsv): - runTestSet(kafka_test, testSet, nameSalt, pressure, skipProxy, allowedTestsCsv) +def run_test_set_with_parameters(kafka_test: KafkaTest, testSet, nameSalt, pressure, skipProxy, cloud_platform, allowedTestsCsv): + runTestSet(kafka_test, testSet, nameSalt, pressure, skipProxy, cloud_platform, allowedTestsCsv) def __parseCloudPlatform() -> CloudPlatform: @@ -570,11 +569,11 @@ def __parseCloudPlatform() -> CloudPlatform: credentialPath, parameters, testVersion, - enableSSL, - snowflakeCloudPlatform), + enableSSL), testSet, nameSalt + str(idx), pressure, skipProxy, + snowflakeCloudPlatform, allowedTestsCsv) )