From 45733be4fe4fb3927e228e0895b950953317e44d Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Tue, 2 Jul 2024 11:24:55 -0400 Subject: [PATCH] Do not build shadowjar and disable javadoc for it modules (#31741) * add missing transitive dependencies * Fix analyzeClassesDependencies --- it/cassandra/build.gradle | 14 ++++++++--- .../cassandra/CassandraResourceManager.java | 4 ++-- it/common/build.gradle | 2 -- it/conditions/build.gradle | 3 +-- it/elasticsearch/build.gradle | 12 ++++++++-- it/google-cloud-platform/build.gradle | 23 +++++++++++++++---- it/jdbc/build.gradle | 14 ++++++++--- it/kafka/build.gradle | 13 ++++++++--- it/mongodb/build.gradle | 16 +++++++++---- it/neo4j/build.gradle | 11 ++++++--- it/splunk/build.gradle | 17 ++++++++++---- it/testcontainers/build.gradle | 8 ++++--- it/truthmatchers/build.gradle | 9 ++++---- 13 files changed, 105 insertions(+), 41 deletions(-) diff --git a/it/cassandra/build.gradle b/it/cassandra/build.gradle index 65ff622061eb..d7d94e11b3f6 100644 --- a/it/cassandra/build.gradle +++ b/it/cassandra/build.gradle @@ -18,22 +18,30 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.cassandra', + automaticModuleName: 'org.apache.beam.it.cassandra', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Cassandra" ext.summary = "Integration test utilities for Cassandra." dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") - implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation project(path: ":it:truthmatchers") + implementation library.java.failsafe + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_cassandra + implementation library.java.vendored_guava_32_1_2_jre implementation 'com.datastax.oss:java-driver-core:4.15.0' // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + implementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.mockito_core testRuntimeOnly library.java.slf4j_simple } diff --git a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java index f7bf15eb7646..e116eeaa0b24 100644 --- a/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java +++ b/it/cassandra/src/main/java/org/apache/beam/it/cassandra/CassandraResourceManager.java @@ -24,8 +24,6 @@ import com.datastax.oss.driver.api.core.cql.ResultSet; import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; import java.net.InetSocketAddress; @@ -36,6 +34,8 @@ import org.apache.beam.it.common.ResourceManager; import org.apache.beam.it.common.utils.ExceptionUtils; import org.apache.beam.it.testcontainers.TestContainerResourceManager; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.CassandraContainer; diff --git a/it/common/build.gradle b/it/common/build.gradle index 7d1159cd99b7..c92ec551cb25 100644 --- a/it/common/build.gradle +++ b/it/common/build.gradle @@ -20,8 +20,6 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( exportJavadoc: false, automaticModuleName: 'org.apache.beam.it.common', - validateShadowJar: false, - shadowClosure: {}, ) description = "Apache Beam :: IT :: Common" diff --git a/it/conditions/build.gradle b/it/conditions/build.gradle index 3a0f29e05f9a..04d6635842b2 100644 --- a/it/conditions/build.gradle +++ b/it/conditions/build.gradle @@ -19,8 +19,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.it.conditions', - validateShadowJar: false, - shadowClosure: {}, + exportJavadoc: false, ) description = "Apache Beam :: IT :: Conditions" diff --git a/it/elasticsearch/build.gradle b/it/elasticsearch/build.gradle index d1f3707bb4fc..035694e5f38a 100644 --- a/it/elasticsearch/build.gradle +++ b/it/elasticsearch/build.gradle @@ -18,7 +18,8 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.elasticsearch', + automaticModuleName: 'org.apache.beam.it.elasticsearch', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Elasticsearch" @@ -27,8 +28,13 @@ ext.summary = "Integration test utilities for Elasticsearch." def elastic_search_version = "7.9.2" dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation library.java.http_core + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_elasticsearch + implementation library.java.vendored_guava_32_1_2_jre implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:$elastic_search_version" implementation "org.elasticsearch.client:elasticsearch-rest-client:$elastic_search_version" implementation "org.elasticsearch:elasticsearch:$elastic_search_version" @@ -36,6 +42,8 @@ dependencies { testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + testImplementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.mockito_inline testImplementation library.java.commons_lang3 testRuntimeOnly library.java.slf4j_simple diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle index f83f24398409..f5fbcd8ba5d5 100644 --- a/it/google-cloud-platform/build.gradle +++ b/it/google-cloud-platform/build.gradle @@ -15,12 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import groovy.json.JsonOutput + import org.apache.beam.gradle.IoPerformanceTestUtilities plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.gcp', + automaticModuleName: 'org.apache.beam.it.gcp', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Google Cloud Platform" @@ -30,10 +31,17 @@ dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":runners:google-cloud-dataflow-java") - implementation project(path: ":it:conditions", configuration: "shadow") - implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:conditions") + implementation project(path: ":it:truthmatchers") implementation project(path: ":sdks:java:testing:test-utils") + implementation library.java.failsafe + implementation library.java.google_api_services_dataflow + implementation library.java.junit implementation library.java.slf4j_api + implementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } implementation library.java.vendored_guava_32_1_2_jre implementation library.java.jackson_core implementation library.java.jackson_databind @@ -45,13 +53,18 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.9' implementation library.java.gax implementation library.java.google_api_common + implementation library.java.google_api_client + implementation library.java.google_api_services_bigquery + implementation library.java.google_auth_library_credentials + implementation library.java.google_auth_library_oauth2_http + implementation library.java.google_http_client + implementation library.java.guava implementation library.java.protobuf_java_util implementation library.java.protobuf_java implementation library.java.threetenbp implementation 'org.awaitility:awaitility:4.2.0' implementation 'joda-time:joda-time:2.10.10' // Google Cloud Dependencies - implementation library.java.google_api_services_bigquery implementation library.java.google_cloud_core implementation 'com.google.cloud:google-cloud-storage' implementation 'com.google.cloud:google-cloud-bigquery' diff --git a/it/jdbc/build.gradle b/it/jdbc/build.gradle index 72cc8794159f..cb99f818661f 100644 --- a/it/jdbc/build.gradle +++ b/it/jdbc/build.gradle @@ -18,26 +18,34 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.jdbc', - enableSpotbugs: false, + automaticModuleName: 'org.apache.beam.it.jdbc', + exportJavadoc: false, + enableSpotbugs: false, ) description = "Apache Beam :: IT :: Jdbc" ext.summary = "Integration test utilities for Jdbc." dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation library.java.commons_lang3 + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_jdbc implementation library.java.testcontainers_postgresql implementation library.java.testcontainers_mysql implementation library.java.testcontainers_oracle implementation library.java.testcontainers_mssqlserver + implementation library.java.vendored_guava_32_1_2_jre testImplementation project(":it:truthmatchers") // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + testImplementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.mockito_core testImplementation library.java.mockito_inline testRuntimeOnly library.java.slf4j_simple diff --git a/it/kafka/build.gradle b/it/kafka/build.gradle index 158de54d89e4..0496c423bebd 100644 --- a/it/kafka/build.gradle +++ b/it/kafka/build.gradle @@ -19,21 +19,27 @@ import org.apache.beam.gradle.IoPerformanceTestUtilities plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.kafka', + automaticModuleName: 'org.apache.beam.it.kafka', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Kafka" ext.summary = "Integration test utilities for Kafka." dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_kafka implementation library.java.kafka_clients - + implementation library.java.vendored_guava_32_1_2_jre // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + testImplementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.mockito_core testRuntimeOnly library.java.slf4j_simple // TODO: Remove the below dependencies after KafkaIOLT has been moved to the right directory. @@ -44,6 +50,7 @@ dependencies { testImplementation project(path: ":runners:google-cloud-dataflow-java") testImplementation project(path: ":sdks:java:testing:test-utils") testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration") + testImplementation library.java.google_api_services_dataflow } tasks.register( diff --git a/it/mongodb/build.gradle b/it/mongodb/build.gradle index 6b4fa400abd3..a24293b218bf 100644 --- a/it/mongodb/build.gradle +++ b/it/mongodb/build.gradle @@ -18,24 +18,32 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.mongodb', + automaticModuleName: 'org.apache.beam.it.mongodb', + exportJavadoc: false, ) description = "Apache Beam :: IT :: MongoDB" ext.summary = "Integration test utilities for MongoDB." dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") - implementation project(path: ":it:conditions", configuration: "shadow") - implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation project(path: ":it:conditions") + implementation project(path: ":it:truthmatchers") + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_mongodb + implementation library.java.google_code_gson implementation library.java.mongo_java_driver + implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.mockito_core // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + testImplementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.commons_lang3 testRuntimeOnly library.java.slf4j_simple } diff --git a/it/neo4j/build.gradle b/it/neo4j/build.gradle index 25650739646c..369f1c2ab008 100644 --- a/it/neo4j/build.gradle +++ b/it/neo4j/build.gradle @@ -18,15 +18,20 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( - automaticModuleName: 'org.apache.beam.it.neo4j', + automaticModuleName: 'org.apache.beam.it.neo4j', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Neo4j" ext.summary = "Integration test utilities for Neo4j." dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") - implementation project(path: ":it:conditions", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation project(path: ":it:conditions") + implementation library.java.guava + implementation library.java.slf4j_api + implementation library.java.testcontainers_base implementation library.java.testcontainers_neo4j implementation "org.neo4j.driver:neo4j-java-driver:4.4.3" diff --git a/it/splunk/build.gradle b/it/splunk/build.gradle index deacc1051b44..7eeeb3a2cc3d 100644 --- a/it/splunk/build.gradle +++ b/it/splunk/build.gradle @@ -19,6 +19,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.it.splunk', + exportJavadoc: false, ) description = "Apache Beam :: IT :: Splunk" @@ -32,19 +33,27 @@ repositories { } dependencies { - implementation project(path: ":it:testcontainers", configuration: "shadow") - implementation project(path: ":it:conditions", configuration: "shadow") - implementation project(path: ":it:truthmatchers", configuration: "shadow") + implementation project(path: ":it:common") + implementation project(path: ":it:testcontainers") + implementation project(path: ":it:conditions") + implementation project(path: ":it:truthmatchers") implementation project(path: ":sdks:java:io:splunk") + implementation library.java.http_core + implementation library.java.http_client + provided library.java.json_org + implementation library.java.slf4j_api + implementation library.java.testcontainers_base + implementation library.java.vendored_guava_32_1_2_jre implementation 'com.splunk:splunk:1.9.4' implementation 'org.awaitility:awaitility:4.2.0' - provided library.java.json_org testImplementation library.java.mockito_core // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + testImplementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.commons_lang3 testRuntimeOnly library.java.slf4j_simple } \ No newline at end of file diff --git a/it/testcontainers/build.gradle b/it/testcontainers/build.gradle index f29977fd24a5..79a06f0760cf 100644 --- a/it/testcontainers/build.gradle +++ b/it/testcontainers/build.gradle @@ -19,21 +19,23 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.it.testcontainers', + exportJavadoc: false, enableSpotbugs: false, - validateShadowJar: false, - shadowClosure: {}, ) description = "Apache Beam :: IT :: Testcontainers" ext.summary = "Integration test utilities for Testcontainers." dependencies { - implementation project(path: ":it:common", configuration: "shadow") + implementation project(path: ":it:common") + implementation library.java.slf4j_api implementation library.java.testcontainers_base // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } + implementation library.java.guava + permitUsedUndeclared library.java.guava testImplementation library.java.mockito_core testRuntimeOnly library.java.slf4j_simple } \ No newline at end of file diff --git a/it/truthmatchers/build.gradle b/it/truthmatchers/build.gradle index 8363523dced3..fd54ad591a0a 100644 --- a/it/truthmatchers/build.gradle +++ b/it/truthmatchers/build.gradle @@ -20,19 +20,18 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( exportJavadoc: false, automaticModuleName: 'org.apache.beam.it.truthmatchers', - validateShadowJar: false, - shadowClosure: {}, ) description = "Apache Beam :: IT :: Truth matchers" ext.summary = "Truth matchers for integration tests." dependencies { - implementation project(path: ":it:common", configuration: "shadow") + implementation project(path: ":it:common") // TODO: excluding Guava until Truth updates it to >32.1.x implementation(library.java.truth) { exclude group: 'com.google.guava', module: 'guava' } - implementation library.java.vendored_guava_32_1_2_jre - permitUnusedDeclared library.java.vendored_guava_32_1_2_jre + implementation library.java.guava + permitUsedUndeclared library.java.guava + implementation library.java.google_code_gson } \ No newline at end of file