From 89691505107b5ae818475696bf5989de1c32a07c Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 4 Jan 2024 16:29:02 -0500 Subject: [PATCH 1/5] chore: relocate brazil transform --- .brazil.json | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .brazil.json diff --git a/.brazil.json b/.brazil.json new file mode 100644 index 00000000000..40566118f5c --- /dev/null +++ b/.brazil.json @@ -0,0 +1,32 @@ +{ + "dependencies": { + "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.*": "KotlinStdlibCommon-1.9.x", + "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.*": "KotlinStdlibJdk8-1.9.x", + "org.jetbrains.kotlin:kotlin-stdlib:1.9.*": "KotlinStdlib-1.9.x", + "org.jetbrains.kotlinx:atomicfu-jvm:0.23.1": "AtomicfuJvm-0.23.1", + "org.jetbrains.kotlinx:atomicfu:0.23.1": "Atomicfu-0.23.1", + "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.7.*": "KotlinxCoroutinesCoreJvm-1.7.x", + "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.*": "KotlinxCoroutinesCore-1.7.x", + "org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.7.*": "KotlinxCoroutinesJdk8-1.7.x" + }, + "packageHandlingRules": { + "versioning": { + "defaultVersionLayout": "{MAJOR}.0.x" + }, + "ignore": [ + "aws.sdk.kotlin:bom", + "aws.sdk.kotlin.crt:aws-crt-kotlin-android", + "aws.sdk.kotlin:testing", + "aws.sdk.kotlin:version-catalog" + ], + "resolvesConflictDependencies": { + "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.7.*": [ + "KotlinStdlibCommon-1.9.x", + "KotlinStdlibJdk8-1.9.x" + ], + "org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.7.*": [ + "KotlinStdlibJdk8-1.9.x" + ] + } + } +} From 54c52dabbc878469e209d9ac4b62520f885561b2 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 8 Jan 2024 15:56:26 -0500 Subject: [PATCH 2/5] refactor sdk codegen into it's own composite build --- build-support/build.gradle.kts | 49 ++++ build-support/settings.gradle.kts | 13 ++ .../aws/sdk/kotlin/gradle/sdk/AwsService.kt | 125 ++++++++++ .../aws/sdk/kotlin/gradle/sdk/Bootstrap.kt | 14 ++ .../sdk/kotlin/gradle/sdk/BootstrapConfig.kt | 34 +++ .../aws/sdk/kotlin/gradle/sdk/Membership.kt | 32 +++ .../aws/sdk/kotlin/gradle/sdk/Naming.kt | 39 ++++ .../sdk/kotlin/gradle/sdk/PackageSettings.kt | 46 ++++ .../kotlin/aws/sdk/kotlin/gradle/sdk/Util.kt | 42 ++++ .../sdk/kotlin/gradle/sdk/AwsServiceTest.kt | 101 ++++++++ .../sdk/kotlin/gradle/sdk/MembershipTest.kt | 38 +++ .../aws/sdk/kotlin/gradle/sdk/NamingTest.kt | 35 +++ codegen/sdk/build.gradle.kts | 219 ++---------------- gradle/libs.versions.toml | 2 +- settings.gradle.kts | 2 + 15 files changed, 591 insertions(+), 200 deletions(-) create mode 100644 build-support/build.gradle.kts create mode 100644 build-support/settings.gradle.kts create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Bootstrap.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/BootstrapConfig.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Membership.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/PackageSettings.kt create mode 100644 build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Util.kt create mode 100644 build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/AwsServiceTest.kt create mode 100644 build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt create mode 100644 build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt diff --git a/build-support/build.gradle.kts b/build-support/build.gradle.kts new file mode 100644 index 00000000000..ac9849affcd --- /dev/null +++ b/build-support/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + `kotlin-dsl` + `java-gradle-plugin` + alias(libs.plugins.kotlinx.serialization) +} + +group = "aws.sdk.kotlin" + +repositories { + mavenCentral() +} + +dependencies { + compileOnly(kotlin("gradle-plugin")) + compileOnly(kotlin("gradle-plugin-api")) + + implementation(libs.smithy.model) + implementation(libs.smithy.aws.traits) + implementation(libs.kotlinx.serialization.json) + + testImplementation(libs.junit.jupiter) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.kotlin.test.junit5) +} + +gradlePlugin { + plugins { + create("sdk-bootstrap") { + id = "sdk-bootstrap" + implementationClass = "aws.sdk.kotlin.gradle.sdk.Bootstrap" + } + } +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} diff --git a/build-support/settings.gradle.kts b/build-support/settings.gradle.kts new file mode 100644 index 00000000000..307522941e6 --- /dev/null +++ b/build-support/settings.gradle.kts @@ -0,0 +1,13 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +rootProject.name = "build-support" + +dependencyResolutionManagement { + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt new file mode 100644 index 00000000000..be63eb9a713 --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt @@ -0,0 +1,125 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import org.gradle.api.Project +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.ServiceShape +import java.io.File + +/** + * Represents information needed to generate a smithy projection JSON stanza + */ +data class AwsService( + /** + * The service shape ID name + */ + val serviceShapeId: String, + + /** + * The package name to use for the service when generating smithy-build.json + */ + val packageName: String, + + /** + * The package version (this should match the sdk version of the project) + */ + val packageVersion: String, + + /** + * The path to the model file in aws-sdk-kotlin + */ + val modelFile: File, + + /** + * The name of the projection to generate + */ + val projectionName: String, + + /** + * The sdkId value from the service trait + */ + val sdkId: String, + + /** + * The model version from the service shape + */ + val version: String, + + /** + * A description of the service (taken from the title trait) + */ + val description: String? = null, + +) + +/** + * Get the artifact name to use for the service derived from the sdkId. This will be the `A` in the GAV coordinates + * and the directory name under `services/`. + */ +val AwsService.artifactName: String + get() = sdkIdToArtifactName(sdkId) + +/** + * Returns a lambda for a service model file that respects the given bootstrap config + * + * @param project the codegen gradle project + * @param bootstrap the [BootstrapConfig] used to include/exclude a service based on the given config + */ +fun fileToService( + project: Project, + bootstrap: BootstrapConfig, +): (File) -> AwsService? = { file: File -> + val sdkVersion = project.findProperty("sdkVersion") as? String ?: error("expected sdkVersion to be set on project ${project.name}") + val filename = file.nameWithoutExtension + // TODO - Can't enable validation without being able to recognize all traits which requires additional deps on classpath + // This is _OK_ for the build because the CLI will do validation with the correct classpath but for unit tests + // it catches some errors that were difficult to track down. Would be nice to enable + val model = Model.assembler() + .discoverModels() // FIXME - why needed in tests but not in actual gradle build? + .addImport(file.absolutePath) + .assemble() + .result + .get() + val services: List = model.shapes(ServiceShape::class.java).sorted().toList() + val service = services.singleOrNull() ?: error("Expected one service per aws model, but found ${services.size} in ${file.absolutePath}: ${services.map { it.id }}") + val protocolName = service.protocolName() + + val serviceTrait = service + .findTrait(software.amazon.smithy.aws.traits.ServiceTrait.ID) + .map { it as software.amazon.smithy.aws.traits.ServiceTrait } + .orNull() + ?: error("Expected aws.api#service trait attached to model ${file.absolutePath}") + + val sdkId = serviceTrait.sdkId + val packageName = packageNameForService(sdkId) + val packageDescription = "The AWS SDK for Kotlin client for $sdkId" + + when { + !bootstrap.serviceMembership.isMember(filename, packageName) -> { + project.logger.info("skipping ${file.absolutePath}, $filename/$packageName not a member of ${bootstrap.serviceMembership}") + null + } + + !bootstrap.protocolMembership.isMember(protocolName) -> { + project.logger.info("skipping ${file.absolutePath}, $protocolName not a member of $${bootstrap.protocolMembership}") + null + } + + else -> { + project.logger.info("discovered service: ${serviceTrait.sdkId}") + AwsService( + serviceShapeId = service.id.toString(), + packageName = packageNamespaceForService(sdkId), + packageVersion = sdkVersion, + modelFile = file, + projectionName = filename, + sdkId = sdkId, + version = service.version, + description = packageDescription, + ) + } + } +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Bootstrap.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Bootstrap.kt new file mode 100644 index 00000000000..fb2bde22200 --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Bootstrap.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import org.gradle.api.Plugin +import org.gradle.api.Project + +// Dummy plugin, we use a plugin because it's easiest with an included build to apply to a buildscript and get +// the buildscript classpath correct. +class Bootstrap : Plugin { + override fun apply(project: Project) {} +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/BootstrapConfig.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/BootstrapConfig.kt new file mode 100644 index 00000000000..a1e5d867c55 --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/BootstrapConfig.kt @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import org.gradle.kotlin.dsl.provideDelegate + +/** + * Settings related to bootstrapping codegen tasks for AWS service code generation. + * + * Services and protocols can be included or excluded by `+` or `-` prefix. If no prefix is found then it is + * considered included (implicit `+`). + * + * @param services the service names to bootstrap. Services are named by either their model filename without + * the extension or by their artifact/package name. + * @param protocols the names of protocols to bootstrap + */ +class BootstrapConfig( + services: String? = null, + protocols: String? = null, +) { + companion object { + /** + * A bootstrap configuration that includes everything by default + */ + val ALL: BootstrapConfig = BootstrapConfig() + } + + val serviceMembership: Membership by lazy { parseMembership(services) } + val protocolMembership: Membership by lazy { parseMembership(protocols) } + override fun toString(): String = + "BootstrapConfig(serviceMembership=$serviceMembership, protocolMembership=$protocolMembership)" +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Membership.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Membership.kt new file mode 100644 index 00000000000..b1d62a28075 --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Membership.kt @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +/** + * Service and protocol membership for SDK generation + */ +data class Membership(val inclusions: Set = emptySet(), val exclusions: Set = emptySet()) + +fun Membership.isMember(vararg memberNames: String): Boolean = + memberNames.none(exclusions::contains) && (inclusions.isEmpty() || memberNames.any(inclusions::contains)) +fun parseMembership(rawList: String?): Membership { + if (rawList == null) return Membership() + + val inclusions = mutableSetOf() + val exclusions = mutableSetOf() + + rawList.split(",").map { it.trim() }.forEach { item -> + when { + item.startsWith('-') -> exclusions.add(item.substring(1)) + item.startsWith('+') -> inclusions.add(item.substring(1)) + else -> inclusions.add(item) + } + } + + val conflictingMembers = inclusions.intersect(exclusions) + require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" } + + return Membership(inclusions, exclusions) +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt new file mode 100644 index 00000000000..653f8b98a01 --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt @@ -0,0 +1,39 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +// The root namespace prefix for SDKs +const val SDK_PACKAGE_NAME_PREFIX: String = "aws.sdk.kotlin.services." + +/** + * Get the package name to use for a service from it's `sdkId` + */ +fun packageNameForService(sdkId: String): String = + sdkId.replace(" ", "") + .replace("-", "") + .lowercase() + .kotlinNamespace() + +/** + * Get the package namespace for a service from it's `sdkId` + */ +fun packageNamespaceForService(sdkId: String): String = "$SDK_PACKAGE_NAME_PREFIX${packageNameForService(sdkId)}" + +/** + * Remove characters invalid for Kotlin package namespace identifier + */ +fun String.kotlinNamespace(): String = split(".") + .joinToString(separator = ".") { segment -> segment.filter { it.isLetterOrDigit() } } + +/** + * Convert an sdkID to the module/artifact name to use + */ +internal fun sdkIdToArtifactName(sdkId: String): String = sdkId.replace(" ", "").replace("-", "").lowercase() + +/** + * Maps an sdkId from a model to the local filename to use. This logic has to match the logic used by + * catapult! See AwsSdkCatapultWorkspaceTools:lib/source/merge/smithy-model-handler.ts + */ +fun sdkIdToModelFilename(sdkId: String): String = sdkId.trim().replace("""[\s]+""".toRegex(), "-").lowercase() diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/PackageSettings.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/PackageSettings.kt new file mode 100644 index 00000000000..fe4489cb9db --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/PackageSettings.kt @@ -0,0 +1,46 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.Required +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import java.io.File + +/** + * Container for AWS service package settings. + * Each service can optionally have a `services//package.json` file that is used + * to control some aspect of code generation specific to that service + */ +@Serializable +data class PackageSettings( + /** + * The sdkId of the service. This is used as a check that the package settings are used on the correct service + */ + @Required + val sdkId: String, + + /** + * Whether to enable generating an auth scheme resolver based on endpoint resolution (rare). + */ + val enableEndpointAuthProvider: Boolean = false, +) { + companion object { + + /** + * Parse package settings from the given file path if it exists, otherwise return the default settings with + * the given sdkId. + */ + @OptIn(ExperimentalSerializationApi::class) + fun fromFile(sdkId: String, packageSettingsFile: File): PackageSettings { + if (!packageSettingsFile.exists()) return PackageSettings(sdkId) + val settings = Json.decodeFromStream(packageSettingsFile.inputStream()) + check(sdkId == settings.sdkId) { "${packageSettingsFile.absolutePath} `sdkId` from settings (${settings.sdkId}) does not match expected `$sdkId`" } + return settings + } + } +} diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Util.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Util.kt new file mode 100644 index 00000000000..2f7d1e8c0af --- /dev/null +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Util.kt @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import software.amazon.smithy.aws.traits.protocols.AwsJson1_0Trait +import software.amazon.smithy.aws.traits.protocols.AwsJson1_1Trait +import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait +import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait +import software.amazon.smithy.aws.traits.protocols.RestJson1Trait +import software.amazon.smithy.aws.traits.protocols.RestXmlTrait +import software.amazon.smithy.model.shapes.ServiceShape + +private const val DEPRECATED_SHAPES_CUTOFF_DATE: String = "2023-11-28" + +public val REMOVE_DEPRECATED_SHAPES_TRANSFORM: String = """ + { + "name": "awsSmithyKotlinRemoveDeprecatedShapes", + "args": { + "until": "$DEPRECATED_SHAPES_CUTOFF_DATE" + } + } +""".trimIndent() + +/** + * Convert an Optional to T? + */ +fun java.util.Optional.orNull(): T? = this.orElse(null) + +/** + * Returns the trait name of the protocol of the service + */ +fun ServiceShape.protocolName(): String = + listOf( + RestJson1Trait.ID, + RestXmlTrait.ID, + AwsJson1_0Trait.ID, + AwsJson1_1Trait.ID, + AwsQueryTrait.ID, + Ec2QueryTrait.ID, + ).first { hasTrait(it) }.name diff --git a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/AwsServiceTest.kt b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/AwsServiceTest.kt new file mode 100644 index 00000000000..b014f8d091f --- /dev/null +++ b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/AwsServiceTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import org.gradle.kotlin.dsl.extra +import org.gradle.testfixtures.ProjectBuilder +import org.junit.jupiter.api.io.TempDir +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class AwsServiceTest { + + val modelContents = """ + ${"$"}version: "2.0" + namespace gradle.test + + use aws.api#service + use aws.protocols#awsJson1_0 + + @service(sdkId: "Test Gradle") + @awsJson1_0 + service TestService{ + operations: [], + version: "1-alpha" + } + """.trimIndent() + + private data class TestResult( + val model: File, + val actual: AwsService?, + ) + + private fun testWith( + tempDir: File, + bootstrap: BootstrapConfig, + ): TestResult { + val project = ProjectBuilder.builder() + .build() + project.extra.set("sdkVersion", "1.2.3") + + // NOTE: Model assembler requires the correct .json or .smithy extension for the file contents + val model = tempDir.resolve("test-gradle.smithy") + model.writeText(modelContents) + + val lambda = fileToService(project, bootstrap) + val actual = lambda(model) + return TestResult(model, actual) + } + + @Test + fun testFileToService(@TempDir tempDir: File) { + val tests = listOf( + BootstrapConfig.ALL, + // filename + BootstrapConfig("+test-gradle"), + BootstrapConfig("test-gradle"), + // artifact name + BootstrapConfig("+testgradle"), + BootstrapConfig("testgradle"), + // protocol + BootstrapConfig(null, "awsJson1_0"), + ) + + tests.forEach { bootstrap -> + val result = testWith(tempDir, bootstrap) + val expected = AwsService( + "gradle.test#TestService", + "aws.sdk.kotlin.services.testgradle", + "1.2.3", + result.model, + "test-gradle", + "Test Gradle", + "1-alpha", + "The AWS SDK for Kotlin client for Test Gradle", + ) + assertEquals(expected, result.actual) + } + } + + @Test + fun testFileToServiceExclude(@TempDir tempDir: File) { + val tests = listOf( + // explicit exclude + BootstrapConfig("-test-gradle"), + BootstrapConfig("-testgradle"), + // explicit include without service under test + BootstrapConfig("other"), + // protocol exclude + BootstrapConfig(null, "-awsJson1_0"), + ) + + tests.forEach { bootstrap -> + val result = testWith(tempDir, bootstrap) + assertNull(result.actual, "expected null for bootstrap with $bootstrap") + } + } +} diff --git a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt new file mode 100644 index 00000000000..04f309972a9 --- /dev/null +++ b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import kotlin.test.* + +class MembershipTest { + @Test + fun testIsMember() { + val unit = Membership(setOf("i1", "i2"), setOf("e1")) + assertTrue(unit.isMember("i1")) + assertTrue(unit.isMember("i2")) + assertFalse(unit.isMember("e1")) + + // test implicit include + assertTrue(Membership().isMember("i3")) + } + + @Test + fun testParse() { + val expected = Membership( + setOf("i1", "i2"), + setOf("e1"), + ) + + val actual = parseMembership("+i1,-e1,i2") + assertEquals(expected, actual) + } + + @Test + fun testParseWithConflict() { + assertFails { + parseMembership("+i1,-i1") + } + } +} diff --git a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt new file mode 100644 index 00000000000..66e750fa810 --- /dev/null +++ b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.gradle.sdk + +import kotlin.test.Test +import kotlin.test.assertEquals + +class NamingTest { + @Test + fun testPackageNameForService() { + assertEquals("foobar", packageNameForService("Foo Bar")) + assertEquals("foobar", packageNameForService("Foo Bar")) + assertEquals("foobar", packageNameForService("FoO-BaR")) + } + + @Test + fun testKotlinNamespace() { + assertEquals("foobar", "foo b-ar".kotlinNamespace()) + assertEquals("foobar", " foo bar ".kotlinNamespace()) + assertEquals("foo.bar", "foo.b-ar".kotlinNamespace()) + } + + @Test + fun testSdkIdToArtifactName() { + } + + @Test + fun testSdkIdToModelFilename() { + assertEquals("foo", sdkIdToModelFilename("Foo")) + assertEquals("foo-bar", sdkIdToModelFilename("Foo Bar")) + assertEquals("foo-bar", sdkIdToModelFilename(" Foo Bar ")) + } +} diff --git a/codegen/sdk/build.gradle.kts b/codegen/sdk/build.gradle.kts index a8018235a1f..1a5800380e6 100644 --- a/codegen/sdk/build.gradle.kts +++ b/codegen/sdk/build.gradle.kts @@ -3,87 +3,27 @@ * SPDX-License-Identifier: Apache-2.0 */ +import aws.sdk.kotlin.gradle.codegen.* import aws.sdk.kotlin.gradle.codegen.dsl.SmithyProjection import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath +import aws.sdk.kotlin.gradle.sdk.* +import aws.sdk.kotlin.gradle.util.typedProp import software.amazon.smithy.model.Model -import software.amazon.smithy.model.node.Node import software.amazon.smithy.model.shapes.ServiceShape import java.nio.file.Paths import java.util.* import kotlin.streams.toList plugins { - kotlin("jvm") // FIXME - configuration doesn't resolve without this + kotlin("jvm") // FIXME - codegen configuration doesn't resolve without this id("aws.sdk.kotlin.gradle.smithybuild") + id("sdk-bootstrap") } +val sdkVersion: String by project description = "AWS SDK codegen tasks" -// get a project property by name if it exists (including from local.properties) -fun getProperty(name: String): String? { - if (project.hasProperty(name)) { - return project.properties[name].toString() - } else if (project.ext.has(name)) { - return project.ext[name].toString() - } - - val localProperties = Properties() - val propertiesFile: File = rootProject.file("local.properties") - if (propertiesFile.exists()) { - propertiesFile.inputStream().use { localProperties.load(it) } - - if (localProperties.containsKey(name)) { - return localProperties[name].toString() - } - } - return null -} - -// Represents information needed to generate a smithy projection JSON stanza -data class AwsService( - /** - * The service shape ID name - */ - val serviceShapeId: String, - /** - * The package name to use for the service when generating smithy-build.json - */ - val packageName: String, - - /** - * The package version (this should match the sdk version of the project) - */ - val packageVersion: String, - - /** - * The path to the model file in aws-sdk-kotlin - */ - val modelFile: File, - - /** - * The name of the projection to generate - */ - val projectionName: String, - - /** - * The sdkId value from the service trait - */ - val sdkId: String, - - /** - * The model version from the service shape - */ - val version: String, - - /** - * A description of the service (taken from the title trait) - */ - val description: String? = null, - -) - val servicesProvider: Provider> = project.provider { discoverServices() } // Manually create the projections rather than using the extension to avoid unnecessary configuration evaluation. @@ -100,18 +40,9 @@ fun awsServiceProjections(): Provider> { importPaths.add(service.modelExtrasDir) } imports = importPaths - transforms = (transformsForService(service) ?: emptyList()) + removeDeprecatedShapesTransform("2023-11-28") - - val packageSettingsFile = file(service.packageSettings) - val packageSettings = if (packageSettingsFile.exists()) { - val node = Node.parse(packageSettingsFile.inputStream()).asObjectNode().get() - node.expectMember("sdkId", "${packageSettingsFile.absolutePath} does not contain member `sdkId`") - val packageSdkId = node.getStringMember("sdkId").get().value - check(service.sdkId == packageSdkId) { "${packageSettingsFile.absolutePath} `sdkId` ($packageSdkId) does not match expected `${service.sdkId}`" } - node - } else { - Node.objectNode() - } + transforms = (transformsForService(service) ?: emptyList()) + REMOVE_DEPRECATED_SHAPES_TRANSFORM + + val packageSettings = PackageSettings.fromFile(service.sdkId, file(service.packageSettings)) smithyKotlinPlugin { serviceShapeId = service.serviceShapeId @@ -124,7 +55,7 @@ fun awsServiceProjections(): Provider> { generateDefaultBuildFiles = false } apiSettings { - enableEndpointAuthProvider = packageSettings.getBooleanMember("enableEndpointAuthProvider").orNull()?.value + enableEndpointAuthProvider = packageSettings.enableEndpointAuthProvider } } } @@ -157,69 +88,10 @@ fun transformsForService(service: AwsService): List? { } } -fun removeDeprecatedShapesTransform(removeDeprecatedShapesUntil: String): String = """ - { - "name": "awsSmithyKotlinRemoveDeprecatedShapes", - "args": { - "until": "$removeDeprecatedShapesUntil" - } - } -""".trimIndent() - -// The root namespace prefix for SDKs -val sdkPackageNamePrefix = "aws.sdk.kotlin.services." - -val sdkVersion: String by project - -val serviceMembership: Membership by lazy { parseMembership(getProperty("aws.services")) } -val protocolMembership: Membership by lazy { parseMembership(getProperty("aws.protocols")) } - -fun fileToService(applyFilters: Boolean): (File) -> AwsService? = { file: File -> - val filename = file.nameWithoutExtension - val model = Model.assembler().addImport(file.absolutePath).assemble().result.get() - val services: List = model.shapes(ServiceShape::class.java).sorted().toList() - val service = services.singleOrNull() ?: error("Expected one service per aws model, but found ${services.size} in ${file.absolutePath}: ${services.map { it.id }}") - val protocol = service.protocol() - - val serviceTrait = service - .findTrait(software.amazon.smithy.aws.traits.ServiceTrait.ID) - .map { it as software.amazon.smithy.aws.traits.ServiceTrait } - .orNull() - ?: error("Expected aws.api#service trait attached to model ${file.absolutePath}") - - val sdkId = serviceTrait.sdkId - val packageName = sdkId.replace(" ", "") - .replace("-", "") - .lowercase() - .kotlinNamespace() - val packageDescription = "The AWS Kotlin client for $sdkId" - - when { - applyFilters && !serviceMembership.isMember(filename, packageName) -> { - logger.info("skipping ${file.absolutePath}, $filename/$packageName not a member of $serviceMembership") - null - } - - applyFilters && !protocolMembership.isMember(protocol) -> { - logger.info("skipping ${file.absolutePath}, $protocol not a member of $protocolMembership") - null - } - - else -> { - logger.info("discovered service: ${serviceTrait.sdkId}") - AwsService( - serviceShapeId = service.id.toString(), - packageName = "$sdkPackageNamePrefix$packageName", - packageVersion = sdkVersion, - modelFile = file, - projectionName = filename, - sdkId = sdkId, - version = service.version, - description = packageDescription, - ) - } - } -} +val bootstrap = BootstrapConfig( + typedProp("aws.services"), + typedProp("aws.protocols"), +) /** * Returns an AwsService model for every JSON file found in directory defined by property `modelsDirProp` @@ -229,64 +101,17 @@ fun fileToService(applyFilters: Boolean): (File) -> AwsService? = { file: File - fun discoverServices(applyFilters: Boolean = true): List { println("discover services called") val modelsDir: String by project - return fileTree(project.file(modelsDir)).mapNotNull(fileToService(applyFilters)) + val bootstrapConfig = bootstrap.takeIf { applyFilters } ?: BootstrapConfig.ALL + return fileTree(project.file(modelsDir)).mapNotNull(fileToService(project, bootstrapConfig)) } -// Returns the trait name of the protocol of the service -fun ServiceShape.protocol(): String = - listOf( - "aws.protocols#awsJson1_0", - "aws.protocols#awsJson1_1", - "aws.protocols#awsQuery", - "aws.protocols#ec2Query", - "aws.protocols#restJson1", - "aws.protocols#restXml", - ).first { protocol -> findTrait(protocol).isPresent }.split("#")[1] - -// Class and functions for service and protocol membership for SDK generation -data class Membership(val inclusions: Set = emptySet(), val exclusions: Set = emptySet()) - -fun Membership.isMember(vararg memberNames: String): Boolean = - memberNames.none(exclusions::contains) && (inclusions.isEmpty() || memberNames.any(inclusions::contains)) - -fun parseMembership(rawList: String?): Membership { - if (rawList == null) return Membership() - - val inclusions = mutableSetOf() - val exclusions = mutableSetOf() - - rawList.split(",").map { it.trim() }.forEach { item -> - when { - item.startsWith('-') -> exclusions.add(item.substring(1)) - item.startsWith('+') -> inclusions.add(item.substring(1)) - else -> inclusions.add(item) - } - } - - val conflictingMembers = inclusions.intersect(exclusions) - require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" } - - return Membership(inclusions, exclusions) -} - -fun java.util.Optional.orNull(): T? = this.orElse(null) - -/** - * Remove characters invalid for Kotlin package namespace identifier - */ -fun String.kotlinNamespace(): String = split(".") - .joinToString(separator = ".") { segment -> segment.filter { it.isLetterOrDigit() } } - /** * The project directory under `aws-sdk-kotlin/services` * * NOTE: this will also be the artifact name in the GAV coordinates */ val AwsService.destinationDir: String - get() { - val sanitizedName = sdkId.replace(" ", "").replace("-", "").lowercase() - return rootProject.file("services/$sanitizedName").absolutePath - } + get() = rootProject.file("services/$artifactName").absolutePath /** * Service specific model extras @@ -307,7 +132,7 @@ val AwsService.packageSettings: String get() = rootProject.file("$destinationDir/package.json").absolutePath fun forwardProperty(name: String) { - getProperty(name)?.let { + typedProp(name)?.let { System.setProperty(name, it) } } @@ -335,7 +160,6 @@ val stageSdks = tasks.register("stageSdks") { val discoveredServices = servicesProvider.get() println("discoveredServices = ${discoveredServices.joinToString { it.sdkId }}") discoveredServices.forEach { - // val projectionOutputDir = smithyBuild.projections.getByName(it.projectionName).projectionRootDir val projectionOutputDir = smithyBuild.smithyKotlinProjectionPath(it.projectionName).get() logger.info("copying $projectionOutputDir to ${it.destinationDir}") copy { @@ -379,10 +203,7 @@ data class SourceModel( * The model filename in aws-sdk-kotlin */ val destFilename: String - get() { - val name = sdkId.replace(" ", "-").lowercase() - return "$name.json" - } + get() = "${sdkIdToModelFilename(sdkId)}.json" } fun discoverSourceModels(repoPath: String): List { @@ -407,7 +228,7 @@ fun discoverAwsModelsRepoPath(): String? { val discovered = rootProject.file("../aws-models") if (discovered.exists()) return discovered.absolutePath - return getProperty("awsModelsDir")?.let { File(it) }?.absolutePath + return typedProp("awsModelsDir")?.let { File(it) }?.absolutePath } /** diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 310207dba9f..a163a0deb39 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -20,7 +20,7 @@ smithy-gradle-version = "0.7.0" junit-version = "5.10.1" kotest-version = "5.8.0" kotlinx-benchmark-version = "0.4.9" -kotlinx-serialization-version = "1.6.0" +kotlinx-serialization-version = "1.6.2" mockk-version = "1.13.7" slf4j-version = "2.0.9" diff --git a/settings.gradle.kts b/settings.gradle.kts index 9c5e6ed92b1..3a672c596d5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -27,6 +27,8 @@ dependencyResolutionManagement { rootProject.name = "aws-sdk-kotlin" +includeBuild("build-support") + include(":dokka-aws") include(":bom") include(":codegen:sdk") From 8f9d47f9feb4a099372c47f5161908fc24de7e01 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 9 Jan 2024 08:25:27 -0500 Subject: [PATCH 3/5] remove print in favor of logging --- codegen/sdk/build.gradle.kts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/codegen/sdk/build.gradle.kts b/codegen/sdk/build.gradle.kts index 1a5800380e6..b379c589989 100644 --- a/codegen/sdk/build.gradle.kts +++ b/codegen/sdk/build.gradle.kts @@ -29,7 +29,7 @@ val servicesProvider: Provider> = project.provider { discoverSe // Manually create the projections rather than using the extension to avoid unnecessary configuration evaluation. // Otherwise we would be reading the models from disk on every gradle invocation for unrelated projects/tasks fun awsServiceProjections(): Provider> { - println("AWS service projection provider called") + logger.info("AWS service projection provider called") val p = servicesProvider.map { it.map { service -> SmithyProjection( @@ -99,10 +99,12 @@ val bootstrap = BootstrapConfig( * membership tests */ fun discoverServices(applyFilters: Boolean = true): List { - println("discover services called") + logger.info("discover services called") val modelsDir: String by project val bootstrapConfig = bootstrap.takeIf { applyFilters } ?: BootstrapConfig.ALL - return fileTree(project.file(modelsDir)).mapNotNull(fileToService(project, bootstrapConfig)) + return fileTree(project.file(modelsDir)).mapNotNull(fileToService(project, bootstrapConfig)).also { + logger.lifecycle("discovered ${it.size} services") + } } /** @@ -146,6 +148,7 @@ dependencies { } tasks.generateSmithyProjections { + inputs.property("bootstrapConfigHash", bootstrap.hashCode()) doFirst { forwardProperty("aws.partitions_file") forwardProperty("aws.user_agent.add_metadata") @@ -158,7 +161,7 @@ val stageSdks = tasks.register("stageSdks") { dependsOn(tasks.generateSmithyProjections) doLast { val discoveredServices = servicesProvider.get() - println("discoveredServices = ${discoveredServices.joinToString { it.sdkId }}") + logger.lifecycle("discoveredServices = ${discoveredServices.joinToString { it.sdkId }}") discoveredServices.forEach { val projectionOutputDir = smithyBuild.smithyKotlinProjectionPath(it.projectionName).get() logger.info("copying $projectionOutputDir to ${it.destinationDir}") From bcc1119473abbfa71af4054de09cfa52f2b47c19 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Jan 2024 10:49:11 -0500 Subject: [PATCH 4/5] fix lint --- .../src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt index be63eb9a713..1eddfc0ed57 100644 --- a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt @@ -8,6 +8,7 @@ import org.gradle.api.Project import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.ServiceShape import java.io.File +import kotlin.streams.toList /** * Represents information needed to generate a smithy projection JSON stanza From a11fe0db60f9439cb383b5a326fd6b8ee9a82df6 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Jan 2024 12:25:06 -0500 Subject: [PATCH 5/5] feedback --- .../src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt | 2 +- .../src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt | 2 +- .../test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt | 6 ++++++ .../src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt | 3 +++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt index 1eddfc0ed57..da4efcfface 100644 --- a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/AwsService.kt @@ -85,7 +85,7 @@ fun fileToService( .result .get() val services: List = model.shapes(ServiceShape::class.java).sorted().toList() - val service = services.singleOrNull() ?: error("Expected one service per aws model, but found ${services.size} in ${file.absolutePath}: ${services.map { it.id }}") + val service = services.singleOrNull() ?: error("Expected one service per aws model, but found ${services.size} in ${file.absolutePath}: ${services.joinToString { it.id.toString() }}") val protocolName = service.protocolName() val serviceTrait = service diff --git a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt index 653f8b98a01..8471edd7d29 100644 --- a/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt +++ b/build-support/src/main/kotlin/aws/sdk/kotlin/gradle/sdk/Naming.kt @@ -28,7 +28,7 @@ fun String.kotlinNamespace(): String = split(".") .joinToString(separator = ".") { segment -> segment.filter { it.isLetterOrDigit() } } /** - * Convert an sdkID to the module/artifact name to use + * Convert an sdkId to the module/artifact name to use */ internal fun sdkIdToArtifactName(sdkId: String): String = sdkId.replace(" ", "").replace("-", "").lowercase() diff --git a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt index 04f309972a9..b5003a6ab4e 100644 --- a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt +++ b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/MembershipTest.kt @@ -18,6 +18,12 @@ class MembershipTest { assertTrue(Membership().isMember("i3")) } + @Test + fun testExcludePrecedence() { + val unit = Membership(setOf("m1"), setOf("m1")) + assertFalse(unit.isMember("m1")) + } + @Test fun testParse() { val expected = Membership( diff --git a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt index 66e750fa810..521ada03fc0 100644 --- a/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt +++ b/build-support/src/test/kotlin/aws/sdk/kotlin/gradle/sdk/NamingTest.kt @@ -24,6 +24,9 @@ class NamingTest { @Test fun testSdkIdToArtifactName() { + assertEquals("foobar", sdkIdToArtifactName("foo bar")) + assertEquals("foobar", sdkIdToArtifactName("foo-bar")) + assertEquals("foobar", sdkIdToArtifactName("fOo -Bar")) } @Test