From 472958442b2f1bcbe498bcf2101f1fa48d459c47 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 02:22:15 -0500 Subject: [PATCH 01/64] misc: add rules engine codegen tests --- settings.gradle.kts | 1 + tests/codegen/rules-engine/build.gradle.kts | 158 ++++++++++++++++++ .../operation-context-params.smithy | 58 +++++++ 3 files changed, 217 insertions(+) create mode 100644 tests/codegen/rules-engine/build.gradle.kts create mode 100644 tests/codegen/rules-engine/operation-context-params.smithy diff --git a/settings.gradle.kts b/settings.gradle.kts index 203fcef7b8d..6f53bf06951 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -53,6 +53,7 @@ include(":hll:hll-mapping-core") include(":services") include(":tests") include(":tests:codegen:event-stream") +include("tests:codegen:rules-engine") include(":tests:e2e-test-util") // generated services diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts new file mode 100644 index 00000000000..ef35526b380 --- /dev/null +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -0,0 +1,158 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) +} + +description = "Smithy rules engine codegen integration test suite" + +data class Test( + val projectionName: String, + val protocolName: String, + val modelTemplate: File, +) { + val model: File + get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile +} + +val tests = listOf( + Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +) + +fun fillInModel(output: File, protocolName: String, template: File) { + val input = template.readText() + val opTraits = when (protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + val replaced = input + .replace("{protocol-name}", protocolName) + .replace("{op-traits}", opTraits) + + output.parentFile.mkdirs() + output.writeText(replaced) +} + +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" +smithyBuild { + tests.forEach { test -> + + projections.register(test.projectionName) { + imports = listOf(test.model.absolutePath) + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["$testServiceShapeId"] + } + } + """, + ) + + smithyKotlinPlugin { + serviceShapeId = testServiceShapeId + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} + +val codegen by configurations.getting +dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libs.smithy.cli) + codegen(libs.smithy.model) +} + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } + } +} + +tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } +} + +val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", +) + +kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } +} + +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } +} + +tasks.withType { + dependsOn(tasks.generateSmithyProjections) + // generated clients have quite a few warnings + kotlinOptions.allWarningsAsErrors = false +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +dependencies { + + implementation(libs.kotlinx.coroutines.core) + + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlin.test.junit5) + testImplementation(libs.kotlinx.coroutines.test) + + testImplementation(libs.smithy.kotlin.smithy.test) + testImplementation(libs.smithy.kotlin.aws.signing.default) + testImplementation(libs.smithy.kotlin.telemetry.api) + + // have to manually add all the dependencies of the generated client(s) + // doing it this way (as opposed to doing what we do for protocol-tests) allows + // the tests to work without a publish to maven-local step at the cost of maintaining + // this set of dependencies manually + // <-- BEGIN GENERATED DEPENDENCY LIST --> + implementation(libs.bundles.smithy.kotlin.service.client) + implementation(libs.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libs.smithy.kotlin.aws.json.protocols) + implementation(libs.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + // <-- END GENERATED DEPENDENCY LIST --> +} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/operation-context-params.smithy new file mode 100644 index 00000000000..c63a2756e39 --- /dev/null +++ b/tests/codegen/rules-engine/operation-context-params.smithy @@ -0,0 +1,58 @@ +$version: "2.0" +namespace aws.sdk.kotlin.test + +use aws.protocols#awsJson1_0 +use smithy.rules#operationContextParams +use smithy.rules#endpointRuleSet +use aws.api#service + +@awsJson1_0 +@service(sdkId: "OperationContextParamsTest") +@endpointRuleSet( + version: "1.0", + parameters: { + "ObjectKeys": { + "type": "stringArray", + "documentation": "A string array.", + "required": true + } + }, + rules: [ + { + "type": "endpoint", + "conditions": [], + "endpoint": { + "url": "https://static.endpoint" + } + } + ] +) +service TestService { + operations: [DeleteObjects], + version: "1" +} + +@operationContextParams( + ObjectKeys: { + path: "Delete.Objects[*].[Key][]" + } +) +operation DeleteObjects { + input: DeleteObjectsRequest +} + +structure DeleteObjectsRequest { + Delete: Delete +} + +structure Delete { + Objects: ObjectIdentifierList +} + +list ObjectIdentifierList { + member: ObjectIdentifier +} + +structure ObjectIdentifier { + Key: String +} From 7b7308836e00da347a3a3d2673127cdc0c631a13 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 10:32:10 -0500 Subject: [PATCH 02/64] Use snapshot version of smithy kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d86529cab72..533963fcac2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.19" -smithy-kotlin-codegen-version = "0.33.19" +smithy-kotlin-runtime-version = "1.3.20-SNAPSHOT" +smithy-kotlin-codegen-version = "0.33.20-SNAPSHOT" # codegen smithy-version = "1.51.0" From dbfb9735643e8201a053cba5bd2a0fe081857ded Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 11:32:03 -0500 Subject: [PATCH 03/64] debugging: throw exception to see what metrics are strings --- .../runtime/http/interceptors/BusinessMetricsInterceptor.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt index 45acf59e5d4..ddfe0ab7808 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt @@ -20,6 +20,7 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder public class BusinessMetricsInterceptor : HttpInterceptor { override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { context.executionContext.getOrNull(BusinessMetrics)?.let { metrics -> + throw Exception("Metrics: $metrics") val metricsString = formatMetrics(metrics) val currentUserAgentHeader = context.protocolRequest.headers[USER_AGENT] val modifiedRequest = context.protocolRequest.toBuilder() From b15358b8eff089c36c8b9aad579c5f586c0c1160 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 11:39:05 -0500 Subject: [PATCH 04/64] debugging: remove exception --- .../runtime/http/interceptors/BusinessMetricsInterceptor.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt index ddfe0ab7808..45acf59e5d4 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt @@ -20,7 +20,6 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder public class BusinessMetricsInterceptor : HttpInterceptor { override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { context.executionContext.getOrNull(BusinessMetrics)?.let { metrics -> - throw Exception("Metrics: $metrics") val metricsString = formatMetrics(metrics) val currentUserAgentHeader = context.protocolRequest.headers[USER_AGENT] val modifiedRequest = context.protocolRequest.toBuilder() From dacc3458ed2ad86a5f74b2b1db6105bc1197cbda Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 12:19:53 -0500 Subject: [PATCH 05/64] Don't depend on snapshot version of smithy kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 533963fcac2..d003593d62b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.20-SNAPSHOT" -smithy-kotlin-codegen-version = "0.33.20-SNAPSHOT" +smithy-kotlin-runtime-version = "1.3.20" +smithy-kotlin-codegen-version = "0.33.20" # codegen smithy-version = "1.51.0" From bbafc3fac438d014c592564aa04a1d25c65e668d Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 12:20:40 -0500 Subject: [PATCH 06/64] Use released version --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d003593d62b..d86529cab72 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.20" -smithy-kotlin-codegen-version = "0.33.20" +smithy-kotlin-runtime-version = "1.3.19" +smithy-kotlin-codegen-version = "0.33.19" # codegen smithy-version = "1.51.0" From a79aa1ea29244733565c1b07d1f3ed1cb9794d01 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 13:16:25 -0500 Subject: [PATCH 07/64] PR feedback --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 6f53bf06951..c686feb4989 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -53,7 +53,7 @@ include(":hll:hll-mapping-core") include(":services") include(":tests") include(":tests:codegen:event-stream") -include("tests:codegen:rules-engine") +include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") // generated services From e61efbbb7b642f5f703ef87541db7fa0233a177b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 7 Nov 2024 11:54:21 -0500 Subject: [PATCH 08/64] Checkpoint --- tests/codegen/common.gradle.kts | 158 ++++++++++++++++++ tests/codegen/event-stream/build.gradle.kts | 10 +- ...ent-stream-initial-request-response.smithy | 2 +- .../event-stream-model-template.smithy | 2 +- .../src/test/kotlin/HttpEventStreamTests.kt | 6 +- .../src/test/kotlin/RpcEventStreamTests.kt | 12 +- tests/codegen/rules-engine/build.gradle.kts | 4 +- 7 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 tests/codegen/common.gradle.kts diff --git a/tests/codegen/common.gradle.kts b/tests/codegen/common.gradle.kts new file mode 100644 index 00000000000..7e3409bf865 --- /dev/null +++ b/tests/codegen/common.gradle.kts @@ -0,0 +1,158 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) +} + +description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize + +data class Test( + val projectionName: String, + val protocolName: String, + val modelTemplate: File, +) { + val model: File + get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile +} + +val tests = listOf( // TODO: Don't commonize + Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +) + +fun fillInModel(output: File, protocolName: String, template: File) { + val input = template.readText() + val opTraits = when (protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + val replaced = input + .replace("{protocol-name}", protocolName) + .replace("{op-traits}", opTraits) + + output.parentFile.mkdirs() + output.writeText(replaced) +} + +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" +smithyBuild { + tests.forEach { test -> + + projections.register(test.projectionName) { + imports = listOf(test.model.absolutePath) + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["$testServiceShapeId"] + } + } + """, + ) + + smithyKotlinPlugin { + serviceShapeId = testServiceShapeId + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} + +val codegen by configurations.getting +dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libs.smithy.cli) + codegen(libs.smithy.model) +} + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } + } +} + +tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } +} + +val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", +) + +kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } +} + +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } +} + +tasks.withType { + dependsOn(tasks.generateSmithyProjections) + // generated clients have quite a few warnings + kotlinOptions.allWarningsAsErrors = false +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +dependencies { + + implementation(libs.kotlinx.coroutines.core) + + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlin.test.junit5) + testImplementation(libs.kotlinx.coroutines.test) + + testImplementation(libs.smithy.kotlin.smithy.test) + testImplementation(libs.smithy.kotlin.aws.signing.default) + testImplementation(libs.smithy.kotlin.telemetry.api) + + // have to manually add all the dependencies of the generated client(s) + // doing it this way (as opposed to doing what we do for protocol-tests) allows + // the tests to work without a publish to maven-local step at the cost of maintaining + // this set of dependencies manually + // <-- BEGIN GENERATED DEPENDENCY LIST --> + implementation(libs.bundles.smithy.kotlin.service.client) + implementation(libs.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libs.smithy.kotlin.aws.json.protocols) + implementation(libs.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + // <-- END GENERATED DEPENDENCY LIST --> +} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 837d12b2d3c..8c6ae318868 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -14,7 +14,7 @@ plugins { description = "Event stream codegen integration test suite" -data class EventStreamTest( +data class Test( val projectionName: String, val protocolName: String, val modelTemplate: File, @@ -24,8 +24,8 @@ data class EventStreamTest( } val tests = listOf( - EventStreamTest("restJson1", "restJson1", file("event-stream-model-template.smithy")), - EventStreamTest("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), + Test("restJson1", "restJson1", file("event-stream-model-template.smithy")), + Test("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), ) fun fillInModel(output: File, protocolName: String, template: File) { @@ -42,7 +42,7 @@ fun fillInModel(output: File, protocolName: String, template: File) { output.writeText(replaced) } -val testServiceShapeId = "aws.sdk.kotlin.test.eventstream#TestService" +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> @@ -61,7 +61,7 @@ smithyBuild { smithyKotlinPlugin { serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.eventstream.${test.projectionName.lowercase()}" + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false diff --git a/tests/codegen/event-stream/event-stream-initial-request-response.smithy b/tests/codegen/event-stream/event-stream-initial-request-response.smithy index 917b58ac292..180311c8eb5 100644 --- a/tests/codegen/event-stream/event-stream-initial-request-response.smithy +++ b/tests/codegen/event-stream/event-stream-initial-request-response.smithy @@ -1,4 +1,4 @@ -namespace aws.sdk.kotlin.test.eventstream +namespace aws.sdk.kotlin.test use aws.protocols#awsJson1_1 use aws.api#service diff --git a/tests/codegen/event-stream/event-stream-model-template.smithy b/tests/codegen/event-stream/event-stream-model-template.smithy index f3d91364235..8150dcec2db 100644 --- a/tests/codegen/event-stream/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/event-stream-model-template.smithy @@ -1,4 +1,4 @@ -namespace aws.sdk.kotlin.test.eventstream +namespace aws.sdk.kotlin.test use aws.protocols#{protocol-name} use aws.api#service diff --git a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt index 8402dd270fa..1bb5ec2bda9 100644 --- a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt +++ b/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt @@ -4,9 +4,9 @@ */ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.eventstream.restjson1.model.* -import aws.sdk.kotlin.test.eventstream.restjson1.serde.deserializeTestStreamOpOperationBody -import aws.sdk.kotlin.test.eventstream.restjson1.serde.serializeTestStreamOpOperationBody +import aws.sdk.kotlin.test.restjson1.model.* +import aws.sdk.kotlin.test.restjson1.serde.deserializeTestStreamOpOperationBody +import aws.sdk.kotlin.test.restjson1.serde.serializeTestStreamOpOperationBody import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt index 0b7e4b94651..08cbecbda2a 100644 --- a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt +++ b/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt @@ -3,12 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.eventstream.awsjson11.model.MessageWithString -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStream -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStreamOperationWithInitialRequestResponseRequest -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStreamOperationWithInitialRequestResponseResponse -import aws.sdk.kotlin.test.eventstream.awsjson11.serde.deserializeTestStreamOperationWithInitialRequestResponseOperationBody -import aws.sdk.kotlin.test.eventstream.awsjson11.serde.serializeTestStreamOperationWithInitialRequestResponseOperationBody +import aws.sdk.kotlin.test.awsjson11.model.MessageWithString +import aws.sdk.kotlin.test.awsjson11.model.TestStream +import aws.sdk.kotlin.test.awsjson11.model.TestStreamOperationWithInitialRequestResponseRequest +import aws.sdk.kotlin.test.awsjson11.model.TestStreamOperationWithInitialRequestResponseResponse +import aws.sdk.kotlin.test.awsjson11.serde.deserializeTestStreamOperationWithInitialRequestResponseOperationBody +import aws.sdk.kotlin.test.awsjson11.serde.serializeTestStreamOperationWithInitialRequestResponseOperationBody import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index ef35526b380..7e3409bf865 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -12,7 +12,7 @@ plugins { alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) } -description = "Smithy rules engine codegen integration test suite" +description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize data class Test( val projectionName: String, @@ -23,7 +23,7 @@ data class Test( get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile } -val tests = listOf( +val tests = listOf( // TODO: Don't commonize Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), ) From fbb3dd51c2b77f4fcaa8ab1a89eea0ccaf7db1d7 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 12:47:28 -0500 Subject: [PATCH 09/64] Checkoint 2 --- settings.gradle.kts | 5 +- tests/codegen/build.gradle.kts | 80 ++++++++++ tests/codegen/common.gradle.kts | 158 -------------------- tests/codegen/event-stream/build.gradle.kts | 79 ++++------ tests/codegen/rules-engine/build.gradle.kts | 78 ++++------ 5 files changed, 137 insertions(+), 263 deletions(-) create mode 100644 tests/codegen/build.gradle.kts delete mode 100644 tests/codegen/common.gradle.kts diff --git a/settings.gradle.kts b/settings.gradle.kts index 7f35e266c03..7a7751ebeb8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,11 +52,12 @@ include(":hll:hll-codegen") include(":hll:hll-mapping-core") include(":services") include(":tests") +include(":tests:codegen") include(":tests:codegen:event-stream") include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") -include(":tests:codegen:smoke-tests") -include(":tests:codegen:smoke-tests:services") +//include(":tests:codegen:smoke-tests") +//include(":tests:codegen:smoke-tests:services") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts new file mode 100644 index 00000000000..7fffa0f4fb0 --- /dev/null +++ b/tests/codegen/build.gradle.kts @@ -0,0 +1,80 @@ +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +//import aws.sdk.kotlin.gradle.kmp.kotlin + +plugins { + alias(libs.plugins.kotlin.jvm) apply false + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) apply false +// alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false +} + +val librares = libs + +subprojects { + apply(plugin = librares.plugins.kotlin.jvm.get().pluginId) + apply(plugin = librares.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) +// apply(plugin = librares.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) + + val codegen by configurations + + dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(librares.smithy.cli) + codegen(librares.smithy.model) + } + + tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = librares.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } + } + +// val optinAnnotations = listOf( +// "kotlin.RequiresOptIn", +// "aws.smithy.kotlin.runtime.InternalApi", +// "aws.sdk.kotlin.runtime.InternalSdkApi", +// ) +// kotlin.sourceSets.all { +// optinAnnotations.forEach { languageSettings.optIn(it) } +// } + +// kotlin.sourceSets.getByName("test") { +// smithyBuild.projections.forEach { +// kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) +// } +// } + + tasks.withType { + dependsOn(tasks.generateSmithyProjections) + kotlinOptions.allWarningsAsErrors = false + } + + val implementation by configurations + val testImplementation by configurations + val api by configurations + dependencies { + implementation(librares.kotlinx.coroutines.core) + + testImplementation(librares.kotlin.test) + testImplementation(librares.kotlin.test.junit5) + testImplementation(librares.kotlinx.coroutines.test) + testImplementation(librares.smithy.kotlin.smithy.test) + testImplementation(librares.smithy.kotlin.aws.signing.default) + testImplementation(librares.smithy.kotlin.telemetry.api) + + /* We have to manually add all the dependencies of the generated client(s). + Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a + publish to maven-local step at the cost of maintaining this set of dependencies manually. */ + implementation(librares.bundles.smithy.kotlin.service.client) + implementation(librares.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(librares.smithy.kotlin.aws.json.protocols) + implementation(librares.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + } +} diff --git a/tests/codegen/common.gradle.kts b/tests/codegen/common.gradle.kts deleted file mode 100644 index 7e3409bf865..00000000000 --- a/tests/codegen/common.gradle.kts +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir - -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} - -description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize - -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, -) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile -} - -val tests = listOf( // TODO: Don't commonize - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), -) - -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" -smithyBuild { - tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) - transforms = listOf( - """ - { - "name": "includeServices", - "args": { - "services": ["$testServiceShapeId"] - } - } - """, - ) - - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" - packageVersion = "1.0" - buildSettings { - generateFullProject = false - generateDefaultBuildFiles = false - optInAnnotations = listOf( - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", - ) - } - } - } - } -} - -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 8c6ae318868..2cb84424b19 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -7,10 +7,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} description = "Event stream codegen integration test suite" @@ -76,27 +72,12 @@ smithyBuild { } } -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - tasks.generateSmithyBuild { doFirst { tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } } } -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - val optinAnnotations = listOf( "kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi", @@ -112,12 +93,6 @@ kotlin.sourceSets.getByName("test") { } } -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - tasks.test { useJUnitPlatform() testLogging { @@ -129,30 +104,30 @@ tasks.test { } } -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} +//dependencies { +// +// implementation(libs.kotlinx.coroutines.core) +// +// testImplementation(libs.kotlin.test) +// testImplementation(libs.kotlin.test.junit5) +// testImplementation(libs.kotlinx.coroutines.test) +// +// testImplementation(libs.smithy.kotlin.smithy.test) +// testImplementation(libs.smithy.kotlin.aws.signing.default) +// testImplementation(libs.smithy.kotlin.telemetry.api) +// +// // have to manually add all the dependencies of the generated client(s) +// // doing it this way (as opposed to doing what we do for protocol-tests) allows +// // the tests to work without a publish to maven-local step at the cost of maintaining +// // this set of dependencies manually +// // <-- BEGIN GENERATED DEPENDENCY LIST --> +// implementation(libs.bundles.smithy.kotlin.service.client) +// implementation(libs.smithy.kotlin.aws.event.stream) +// implementation(project(":aws-runtime:aws-http")) +// implementation(libs.smithy.kotlin.aws.json.protocols) +// implementation(libs.smithy.kotlin.serde.json) +// api(project(":aws-runtime:aws-config")) +// api(project(":aws-runtime:aws-core")) +// api(project(":aws-runtime:aws-endpoint")) +// // <-- END GENERATED DEPENDENCY LIST --> +//} diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 7e3409bf865..a00f4205b00 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -7,10 +7,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize @@ -75,26 +71,12 @@ smithyBuild { } } -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - tasks.generateSmithyBuild { doFirst { tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } } } -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} val optinAnnotations = listOf( "kotlin.RequiresOptIn", @@ -112,12 +94,6 @@ kotlin.sourceSets.getByName("test") { } } -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - tasks.test { useJUnitPlatform() testLogging { @@ -129,30 +105,30 @@ tasks.test { } } -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} +//dependencies { +// +// implementation(libs.kotlinx.coroutines.core) +// +// testImplementation(libs.kotlin.test) +// testImplementation(libs.kotlin.test.junit5) +// testImplementation(libs.kotlinx.coroutines.test) +// +// testImplementation(libs.smithy.kotlin.smithy.test) +// testImplementation(libs.smithy.kotlin.aws.signing.default) +// testImplementation(libs.smithy.kotlin.telemetry.api) +// +// // have to manually add all the dependencies of the generated client(s) +// // doing it this way (as opposed to doing what we do for protocol-tests) allows +// // the tests to work without a publish to maven-local step at the cost of maintaining +// // this set of dependencies manually +// // <-- BEGIN GENERATED DEPENDENCY LIST --> +// implementation(libs.bundles.smithy.kotlin.service.client) +// implementation(libs.smithy.kotlin.aws.event.stream) +// implementation(project(":aws-runtime:aws-http")) +// implementation(libs.smithy.kotlin.aws.json.protocols) +// implementation(libs.smithy.kotlin.serde.json) +// api(project(":aws-runtime:aws-config")) +// api(project(":aws-runtime:aws-core")) +// api(project(":aws-runtime:aws-endpoint")) +// // <-- END GENERATED DEPENDENCY LIST --> +//} From 136c9dbfdeec9212101205937f7b07853474f8da Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 16:49:56 -0500 Subject: [PATCH 10/64] It works now --- settings.gradle.kts | 4 +- tests/codegen/build.gradle.kts | 128 ++++++++++-------- tests/codegen/event-stream/build.gradle.kts | 60 +------- .../kotlin/HttpEventStreamTests.kt | 0 .../kotlin/RpcEventStreamTests.kt | 0 tests/codegen/rules-engine/build.gradle.kts | 53 +------- tests/codegen/smoke-tests/build.gradle.kts | 38 ++---- .../smoke-tests/services/build.gradle.kts | 24 ---- .../kotlin/SmokeTestE2ETest.kt | 1 + .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 12 files changed, 90 insertions(+), 218 deletions(-) rename tests/codegen/event-stream/src/{test => commonTest}/kotlin/HttpEventStreamTests.kt (100%) rename tests/codegen/event-stream/src/{test => commonTest}/kotlin/RpcEventStreamTests.kt (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/kotlin/SmokeTestE2ETest.kt (96%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-success.smithy (100%) diff --git a/settings.gradle.kts b/settings.gradle.kts index 7a7751ebeb8..06a685d8512 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -56,8 +56,8 @@ include(":tests:codegen") include(":tests:codegen:event-stream") include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") -//include(":tests:codegen:smoke-tests") -//include(":tests:codegen:smoke-tests:services") +include(":tests:codegen:smoke-tests") +include(":tests:codegen:smoke-tests:services") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 7fffa0f4fb0..0fda97f3cd4 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -1,80 +1,92 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -//import aws.sdk.kotlin.gradle.kmp.kotlin plugins { - alias(libs.plugins.kotlin.jvm) apply false - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) apply false -// alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) + alias(libs.plugins.kotlin.multiplatform) } -val librares = libs +kotlin { + jvm() +} + +val libraries = libs subprojects { - apply(plugin = librares.plugins.kotlin.jvm.get().pluginId) - apply(plugin = librares.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) -// apply(plugin = librares.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) + apply(plugin = libraries.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) + apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) - val codegen by configurations + val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } + } - dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(librares.smithy.cli) - codegen(librares.smithy.model) + tasks.withType { + dependsOn(tasks.generateSmithyProjections) + kotlinOptions.allWarningsAsErrors = false } tasks.generateSmithyProjections { doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = librares.versions.smithy.kotlin.runtime.version.get() + // Ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libraries.versions.smithy.kotlin.runtime.version.get() System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) } } -// val optinAnnotations = listOf( -// "kotlin.RequiresOptIn", -// "aws.smithy.kotlin.runtime.InternalApi", -// "aws.sdk.kotlin.runtime.InternalSdkApi", -// ) -// kotlin.sourceSets.all { -// optinAnnotations.forEach { languageSettings.optIn(it) } -// } - -// kotlin.sourceSets.getByName("test") { -// smithyBuild.projections.forEach { -// kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) -// } -// } - - tasks.withType { - dependsOn(tasks.generateSmithyProjections) - kotlinOptions.allWarningsAsErrors = false - } - - val implementation by configurations - val testImplementation by configurations - val api by configurations + val codegen by configurations dependencies { - implementation(librares.kotlinx.coroutines.core) + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libraries.smithy.cli) + codegen(libraries.smithy.model) + } - testImplementation(librares.kotlin.test) - testImplementation(librares.kotlin.test.junit5) - testImplementation(librares.kotlinx.coroutines.test) - testImplementation(librares.smithy.kotlin.smithy.test) - testImplementation(librares.smithy.kotlin.aws.signing.default) - testImplementation(librares.smithy.kotlin.telemetry.api) + kotlin { + jvm() + sourceSets { + commonMain { + dependencies { + implementation(project(":codegen:aws-sdk-codegen")) + implementation(libraries.smithy.kotlin.codegen) - /* We have to manually add all the dependencies of the generated client(s). - Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a - publish to maven-local step at the cost of maintaining this set of dependencies manually. */ - implementation(librares.bundles.smithy.kotlin.service.client) - implementation(librares.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(librares.smithy.kotlin.aws.json.protocols) - implementation(librares.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) + /* We have to manually add all the dependencies of the generated client(s). + Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a + publish to maven-local step at the cost of maintaining this set of dependencies manually. */ + implementation(libraries.kotlinx.coroutines.core) + implementation(libraries.bundles.smithy.kotlin.service.client) + implementation(libraries.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libraries.smithy.kotlin.aws.json.protocols) + implementation(libraries.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + } + } + commonTest { + dependencies { + implementation(libraries.kotlin.test) + implementation(libraries.kotlinx.coroutines.test) + implementation(libraries.smithy.kotlin.smithy.test) + implementation(libraries.smithy.kotlin.aws.signing.default) + implementation(libraries.smithy.kotlin.telemetry.api) + } + } + jvmTest { + tasks.withType { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } + } + } + } } } diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 2cb84424b19..89f96fdf1ce 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -3,11 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir - description = "Event stream codegen integration test suite" data class Test( @@ -78,56 +76,12 @@ tasks.generateSmithyBuild { } } -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL +kotlin { + sourceSets { + commonTest { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } + } } } - -//dependencies { -// -// implementation(libs.kotlinx.coroutines.core) -// -// testImplementation(libs.kotlin.test) -// testImplementation(libs.kotlin.test.junit5) -// testImplementation(libs.kotlinx.coroutines.test) -// -// testImplementation(libs.smithy.kotlin.smithy.test) -// testImplementation(libs.smithy.kotlin.aws.signing.default) -// testImplementation(libs.smithy.kotlin.telemetry.api) -// -// // have to manually add all the dependencies of the generated client(s) -// // doing it this way (as opposed to doing what we do for protocol-tests) allows -// // the tests to work without a publish to maven-local step at the cost of maintaining -// // this set of dependencies manually -// // <-- BEGIN GENERATED DEPENDENCY LIST --> -// implementation(libs.bundles.smithy.kotlin.service.client) -// implementation(libs.smithy.kotlin.aws.event.stream) -// implementation(project(":aws-runtime:aws-http")) -// implementation(libs.smithy.kotlin.aws.json.protocols) -// implementation(libs.smithy.kotlin.serde.json) -// api(project(":aws-runtime:aws-config")) -// api(project(":aws-runtime:aws-core")) -// api(project(":aws-runtime:aws-endpoint")) -// // <-- END GENERATED DEPENDENCY LIST --> -//} diff --git a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt diff --git a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index a00f4205b00..43e49004278 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir @@ -77,58 +76,8 @@ tasks.generateSmithyBuild { } } - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { +kotlin.sourceSets.getByName("jvmTest") { smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -//dependencies { -// -// implementation(libs.kotlinx.coroutines.core) -// -// testImplementation(libs.kotlin.test) -// testImplementation(libs.kotlin.test.junit5) -// testImplementation(libs.kotlinx.coroutines.test) -// -// testImplementation(libs.smithy.kotlin.smithy.test) -// testImplementation(libs.smithy.kotlin.aws.signing.default) -// testImplementation(libs.smithy.kotlin.telemetry.api) -// -// // have to manually add all the dependencies of the generated client(s) -// // doing it this way (as opposed to doing what we do for protocol-tests) allows -// // the tests to work without a publish to maven-local step at the cost of maintaining -// // this set of dependencies manually -// // <-- BEGIN GENERATED DEPENDENCY LIST --> -// implementation(libs.bundles.smithy.kotlin.service.client) -// implementation(libs.smithy.kotlin.aws.event.stream) -// implementation(project(":aws-runtime:aws-http")) -// implementation(libs.smithy.kotlin.aws.json.protocols) -// implementation(libs.smithy.kotlin.serde.json) -// api(project(":aws-runtime:aws-config")) -// api(project(":aws-runtime:aws-core")) -// api(project(":aws-runtime:aws-endpoint")) -// // <-- END GENERATED DEPENDENCY LIST --> -//} diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 77a74afceba..7f4449c169e 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -9,9 +9,14 @@ import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath description = "Tests for smoke tests runners" -plugins { - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) - alias(libs.plugins.kotlin.jvm) +kotlin { + sourceSets { + jvmTest { + dependencies { + implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml + } + } + } } val projections = listOf( @@ -20,29 +25,12 @@ val projections = listOf( Projection("exceptionService", "smoke-tests-exception.smithy", "smithy.kotlin.traits#ExceptionService"), ) -configureProject() configureProjections() configureTasks() -fun configureProject() { - val codegen by configurations.getting - - dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) - - implementation(project(":codegen:aws-sdk-codegen")) - implementation(libs.smithy.kotlin.codegen) - - testImplementation(libs.kotlin.test) - testImplementation(gradleTestKit()) - } -} - fun configureProjections() { smithyBuild { - val pathToSmithyModels = "src/test/resources/" + val pathToSmithyModels = "src/jvmTest/resources/" this@Build_gradle.projections.forEach { projection -> projections.register(projection.name) { @@ -105,14 +93,6 @@ fun configureTasks() { tasks.withType { dependsOn(tasks.getByName("stageServices")) mustRunAfter(tasks.getByName("stageServices")) - - testLogging { - events("passed", "skipped", "failed") - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - showExceptions = true - showCauses = true - showStackTraces = true - } } } diff --git a/tests/codegen/smoke-tests/services/build.gradle.kts b/tests/codegen/smoke-tests/services/build.gradle.kts index 2238ac4b0d1..f48b8b7b5dd 100644 --- a/tests/codegen/smoke-tests/services/build.gradle.kts +++ b/tests/codegen/smoke-tests/services/build.gradle.kts @@ -1,33 +1,9 @@ -import aws.sdk.kotlin.gradle.kmp.kotlin - -plugins { - alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false -} - -// capture locally - scope issue with custom KMP plugin -val libraries = libs - subprojects { - apply { - plugin(libraries.plugins.kotlin.multiplatform.get().pluginId) - plugin(libraries.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) - } - kotlin { - - jvm() - sourceSets { - all { - languageSettings.optIn("kotlin.RequiresOptIn") - languageSettings.optIn("aws.smithy.kotlin.runtime.InternalApi") - languageSettings.optIn("aws.sdk.kotlin.runtime.InternalSdkApi") - } - commonMain { kotlin.srcDir("generated-src/main/kotlin") } - commonTest { kotlin.srcDir("generated-src/test/kotlin") } diff --git a/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt similarity index 96% rename from tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt index 61f38fe62c1..7dc76031929 100644 --- a/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt @@ -61,6 +61,7 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) + .withGradleVersion("8.5") // FIXME: Use lib.versions - or find a way to use `gradleTestKit()` val buildResult = if (expectingFailure) task.buildAndFail() else task.build() diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy From cb4ab1269b49ccf86c3c4e0cecaff64e11867f4c Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 20:42:19 -0500 Subject: [PATCH 11/64] Clean up build scripts --- tests/codegen/event-stream/build.gradle.kts | 95 +++++++++++-------- ...ent-stream-initial-request-response.smithy | 0 .../event-stream-model-template.smithy | 6 +- tests/codegen/rules-engine/build.gradle.kts | 71 ++++---------- .../operation-context-params.smithy | 0 tests/codegen/smoke-tests/build.gradle.kts | 73 +++++++------- 6 files changed, 114 insertions(+), 131 deletions(-) rename tests/codegen/event-stream/{ => src/commonTest/resources}/event-stream-initial-request-response.smithy (100%) rename tests/codegen/event-stream/{ => src/commonTest/resources}/event-stream-model-template.smithy (95%) rename tests/codegen/rules-engine/{ => src/commonTest/resources}/operation-context-params.smithy (100%) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 89f96fdf1ce..e9fbb56f7b6 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -6,56 +6,57 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -description = "Event stream codegen integration test suite" +description = "AWS SDK for Kotlin codegen event stream integration test suite" -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, +apply(from = rootProject.file("buildSrc/shared.gradle.kts")) + +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", ) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile } val tests = listOf( - Test("restJson1", "restJson1", file("event-stream-model-template.smithy")), - Test("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), + CodegenTest( + "restJson1", + Model("event-stream-model-template.smithy"), + "aws.sdk.kotlin.test#TestService", + "restJson1" + ), + CodegenTest( + "awsJson11", + Model("event-stream-initial-request-response.smithy"), + "aws.sdk.kotlin.test#TestService", + "awsJson1_1" + ), ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) transforms = listOf( """ { "name": "includeServices", "args": { - "services": ["$testServiceShapeId"] + "services": ["${test.serviceShapeId}"] } } """, ) - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -70,18 +71,36 @@ smithyBuild { } } -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - kotlin { sourceSets { - commonTest { + commonTest { // TODO: CHANGE TO JUST TEST WHEN NON-KMPing the project smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } } } + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test) } + } +} + +fun fillInModel(test: CodegenTest) { + val modelFile = test.model.file + val model = modelFile.readText() + + val opTraits = + when (test.protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + + val replaced = model + .replace("{protocol-name}", test.protocolName!!) + .replace("{op-traits}", opTraits) + + modelFile.parentFile.mkdirs() + modelFile.writeText(replaced) +} diff --git a/tests/codegen/event-stream/event-stream-initial-request-response.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy similarity index 100% rename from tests/codegen/event-stream/event-stream-initial-request-response.smithy rename to tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy diff --git a/tests/codegen/event-stream/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy similarity index 95% rename from tests/codegen/event-stream/event-stream-model-template.smithy rename to tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 43e49004278..70d57ec3c8c 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -4,58 +4,35 @@ */ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +description = "AWS SDK for Kotlin codegen rules engine integration test suite" -description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", ) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile } -val tests = listOf( // TODO: Don't commonize - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +val tests = listOf( + CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) - transforms = listOf( - """ - { - "name": "includeServices", - "args": { - "services": ["$testServiceShapeId"] - } - } - """, - ) - + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -69,15 +46,3 @@ smithyBuild { } } } - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -kotlin.sourceSets.getByName("jvmTest") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy similarity index 100% rename from tests/codegen/rules-engine/operation-context-params.smithy rename to tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 7f4449c169e..727a0c56fb5 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -7,7 +7,7 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath -description = "Tests for smoke tests runners" +description = "AWS SDK for Kotlin codegen smoke tests integration test suite" kotlin { sourceSets { @@ -19,10 +19,25 @@ kotlin { } } -val projections = listOf( - Projection("successService", "smoke-tests-success.smithy", "smithy.kotlin.traits#SuccessService"), - Projection("failureService", "smoke-tests-failure.smithy", "smithy.kotlin.traits#FailureService"), - Projection("exceptionService", "smoke-tests-exception.smithy", "smithy.kotlin.traits#ExceptionService"), +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +data class Model( + val fileName: String, + val path: String = "src/jvmTest/resources/", +) { + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile +} + +val tests = listOf( + CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), + CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), + CodegenTest("exceptionService", Model("smoke-tests-exception.smithy"), "smithy.kotlin.traits#ExceptionService"), ) configureProjections() @@ -30,14 +45,12 @@ configureTasks() fun configureProjections() { smithyBuild { - val pathToSmithyModels = "src/jvmTest/resources/" - - this@Build_gradle.projections.forEach { projection -> - projections.register(projection.name) { - imports = listOf(layout.projectDirectory.file(pathToSmithyModels + projection.modelFile).asFile.absolutePath) + this@Build_gradle.tests.forEach { test -> + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) smithyKotlinPlugin { - serviceShapeId = projection.serviceShapeId - packageName = "aws.sdk.kotlin.test" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -51,26 +64,21 @@ fun configureProjections() { } } } - - tasks.withType { - dependsOn(tasks.generateSmithyProjections) - kotlinOptions.allWarningsAsErrors = false - } } fun configureTasks() { tasks.register("stageServices") { dependsOn(tasks.generateSmithyProjections) - doLast { - this@Build_gradle.projections.forEach { projection -> - val projectionPath = smithyBuild.smithyKotlinProjectionPath(projection.name).get() - val destinationPath = layout.projectDirectory.asFile.absolutePath + "/services/${projection.name}" + this@Build_gradle.tests.forEach { test -> + val projectionPath = smithyBuild.smithyKotlinProjectionPath(test.name).get() + val destinationPath = layout.projectDirectory.asFile.absolutePath + "/services/${test.name}" copy { from("$projectionPath/src") into("$destinationPath/generated-src") } + copy { from("$projectionPath/build.gradle.kts") into(destinationPath) @@ -79,28 +87,19 @@ fun configureTasks() { } } + tasks.withType { + dependsOn(tasks.getByName("stageServices")) + mustRunAfter(tasks.getByName("stageServices")) + } + tasks.build { dependsOn(tasks.getByName("stageServices")) mustRunAfter(tasks.getByName("stageServices")) } tasks.clean { - this@Build_gradle.projections.forEach { projection -> - delete("services/${projection.name}") + this@Build_gradle.tests.forEach { test -> + delete("services/${test.name}") } } - - tasks.withType { - dependsOn(tasks.getByName("stageServices")) - mustRunAfter(tasks.getByName("stageServices")) - } } - -/** - * Holds metadata about a smithy projection - */ -data class Projection( - val name: String, - val modelFile: String, - val serviceShapeId: String, -) From 41ad1b4d2272ff2767edd1c223bbd4cd4811dab1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 21:50:01 -0500 Subject: [PATCH 12/64] Commonize some more --- .../.kotlin/errors/errors-1731120437202.log | 4 +++ buildSrc/build.gradle.kts | 7 ++++++ buildSrc/settings.gradle.kts | 7 ++++++ .../src/main/kotlin/shared/CodegenTest.kt | 19 ++++++++++++++ tests/codegen/event-stream/build.gradle.kts | 25 ++++--------------- tests/codegen/rules-engine/build.gradle.kts | 21 +++------------- tests/codegen/smoke-tests/build.gradle.kts | 21 +++------------- .../kotlin/SmokeTestE2ETest.kt | 0 .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 11 files changed, 50 insertions(+), 54 deletions(-) create mode 100644 buildSrc/.kotlin/errors/errors-1731120437202.log create mode 100644 buildSrc/build.gradle.kts create mode 100644 buildSrc/settings.gradle.kts create mode 100644 buildSrc/src/main/kotlin/shared/CodegenTest.kt rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/kotlin/SmokeTestE2ETest.kt (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-success.smithy (100%) diff --git a/buildSrc/.kotlin/errors/errors-1731120437202.log b/buildSrc/.kotlin/errors/errors-1731120437202.log new file mode 100644 index 00000000000..1219b509f09 --- /dev/null +++ b/buildSrc/.kotlin/errors/errors-1731120437202.log @@ -0,0 +1,4 @@ +kotlin version: 2.0.21 +error message: The daemon has terminated unexpectedly on startup attempt #1 with error code: 0. The daemon process output: + 1. Kotlin compile daemon is ready + diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000000..aadc709bb1e --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + alias(libs.plugins.kotlin.jvm) +} + +repositories { + mavenCentral() +} \ No newline at end of file diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts new file mode 100644 index 00000000000..fa8bc749264 --- /dev/null +++ b/buildSrc/settings.gradle.kts @@ -0,0 +1,7 @@ +dependencyResolutionManagement { + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/shared/CodegenTest.kt new file mode 100644 index 00000000000..8bb596db03d --- /dev/null +++ b/buildSrc/src/main/kotlin/shared/CodegenTest.kt @@ -0,0 +1,19 @@ +package shared + +/** + * An AWS SDK for Kotlin codegen test + */ +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +/** + * A smithy model file + */ +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", +) \ No newline at end of file diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index e9fbb56f7b6..0bf44dbd89f 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -5,25 +5,10 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen event stream integration test suite" - -apply(from = rootProject.file("buildSrc/shared.gradle.kts")) - -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/commonTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} +description = "AWS SDK for Kotlin's event stream codegen test suite" val tests = listOf( CodegenTest( @@ -43,7 +28,7 @@ val tests = listOf( smithyBuild { tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) transforms = listOf( """ { @@ -88,7 +73,7 @@ tasks.generateSmithyBuild { } fun fillInModel(test: CodegenTest) { - val modelFile = test.model.file + val modelFile = layout.projectDirectory.file(test.model.path + test.model.fileName).asFile val model = modelFile.readText() val opTraits = diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 70d57ec3c8c..c1dd1571cb6 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -4,23 +4,10 @@ */ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen rules engine integration test suite" - -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/commonTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} +description = "AWS SDK for Kotlin's rules engine codegen test suite" val tests = listOf( CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") @@ -29,7 +16,7 @@ val tests = listOf( smithyBuild { tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) smithyKotlinPlugin { serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 727a0c56fb5..9a5dc15ee07 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -6,8 +6,10 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen smoke tests integration test suite" +description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { @@ -19,21 +21,6 @@ kotlin { } } -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/jvmTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} - val tests = listOf( CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), @@ -47,7 +34,7 @@ fun configureProjections() { smithyBuild { this@Build_gradle.tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) smithyKotlinPlugin { serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" diff --git a/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy From 1a2d9899c7267fa8d2a919e6b0f5013580d1f800 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 22:29:58 -0500 Subject: [PATCH 13/64] Checkpoint 3 --- tests/codegen/event-stream/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 0bf44dbd89f..1b737eb930b 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -58,7 +58,7 @@ smithyBuild { kotlin { sourceSets { - commonTest { // TODO: CHANGE TO JUST TEST WHEN NON-KMPing the project + commonTest { smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } From 4d24a24bb12561b8c74d7b807727ffa991e38322 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:10:08 -0500 Subject: [PATCH 14/64] Remove rules engine codegen tests --- tests/codegen/rules-engine/build.gradle.kts | 35 ----------- .../resources/operation-context-params.smithy | 58 ------------------- 2 files changed, 93 deletions(-) delete mode 100644 tests/codegen/rules-engine/build.gradle.kts delete mode 100644 tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts deleted file mode 100644 index c1dd1571cb6..00000000000 --- a/tests/codegen/rules-engine/build.gradle.kts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import shared.CodegenTest -import shared.Model - -description = "AWS SDK for Kotlin's rules engine codegen test suite" - -val tests = listOf( - CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") -) - -smithyBuild { - tests.forEach { test -> - projections.register(test.name) { - imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) - smithyKotlinPlugin { - serviceShapeId = test.serviceShapeId - packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" - packageVersion = "1.0" - buildSettings { - generateFullProject = false - generateDefaultBuildFiles = false - optInAnnotations = listOf( - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", - ) - } - } - } - } -} diff --git a/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy deleted file mode 100644 index c63a2756e39..00000000000 --- a/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy +++ /dev/null @@ -1,58 +0,0 @@ -$version: "2.0" -namespace aws.sdk.kotlin.test - -use aws.protocols#awsJson1_0 -use smithy.rules#operationContextParams -use smithy.rules#endpointRuleSet -use aws.api#service - -@awsJson1_0 -@service(sdkId: "OperationContextParamsTest") -@endpointRuleSet( - version: "1.0", - parameters: { - "ObjectKeys": { - "type": "stringArray", - "documentation": "A string array.", - "required": true - } - }, - rules: [ - { - "type": "endpoint", - "conditions": [], - "endpoint": { - "url": "https://static.endpoint" - } - } - ] -) -service TestService { - operations: [DeleteObjects], - version: "1" -} - -@operationContextParams( - ObjectKeys: { - path: "Delete.Objects[*].[Key][]" - } -) -operation DeleteObjects { - input: DeleteObjectsRequest -} - -structure DeleteObjectsRequest { - Delete: Delete -} - -structure Delete { - Objects: ObjectIdentifierList -} - -list ObjectIdentifierList { - member: ObjectIdentifier -} - -structure ObjectIdentifier { - Key: String -} From c514cbaf3c68eaeb83bd304808bc2525a4256204 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:18:55 -0500 Subject: [PATCH 15/64] Setup checksums codegen tests --- settings.gradle.kts | 1 + tests/codegen/checksums/build.gradle.kts | 31 +++++ .../src/commonTest/resources/checksums.smithy | 115 ++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 tests/codegen/checksums/build.gradle.kts create mode 100644 tests/codegen/checksums/src/commonTest/resources/checksums.smithy diff --git a/settings.gradle.kts b/settings.gradle.kts index 06a685d8512..dc2418bfe1d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -58,6 +58,7 @@ include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") include(":tests:codegen:smoke-tests") include(":tests:codegen:smoke-tests:services") +include(":tests:codegen:checksums") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts new file mode 100644 index 00000000000..829c453ad1c --- /dev/null +++ b/tests/codegen/checksums/build.gradle.kts @@ -0,0 +1,31 @@ + +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import shared.CodegenTest +import shared.Model + +description = "AWS SDK for Kotlin's checksums codegen test suite" + +val tests = listOf( + CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), +) + +smithyBuild { + this@Build_gradle.tests.forEach { test -> + projections.register(test.name) { + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) + smithyKotlinPlugin { + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/resources/checksums.smithy b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy new file mode 100644 index 00000000000..01feff05e4a --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy @@ -0,0 +1,115 @@ +$version: "2" +namespace aws.sdk.kotlin.test + +use aws.api#service +use aws.auth#sigv4 +use aws.protocols#httpChecksum +use aws.protocols#restJson1 +use smithy.rules#endpointRuleSet + +@service(sdkId: "dontcare") +@restJson1 +@sigv4(name: "dontcare") +@auth([sigv4]) +@endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } +}) +service TestService { + version: "2023-01-01", + operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] +} + +@http(uri: "/HttpChecksumOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumOperation { + input: SomeInput, + output: SomeOutput +} + +@input +structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + + @httpPayload + @required + body: Blob +} + +@output +structure SomeOutput {} + +@http(uri: "/HttpChecksumStreamingOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumStreamingOperation { + input: SomeStreamingInput, + output: SomeStreamingOutput +} + +@streaming +blob StreamingBlob + +@input +structure SomeStreamingInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: StreamingBlob +} + +@output +structure SomeStreamingOutput {} + +enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 +} + +enum ValidationMode { + ENABLED +} \ No newline at end of file From 27963179326cdbe0f3fd544fce745acfc45fb4c2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:26:19 -0500 Subject: [PATCH 16/64] Update smithy IDL version, codegen tests pass now --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d003593d62b..b283ae0aaa2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ smithy-kotlin-runtime-version = "1.3.20" smithy-kotlin-codegen-version = "0.33.20" # codegen -smithy-version = "1.51.0" +smithy-version = "1.52.0" # testing ddb-local-version = "2.5.2" From 6d5324377804df03e4c82abee588df10dd5686ca Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:44:39 -0500 Subject: [PATCH 17/64] Setup dummy unit tests --- tests/codegen/checksums/build.gradle.kts | 14 +++++++++++++- .../src/commonTest/kotlin/ChecksumConfigTests.kt | 9 +++++++++ .../src/commonTest/kotlin/ChecksumResponseTests.kt | 9 +++++++++ .../src/commonTest/kotlin/RequestChecksumTests.kt | 9 +++++++++ .../kotlin/StreamingRequestChecksumTests.kt | 9 +++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 829c453ad1c..aa1dd9a5413 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -1,5 +1,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir import shared.CodegenTest import shared.Model @@ -28,4 +29,15 @@ smithyBuild { } } } -} \ No newline at end of file +} + +// TODO: Commonize this to other codegen tests +kotlin { + sourceSets { + commonTest { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt new file mode 100644 index 00000000000..09447ca6cd9 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class ChecksumConfigTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt new file mode 100644 index 00000000000..a261643a544 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class ChecksumResponseTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt new file mode 100644 index 00000000000..6062830981c --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class RequestChecksumTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt new file mode 100644 index 00000000000..9c7d178b85a --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class StreamingRequestChecksumTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file From 40cfb17fce579de33a9649e9bb08bdf58b560c99 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 23:14:45 -0500 Subject: [PATCH 18/64] Basic setup for client config tests --- tests/codegen/checksums/build.gradle.kts | 3 +- ...sumConfigTests.kt => ClientConfigTests.kt} | 2 +- .../commonTest/resources/client-config.smithy | 55 +++++++++++++++++++ .../{checksums.smithy => kitchen-sink.smithy} | 0 4 files changed, 58 insertions(+), 2 deletions(-) rename tests/codegen/checksums/src/commonTest/kotlin/{ChecksumConfigTests.kt => ClientConfigTests.kt} (84%) create mode 100644 tests/codegen/checksums/src/commonTest/resources/client-config.smithy rename tests/codegen/checksums/src/commonTest/resources/{checksums.smithy => kitchen-sink.smithy} (100%) diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index aa1dd9a5413..159f70e3c7f 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,7 +7,8 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" val tests = listOf( - CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt similarity index 84% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt rename to tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index 09447ca6cd9..e8b26deeaba 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,7 +1,7 @@ import aws.sdk.kotlin.test.checksums.* import kotlin.test.Test -class ChecksumConfigTests { +class ClientConfigTests { @Test fun test() { TestClient {}.use { client -> client.close() } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy new file mode 100644 index 00000000000..0782d0336c3 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -0,0 +1,55 @@ +$version: "2" +namespace aws.sdk.kotlin.test + +use aws.api#service +use aws.auth#sigv4 +use aws.protocols#httpChecksum +use aws.protocols#restJson1 +use smithy.rules#endpointRuleSet + +@service(sdkId: "dontcare") +@restJson1 +@sigv4(name: "dontcare") +@auth([sigv4]) +@endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } +}) +service ClientConfigTestService { + version: "2023-01-01", + operations: [ChecksumsNotRequiredOperation] +} + +@httpChecksum( + requestChecksumRequired: false, + requestAlgorithmMember: "checksumAlgorithm", +) +@http(method: "POST", uri: "/test-eventstream", code: 200) +operation ChecksumsNotRequiredOperation { + input: SomeInput, + output: SomeOutput +} + +@input +structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpPayload + @required + body: String +} + +@output +structure SomeOutput {} + +enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/resources/checksums.smithy b/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy similarity index 100% rename from tests/codegen/checksums/src/commonTest/resources/checksums.smithy rename to tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy From 410c08578ffd2b7f8bf94738dc0ab3a9d1a4c9ef Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 00:43:57 -0500 Subject: [PATCH 19/64] Added requestChecksumCalculation config option --- aws-runtime/aws-config/api/aws-config.api | 7 +++++ .../config/AbstractAwsSdkClientFactory.kt | 7 +++++ .../kotlin/runtime/config/AwsSdkSetting.kt | 6 ++++ .../ResolveRequestChecksumCalculation.kt | 28 +++++++++++++++++++ .../runtime/config/profile/AwsProfile.kt | 7 +++++ 5 files changed, 55 insertions(+) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 05589280306..4b05bd4e8d9 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -243,6 +243,7 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSetting { public final fun getAwsMaxAttempts ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsProfile ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRegion ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; + public final fun getAwsRequestChecksumCalculation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestMinCompressionSizeBytes ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRetryMode ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleArn ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; @@ -260,6 +261,11 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSettingKt { public static final fun resolveEndpointUrl (Laws/sdk/kotlin/runtime/config/AwsSdkSetting;Laws/smithy/kotlin/runtime/util/PlatformProvider;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/net/url/Url; } +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculationKt { + public static final fun resolveRequestChecksumCalculation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/config/compression/RequestCompressionResolversKt { public static final fun resolveDisableRequestCompression (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveDisableRequestCompression$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; @@ -467,6 +473,7 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt { public static synthetic fun getLongOrNull$default (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/Long; public static final fun getMaxAttempts (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Integer; public static final fun getRegion (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; + public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestMinCompressionSizeBytes (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Long; public static final fun getRetryMode (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RetryMode; public static final fun getRoleArn (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index ed5f4b0cd99..662efd8616b 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.runtime.config import aws.sdk.kotlin.runtime.client.AwsSdkClientConfig +import aws.sdk.kotlin.runtime.config.checksums.resolveRequestChecksumCalculation import aws.sdk.kotlin.runtime.config.compression.resolveDisableRequestCompression import aws.sdk.kotlin.runtime.config.compression.resolveRequestMinCompressionSizeBytes import aws.sdk.kotlin.runtime.config.endpoints.resolveUseDualStack @@ -22,6 +23,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.SigV4aClientConfig import aws.smithy.kotlin.runtime.client.* import aws.smithy.kotlin.runtime.client.config.ClientSettings import aws.smithy.kotlin.runtime.client.config.CompressionClientConfig +import aws.smithy.kotlin.runtime.client.config.HttpChecksumClientConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.telemetry.TelemetryConfig import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider @@ -94,6 +96,11 @@ public abstract class AbstractAwsSdkClientFactory< config.sigV4aSigningRegionSet ?: resolveSigV4aSigningRegionSet(platform, profile) } + if (config is HttpChecksumClientConfig.Builder) { + config.requestChecksumCalculation = + config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + } + finalizeConfig(builder) finalizeEnvironmentalConfig(builder, sharedConfig, profile) } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index 4233773f7b8..dff28de934d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -208,6 +208,12 @@ public object AwsSdkSetting { */ public val AwsSigV4aSigningRegionSet: EnvironmentSetting = strEnvSetting("aws.sigV4aSigningRegionSet", "AWS_SIGV4A_SIGNING_REGION_SET") + + /** + * todo + */ + public val AwsRequestChecksumCalculation: EnvironmentSetting = + strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") } /** diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt new file mode 100644 index 00000000000..13f5ded6837 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -0,0 +1,28 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider +import java.util.* + +/** + * todo + */ +@InternalSdkApi +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): RequestChecksumCalculation? { + AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { + try { + return RequestChecksumCalculation.valueOf(it.uppercase(Locale.getDefault())) + } catch (_: IllegalArgumentException) { + throw ConfigurationException("'$it' is not a valid value for request checksum calculation. Valid values are: 'WHEN_SUPPORTED' & 'WHEN_REQUIRED'") + } + } + return null +} + diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 8a4e31016c5..487afda2da3 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -167,6 +167,13 @@ public val AwsProfile.requestMinCompressionSizeBytes: Long? public val AwsProfile.sigV4aSigningRegionSet: String? get() = getOrNull("sigv4a_signing_region_set") +/** + * todo + */ +@InternalSdkApi +public val AwsProfile.requestChecksumCalculation: String? + get() = getOrNull("request_checksum_calculation") + /** * Parse a config value as a boolean, ignoring case. */ From 472fae966861c037582591c750d30fa3db9777a3 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 01:27:21 -0500 Subject: [PATCH 20/64] RequestChecksum not required client config tests --- tests/codegen/checksums/build.gradle.kts | 10 +++ .../commonTest/kotlin/ClientConfigTests.kt | 64 +++++++++++++++++-- .../commonTest/resources/client-config.smithy | 2 +- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 159f70e3c7f..b094022daac 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -6,6 +6,16 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" +kotlin { + sourceSets { + commonTest { + dependencies { + implementation(libs.smithy.kotlin.http.test) + } + } + } +} + val tests = listOf( CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index e8b26deeaba..ef4475f9836 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,9 +1,65 @@ -import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.clientconfig.* +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import kotlinx.coroutines.runBlocking import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import aws.smithy.kotlin.runtime.httptest.TestEngine class ClientConfigTests { @Test - fun test() { - TestClient {}.use { client -> client.close() } + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertTrue(testInterceptor.containsChecksum) + } + + @Test + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse(testInterceptor.containsChecksum) + } +} + +private class TestInterceptor : HttpInterceptor { + var containsChecksum = false + + override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { + containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") + return context.protocolRequest } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 0782d0336c3..7323601f050 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -27,7 +27,7 @@ service ClientConfigTestService { requestChecksumRequired: false, requestAlgorithmMember: "checksumAlgorithm", ) -@http(method: "POST", uri: "/test-eventstream", code: 200) +@http(method: "POST", uri: "/test-checksums", code: 200) operation ChecksumsNotRequiredOperation { input: SomeInput, output: SomeOutput From e9bf8e5be22dcba69163cc8e6dbc4a419e5ff6c7 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 01:48:07 -0500 Subject: [PATCH 21/64] RequestChecksum required client config tests --- .../commonTest/kotlin/ClientConfigTests.kt | 113 +++++++++++++----- .../commonTest/resources/client-config.smithy | 25 +++- 2 files changed, 107 insertions(+), 31 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index ef4475f9836..ba456125d8a 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -10,48 +10,101 @@ import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue import aws.smithy.kotlin.runtime.httptest.TestEngine +import org.junit.jupiter.api.Nested +import kotlin.test.Ignore class ClientConfigTests { - @Test - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + @Nested + inner class RequestChecksumNotRequired { + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() - ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } } + + assertTrue(testInterceptor.containsChecksum) } - assertTrue(testInterceptor.containsChecksum) + @Test + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse(testInterceptor.containsChecksum) + } } - @Test - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + @Nested + inner class RequestChecksumRequired { + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() - ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } } + + assertTrue(testInterceptor.containsChecksum) } - assertFalse(testInterceptor.containsChecksum) + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue(testInterceptor.containsChecksum) + } } } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 7323601f050..20db2263102 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -20,7 +20,7 @@ use smithy.rules#endpointRuleSet }) service ClientConfigTestService { version: "2023-01-01", - operations: [ChecksumsNotRequiredOperation] + operations: [ChecksumsNotRequiredOperation, ChecksumsRequiredOperation] } @httpChecksum( @@ -33,6 +33,16 @@ operation ChecksumsNotRequiredOperation { output: SomeOutput } +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", +) +@http(method: "POST", uri: "/test-checksums-2", code: 200) +operation ChecksumsRequiredOperation { + input: AnotherInput, + output: AnotherOutput +} + @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -46,6 +56,19 @@ structure SomeInput { @output structure SomeOutput {} +@input +structure AnotherInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpPayload + @required + body: String +} + +@output +structure AnotherOutput {} + enum ChecksumAlgorithm { CRC32 CRC32C From 5133a60c45d1281e0ae8fe5cd60a9d5beee930b3 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 02:51:04 -0500 Subject: [PATCH 22/64] Added responseChecksumValidation --- aws-runtime/aws-config/api/aws-config.api | 7 +++++ .../config/AbstractAwsSdkClientFactory.kt | 4 +++ .../kotlin/runtime/config/AwsSdkSetting.kt | 6 ++++ .../ResolveRequestChecksumCalculation.kt | 11 +++---- .../ResolveResponseChecksumValidation.kt | 29 +++++++++++++++++++ .../runtime/config/profile/AwsProfile.kt | 7 +++++ buildSrc/build.gradle.kts | 2 +- buildSrc/settings.gradle.kts | 2 +- .../src/main/kotlin/shared/CodegenTest.kt | 2 +- .../kotlin/ChecksumResponseTests.kt | 2 +- .../commonTest/kotlin/ClientConfigTests.kt | 16 +++++----- .../commonTest/kotlin/RequestChecksumTests.kt | 2 +- .../kotlin/StreamingRequestChecksumTests.kt | 2 +- tests/codegen/event-stream/build.gradle.kts | 4 +-- 14 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 4b05bd4e8d9..6a9be2adba5 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -245,6 +245,7 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSetting { public final fun getAwsRegion ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestChecksumCalculation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestMinCompressionSizeBytes ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; + public final fun getAwsResponseChecksumValidation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRetryMode ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleArn ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleSessionName ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; @@ -266,6 +267,11 @@ public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksu public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; } +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidationKt { + public static final fun resolveResponseChecksumValidation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public static synthetic fun resolveResponseChecksumValidation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/config/compression/RequestCompressionResolversKt { public static final fun resolveDisableRequestCompression (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveDisableRequestCompression$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; @@ -475,6 +481,7 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt { public static final fun getRegion (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestMinCompressionSizeBytes (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Long; + public static final fun getResponseChecksumValidation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRetryMode (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RetryMode; public static final fun getRoleArn (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getSdkUserAgentAppId (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 662efd8616b..190f41a0b12 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -7,6 +7,7 @@ package aws.sdk.kotlin.runtime.config import aws.sdk.kotlin.runtime.client.AwsSdkClientConfig import aws.sdk.kotlin.runtime.config.checksums.resolveRequestChecksumCalculation +import aws.sdk.kotlin.runtime.config.checksums.resolveResponseChecksumValidation import aws.sdk.kotlin.runtime.config.compression.resolveDisableRequestCompression import aws.sdk.kotlin.runtime.config.compression.resolveRequestMinCompressionSizeBytes import aws.sdk.kotlin.runtime.config.endpoints.resolveUseDualStack @@ -99,6 +100,9 @@ public abstract class AbstractAwsSdkClientFactory< if (config is HttpChecksumClientConfig.Builder) { config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + + config.responseChecksumValidation = + config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } finalizeConfig(builder) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index dff28de934d..9fe989f58ed 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -214,6 +214,12 @@ public object AwsSdkSetting { */ public val AwsRequestChecksumCalculation: EnvironmentSetting = strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") + + /** + * todo + */ + public val AwsResponseChecksumValidation: EnvironmentSetting = + strEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") } /** diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index 13f5ded6837..a06d61103c2 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -5,7 +5,7 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider @@ -15,14 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): RequestChecksumCalculation? { +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { try { - return RequestChecksumCalculation.valueOf(it.uppercase(Locale.getDefault())) + return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) } catch (_: IllegalArgumentException) { - throw ConfigurationException("'$it' is not a valid value for request checksum calculation. Valid values are: 'WHEN_SUPPORTED' & 'WHEN_REQUIRED'") + throw ConfigurationException( + "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + ) } } return null } - diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt new file mode 100644 index 00000000000..446480d527e --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -0,0 +1,29 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider +import java.util.* + +/** + * todo + */ +@InternalSdkApi +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { + AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation?.let { + try { + return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) + } catch (_: IllegalArgumentException) { + throw ConfigurationException( + "'$it' is not a valid value for response checksum validation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + ) + } + } + return null +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 487afda2da3..472228604a9 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -174,6 +174,13 @@ public val AwsProfile.sigV4aSigningRegionSet: String? public val AwsProfile.requestChecksumCalculation: String? get() = getOrNull("request_checksum_calculation") +/** + * todo + */ +@InternalSdkApi +public val AwsProfile.responseChecksumValidation: String? + get() = getOrNull("response_checksum_validation") + /** * Parse a config value as a boolean, ignoring case. */ diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index aadc709bb1e..dccd6941e76 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -4,4 +4,4 @@ plugins { repositories { mavenCentral() -} \ No newline at end of file +} diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts index fa8bc749264..b5a0fabf664 100644 --- a/buildSrc/settings.gradle.kts +++ b/buildSrc/settings.gradle.kts @@ -4,4 +4,4 @@ dependencyResolutionManagement { from(files("../gradle/libs.versions.toml")) } } -} \ No newline at end of file +} diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/shared/CodegenTest.kt index 8bb596db03d..f9a03d715e9 100644 --- a/buildSrc/src/main/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/shared/CodegenTest.kt @@ -16,4 +16,4 @@ data class CodegenTest( data class Model( val fileName: String, val path: String = "src/commonTest/resources/", -) \ No newline at end of file +) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt index a261643a544..fd3c8965828 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt @@ -6,4 +6,4 @@ class ChecksumResponseTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index ba456125d8a..d98a3f6a6ba 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -2,16 +2,16 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Nested +import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue -import aws.smithy.kotlin.runtime.httptest.TestEngine -import org.junit.jupiter.api.Nested -import kotlin.test.Ignore class ClientConfigTests { @Nested @@ -22,7 +22,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -43,7 +43,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -68,7 +68,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -90,7 +90,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt index 6062830981c..15d8764dc2c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt @@ -6,4 +6,4 @@ class RequestChecksumTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt index 9c7d178b85a..25d8210cf10 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt @@ -6,4 +6,4 @@ class StreamingRequestChecksumTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 1b737eb930b..71edc4f41fb 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -15,13 +15,13 @@ val tests = listOf( "restJson1", Model("event-stream-model-template.smithy"), "aws.sdk.kotlin.test#TestService", - "restJson1" + "restJson1", ), CodegenTest( "awsJson11", Model("event-stream-initial-request-response.smithy"), "aws.sdk.kotlin.test#TestService", - "awsJson1_1" + "awsJson1_1", ), ) From 4ce5da7788867e69db8829e07f102d95e952f1ee Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 11:34:26 -0500 Subject: [PATCH 23/64] TODO ResponseChecksumValidation tests --- .../checksums/src/commonTest/kotlin/ClientConfigTests.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index d98a3f6a6ba..3770cfd6ee5 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -106,6 +106,11 @@ class ClientConfigTests { assertTrue(testInterceptor.containsChecksum) } } + + @Nested + inner class ResponseChecksumValidation { + // TODO + } } private class TestInterceptor : HttpInterceptor { From da31deb21a4b33fa0fb5eebbdcaffb6deed7e0ab Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:16:45 -0500 Subject: [PATCH 24/64] Added ResponseChecksumValidation codegen tests --- .../commonTest/kotlin/ClientConfigTests.kt | 151 +++++++++++++++--- .../commonTest/resources/client-config.smithy | 31 ++-- 2 files changed, 152 insertions(+), 30 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index 3770cfd6ee5..aeebef9d6f4 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,11 +1,16 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* +import aws.sdk.kotlin.test.clientconfig.model.ValidationMode import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Nested import kotlin.test.Ignore @@ -18,12 +23,12 @@ class ClientConfigTests { inner class RequestChecksumNotRequired { @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -35,16 +40,16 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } @Test - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -56,7 +61,7 @@ class ClientConfigTests { } } - assertFalse(testInterceptor.containsChecksum) + assertFalse(requestInterceptor.containsChecksum) } } @@ -64,12 +69,12 @@ class ClientConfigTests { inner class RequestChecksumRequired { @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -81,17 +86,17 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -103,21 +108,129 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } } @Nested inner class ResponseChecksumValidation { - // TODO + @Test + @Ignore // todo - unignore + fun responseChecksumValidationResponseChecksumValidationWhenSupported(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertTrue(responseChecksumValidated) + } + + @Test + fun responseChecksumValidationResponseChecksumValidationWhenRequired(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertFalse(responseChecksumValidated) + } + + @Test + @Ignore // todo - unignore + fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeMember(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertTrue(responseChecksumValidated) + } } } -private class TestInterceptor : HttpInterceptor { +private class RequestInterceptor : HttpInterceptor { var containsChecksum = false override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { - containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") + containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") // default checksum algorithm return context.protocolRequest } } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 20db2263102..e1d1f24e21d 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -33,16 +33,6 @@ operation ChecksumsNotRequiredOperation { output: SomeOutput } -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", -) -@http(method: "POST", uri: "/test-checksums-2", code: 200) -operation ChecksumsRequiredOperation { - input: AnotherInput, - output: AnotherOutput -} - @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -56,11 +46,26 @@ structure SomeInput { @output structure SomeOutput {} +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32"] +) +@http(method: "POST", uri: "/test-checksums-2", code: 200) +operation ChecksumsRequiredOperation { + input: AnotherInput, + output: AnotherOutput +} + @input structure AnotherInput { @httpHeader("x-amz-request-algorithm") checksumAlgorithm: ChecksumAlgorithm + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + @httpPayload @required body: String @@ -75,4 +80,8 @@ enum ChecksumAlgorithm { CRC64NVME SHA1 SHA256 -} \ No newline at end of file +} + +enum ValidationMode { + ENABLED +} From 6037580603d7e24f1accf48db3af28edaab43aba Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:19:32 -0500 Subject: [PATCH 25/64] Quick self review --- .../src/commonTest/kotlin/ClientConfigTests.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index aeebef9d6f4..edd61d974ce 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -128,7 +128,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -141,7 +141,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true @@ -164,7 +164,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -177,7 +177,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true @@ -201,7 +201,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -214,7 +214,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true From 2f6e10ba3d1e18a754cc9417118aad00ffe28324 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:23:37 -0500 Subject: [PATCH 26/64] Self review V2 --- .../checksums/src/commonTest/kotlin/ClientConfigTests.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index edd61d974ce..ddeb01a26b0 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -189,7 +189,7 @@ class ClientConfigTests { @Test @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeMember(): Unit = runBlocking { + fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { var responseChecksumValidated = false ClientConfigTestClient { @@ -215,6 +215,7 @@ class ClientConfigTests { try { client.checksumsRequiredOperation { body = "Hello" + validationMode = ValidationMode.Enabled } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true From d085dec970a29f20aa6fb438c6ebf5189eae24e8 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:27:17 -0500 Subject: [PATCH 27/64] Add todos for business metrics --- .../sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 190f41a0b12..1732b398459 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -98,9 +98,11 @@ public abstract class AbstractAwsSdkClientFactory< } if (config is HttpChecksumClientConfig.Builder) { + // TODO - business metric config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + // TODO - business metric config.responseChecksumValidation = config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } From 0bafdcccd809a9e524e957028cb074ccafdb796a Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 25 Nov 2024 10:05:41 -0500 Subject: [PATCH 28/64] Unit tests pass --- .../config/AbstractAwsSdkClientFactory.kt | 2 - .../ResolveRequestChecksumCalculation.kt | 16 +- .../ResolveResponseChecksumValidation.kt | 18 +- .../FlexibleChecksumsRequest.kt | 82 +++- .../FlexibleChecksumsResponse.kt | 56 ++- .../e2eTest/src/MutliRegionAccessPointTest.kt | 222 +++++----- services/s3/e2eTest/src/PaginatorTest.kt | 3 +- services/s3/e2eTest/src/S3ChecksumTest.kt | 58 +++ services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3PresignerTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 61 ++- tests/codegen/checksums/build.gradle.kts | 5 +- .../kotlin/ChecksumBusinessMetricsTest.kt | 163 ++++++++ .../commonTest/kotlin/ChecksumConfigTest.kt | 387 ++++++++++++++++++ .../commonTest/kotlin/ChecksumRequestTest.kt | 126 ++++++ .../commonTest/kotlin/ChecksumResponseTest.kt | 243 +++++++++++ .../kotlin/ChecksumResponseTests.kt | 9 - .../kotlin/ChecksumStreamingRequestTest.kt | 155 +++++++ .../commonTest/kotlin/ClientConfigTests.kt | 237 ----------- .../commonTest/kotlin/RequestChecksumTests.kt | 9 - .../kotlin/StreamingRequestChecksumTests.kt | 9 - .../kotlin/utils/ChecksumTestUtils.kt | 92 +++++ ...lient-config.smithy => config-test.smithy} | 0 ...nk.smithy => request-response-test.smithy} | 2 +- tests/codegen/smoke-tests/build.gradle.kts | 1 + .../src/commonTest/kotlin/SmokeTestE2ETest.kt | 7 + 26 files changed, 1542 insertions(+), 425 deletions(-) create mode 100644 services/s3/e2eTest/src/S3ChecksumTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt rename tests/codegen/checksums/src/commonTest/resources/{client-config.smithy => config-test.smithy} (100%) rename tests/codegen/checksums/src/commonTest/resources/{kitchen-sink.smithy => request-response-test.smithy} (99%) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 1732b398459..190f41a0b12 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -98,11 +98,9 @@ public abstract class AbstractAwsSdkClientFactory< } if (config is HttpChecksumClientConfig.Builder) { - // TODO - business metric config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) - // TODO - business metric config.responseChecksumValidation = config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index a06d61103c2..d731b5d3006 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -15,15 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { - AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { - try { - return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) - } catch (_: IllegalArgumentException) { - throw ConfigurationException( +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation + return unparsedString?.let { + when (unparsedString.uppercase()) { + "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", ) } - } - return null + } ?: ChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index 446480d527e..dbf4aa9b377 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -15,15 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { - AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation?.let { - try { - return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) - } catch (_: IllegalArgumentException) { - throw ConfigurationException( - "'$it' is not a valid value for response checksum validation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation + return unparsedString?.let { + when (unparsedString.uppercase()) { + "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", ) } - } - return null + } ?: ChecksumConfigOption.WHEN_SUPPORTED } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 8799ac5364e..f00838fca03 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.codegen.customization.flexiblechecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.core.KotlinWriter import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes import software.amazon.smithy.kotlin.codegen.core.withBlock @@ -13,10 +14,15 @@ import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.HttpPayloadTrait +import software.amazon.smithy.model.traits.StreamingTrait /** * Adds a middleware that enables sending flexible checksums during an HTTP request @@ -26,8 +32,61 @@ class FlexibleChecksumsRequest : KotlinIntegration { .shapes() .any { it.hasTrait() } + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ConfigProperty { + name = "requestChecksumCalculation" + symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + useNestedBuilderBaseClass() + documentation = "" // todo + propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + }, + ) + + private val operationsWithStreamingPayloads = mutableListOf() + + override fun preprocessModel(model: Model, settings: KotlinSettings): Model { + model.operationShapes.forEach { operationShape -> + + val operationInput = model.expectShape(operationShape.inputShape) + + operationInput.members().find { it.hasTrait() }?.let { httpPayload -> + if (model.getShape(httpPayload.target).get().hasTrait()) { + operationsWithStreamingPayloads.add(operationShape.id) + } + } + } + + return model + } + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsRequestMiddleware + resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics + + private val configBusinessMetrics = object : ProtocolMiddleware { + override val name: String = "requestChecksumCalculationBusinessMetric" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.requestChecksumCalculation) {", "}") { + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + } + } + } private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { override val name: String = "FlexibleChecksumsRequest" @@ -42,22 +101,25 @@ class FlexibleChecksumsRequest : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) - val httpChecksumTrait = op.getTrait()!! val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + val streamingPayload = operationsWithStreamingPayloads.contains(op.id) - writer.withBlock("op.interceptors.add(#T<#T>() {", "})", interceptorSymbol, inputSymbol) { - writer.write("input.#L?.value", requestAlgorithmMemberName) - } - writer.withBlock("input.#L?.let {", "}", requestAlgorithmMemberName) { - writer.write("op.context[#T.ChecksumAlgorithm] = it.value", RuntimeTypes.HttpClient.Operation.HttpOperationContext) + writer.withBlock( + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + ) { + writer.write("requestChecksumRequired = #L,", requestChecksumRequired) + writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") + writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) + writer.write("streamingPayload = #L,", streamingPayload) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index c641765a3c3..95f27f24fae 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -6,14 +6,13 @@ package aws.sdk.kotlin.codegen.customization.flexiblechecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.KotlinWriter -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes -import software.amazon.smithy.kotlin.codegen.core.defaultName -import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape @@ -27,8 +26,41 @@ class FlexibleChecksumsResponse : KotlinIntegration { .shapes() .any { it.hasTrait() } + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ConfigProperty { + name = "responseChecksumValidation" + symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + useNestedBuilderBaseClass() + documentation = "" // todo + propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + }, + ) + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsResponseMiddleware + resolved + flexibleChecksumsResponseMiddleware + configBusinessMetrics + + private val configBusinessMetrics = object : ProtocolMiddleware { + override val name: String = "responseChecksumValidationBusinessMetric" + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.responseChecksumValidation) {", "}") { + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + } + } + } private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { override val name: String = "FlexibleChecksumsResponse" @@ -43,21 +75,19 @@ class FlexibleChecksumsResponse : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) - val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor - val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } + val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) writer.withBlock( - "op.interceptors.add(#T<#T> {", - "})", - interceptorSymbol, - inputSymbol, + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, ) { - writer.write("it.#L?.value == \"ENABLED\"", requestValidationModeMember.defaultName()) + writer.write("responseValidation = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("responseChecksumValidation = config.responseChecksumValidation,") } } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 49630763536..2bb27b33c6e 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,111 +1,111 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.e2etest - -import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.deleteObject -import aws.sdk.kotlin.services.s3.putObject -import aws.sdk.kotlin.services.s3.withConfig -import aws.sdk.kotlin.services.s3control.S3ControlClient -import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.TestInstance -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith - -private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -class MutliRegionAccessPointTest { - private val s3West = S3Client { region = "us-west-2" } - private val s3East = s3West.withConfig { region = "us-east-2" } - private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } - private val s3Control = S3ControlClient { region = "us-west-2" } - - private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" - private val objectKey = "test.txt" - - private lateinit var accountId: String - private lateinit var multiRegionAccessPointArn: String - private lateinit var usWestBucket: String - private lateinit var usEastBucket: String - - @BeforeAll - private fun setUp(): Unit = runBlocking { - accountId = getAccountId() - usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) - - createMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - accountId, - ) - - multiRegionAccessPointArn = - getMultiRegionAccessPointArn( - s3Control, - multiRegionAccessPoint, - accountId, - ) - } - - @AfterAll - private fun cleanUp(): Unit = runBlocking { - if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { - deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) - } - - deleteBucketAndAllContents(s3West, usWestBucket) - deleteBucketAndAllContents(s3East, usEastBucket) - - s3West.close() - s3East.close() - s3SigV4a.close() - s3Control.close() - } - - @Test - fun testMultiRegionAccessPointOperation(): Unit = runBlocking { - s3SigV4a.putObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - - s3SigV4a.deleteObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - } - - @Test - fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { - val ex = assertFailsWith { - s3West.putObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - } - - assertEquals( - ex.message, - "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", - ) - } -} +///* +// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// * SPDX-License-Identifier: Apache-2.0 +// */ +//package aws.sdk.kotlin.e2etest +// +//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +//import aws.sdk.kotlin.services.s3.S3Client +//import aws.sdk.kotlin.services.s3.deleteObject +//import aws.sdk.kotlin.services.s3.putObject +//import aws.sdk.kotlin.services.s3.withConfig +//import aws.sdk.kotlin.services.s3control.S3ControlClient +//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +//import kotlinx.coroutines.runBlocking +//import org.junit.jupiter.api.AfterAll +//import org.junit.jupiter.api.BeforeAll +//import org.junit.jupiter.api.TestInstance +//import kotlin.test.Test +//import kotlin.test.assertEquals +//import kotlin.test.assertFailsWith +// +//private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//class MutliRegionAccessPointTest { +// private val s3West = S3Client { region = "us-west-2" } +// private val s3East = s3West.withConfig { region = "us-east-2" } +// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } +// private val s3Control = S3ControlClient { region = "us-west-2" } +// +// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" +// private val objectKey = "test.txt" +// +// private lateinit var accountId: String +// private lateinit var multiRegionAccessPointArn: String +// private lateinit var usWestBucket: String +// private lateinit var usEastBucket: String +// +// @BeforeAll +// private fun setUp(): Unit = runBlocking { +// accountId = getAccountId() +// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) +// usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) +// +// createMultiRegionAccessPoint( +// s3Control, +// multiRegionAccessPoint, +// usWestBucket, +// usEastBucket, +// accountId, +// ) +// +// multiRegionAccessPointArn = +// getMultiRegionAccessPointArn( +// s3Control, +// multiRegionAccessPoint, +// accountId, +// ) +// } +// +// @AfterAll +// private fun cleanUp(): Unit = runBlocking { +// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { +// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) +// } +// +// deleteBucketAndAllContents(s3West, usWestBucket) +// deleteBucketAndAllContents(s3East, usEastBucket) +// +// s3West.close() +// s3East.close() +// s3SigV4a.close() +// s3Control.close() +// } +// +// @Test +// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { +// s3SigV4a.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// +// s3SigV4a.deleteObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// @Test +// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { +// val ex = assertFailsWith { +// s3West.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// assertEquals( +// ex.message, +// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", +// ) +// } +//} diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index e0119620bc9..e4555025642 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -11,7 +11,6 @@ import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.paginators.listPartsPaginated import aws.sdk.kotlin.services.s3.uploadPart import aws.smithy.kotlin.runtime.content.ByteStream -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList import kotlinx.coroutines.flow.transform import kotlinx.coroutines.runBlocking @@ -33,7 +32,7 @@ class PaginatorTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt new file mode 100644 index 00000000000..86da21a40a1 --- /dev/null +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -0,0 +1,58 @@ +//package aws.sdk.kotlin.e2etest +// +//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +//import aws.sdk.kotlin.services.s3.S3Client +//import aws.sdk.kotlin.services.s3.deleteObject +//import aws.sdk.kotlin.services.s3.putObject +//import aws.sdk.kotlin.services.s3.withConfig +//import kotlinx.coroutines.runBlocking +//import org.junit.jupiter.api.AfterAll +//import org.junit.jupiter.api.BeforeAll +//import org.junit.jupiter.api.Test +//import org.junit.jupiter.api.TestInstance +// +//// TODO: Test delete objects, get object, upload part. +//// TODO: Currently a lot of tests are failing. Is it because of my code changes or because of only a certain bucket being allowlisted. +//// TODO: Will I have to get rid of the weird streaming behavior I'm seeing ? +// +//// TODO: Errors I'm seeing are: HTTP body type is not supported - Request signature does not match the signature you provided - Checksum type mismatch (expected null but was crc32) - Missing header "transfer-encoding" +//// TODO: Use allow listed bucket +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//class S3ChecksumTest { +// private val s3West = S3Client { region = "us-west-2" } +// private val objectKey = "test.txt" +// private lateinit var accountId: String +// private lateinit var usWestBucket: String +// +// @BeforeAll +// private fun setUp(): Unit = runBlocking { +// accountId = getAccountId() +// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) +// } +// +// @AfterAll +// private fun cleanUp(): Unit = runBlocking { +// deleteBucketAndAllContents(s3West, usWestBucket) +// s3West.close() +// } +// +// @Test +// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { +// s3SigV4a.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// +// s3SigV4a.deleteObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +//} \ No newline at end of file diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 3aecf3195ae..5f6a1ab7c46 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -50,7 +50,7 @@ class S3BucketOpsIntegrationTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3PresignerTest.kt b/services/s3/e2eTest/src/S3PresignerTest.kt index 38e8a39ee65..9b479521c2f 100644 --- a/services/s3/e2eTest/src/S3PresignerTest.kt +++ b/services/s3/e2eTest/src/S3PresignerTest.kt @@ -34,7 +34,7 @@ class S3PresignerTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 6247ac2f129..8e125ea6c7d 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -39,7 +39,7 @@ object S3TestUtils { private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" - suspend fun getTestBucket( + suspend fun getTestBucketWithPrefix( client: S3Client, region: String? = null, accountId: String? = null, @@ -98,6 +98,65 @@ object S3TestUtils { testBucket } + suspend fun getTestBucket( + client: S3Client, + region: String? = null, + accountId: String? = null, + ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) + + suspend fun getBucket( + client: S3Client, + prefix: String, + region: String? = null, + accountId: String? = null, + ): String = withTimeout(60.seconds) { + val buckets = client.listBuckets() + .buckets + ?.mapNotNull { it.name } + + var testBucket = buckets?.firstOrNull { bucketName -> + bucketName.startsWith(prefix) && + region?.let { + client.getBucketLocation { + bucket = bucketName + expectedBucketOwner = accountId + }.locationConstraint?.value == region + } ?: true + } + + if (testBucket == null) { + testBucket = prefix + UUID.randomUUID() + println("Creating S3 bucket: $testBucket") + + client.createBucket { + bucket = testBucket + createBucketConfiguration { + locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) + } + } + + client.waitUntilBucketExists { bucket = testBucket } + } else { + println("Using existing S3 bucket: $testBucket") + } + + client.putBucketLifecycleConfiguration { + bucket = testBucket + lifecycleConfiguration { + rules = listOf( + LifecycleRule { + expiration { days = 1 } + filter { this.prefix = "" } + status = ExpirationStatus.Enabled + id = "delete-old" + }, + ) + } + } + + testBucket + } + suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { var testBucket = client.listBuckets() .buckets diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index b094022daac..cd378296c9b 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,6 +7,7 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" kotlin { + // TODO: This should be part of the shared gradle file ! sourceSets { commonTest { dependencies { @@ -17,8 +18,8 @@ kotlin { } val tests = listOf( - CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), - CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), + CodegenTest("checksums", Model("request-response-test.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("clientConfig", Model("config-test.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt new file mode 100644 index 00000000000..b22e32aa124 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -0,0 +1,163 @@ +import aws.sdk.kotlin.test.checksums.TestClient +import aws.sdk.kotlin.test.checksums.httpChecksumOperation +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.BusinessMetricsReader +import kotlin.test.Test +import kotlin.test.assertTrue + +class ChecksumBusinessMetricsTest { + @Test + fun defaultConfigBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun whenSupportedBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun whenRequiredBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun crc32(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun crc32c(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32C, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun sha1(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA1, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun sha256(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA256, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt new file mode 100644 index 00000000000..618946886bd --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -0,0 +1,387 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.clientconfig.* +import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import utils.HeaderSetter +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +// TODO - Simplify this + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. + */ +class RequestChecksumRequired { + @Test + fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **false**. + */ +class RequestChecksumNotRequired { + @Test + fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestAlgorithmMember`. + */ +class UserSelectedChecksumAlgorithm { + @Test + fun userSelectedChecksumAlgorithmIsUsed(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-sha256" to null), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests user provided checksum calculations. + */ +class UserProvidedChecksumHeader { + @Test + fun userProvidedChecksumIsUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-crc64nvme" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc64nvme" to "foo"), + forbiddenHeaders = mapOf("x-amz-checksum-sha256" to "foo"), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + + assertFalse( + headerReader.containsForbiddenHeaders, + ) + } + + @Test + fun unmodeledChecksumIsUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-some-future-algorithm" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-some-future-algorithm" to "foo"), + forbiddenHeaders = mapOf( + "x-amz-checksum-crc32" to "foo", + "x-amz-checksum-sha256" to "foo", + ), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun userProvidedMd5IsNotUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-md5" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + forbiddenHeaders = mapOf( + "x-amz-checksum-md5" to "foo", + ), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. + */ +class ResponseChecksumValidation { + @Test + fun responseChecksumValidationWhenSupported(): Unit = runBlocking { + assertFailsWith { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } + } + + @Test + fun responseChecksumValidationWhenRequired(): Unit = runBlocking { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } + + @Test + fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { + assertFailsWith { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + validationMode = ValidationMode.Enabled + } + } + } + } + + @Test + fun compositeChecksumsAreNotValidated(): Unit = runBlocking { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append( + "x-amz-checksum-crc32", + "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1" + ) + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt new file mode 100644 index 00000000000..dc0c0cad6b5 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -0,0 +1,126 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import kotlin.test.Test +import kotlin.test.assertTrue + +// TODO - Simplify +// TODO - Test empty payload + +class ChecksumRequestTest { + @Test + fun crc32(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "CRC32", + "x-amz-checksum-crc32" to "i9aeUg==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" // TODO - region is unnecessary ..... + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun crc32c(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "CRC32C", + "x-amz-checksum-crc32c" to "crUfeA==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun sha1(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "SHA1", + "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun sha256(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "SHA256", + "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt new file mode 100644 index 00000000000..d21d89e07bc --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -0,0 +1,243 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpCall +import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.http.toHttpBody +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertFailsWith + +class SuccessfulChecksumResponseTest { + @Test + fun crc32(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "i9aeUg==") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun crc32c(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32c", "crUfeA==") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun sha1(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun sha256(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } +} + +class FailedChecksumResponseTest { + @Test + fun crc32(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun crc32c(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun sha1(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun sha256(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt deleted file mode 100644 index fd3c8965828..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class ChecksumResponseTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt new file mode 100644 index 00000000000..633e26530e2 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt @@ -0,0 +1,155 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import utils.TrailerReader +import kotlin.test.Test +import kotlin.test.assertTrue + +// TODO - Simplify + +class ChecksumStreamingRequestTest { + @Test + fun crc32(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-crc32", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-crc32" to "i9aeUg==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun crc32c(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-crc32c", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-crc32c" to "crUfeA==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun sha1(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-sha1", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun sha256(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-sha256", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt deleted file mode 100644 index ddeb01a26b0..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ /dev/null @@ -1,237 +0,0 @@ -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.clientconfig.* -import aws.sdk.kotlin.test.clientconfig.model.ValidationMode -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption -import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Nested -import kotlin.test.Ignore -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ClientConfigTests { - @Nested - inner class RequestChecksumNotRequired { - @Test - @Ignore // todo: un-ignore - fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - - @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertFalse(requestInterceptor.containsChecksum) - } - } - - @Nested - inner class RequestChecksumRequired { - @Test - @Ignore // todo: un-ignore - fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - - @Test - @Ignore // todo: un-ignore - fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - } - - @Nested - inner class ResponseChecksumValidation { - @Test - @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenSupported(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertTrue(responseChecksumValidated) - } - - @Test - fun responseChecksumValidationResponseChecksumValidationWhenRequired(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertFalse(responseChecksumValidated) - } - - @Test - @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - validationMode = ValidationMode.Enabled - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertTrue(responseChecksumValidated) - } - } -} - -private class RequestInterceptor : HttpInterceptor { - var containsChecksum = false - - override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { - containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") // default checksum algorithm - return context.protocolRequest - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt deleted file mode 100644 index 15d8764dc2c..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class RequestChecksumTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt deleted file mode 100644 index 25d8210cf10..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class StreamingRequestChecksumTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt new file mode 100644 index 00000000000..b010c1ffd2b --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt @@ -0,0 +1,92 @@ +package utils + +import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric +import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.toBuilder +import kotlinx.coroutines.runBlocking + +/** + * Checks if the specified headers are set in an HTTP request. + */ +internal class HeaderReader( + private val expectedHeaders: Map, + private val forbiddenHeaders: Map = emptyMap(), +) : HttpInterceptor { + var containsExpectedHeaders = true + var containsForbiddenHeaders = false + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + expectedHeaders.forEach { header -> + val containsHeader = context.protocolRequest.headers.contains(header.key) + val headerValueMatches = header.value?.let { headerValue -> + context.protocolRequest.headers[header.key] == headerValue + } ?: true + + if (!containsHeader || !headerValueMatches) { + containsExpectedHeaders = false + return + } + } + + forbiddenHeaders.forEach { header -> + if (context.protocolRequest.headers.contains(header.key)) { + containsForbiddenHeaders = true + return + } + } + } +} + +/** + * Checks if the specified trailer headers are set in an HTTP request. + */ +internal class TrailerReader( + private val expectedTrailers: Map, +) : HttpInterceptor { + var containsExpectedTrailers = true + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + expectedTrailers.forEach { trailer -> + val containsTrailer = context.protocolRequest.trailingHeaders.contains(trailer.key) + val trailerValueMatches = trailer.value?.let { trailerValue -> + runBlocking { + context.protocolRequest.trailingHeaders[trailer.key]?.await() == trailerValue + } + } ?: true + + if (!containsTrailer || !trailerValueMatches) { + containsExpectedTrailers = false + return + } + } + } +} + +/** + * Sets the specified checksum header and value in an HTTP request. + */ +internal class HeaderSetter( + private val headers: Map, +) : HttpInterceptor { + override suspend fun modifyBeforeRetryLoop(context: ProtocolRequestInterceptorContext): HttpRequest { + val request = context.protocolRequest.toBuilder() + headers.forEach { + request.headers[it.key] = it.value + } + return request.build() + } +} + +internal class BusinessMetricsReader( + private val expectedBusinessMetrics: Set, +) : HttpInterceptor { + var containsExpectedBusinessMetrics = true + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) + } +} diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/config-test.smithy similarity index 100% rename from tests/codegen/checksums/src/commonTest/resources/client-config.smithy rename to tests/codegen/checksums/src/commonTest/resources/config-test.smithy diff --git a/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy b/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy similarity index 99% rename from tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy rename to tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy index 01feff05e4a..aa5dc5bc9af 100644 --- a/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy @@ -71,7 +71,7 @@ structure SomeInput { structure SomeOutput {} @http(uri: "/HttpChecksumStreamingOperation", method: "POST") -@optionalAuth +@auth([sigv4]) @httpChecksum( requestChecksumRequired: true, requestAlgorithmMember: "checksumAlgorithm", diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 9a5dc15ee07..87aaab703e4 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -14,6 +14,7 @@ description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { jvmTest { + // todo: apply jvm plugin just here ? dependencies { implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml } diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index 7dc76031929..d68cdabcdc1 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -4,8 +4,11 @@ import org.gradle.testkit.runner.GradleRunner import java.io.File import kotlin.test.* +// TODO: Turn on tests again + class SmokeTestE2ETest { @Test + @Ignore fun successService() { val smokeTestRunnerOutput = runSmokeTests("successService") @@ -14,6 +17,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun failureService() { val smokeTestRunnerOutput = runSmokeTests("failureService") @@ -21,6 +25,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun exceptionService() { val smokeTestRunnerOutput = runSmokeTests("exceptionService", expectingFailure = true) @@ -31,6 +36,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun successServiceSkipTags() { val envVars = mapOf(AWS_SKIP_TAGS to "success") val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -40,6 +46,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun successServiceServiceFilter() { val envVars = mapOf(AWS_SERVICE_FILTER to "Failure") // Only run tests for services with this SDK ID val smokeTestRunnerOutput = runSmokeTests("successService", envVars) From fc84b611ca377d621af974449735de680ee33124 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 26 Nov 2024 17:29:39 -0500 Subject: [PATCH 29/64] E2E tests pass --- .../FlexibleChecksumsRequest.kt | 22 -- gradle/libs.versions.toml | 2 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 222 +++++++++--------- services/s3/e2eTest/src/PaginatorTest.kt | 5 + services/s3/e2eTest/src/S3ChecksumTest.kt | 186 ++++++++++----- services/s3/e2eTest/src/S3IntegrationTest.kt | 11 +- services/s3/e2eTest/src/S3TestUtils.kt | 66 ++++-- .../commonTest/kotlin/ChecksumConfigTest.kt | 3 +- .../commonTest/kotlin/ChecksumResponseTest.kt | 1 - .../kotlin/ChecksumStreamingRequestTest.kt | 155 ------------ 10 files changed, 293 insertions(+), 380 deletions(-) delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index f00838fca03..11c0f8a43fd 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -19,10 +19,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.shapes.StructureShape -import software.amazon.smithy.model.traits.HttpPayloadTrait -import software.amazon.smithy.model.traits.StreamingTrait /** * Adds a middleware that enables sending flexible checksums during an HTTP request @@ -44,23 +41,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { }, ) - private val operationsWithStreamingPayloads = mutableListOf() - - override fun preprocessModel(model: Model, settings: KotlinSettings): Model { - model.operationShapes.forEach { operationShape -> - - val operationInput = model.expectShape(operationShape.inputShape) - - operationInput.members().find { it.hasTrait() }?.let { httpPayload -> - if (model.getShape(httpPayload.target).get().hasTrait()) { - operationsWithStreamingPayloads.add(operationShape.id) - } - } - } - - return model - } - override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics @@ -109,7 +89,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired - val streamingPayload = operationsWithStreamingPayloads.contains(op.id) writer.withBlock( "op.interceptors.add(#T(", @@ -119,7 +98,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { writer.write("requestChecksumRequired = #L,", requestChecksumRequired) writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) - writer.write("streamingPayload = #L,", streamingPayload) } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b283ae0aaa2..b09842500f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ smithy-kotlin-runtime-version = "1.3.20" smithy-kotlin-codegen-version = "0.33.20" # codegen -smithy-version = "1.52.0" +smithy-version = "1.53.0" # testing ddb-local-version = "2.5.2" diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 2bb27b33c6e..49630763536 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,111 +1,111 @@ -///* -// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// * SPDX-License-Identifier: Apache-2.0 -// */ -//package aws.sdk.kotlin.e2etest -// -//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -//import aws.sdk.kotlin.services.s3.S3Client -//import aws.sdk.kotlin.services.s3.deleteObject -//import aws.sdk.kotlin.services.s3.putObject -//import aws.sdk.kotlin.services.s3.withConfig -//import aws.sdk.kotlin.services.s3control.S3ControlClient -//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -//import kotlinx.coroutines.runBlocking -//import org.junit.jupiter.api.AfterAll -//import org.junit.jupiter.api.BeforeAll -//import org.junit.jupiter.api.TestInstance -//import kotlin.test.Test -//import kotlin.test.assertEquals -//import kotlin.test.assertFailsWith -// -//private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" -// -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//class MutliRegionAccessPointTest { -// private val s3West = S3Client { region = "us-west-2" } -// private val s3East = s3West.withConfig { region = "us-east-2" } -// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } -// private val s3Control = S3ControlClient { region = "us-west-2" } -// -// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" -// private val objectKey = "test.txt" -// -// private lateinit var accountId: String -// private lateinit var multiRegionAccessPointArn: String -// private lateinit var usWestBucket: String -// private lateinit var usEastBucket: String -// -// @BeforeAll -// private fun setUp(): Unit = runBlocking { -// accountId = getAccountId() -// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) -// usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) -// -// createMultiRegionAccessPoint( -// s3Control, -// multiRegionAccessPoint, -// usWestBucket, -// usEastBucket, -// accountId, -// ) -// -// multiRegionAccessPointArn = -// getMultiRegionAccessPointArn( -// s3Control, -// multiRegionAccessPoint, -// accountId, -// ) -// } -// -// @AfterAll -// private fun cleanUp(): Unit = runBlocking { -// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { -// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) -// } -// -// deleteBucketAndAllContents(s3West, usWestBucket) -// deleteBucketAndAllContents(s3East, usEastBucket) -// -// s3West.close() -// s3East.close() -// s3SigV4a.close() -// s3Control.close() -// } -// -// @Test -// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { -// s3SigV4a.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// -// s3SigV4a.deleteObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// @Test -// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { -// val ex = assertFailsWith { -// s3West.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// assertEquals( -// ex.message, -// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", -// ) -// } -//} +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.deleteObject +import aws.sdk.kotlin.services.s3.putObject +import aws.sdk.kotlin.services.s3.withConfig +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MutliRegionAccessPointTest { + private val s3West = S3Client { region = "us-west-2" } + private val s3East = s3West.withConfig { region = "us-east-2" } + private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } + private val s3Control = S3ControlClient { region = "us-west-2" } + + private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + private val objectKey = "test.txt" + + private lateinit var accountId: String + private lateinit var multiRegionAccessPointArn: String + private lateinit var usWestBucket: String + private lateinit var usEastBucket: String + + @BeforeAll + private fun setUp(): Unit = runBlocking { + accountId = getAccountId() + usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) + + multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) + } + + @AfterAll + private fun cleanUp(): Unit = runBlocking { + if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { + deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) + } + + deleteBucketAndAllContents(s3West, usWestBucket) + deleteBucketAndAllContents(s3East, usEastBucket) + + s3West.close() + s3East.close() + s3SigV4a.close() + s3Control.close() + } + + @Test + fun testMultiRegionAccessPointOperation(): Unit = runBlocking { + s3SigV4a.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + + s3SigV4a.deleteObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + @Test + fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { + val ex = assertFailsWith { + s3West.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + assertEquals( + ex.message, + "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", + ) + } +} diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index e4555025642..27285a080bd 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -19,6 +19,7 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import kotlin.test.Ignore import kotlin.test.assertContentEquals import kotlin.time.Duration.Companion.seconds @@ -40,7 +41,11 @@ class PaginatorTest { S3TestUtils.deleteBucketAndAllContents(client, testBucket) } + // FIXME: Enable test when motorcade is ready + // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 + // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" // ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly. + @Ignore @Test fun testListPartsPagination() = runBlocking { val chunk = "!".repeat(5 * 1024 * 1024).encodeToByteArray() // Parts must be at least 5MB diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 86da21a40a1..eff66e890df 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -1,58 +1,128 @@ -//package aws.sdk.kotlin.e2etest -// -//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -//import aws.sdk.kotlin.services.s3.S3Client -//import aws.sdk.kotlin.services.s3.deleteObject -//import aws.sdk.kotlin.services.s3.putObject -//import aws.sdk.kotlin.services.s3.withConfig -//import kotlinx.coroutines.runBlocking -//import org.junit.jupiter.api.AfterAll -//import org.junit.jupiter.api.BeforeAll -//import org.junit.jupiter.api.Test -//import org.junit.jupiter.api.TestInstance -// -//// TODO: Test delete objects, get object, upload part. -//// TODO: Currently a lot of tests are failing. Is it because of my code changes or because of only a certain bucket being allowlisted. -//// TODO: Will I have to get rid of the weird streaming behavior I'm seeing ? -// -//// TODO: Errors I'm seeing are: HTTP body type is not supported - Request signature does not match the signature you provided - Checksum type mismatch (expected null but was crc32) - Missing header "transfer-encoding" -//// TODO: Use allow listed bucket -// -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//class S3ChecksumTest { -// private val s3West = S3Client { region = "us-west-2" } -// private val objectKey = "test.txt" -// private lateinit var accountId: String -// private lateinit var usWestBucket: String -// -// @BeforeAll -// private fun setUp(): Unit = runBlocking { -// accountId = getAccountId() -// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) -// } -// -// @AfterAll -// private fun cleanUp(): Unit = runBlocking { -// deleteBucketAndAllContents(s3West, usWestBucket) -// s3West.close() -// } -// -// @Test -// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { -// s3SigV4a.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// -// s3SigV4a.deleteObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -//} \ No newline at end of file +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads +import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucketByName +import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.* +import java.io.File +import java.io.FileInputStream +import java.util.* + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation::class) +class S3ChecksumTest { + private val s3West = S3Client { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aws-kotlin-sdk+ci@amazon.com --role Admin") + } + private val checksumsTestBucket = "s3-test-bucket-ci-motorcade" + private val testObject = "test-object" + private lateinit var accountId: String + private lateinit var usWestBucket: String + + @BeforeAll + private fun setUp(): Unit = runBlocking { + accountId = getAccountId() + usWestBucket = getTestBucketByName(s3West, checksumsTestBucket, "us-west-2", accountId) + } + + @AfterAll + private fun cleanUp(): Unit = runBlocking { + deleteMultiPartUploads(s3West, checksumsTestBucket) + deleteBucketContents(s3West, checksumsTestBucket) + s3West.close() + } + + @Test + @Order(1) + fun testPutObject(): Unit = runBlocking { + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + body = ByteStream.fromString("Hello World") + } + } + + @Test + @Order(2) + fun testPutObjectWithEmptyBody(): Unit = runBlocking { + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + UUID.randomUUID() + } + } + + @Test + @Order(3) + fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { + val testString = "Hello World" + val tempFile = File.createTempFile("test", ".txt").also { + it.writeText(testString) + it.deleteOnExit() + } + val inputStream = FileInputStream(tempFile) + + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + UUID.randomUUID() + body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) + } + } + + @Test + @Order(4) + fun testMultiPartUpload(): Unit = runBlocking { + // Parts need to be at least 5 MB + val partOne = "Hello".repeat(1_048_576) + val partTwo = "World".repeat(1_048_576) + + val testUploadId = s3West.createMultipartUpload { + bucket = checksumsTestBucket + key = testObject + }.uploadId + + val eTagPartOne = s3West.uploadPart { + bucket = checksumsTestBucket + key = testObject + partNumber = 1 + uploadId = testUploadId + body = ByteStream.fromString(partOne) + }.eTag + + val eTagPartTwo = s3West.uploadPart { + bucket = checksumsTestBucket + key = testObject + partNumber = 2 + uploadId = testUploadId + body = ByteStream.fromString(partTwo) + }.eTag + + s3West.completeMultipartUpload { + bucket = checksumsTestBucket + key = testObject + uploadId = testUploadId + multipartUpload = CompletedMultipartUpload { + parts = listOf( + CompletedPart { + partNumber = 1 + eTag = eTagPartOne + }, + CompletedPart { + partNumber = 2 + eTag = eTagPartTwo + }, + ) + } + } + + // TODO: Get the object and make sure a composite checksum doesn't break us + } +} diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 5f6a1ab7c46..383ca82a381 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -29,12 +29,7 @@ import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestInstance import java.io.File import java.util.UUID -import kotlin.test.Test -import kotlin.test.assertContains -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertFailsWith -import kotlin.test.assertIs +import kotlin.test.* import kotlin.time.Duration.Companion.seconds /** @@ -193,6 +188,10 @@ class S3BucketOpsIntegrationTest { } } + // FIXME: Enable test when motorcade is ready + // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 + // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" + @Ignore @Test fun testMultipartUpload(): Unit = runBlocking { s3WithAllEngines { s3 -> diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 8e125ea6c7d..575ceb1f7fe 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -33,8 +33,10 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" - // The E2E test account only has permission to operate on buckets with the prefix - private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + // The E2E test account only has permission to operate on buckets with the prefix (s3-test-bucket-) + // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". E2E tests will use it and delete it! + // FIXME: Change back to "s3-test-bucket-" after motorcade is released. + private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" @@ -98,15 +100,9 @@ object S3TestUtils { testBucket } - suspend fun getTestBucket( + suspend fun getTestBucketByName( client: S3Client, - region: String? = null, - accountId: String? = null, - ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) - - suspend fun getBucket( - client: S3Client, - prefix: String, + bucket: String, region: String? = null, accountId: String? = null, ): String = withTimeout(60.seconds) { @@ -115,33 +111,33 @@ object S3TestUtils { ?.mapNotNull { it.name } var testBucket = buckets?.firstOrNull { bucketName -> - bucketName.startsWith(prefix) && + bucketName == bucket && region?.let { client.getBucketLocation { - bucket = bucketName + this.bucket = bucketName expectedBucketOwner = accountId }.locationConstraint?.value == region } ?: true } if (testBucket == null) { - testBucket = prefix + UUID.randomUUID() + testBucket = bucket println("Creating S3 bucket: $testBucket") client.createBucket { - bucket = testBucket + this.bucket = testBucket createBucketConfiguration { locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) } } - client.waitUntilBucketExists { bucket = testBucket } + client.waitUntilBucketExists { this.bucket = testBucket } } else { println("Using existing S3 bucket: $testBucket") } client.putBucketLifecycleConfiguration { - bucket = testBucket + this.bucket = testBucket lifecycleConfiguration { rules = listOf( LifecycleRule { @@ -192,12 +188,26 @@ object S3TestUtils { testBucket } - @OptIn(ExperimentalCoroutinesApi::class) suspend fun deleteBucketAndAllContents(client: S3Client, bucketName: String): Unit = coroutineScope { + deleteBucketContents(client, bucketName) + + try { + client.deleteBucket { bucket = bucketName } + + client.waitUntilBucketNotExists { + bucket = bucketName + } + } catch (ex: Exception) { + println("Failed to delete bucket: $bucketName") + throw ex + } + } + + suspend fun deleteBucketContents(client: S3Client, bucketName: String): Unit = coroutineScope { val scope = this try { - println("Deleting S3 bucket: $bucketName") + println("Deleting S3 buckets contents: $bucketName") val dispatcher = Dispatchers.Default.limitedParallelism(64) val jobs = mutableListOf() @@ -216,14 +226,8 @@ object S3TestUtils { } jobs.joinAll() - - client.deleteBucket { bucket = bucketName } - - client.waitUntilBucketNotExists { - bucket = bucketName - } } catch (ex: Exception) { - println("Failed to delete bucket: $bucketName") + println("Failed to delete buckets contents: $bucketName") throw ex } } @@ -373,4 +377,16 @@ object S3TestUtils { accountId = testAccountId }.accessPoints?.any { it.name == multiRegionAccessPointName } ?: false } + + internal suspend fun deleteMultiPartUploads(client: S3Client, bucketName: String) { + client.listMultipartUploads { + bucket = bucketName + }.uploads?.forEach { upload -> + client.abortMultipartUpload { + bucket = bucketName + key = upload.key + uploadId = upload.uploadId + } + } + } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 618946886bd..375b040421f 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -356,6 +356,7 @@ class ResponseChecksumValidation { @Test fun compositeChecksumsAreNotValidated(): Unit = runBlocking { + // TODO: Move elsewhere ClientConfigTestClient { responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( @@ -365,7 +366,7 @@ class ResponseChecksumValidation { Headers { append( "x-amz-checksum-crc32", - "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1" + "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1", ) }, "World!".toHttpBody(), diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt index d21d89e07bc..9c98561a70c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -1,7 +1,6 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.HttpCall import aws.smithy.kotlin.runtime.http.HttpStatusCode diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt deleted file mode 100644 index 633e26530e2..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt +++ /dev/null @@ -1,155 +0,0 @@ -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* -import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking -import utils.HeaderReader -import utils.TrailerReader -import kotlin.test.Test -import kotlin.test.assertTrue - -// TODO - Simplify - -class ChecksumStreamingRequestTest { - @Test - fun crc32(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-crc32", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-crc32" to "i9aeUg==", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun crc32c(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-crc32c", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-crc32c" to "crUfeA==", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun sha1(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-sha1", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun sha256(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-sha256", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } -} From 0f6fa6579c28034380cf51dc7457d86d5ce86da5 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 27 Nov 2024 01:51:01 -0500 Subject: [PATCH 30/64] Self review --- .../kotlin/runtime/config/AwsSdkSetting.kt | 4 +- .../ResolveRequestChecksumCalculation.kt | 16 ++--- .../ResolveResponseChecksumValidation.kt | 16 ++--- .../runtime/config/profile/AwsProfile.kt | 4 +- .../.kotlin/errors/errors-1731120437202.log | 4 -- .../FlexibleChecksumsRequest.kt | 14 ++--- .../FlexibleChecksumsResponse.kt | 12 ++-- .../e2eTest/src/MutliRegionAccessPointTest.kt | 6 +- services/s3/e2eTest/src/PaginatorTest.kt | 2 +- services/s3/e2eTest/src/S3ChecksumTest.kt | 58 +++++++++---------- services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3PresignerTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 29 +++------- settings.gradle.kts | 1 - tests/codegen/checksums/build.gradle.kts | 2 - .../kotlin/ChecksumBusinessMetricsTest.kt | 10 ++-- .../commonTest/kotlin/ChecksumConfigTest.kt | 21 +++---- .../commonTest/kotlin/ChecksumRequestTest.kt | 5 +- .../kotlin/utils/ChecksumTestUtils.kt | 29 +--------- tests/codegen/smoke-tests/build.gradle.kts | 3 +- .../src/commonTest/kotlin/SmokeTestE2ETest.kt | 9 +-- 21 files changed, 96 insertions(+), 153 deletions(-) delete mode 100644 buildSrc/.kotlin/errors/errors-1731120437202.log diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index 9fe989f58ed..edbfe7cd92c 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -210,13 +210,13 @@ public object AwsSdkSetting { strEnvSetting("aws.sigV4aSigningRegionSet", "AWS_SIGV4A_SIGNING_REGION_SET") /** - * todo + * Configures request checksum calculation */ public val AwsRequestChecksumCalculation: EnvironmentSetting = strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") /** - * todo + * Configures response checksum validation */ public val AwsResponseChecksumValidation: EnvironmentSetting = strEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index d731b5d3006..c629b54fab4 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -5,25 +5,25 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider -import java.util.* /** - * todo + * Attempts to resolve requestChecksumCalculation from the specified sources. + * @return requestChecksumCalculation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation return unparsedString?.let { when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", ) } - } ?: ChecksumConfigOption.WHEN_SUPPORTED + } ?: HttpChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index dbf4aa9b377..b2d40c0711a 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -5,25 +5,25 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider -import java.util.* /** - * todo + * Attempts to resolve responseChecksumValidation from the specified sources. + * @return responseChecksumValidation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation return unparsedString?.let { when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", ) } - } ?: ChecksumConfigOption.WHEN_SUPPORTED + } ?: HttpChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 472228604a9..119420a09e3 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -168,14 +168,14 @@ public val AwsProfile.sigV4aSigningRegionSet: String? get() = getOrNull("sigv4a_signing_region_set") /** - * todo + * Configures request checksum calculation */ @InternalSdkApi public val AwsProfile.requestChecksumCalculation: String? get() = getOrNull("request_checksum_calculation") /** - * todo + * Configures response checksum validation */ @InternalSdkApi public val AwsProfile.responseChecksumValidation: String? diff --git a/buildSrc/.kotlin/errors/errors-1731120437202.log b/buildSrc/.kotlin/errors/errors-1731120437202.log deleted file mode 100644 index 1219b509f09..00000000000 --- a/buildSrc/.kotlin/errors/errors-1731120437202.log +++ /dev/null @@ -1,4 +0,0 @@ -kotlin version: 2.0.21 -error message: The daemon has terminated unexpectedly on startup attempt #1 with error code: 0. The daemon process output: - 1. Kotlin compile daemon is ready - diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 11c0f8a43fd..89f6e84be22 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsRequest : KotlinIntegration { listOf( ConfigProperty { name = "requestChecksumCalculation" - symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig useNestedBuilderBaseClass() - documentation = "" // todo - propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + documentation = "Configures request checksum calculation" + propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") }, ) @@ -54,13 +54,13 @@ class FlexibleChecksumsRequest : KotlinIntegration { writer.withBlock("when(config.requestChecksumCalculation) {", "}") { writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -87,7 +87,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired writer.withBlock( @@ -97,7 +97,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { ) { writer.write("requestChecksumRequired = #L,", requestChecksumRequired) writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") - writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) + writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", requestAlgorithmMemberName) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 95f27f24fae..4acf2083b64 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -30,11 +30,11 @@ class FlexibleChecksumsResponse : KotlinIntegration { listOf( ConfigProperty { name = "responseChecksumValidation" - symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig useNestedBuilderBaseClass() - documentation = "" // todo - propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + documentation = "Configures response checksum validation" + propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") }, ) @@ -48,13 +48,13 @@ class FlexibleChecksumsResponse : KotlinIntegration { writer.withBlock("when(config.responseChecksumValidation) {", "}") { writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -86,7 +86,7 @@ class FlexibleChecksumsResponse : KotlinIntegration { "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, ) { - writer.write("responseValidation = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("responseValidationRequired = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("responseChecksumValidation = config.responseChecksumValidation,") } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 49630763536..93a193e439b 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -8,7 +8,7 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucket import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated import aws.sdk.kotlin.services.s3.S3Client @@ -47,8 +47,8 @@ class MutliRegionAccessPointTest { @BeforeAll private fun setUp(): Unit = runBlocking { accountId = getAccountId() - usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + usWestBucket = getBucket(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucket(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) createMultiRegionAccessPoint( s3Control, diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index 27285a080bd..3d1ed2f4919 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -33,7 +33,7 @@ class PaginatorTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index eff66e890df..6dccc573e30 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -3,11 +3,11 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucketByName -import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketByName import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.GetObjectRequest import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.fromInputStream import kotlinx.coroutines.runBlocking @@ -19,33 +19,28 @@ import java.util.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation::class) class S3ChecksumTest { - private val s3West = S3Client { - region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aws-kotlin-sdk+ci@amazon.com --role Admin") - } - private val checksumsTestBucket = "s3-test-bucket-ci-motorcade" + private val client = S3Client { region = "us-west-2" } + private val testBucket = "s3-test-bucket-ci-motorcade" private val testObject = "test-object" - private lateinit var accountId: String - private lateinit var usWestBucket: String @BeforeAll private fun setUp(): Unit = runBlocking { - accountId = getAccountId() - usWestBucket = getTestBucketByName(s3West, checksumsTestBucket, "us-west-2", accountId) + val accountId = getAccountId() + getBucketByName(client, testBucket, "us-west-2", accountId) } @AfterAll private fun cleanUp(): Unit = runBlocking { - deleteMultiPartUploads(s3West, checksumsTestBucket) - deleteBucketContents(s3West, checksumsTestBucket) - s3West.close() + deleteMultiPartUploads(client, testBucket) + deleteBucketContents(client, testBucket) + client.close() } @Test @Order(1) fun testPutObject(): Unit = runBlocking { - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject body = ByteStream.fromString("Hello World") } @@ -54,8 +49,8 @@ class S3ChecksumTest { @Test @Order(2) fun testPutObjectWithEmptyBody(): Unit = runBlocking { - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject + UUID.randomUUID() } } @@ -70,8 +65,8 @@ class S3ChecksumTest { } val inputStream = FileInputStream(tempFile) - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject + UUID.randomUUID() body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) } @@ -84,29 +79,29 @@ class S3ChecksumTest { val partOne = "Hello".repeat(1_048_576) val partTwo = "World".repeat(1_048_576) - val testUploadId = s3West.createMultipartUpload { - bucket = checksumsTestBucket + val testUploadId = client.createMultipartUpload { + bucket = testBucket key = testObject }.uploadId - val eTagPartOne = s3West.uploadPart { - bucket = checksumsTestBucket + val eTagPartOne = client.uploadPart { + bucket = testBucket key = testObject partNumber = 1 uploadId = testUploadId body = ByteStream.fromString(partOne) }.eTag - val eTagPartTwo = s3West.uploadPart { - bucket = checksumsTestBucket + val eTagPartTwo = client.uploadPart { + bucket = testBucket key = testObject partNumber = 2 uploadId = testUploadId body = ByteStream.fromString(partTwo) }.eTag - s3West.completeMultipartUpload { - bucket = checksumsTestBucket + client.completeMultipartUpload { + bucket = testBucket key = testObject uploadId = testUploadId multipartUpload = CompletedMultipartUpload { @@ -123,6 +118,11 @@ class S3ChecksumTest { } } - // TODO: Get the object and make sure a composite checksum doesn't break us + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testObject + }, + ) {} } } diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 383ca82a381..37ec353c9f6 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -45,7 +45,7 @@ class S3BucketOpsIntegrationTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3PresignerTest.kt b/services/s3/e2eTest/src/S3PresignerTest.kt index 9b479521c2f..38e8a39ee65 100644 --- a/services/s3/e2eTest/src/S3PresignerTest.kt +++ b/services/s3/e2eTest/src/S3PresignerTest.kt @@ -34,7 +34,7 @@ class S3PresignerTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 575ceb1f7fe..5f10a9f52ee 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -33,21 +33,22 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" - // The E2E test account only has permission to operate on buckets with the prefix (s3-test-bucket-) - // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". E2E tests will use it and delete it! - // FIXME: Change back to "s3-test-bucket-" after motorcade is released. + // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-" + // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". + // Non-checksum E2E tests will use it and delete it if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` + // TODO: Change back to "s3-test-bucket-" after motorcade is released. private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" - suspend fun getTestBucketWithPrefix( + suspend fun getTestBucket( client: S3Client, region: String? = null, accountId: String? = null, - ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId) + ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) - suspend fun getBucketWithPrefix( + suspend fun getBucket( client: S3Client, prefix: String, region: String? = null, @@ -100,7 +101,7 @@ object S3TestUtils { testBucket } - suspend fun getTestBucketByName( + suspend fun getBucketByName( client: S3Client, bucket: String, region: String? = null, @@ -136,20 +137,6 @@ object S3TestUtils { println("Using existing S3 bucket: $testBucket") } - client.putBucketLifecycleConfiguration { - this.bucket = testBucket - lifecycleConfiguration { - rules = listOf( - LifecycleRule { - expiration { days = 1 } - filter { this.prefix = "" } - status = ExpirationStatus.Enabled - id = "delete-old" - }, - ) - } - } - testBucket } diff --git a/settings.gradle.kts b/settings.gradle.kts index dc2418bfe1d..9bd423a88cd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,7 +54,6 @@ include(":services") include(":tests") include(":tests:codegen") include(":tests:codegen:event-stream") -include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") include(":tests:codegen:smoke-tests") include(":tests:codegen:smoke-tests:services") diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index cd378296c9b..2139fcafa20 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,7 +7,6 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" kotlin { - // TODO: This should be part of the shared gradle file ! sourceSets { commonTest { dependencies { @@ -43,7 +42,6 @@ smithyBuild { } } -// TODO: Commonize this to other codegen tests kotlin { sourceSets { commonTest { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt index b22e32aa124..a125c8e9a4e 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -2,7 +2,7 @@ import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking import utils.BusinessMetricsReader @@ -43,8 +43,8 @@ class ChecksumBusinessMetricsTest { TestClient { httpClient = TestEngine() interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() @@ -66,8 +66,8 @@ class ChecksumBusinessMetricsTest { TestClient { httpClient = TestEngine() interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 375b040421f..3c81d7924c9 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -3,7 +3,7 @@ import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm import aws.sdk.kotlin.test.clientconfig.model.ValidationMode import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.HttpStatusCode @@ -19,8 +19,6 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertTrue -// TODO - Simplify this - /** * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. */ @@ -32,7 +30,7 @@ class RequestChecksumRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -57,7 +55,7 @@ class RequestChecksumRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -87,7 +85,7 @@ class RequestChecksumNotRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -112,7 +110,7 @@ class RequestChecksumNotRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -269,7 +267,7 @@ class ResponseChecksumValidation { fun responseChecksumValidationWhenSupported(): Unit = runBlocking { assertFailsWith { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -298,7 +296,7 @@ class ResponseChecksumValidation { @Test fun responseChecksumValidationWhenRequired(): Unit = runBlocking { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -327,7 +325,7 @@ class ResponseChecksumValidation { fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { assertFailsWith { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -356,9 +354,8 @@ class ResponseChecksumValidation { @Test fun compositeChecksumsAreNotValidated(): Unit = runBlocking { - // TODO: Move elsewhere ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt index dc0c0cad6b5..16eacf58d64 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -8,9 +8,6 @@ import utils.HeaderReader import kotlin.test.Test import kotlin.test.assertTrue -// TODO - Simplify -// TODO - Test empty payload - class ChecksumRequestTest { @Test fun crc32(): Unit = runBlocking { @@ -27,7 +24,7 @@ class ChecksumRequestTest { credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), ) - region = "us-east-1" // TODO - region is unnecessary ..... + region = "us-east-1" }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt index b010c1ffd2b..725e4c11adf 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt @@ -7,7 +7,6 @@ import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder -import kotlinx.coroutines.runBlocking /** * Checks if the specified headers are set in an HTTP request. @@ -41,31 +40,6 @@ internal class HeaderReader( } } -/** - * Checks if the specified trailer headers are set in an HTTP request. - */ -internal class TrailerReader( - private val expectedTrailers: Map, -) : HttpInterceptor { - var containsExpectedTrailers = true - - override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { - expectedTrailers.forEach { trailer -> - val containsTrailer = context.protocolRequest.trailingHeaders.contains(trailer.key) - val trailerValueMatches = trailer.value?.let { trailerValue -> - runBlocking { - context.protocolRequest.trailingHeaders[trailer.key]?.await() == trailerValue - } - } ?: true - - if (!containsTrailer || !trailerValueMatches) { - containsExpectedTrailers = false - return - } - } - } -} - /** * Sets the specified checksum header and value in an HTTP request. */ @@ -81,6 +55,9 @@ internal class HeaderSetter( } } +/** + * Checks if the specified business metrics are set in an HTTP request. + */ internal class BusinessMetricsReader( private val expectedBusinessMetrics: Set, ) : HttpInterceptor { diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 87aaab703e4..1cc3c379724 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -14,9 +14,8 @@ description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { jvmTest { - // todo: apply jvm plugin just here ? dependencies { - implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml + implementation("dev.gradleplugins:gradle-test-kit:7.3.3") } } } diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index d68cdabcdc1..43667e88b2c 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -4,11 +4,8 @@ import org.gradle.testkit.runner.GradleRunner import java.io.File import kotlin.test.* -// TODO: Turn on tests again - class SmokeTestE2ETest { @Test - @Ignore fun successService() { val smokeTestRunnerOutput = runSmokeTests("successService") @@ -17,7 +14,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun failureService() { val smokeTestRunnerOutput = runSmokeTests("failureService") @@ -25,7 +21,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun exceptionService() { val smokeTestRunnerOutput = runSmokeTests("exceptionService", expectingFailure = true) @@ -36,7 +31,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun successServiceSkipTags() { val envVars = mapOf(AWS_SKIP_TAGS to "success") val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -46,7 +40,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun successServiceServiceFilter() { val envVars = mapOf(AWS_SERVICE_FILTER to "Failure") // Only run tests for services with this SDK ID val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -68,7 +61,7 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) - .withGradleVersion("8.5") // FIXME: Use lib.versions - or find a way to use `gradleTestKit()` + .withGradleVersion("8.5") val buildResult = if (expectingFailure) task.buildAndFail() else task.build() From c76aefe30c4ce565d13d3b02e40238323cdfb3d6 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 27 Nov 2024 05:57:26 -0500 Subject: [PATCH 31/64] trigger ci From 3476242b83efea4b280d5763aa3b821505789c63 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:05:01 -0700 Subject: [PATCH 32/64] Fix smoke tests --- tests/codegen/smoke-tests/build.gradle.kts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 1cc3c379724..6448e7f35ae 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -11,14 +11,8 @@ import shared.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" -kotlin { - sourceSets { - jvmTest { - dependencies { - implementation("dev.gradleplugins:gradle-test-kit:7.3.3") - } - } - } +dependencies { + jvmTestImplementation(gradleTestKit()) } val tests = listOf( From b21ec415508912c662b24e29cba0917f7b954d8b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:09:46 -0700 Subject: [PATCH 33/64] Don't specify gradle version in smoke tests --- .../smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index 43667e88b2c..61f38fe62c1 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -61,7 +61,6 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) - .withGradleVersion("8.5") val buildResult = if (expectingFailure) task.buildAndFail() else task.build() From dd2792fae3f9698c4ee46e365bd8c85d4bfd2db0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:30:42 -0700 Subject: [PATCH 34/64] Correctly print enum entries in error message --- .../config/checksums/ResolveRequestChecksumCalculation.kt | 2 +- .../config/checksums/ResolveResponseChecksumValidation.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index c629b54fab4..dd521548b9f 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -22,7 +22,7 @@ public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", ) } } ?: HttpChecksumConfigOption.WHEN_SUPPORTED diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index b2d40c0711a..6c7162c2195 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -22,7 +22,7 @@ public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", ) } } ?: HttpChecksumConfigOption.WHEN_SUPPORTED From 779cd3a489d784bd013dbf01534585b0455da01b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:31:40 -0700 Subject: [PATCH 35/64] Remove test ordering from S3ChecksumTest --- services/s3/e2eTest/src/S3ChecksumTest.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 6dccc573e30..e75173cf691 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -17,7 +17,6 @@ import java.io.FileInputStream import java.util.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestMethodOrder(MethodOrderer.OrderAnnotation::class) class S3ChecksumTest { private val client = S3Client { region = "us-west-2" } private val testBucket = "s3-test-bucket-ci-motorcade" @@ -37,7 +36,6 @@ class S3ChecksumTest { } @Test - @Order(1) fun testPutObject(): Unit = runBlocking { client.putObject { bucket = testBucket @@ -47,7 +45,6 @@ class S3ChecksumTest { } @Test - @Order(2) fun testPutObjectWithEmptyBody(): Unit = runBlocking { client.putObject { bucket = testBucket @@ -56,7 +53,6 @@ class S3ChecksumTest { } @Test - @Order(3) fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { val testString = "Hello World" val tempFile = File.createTempFile("test", ".txt").also { @@ -73,7 +69,6 @@ class S3ChecksumTest { } @Test - @Order(4) fun testMultiPartUpload(): Unit = runBlocking { // Parts need to be at least 5 MB val partOne = "Hello".repeat(1_048_576) From a3a2e095c27571a20b3aeab295053c6cbf91ef34 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:46:34 -0700 Subject: [PATCH 36/64] Add copyright headers to checksum codegen tests --- .../src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumConfigTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumRequestTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumResponseTest.kt | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt index a125c8e9a4e..dcb05a81c56 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 3c81d7924c9..400e5406cb4 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt index 16eacf58d64..f0eef19014d 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt index 9c98561a70c..6c029b5beb7 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials From c80b55cb75dba1b5eb403b429005484dd631dafd Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:53:07 -0700 Subject: [PATCH 37/64] Update buildSrc package setup --- .../main/kotlin/{ => aws/sdk/kotlin}/shared/CodegenTest.kt | 2 +- tests/codegen/checksums/build.gradle.kts | 4 ++-- tests/codegen/event-stream/build.gradle.kts | 4 ++-- tests/codegen/smoke-tests/build.gradle.kts | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename buildSrc/src/main/kotlin/{ => aws/sdk/kotlin}/shared/CodegenTest.kt (91%) diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt similarity index 91% rename from buildSrc/src/main/kotlin/shared/CodegenTest.kt rename to buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt index f9a03d715e9..4ee9580629c 100644 --- a/buildSrc/src/main/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt @@ -1,4 +1,4 @@ -package shared +package aws.sdk.kotlin.shared /** * An AWS SDK for Kotlin codegen test diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 2139fcafa20..6867921e4a2 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -1,8 +1,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 71edc4f41fb..4ce7b11ca4d 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -5,8 +5,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's event stream codegen test suite" diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 6448e7f35ae..86422a486f7 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -6,8 +6,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" From 5e2855c70859c1f74d34737e6db829ab776b1aa2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:19:59 -0700 Subject: [PATCH 38/64] Fix codegen tests not having packages --- .../codegen/checksums}/ChecksumBusinessMetricsTest.kt | 4 +++- .../kotlin/tests/codegen/checksums}/ChecksumConfigTest.kt | 6 ++++-- .../kotlin/tests/codegen/checksums}/ChecksumRequestTest.kt | 4 +++- .../tests/codegen/checksums}/ChecksumResponseTest.kt | 2 ++ .../tests/codegen/checksums}/utils/ChecksumTestUtils.kt | 7 ++++++- .../tests/codegen/eventstream}/HttpEventStreamTests.kt | 2 ++ .../tests/codegen/eventstream}/RpcEventStreamTests.kt | 3 +++ .../tests/codegen/smoketests/SmokeTestE2ETest.kt.kt} | 7 +++++++ 8 files changed, 30 insertions(+), 5 deletions(-) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumBusinessMetricsTest.kt (97%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumConfigTest.kt (98%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumRequestTest.kt (97%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumResponseTest.kt (99%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/utils/ChecksumTestUtils.kt (93%) rename tests/codegen/event-stream/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/eventstream}/HttpEventStreamTests.kt (99%) rename tests/codegen/event-stream/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/eventstream}/RpcEventStreamTests.kt (99%) rename tests/codegen/smoke-tests/src/commonTest/kotlin/{SmokeTestE2ETest.kt => aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt} (94%) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt similarity index 97% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt index dcb05a81c56..065d5f83b76 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt @@ -3,14 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.tests.codegen.checksums.utils.BusinessMetricsReader import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking -import utils.BusinessMetricsReader import kotlin.test.Test import kotlin.test.assertTrue diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt similarity index 98% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index 400e5406cb4..107db032312 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -3,10 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderSetter import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.http.* @@ -17,8 +21,6 @@ import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking -import utils.HeaderReader -import utils.HeaderSetter import kotlin.test.Test import kotlin.test.assertFailsWith import kotlin.test.assertFalse diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt similarity index 97% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt index f0eef19014d..d64794a136d 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt @@ -3,13 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking -import utils.HeaderReader import kotlin.test.Test import kotlin.test.assertTrue diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt similarity index 99% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index 6c029b5beb7..34f74c1f130 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt similarity index 93% rename from tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index 725e4c11adf..a8f1f634a1f 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -1,4 +1,9 @@ -package utils +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.tests.codegen.checksums.utils import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt similarity index 99% rename from tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt index 1bb5ec2bda9..43f2b80339d 100644 --- a/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt +++ b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.eventstream + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.restjson1.model.* import aws.sdk.kotlin.test.restjson1.serde.deserializeTestStreamOpOperationBody diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt similarity index 99% rename from tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt index 08cbecbda2a..3c04b819224 100644 --- a/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt +++ b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt @@ -2,6 +2,9 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ + +package aws.sdk.kotlin.tests.codegen.eventstream + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.awsjson11.model.MessageWithString import aws.sdk.kotlin.test.awsjson11.model.TestStream diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt similarity index 94% rename from tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt index 61f38fe62c1..b3521302b81 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt @@ -1,3 +1,10 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.tests.codegen.smoketests + import aws.sdk.kotlin.codegen.smoketests.AWS_SERVICE_FILTER import aws.sdk.kotlin.codegen.smoketests.AWS_SKIP_TAGS import org.gradle.testkit.runner.GradleRunner From 0405b2034193f2d9f6c655bce83cce8a362035f2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:35:49 -0700 Subject: [PATCH 39/64] Fix event stream codegen test templates --- tests/codegen/event-stream/build.gradle.kts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 4ce7b11ca4d..0de5288432e 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -83,8 +83,14 @@ fun fillInModel(test: CodegenTest) { } val replaced = model - .replace("{protocol-name}", test.protocolName!!) - .replace("{op-traits}", opTraits) + .replace( + "{protocol-name}", + test.protocolName ?: throw IllegalStateException("Please specify a protocol name for the codegen test"), + ) + .replace( + "{op-traits}", + opTraits, + ) modelFile.parentFile.mkdirs() modelFile.writeText(replaced) From eb338f400c06ebb4b049a0d6064dcb71f9498a69 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:39:54 -0700 Subject: [PATCH 40/64] Save test reports for failing JVM CI checks --- .github/workflows/continuous-integration.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index b8cb2e12035..684d610607a 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -40,6 +40,12 @@ jobs: pwd ls -lsa ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace + - name: Save Test Reports + if: failure() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: '**/build/reports' all-platforms: runs-on: ${{ matrix.os }} From a2fb7fb8ec961e626d3cc030b355fa7f625ac782 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 3 Dec 2024 15:01:20 -0700 Subject: [PATCH 41/64] Track smithy kotlin changes --- .../FlexibleChecksumsRequest.kt | 11 +++++++---- .../FlexibleChecksumsResponse.kt | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 89f6e84be22..d79824212ff 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -81,6 +81,8 @@ class FlexibleChecksumsRequest : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) @@ -91,13 +93,14 @@ class FlexibleChecksumsRequest : KotlinIntegration { val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired writer.withBlock( - "op.interceptors.add(#T(", + "op.interceptors.add(#T<#T>(", "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + inputSymbol, ) { - writer.write("requestChecksumRequired = #L,", requestChecksumRequired) - writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") - writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", requestAlgorithmMemberName) + writer.write("#L,", requestChecksumRequired) + writer.write("config.requestChecksumCalculation,") + writer.write("input.#L?.value,", requestAlgorithmMemberName) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 4acf2083b64..afd8c29c20a 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -18,6 +18,13 @@ import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape +/** + * AWS services that can return composite checksums in their responses + */ +private val compositeChecksumServices = setOf( + "s3", +) + /** * Adds a middleware which validates checksums returned in responses if the user has opted-in. */ @@ -75,6 +82,8 @@ class FlexibleChecksumsResponse : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() @@ -82,12 +91,14 @@ class FlexibleChecksumsResponse : KotlinIntegration { val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) writer.withBlock( - "op.interceptors.add(#T(", + "op.interceptors.add(#T<#T>(", "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, + inputSymbol, ) { - writer.write("responseValidationRequired = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) - writer.write("responseChecksumValidation = config.responseChecksumValidation,") + writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("config.responseChecksumValidation,") + writer.write("#L", compositeChecksumServices.contains(ctx.settings.sdkId)) } } } From d498e3010f7af1be346e3d5229e771eaacdcdeaf Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 11:22:47 -0700 Subject: [PATCH 42/64] Fix jvmTest JVM compatibility --- tests/codegen/build.gradle.kts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 0fda97f3cd4..dd0a9537e6b 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -45,7 +45,11 @@ subprojects { } kotlin { - jvm() + jvm { + compilations.all { + kotlinOptions.jvmTarget = "1.8" + } + } sourceSets { commonMain { dependencies { From b534927459ea95e01ad5d7b56b1b98edf9e7b736 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 12:52:42 -0700 Subject: [PATCH 43/64] Drop support for http body dot bytes response checksums --- .../codegen/checksums/ChecksumConfigTest.kt | 16 +++++- .../codegen/checksums/ChecksumResponseTest.kt | 57 +++++++++++++++---- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index 107db032312..e26afa7908c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -19,6 +19,8 @@ import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import kotlin.test.Test @@ -270,6 +272,8 @@ class UserProvidedChecksumHeader { * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. */ class ResponseChecksumValidation { + private val responseBody = "Hello world" + @Test fun responseChecksumValidationWhenSupported(): Unit = runBlocking { assertFailsWith { @@ -282,7 +286,11 @@ class ResponseChecksumValidation { Headers { append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") }, - "World!".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -340,7 +348,11 @@ class ResponseChecksumValidation { Headers { append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") }, - "World!".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index 34f74c1f130..f477a3f1129 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -8,18 +8,19 @@ package aws.sdk.kotlin.tests.codegen.checksums import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.http.Headers -import aws.smithy.kotlin.runtime.http.HttpCall -import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.http.toHttpBody import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith +private val responseBody = "Hello world" + class SuccessfulChecksumResponseTest { @Test fun crc32(): Unit = runBlocking { @@ -31,7 +32,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-crc32", "i9aeUg==") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -58,7 +63,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-crc32c", "crUfeA==") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -85,7 +94,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -112,7 +125,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -142,7 +159,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -171,7 +192,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -200,7 +225,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -229,7 +258,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) From 2761bf93d08000a00ef815c33a6be212ee62e805 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 15:25:27 -0700 Subject: [PATCH 44/64] Left FIXME to use random buckets in motorcade E2E tests --- services/s3/e2eTest/src/S3ChecksumTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index e75173cf691..db8c4026d5a 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -25,6 +25,7 @@ class S3ChecksumTest { @BeforeAll private fun setUp(): Unit = runBlocking { val accountId = getAccountId() + // FIXME: Use randomly generated bucket instead of hardcoded one when motorcade is ready getBucketByName(client, testBucket, "us-west-2", accountId) } From 5328641644fc341b07e20a12ff11ca5095428bc0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 6 Dec 2024 11:08:52 -0700 Subject: [PATCH 45/64] Refactor/fix S3 express integrations --- .../s3/express/S3ExpressIntegration.kt | 35 +------- .../S3ExpressDisableChecksumInterceptor.kt | 49 +++++++++-- .../s3/express/ChecksumRemovalTest.kt | 83 +++++++++++++++++++ 3 files changed, 126 insertions(+), 41 deletions(-) create mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 430c4f8ebc1..03b552141be 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,8 +16,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType -import software.amazon.smithy.kotlin.codegen.utils.dq -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -99,7 +97,6 @@ class S3ExpressIntegration : KotlinIntegration { resolved + listOf( addClientToExecutionContext, addBucketToExecutionContext, - useCrc32Checksum, uploadPartDisableChecksum, ) @@ -132,36 +129,6 @@ class S3ExpressIntegration : KotlinIntegration { } } - /** - * For any operations that require a checksum, set CRC32 if the user has not already configured a checksum. - */ - private val useCrc32Checksum = object : ProtocolMiddleware { - override val name: String = "UseCrc32Checksum" - - override val order: Byte = -1 // Render before flexible checksums - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = !op.isS3UploadPart && - (op.hasTrait() || (op.hasTrait() && op.expectTrait().isRequestChecksumRequired)) - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3.express" - name = "S3ExpressCrc32ChecksumInterceptor" - } - - val httpChecksumTrait = op.getTrait() - - val checksumAlgorithmMember = ctx.model.expectShape(op.input.get()) - .members() - .firstOrNull { it.memberName == httpChecksumTrait?.requestAlgorithmMember?.getOrNull() } - - // S3 models a header name x-amz-sdk-checksum-algorithm representing the name of the checksum algorithm used - val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value - - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) - } - } - /** * Disable all checksums for s3:UploadPart */ @@ -169,7 +136,7 @@ class S3ExpressIntegration : KotlinIntegration { override val name: String = "UploadPartDisableChecksum" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.isS3UploadPart + op.isS3UploadPart && op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 3b10ad3fa69..e78cde99060 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -6,14 +6,18 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder +import aws.smithy.kotlin.runtime.http.HeadersBuilder import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.toBuilder import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext +private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" + /** - * Disable checksums entirely for s3:UploadPart requests. + * Disables checksums for s3:UploadPart requests that use S3 express. */ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { @@ -22,14 +26,45 @@ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { } val logger = coroutineContext.logger() + logger.warn { "Checksums must not be sent with S3 express upload part operation, removing checksum(s)" } + + val request = context.protocolRequest.toBuilder() + + request.headers.removeChecksumHeaders() + request.trailingHeaders.removeChecksumTrailingHeaders() + request.headers.removeChecksumTrailingHeadersFromXAmzTrailer() + + return request.build() + } +} - val configuredChecksumAlgorithm = context.executionContext.getOrNull(HttpOperationContext.ChecksumAlgorithm) +/** + * Removes any checksums sent in the request's headers + */ +internal fun HeadersBuilder.removeChecksumHeaders(): Unit = + names().forEach { name -> + if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + remove(name) + } + } - configuredChecksumAlgorithm?.let { - logger.warn { "Disabling configured checksum $it for S3 Express UploadPart" } - context.executionContext.remove(HttpOperationContext.ChecksumAlgorithm) +/** + * Removes any checksums sent in the request's trailing headers + */ +internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = + names().forEach { name -> + if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + remove(name) } + } - return context.protocolRequest +/** + * Removes any checksums sent in the request's trailing headers from `x-amz-trailer` + */ +internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { + this.getAll("x-amz-trailer")?.forEach { trailingHeader -> + if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX)) { + this.remove("x-amz-trailer", trailingHeader) + } } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt new file mode 100644 index 00000000000..cbf5a9ad4d4 --- /dev/null +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt @@ -0,0 +1,83 @@ +package aws.sdk.kotlin.services.s3.express + +import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ChecksumRemovalTest { + @Test + fun removeChecksumHeaders() { + val headers = HeadersBuilder() + + headers.append("x-amz-checksum-crc32", "foo") + headers.append("x-amz-checksum-sha256", "bar") + + assertTrue( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + + headers.removeChecksumHeaders() + + assertFalse( + headers.contains("x-amz-checksum-crc32"), + ) + assertFalse( + headers.contains("x-amz-checksum-sha256"), + ) + } + + @Test + fun removeChecksumTrailingHeaders() { + val trailingHeaders = DeferredHeadersBuilder() + + trailingHeaders.add("x-amz-checksum-crc32", "foo") + trailingHeaders.add("x-amz-checksum-sha256", "bar") + + assertTrue( + trailingHeaders.contains("x-amz-checksum-crc32"), + ) + assertTrue( + trailingHeaders.contains("x-amz-checksum-sha256"), + ) + + trailingHeaders.removeChecksumTrailingHeaders() + + assertFalse( + trailingHeaders.contains("x-amz-checksum-crc32"), + ) + assertFalse( + trailingHeaders.contains("x-amz-checksum-sha256"), + ) + } + + @Test + fun removeChecksumTrailingHeadersFromXAmzTrailer() { + val headers = HeadersBuilder() + + headers.append("x-amz-trailer", "x-amz-checksum-crc32") + headers.append("x-amz-trailer", "x-amz-trailing-header") + + val xAmzTrailer = headers.getAll("x-amz-trailer") + + assertTrue( + xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + ) + assertTrue( + xAmzTrailer?.contains("x-amz-trailing-header") ?: false, + ) + + headers.removeChecksumTrailingHeadersFromXAmzTrailer() + + assertFalse( + xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + ) + assertTrue( + xAmzTrailer?.contains("x-amz-trailing-header") ?: false, + ) + } +} From ad30f7057602d1bd3c8f024d3bbbf1fcc55dc6ac Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 9 Dec 2024 13:21:33 -0500 Subject: [PATCH 46/64] Clean up S3 integrations and e2e tests --- .../s3/express/S3ExpressIntegration.kt | 3 +- .../S3ExpressCrc32ChecksumInterceptor.kt | 49 ------------------- .../S3ExpressDisableChecksumInterceptor.kt | 2 + .../e2eTest/src/MutliRegionAccessPointTest.kt | 6 +-- services/s3/e2eTest/src/S3TestUtils.kt | 4 +- 5 files changed, 8 insertions(+), 56 deletions(-) delete mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 03b552141be..86e8c568506 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -25,8 +25,7 @@ import software.amazon.smithy.model.transform.ModelTransformer * An integration which handles codegen for S3 Express, such as: * 1. Configure auth scheme by applying a synthetic shape and trait * 2. Add ExpressClient and Bucket to execution context - * 3. Override checksums to use CRC32 instead of MD5 - * 4. Disable all checksums for s3:UploadPart + * 3. Disable all checksums for s3:UploadPart */ class S3ExpressIntegration : KotlinIntegration { companion object { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt deleted file mode 100644 index bad493a2fbd..00000000000 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.AttributeKey -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.coroutines.coroutineContext - -internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" -internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" -private const val CRC32_ALGORITHM_NAME = "CRC32" - -internal class S3ExpressCrc32ChecksumInterceptor( - val checksumAlgorithmHeaderName: String? = null, -) : HttpInterceptor { - override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { - return context.protocolRequest - } - - val logger = coroutineContext.logger() - val req = context.protocolRequest.toBuilder() - - if (!context.executionContext.contains(HttpOperationContext.ChecksumAlgorithm)) { - logger.debug { "Checksum is required and not already configured, enabling CRC32 for S3 Express" } - - // Update the execution context so flexible checksums uses CRC32 - context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME - - // Most checksum headers are handled by the flexible checksums feature. But, S3 models an HTTP header binding for the - // checksum algorithm, which also needs to be overwritten and set to CRC32. - // - // The header is already set by the time this interceptor runs, so it needs to be overwritten and can't be set - // through the normal path. - checksumAlgorithmHeaderName?.let { - req.headers[it] = CRC32_ALGORITHM_NAME - } - } - - return req.build() - } -} diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index e78cde99060..170818208d4 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -15,6 +15,8 @@ import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" +private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** * Disables checksums for s3:UploadPart requests that use S3 express. diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 93a193e439b..49630763536 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -8,7 +8,7 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucket +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated import aws.sdk.kotlin.services.s3.S3Client @@ -47,8 +47,8 @@ class MutliRegionAccessPointTest { @BeforeAll private fun setUp(): Unit = runBlocking { accountId = getAccountId() - usWestBucket = getBucket(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucket(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) createMultiRegionAccessPoint( s3Control, diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 5f10a9f52ee..4139b4d9e33 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -46,9 +46,9 @@ object S3TestUtils { client: S3Client, region: String? = null, accountId: String? = null, - ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) + ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId) - suspend fun getBucket( + suspend fun getBucketWithPrefix( client: S3Client, prefix: String, region: String? = null, From 4a162de181d31dfc178dffd742b844b916a408ea Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 9 Dec 2024 14:00:30 -0500 Subject: [PATCH 47/64] s3 express no checksums in upload part test --- services/s3/e2eTest/src/S3ExpressTest.kt | 70 +++++++++++++++++++++++- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index 5724d3df3ad..c97c58fc2ce 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -4,12 +4,10 @@ */ package aws.sdk.kotlin.e2etest -import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.express.S3_EXPRESS_SESSION_TOKEN_HEADER import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.presigners.presignPutObject -import aws.sdk.kotlin.services.s3.putObject -import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.decodeToString @@ -47,6 +45,7 @@ class S3ExpressTest { @AfterAll fun cleanup(): Unit = runBlocking { testBuckets.forEach { bucket -> + S3TestUtils.deleteMultiPartUploads(client, bucket) S3TestUtils.deleteBucketAndAllContents(client, bucket) } client.close() @@ -136,6 +135,62 @@ class S3ExpressTest { } } + @Test + fun testUploadPartContainsNoChecksums() = runTest { + val testBucket = testBuckets.first() + val testObject = "I-will-be-uploaded-in-parts-!" + + // Parts need to be at least 5 MB + val partOne = "Hello".repeat(1_048_576) + val partTwo = "World".repeat(1_048_576) + + val testUploadId = client.createMultipartUpload { + bucket = testBucket + key = testObject + }.uploadId + + var eTagPartOne: String? + var eTagPartTwo: String? + + client.withConfig { + interceptors += NoChecksumValidatingInterceptor() + }.use { validatingClient -> + eTagPartOne = validatingClient.uploadPart { + bucket = testBucket + key = testObject + partNumber = 1 + uploadId = testUploadId + body = ByteStream.fromString(partOne) + }.eTag + + eTagPartTwo = validatingClient.uploadPart { + bucket = testBucket + key = testObject + partNumber = 2 + uploadId = testUploadId + body = ByteStream.fromString(partTwo) + }.eTag + } + + client.completeMultipartUpload { + bucket = testBucket + key = testObject + uploadId = testUploadId + multipartUpload = CompletedMultipartUpload { + parts = listOf( + CompletedPart { + partNumber = 1 + eTag = eTagPartOne + }, + CompletedPart { + partNumber = 2 + eTag = eTagPartTwo + }, + ) + } + } + } + private class S3ExpressInvocationTrackingInterceptor : HttpInterceptor { var s3ExpressInvocations = 0 @@ -155,4 +210,13 @@ class S3ExpressTest { } } } + + private class NoChecksumValidatingInterceptor : HttpInterceptor { + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + val headers = context.protocolRequest.headers + if (headers.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) { + assertFalse(headers.names().any { it.startsWith("x-amz-checksum-") }) + } + } + } } From d665873b3ebee66d49c014d92dcffb212efa3673 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 11 Dec 2024 16:46:30 -0500 Subject: [PATCH 48/64] PR feedback --- aws-runtime/aws-config/api/aws-config.api | 5 +- .../ResolveFlexibleChecksumsConfig.kt | 42 ++++++ .../ResolveRequestChecksumCalculation.kt | 29 ----- .../ResolveResponseChecksumValidation.kt | 29 ----- aws-runtime/aws-http/api/aws-http.api | 23 ++++ .../S3CompositeChecksumsInterceptor.kt | 47 +++++++ .../S3CompositeChecksumsInterceptorTest.kt | 35 +++++ .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 1 + .../FlexibleChecksumsResponse.kt | 8 -- .../s3/S3CompositeChecksumsIntegration.kt | 44 +++++++ .../s3/express/S3ExpressIntegration.kt | 16 ++- ...tlin.codegen.integration.KotlinIntegration | 1 + .../S3ExpressDisableChecksumInterceptor.kt | 19 ++- .../s3/express/ChecksumRemovalTest.kt | 34 ++--- services/s3/e2eTest/src/S3ChecksumTest.kt | 120 ++++++++++++------ services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 35 +++-- tests/codegen/build.gradle.kts | 1 - .../codegen/checksums/ChecksumConfigTest.kt | 35 +---- .../checksums/utils/ChecksumTestUtils.kt | 11 +- 20 files changed, 356 insertions(+), 181 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt delete mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt delete mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt create mode 100644 aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 6a9be2adba5..1d5c135a3ae 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -262,12 +262,9 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSettingKt { public static final fun resolveEndpointUrl (Laws/sdk/kotlin/runtime/config/AwsSdkSetting;Laws/smithy/kotlin/runtime/util/PlatformProvider;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/net/url/Url; } -public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculationKt { +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfigKt { public static final fun resolveRequestChecksumCalculation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; -} - -public final class aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidationKt { public static final fun resolveResponseChecksumValidation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveResponseChecksumValidation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt new file mode 100644 index 00000000000..bf7049ef2f3 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt @@ -0,0 +1,42 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation +import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider + +/** + * Attempts to resolve requestChecksumCalculation from the specified sources. + * @return requestChecksumCalculation setting if found, the default value if not. + */ +@InternalSdkApi +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation + return parseHttpChecksumConfigOption(unparsedString, "requestChecksumCalculation") +} + +/** + * Attempts to resolve responseChecksumValidation from the specified sources. + * @return responseChecksumValidation setting if found, the default value if not. + */ +@InternalSdkApi +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation + return parseHttpChecksumConfigOption(unparsedString, "responseChecksumValidation") +} + +private fun parseHttpChecksumConfigOption(unparsedString: String?, configOption: String): HttpChecksumConfigOption = + when (unparsedString?.uppercase()) { + null -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$unparsedString' is not a valid value for $configOption. Valid values are: ${HttpChecksumConfigOption.entries}", + ) + } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt deleted file mode 100644 index dd521548b9f..00000000000 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ /dev/null @@ -1,29 +0,0 @@ -package aws.sdk.kotlin.runtime.config.checksums - -import aws.sdk.kotlin.runtime.ConfigurationException -import aws.sdk.kotlin.runtime.InternalSdkApi -import aws.sdk.kotlin.runtime.config.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.profile.AwsProfile -import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.config.resolve -import aws.smithy.kotlin.runtime.util.LazyAsyncValue -import aws.smithy.kotlin.runtime.util.PlatformProvider - -/** - * Attempts to resolve requestChecksumCalculation from the specified sources. - * @return requestChecksumCalculation setting if found, the default value if not. - */ -@InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { - val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation - return unparsedString?.let { - when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", - ) - } - } ?: HttpChecksumConfigOption.WHEN_SUPPORTED -} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt deleted file mode 100644 index 6c7162c2195..00000000000 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ /dev/null @@ -1,29 +0,0 @@ -package aws.sdk.kotlin.runtime.config.checksums - -import aws.sdk.kotlin.runtime.ConfigurationException -import aws.sdk.kotlin.runtime.InternalSdkApi -import aws.sdk.kotlin.runtime.config.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.profile.AwsProfile -import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.config.resolve -import aws.smithy.kotlin.runtime.util.LazyAsyncValue -import aws.smithy.kotlin.runtime.util.PlatformProvider - -/** - * Attempts to resolve responseChecksumValidation from the specified sources. - * @return responseChecksumValidation setting if found, the default value if not. - */ -@InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { - val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation - return unparsedString?.let { - when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", - ) - } - } ?: HttpChecksumConfigOption.WHEN_SUPPORTED -} diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 425d6c7ac51..0b475eb05fa 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,6 +196,29 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } +public final class aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { + public fun ()V + public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +} + public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt new file mode 100644 index 00000000000..16adc9aeef5 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt @@ -0,0 +1,47 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.http.response.toBuilder +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.coroutines.coroutineContext + +private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" + +/** + * Removes any checksum headers that contain composite checksums from an S3 response. + */ +public class S3CompositeChecksumsInterceptor : HttpInterceptor { + override suspend fun modifyBeforeDeserialization(context: ProtocolResponseInterceptorContext): HttpResponse { + val response = context.protocolResponse.toBuilder() + val logger = coroutineContext.logger() + + response.headers.removeCompositeChecksums(logger) + + return response.build() + } +} + +/** + * Removes any checksum headers that contain composite checksums. + */ +internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = + names().forEach { header -> + if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { + logger.warn { "S3 returned a composite checksum, composite checksums are not supported, removing checksum" } + remove(header) + } + } + +/** + * Verifies if a checksum is composite. + */ +private fun String.isCompositeChecksum(): Boolean { + // Ends with "-#" where "#" is a number + val regex = Regex("-(\\d)+$") + return regex.containsMatchIn(this) +} diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt new file mode 100644 index 00000000000..042dcb5b99e --- /dev/null +++ b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt @@ -0,0 +1,35 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.test.runTest +import kotlin.coroutines.coroutineContext +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class S3CompositeChecksumsInterceptorTest { + @Test + fun compositeChecksumRemoval() = runTest { + val headers = HeadersBuilder() + + headers.append("x-amz-checksum-crc32", "foo-1") + headers.append("x-amz-checksum-sha256", "bar") + + assertTrue( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + + headers.removeCompositeChecksums(coroutineContext.logger()) + + assertFalse( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index b4debfc4bd8..70f3a4950ef 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,6 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") + val S3CompositeChecksumsInterceptor = symbol("S3CompositeChecksumsInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index afd8c29c20a..9767d2a9718 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -18,13 +18,6 @@ import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape -/** - * AWS services that can return composite checksums in their responses - */ -private val compositeChecksumServices = setOf( - "s3", -) - /** * Adds a middleware which validates checksums returned in responses if the user has opted-in. */ @@ -98,7 +91,6 @@ class FlexibleChecksumsResponse : KotlinIntegration { ) { writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("config.responseChecksumValidation,") - writer.write("#L", compositeChecksumServices.contains(ctx.settings.sdkId)) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt new file mode 100644 index 00000000000..cef54247f72 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt @@ -0,0 +1,44 @@ +package aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3 + +import aws.sdk.kotlin.codegen.AwsRuntimeTypes.Http.Interceptors.S3CompositeChecksumsInterceptor +import aws.sdk.kotlin.codegen.customization.s3.isS3 +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape + +/** + * Removes composite checksums returned by S3 so that flexible checksums won't validate them. + * Composite checksums are used for multipart uploads and end with "-#" where "#" is a number. + */ +class S3CompositeChecksumsIntegration : KotlinIntegration { + override val order: Byte + get() = -128 + + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.expectShape(settings.service).isS3 + + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = + resolved + s3CompositeChecksumsMiddleware +} + +/** + * Adds [S3CompositeChecksumsInterceptor] to interceptors when operation has flexible checksums. + */ +private val s3CompositeChecksumsMiddleware = object : ProtocolMiddleware { + override val name: String = "S3CompositeChecksumsMiddleware" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.write("op.interceptors.add(#T())", S3CompositeChecksumsInterceptor) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 86e8c568506..067da2da40f 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,6 +16,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType +import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -138,12 +139,25 @@ class S3ExpressIntegration : KotlinIntegration { op.isS3UploadPart && op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val httpChecksumTrait = op.getTrait()!! + + val requestAlgorithmMemberName = httpChecksumTrait.requestAlgorithmMember?.getOrNull()?.let { + val requestAlgorithmMemberShape = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + ctx.symbolProvider.toMemberName(requestAlgorithmMemberShape) + } + val interceptorSymbol = buildSymbol { namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressDisableChecksumInterceptor" } writer.addImport(interceptorSymbol) - writer.write("op.interceptors.add(#T())", interceptorSymbol) + writer.write( + "op.interceptors.add(#T(input.#L?.value != null))", + interceptorSymbol, + requestAlgorithmMemberName, + ) } } diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index de1ab6cd4ac..20c0701ea92 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -12,6 +12,7 @@ aws.sdk.kotlin.codegen.endpoints.AddAwsEndpointFunctions aws.sdk.kotlin.codegen.PresignerGenerator aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsRequest aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsResponse +aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3.S3CompositeChecksumsIntegration aws.sdk.kotlin.codegen.customization.AddAwsSpanAttributes aws.sdk.kotlin.codegen.customization.s3.S3OperationErrorHandler aws.sdk.kotlin.codegen.customization.s3.S3SigningConfig diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 170818208d4..1dd06c25406 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -11,7 +11,7 @@ import aws.smithy.kotlin.runtime.http.HeadersBuilder import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.telemetry.logging.warn import kotlin.coroutines.coroutineContext private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" @@ -21,14 +21,19 @@ private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** * Disables checksums for s3:UploadPart requests that use S3 express. */ -internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { +internal class S3ExpressDisableChecksumInterceptor( + private val userConfiguredChecksum: Boolean, +) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { return context.protocolRequest } - val logger = coroutineContext.logger() - logger.warn { "Checksums must not be sent with S3 express upload part operation, removing checksum(s)" } + if (userConfiguredChecksum) { + coroutineContext.warn { + "Checksums must not be sent with S3 Express UploadPart operation, removing checksum(s)" + } + } val request = context.protocolRequest.toBuilder() @@ -45,7 +50,7 @@ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { */ internal fun HeadersBuilder.removeChecksumHeaders(): Unit = names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { remove(name) } } @@ -55,7 +60,7 @@ internal fun HeadersBuilder.removeChecksumHeaders(): Unit = */ internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { remove(name) } } @@ -65,7 +70,7 @@ internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = */ internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { this.getAll("x-amz-trailer")?.forEach { trailingHeader -> - if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { this.remove("x-amz-trailer", trailingHeader) } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt index cbf5a9ad4d4..c00f78e821f 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt @@ -7,27 +7,31 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue class ChecksumRemovalTest { + // Header capitalization shouldn't matter + private val crc32Header = "x-aMz-cHeCkSum-cRC32" + private val sha256Header = "X-Amz-ChEcKsum-sHa256" + @Test fun removeChecksumHeaders() { val headers = HeadersBuilder() - headers.append("x-amz-checksum-crc32", "foo") - headers.append("x-amz-checksum-sha256", "bar") + headers.append(crc32Header, "foo") + headers.append(sha256Header, "bar") assertTrue( - headers.contains("x-amz-checksum-crc32"), + headers.contains(crc32Header), ) assertTrue( - headers.contains("x-amz-checksum-sha256"), + headers.contains(sha256Header), ) headers.removeChecksumHeaders() assertFalse( - headers.contains("x-amz-checksum-crc32"), + headers.contains(crc32Header), ) assertFalse( - headers.contains("x-amz-checksum-sha256"), + headers.contains(sha256Header), ) } @@ -35,23 +39,23 @@ class ChecksumRemovalTest { fun removeChecksumTrailingHeaders() { val trailingHeaders = DeferredHeadersBuilder() - trailingHeaders.add("x-amz-checksum-crc32", "foo") - trailingHeaders.add("x-amz-checksum-sha256", "bar") + trailingHeaders.add(crc32Header, "foo") + trailingHeaders.add(sha256Header, "bar") assertTrue( - trailingHeaders.contains("x-amz-checksum-crc32"), + trailingHeaders.contains(crc32Header), ) assertTrue( - trailingHeaders.contains("x-amz-checksum-sha256"), + trailingHeaders.contains(sha256Header), ) trailingHeaders.removeChecksumTrailingHeaders() assertFalse( - trailingHeaders.contains("x-amz-checksum-crc32"), + trailingHeaders.contains(crc32Header), ) assertFalse( - trailingHeaders.contains("x-amz-checksum-sha256"), + trailingHeaders.contains(sha256Header), ) } @@ -59,13 +63,13 @@ class ChecksumRemovalTest { fun removeChecksumTrailingHeadersFromXAmzTrailer() { val headers = HeadersBuilder() - headers.append("x-amz-trailer", "x-amz-checksum-crc32") + headers.append("x-amz-trailer", crc32Header) headers.append("x-amz-trailer", "x-amz-trailing-header") val xAmzTrailer = headers.getAll("x-amz-trailer") assertTrue( - xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + xAmzTrailer?.contains(crc32Header) ?: false, ) assertTrue( xAmzTrailer?.contains("x-amz-trailing-header") ?: false, @@ -74,7 +78,7 @@ class ChecksumRemovalTest { headers.removeChecksumTrailingHeadersFromXAmzTrailer() assertFalse( - xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + xAmzTrailer?.contains(crc32Header) ?: false, ) assertTrue( xAmzTrailer?.contains("x-amz-trailing-header") ?: false, diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index db8c4026d5a..4b99ae22e8f 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -8,19 +8,21 @@ import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.model.GetObjectRequest -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.content.* +import aws.smithy.kotlin.runtime.hashing.crc32 +import aws.smithy.kotlin.runtime.testing.RandomTempFile import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.* import java.io.File import java.io.FileInputStream import java.util.* +import kotlin.test.assertEquals @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3ChecksumTest { private val client = S3Client { region = "us-west-2" } private val testBucket = "s3-test-bucket-ci-motorcade" - private val testObject = "test-object" + private fun testKey(): String = "test-object" + UUID.randomUUID() @BeforeAll private fun setUp(): Unit = runBlocking { @@ -38,87 +40,123 @@ class S3ChecksumTest { @Test fun testPutObject(): Unit = runBlocking { + val testBody = "Hello World" + val testKey = testKey() + client.putObject { bucket = testBucket - key = testObject - body = ByteStream.fromString("Hello World") + key = testKey + body = ByteStream.fromString(testBody) + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testPutObjectWithEmptyBody(): Unit = runBlocking { + val testKey = testKey() + val testBody = "" + client.putObject { bucket = testBucket - key = testObject + UUID.randomUUID() + key = testKey + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { - val testString = "Hello World" + val testKey = testKey() + val testBody = "Hello World" + val tempFile = File.createTempFile("test", ".txt").also { - it.writeText(testString) + it.writeText(testBody) it.deleteOnExit() } val inputStream = FileInputStream(tempFile) client.putObject { bucket = testBucket - key = testObject + UUID.randomUUID() - body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) + key = testKey + body = ByteStream.fromInputStream(inputStream, testBody.length.toLong()) + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testMultiPartUpload(): Unit = runBlocking { - // Parts need to be at least 5 MB - val partOne = "Hello".repeat(1_048_576) - val partTwo = "World".repeat(1_048_576) + val testKey = testKey() + + val partSize = 5 * 1024 * 1024 // 5 MB - min part size + val contentSize: Long = 8 * 1024 * 1024 // 2 parts + val file = RandomTempFile(sizeInBytes = contentSize) + + val expectedChecksum = file.readBytes().crc32() val testUploadId = client.createMultipartUpload { bucket = testBucket - key = testObject + key = testKey }.uploadId - val eTagPartOne = client.uploadPart { - bucket = testBucket - key = testObject - partNumber = 1 - uploadId = testUploadId - body = ByteStream.fromString(partOne) - }.eTag + val uploadedParts = file.chunk(partSize).mapIndexed { index, chunk -> + val adjustedIndex = index + 1 // index starts from 0 but partNumber needs to start from 1 - val eTagPartTwo = client.uploadPart { - bucket = testBucket - key = testObject - partNumber = 2 - uploadId = testUploadId - body = ByteStream.fromString(partTwo) - }.eTag + runBlocking { + client.uploadPart { + bucket = testBucket + key = testKey + partNumber = adjustedIndex + uploadId = testUploadId + body = file.asByteStream(chunk) + }.let { + CompletedPart { + partNumber = adjustedIndex + eTag = it.eTag + } + } + } + }.toList() client.completeMultipartUpload { bucket = testBucket - key = testObject + key = testKey uploadId = testUploadId multipartUpload = CompletedMultipartUpload { - parts = listOf( - CompletedPart { - partNumber = 1 - eTag = eTagPartOne - }, - CompletedPart { - partNumber = 2 - eTag = eTagPartTwo - }, - ) + parts = uploadedParts } } client.getObject( GetObjectRequest { bucket = testBucket - key = testObject + key = testKey }, - ) {} + ) { actual -> + val actualChecksum = actual.body!!.toByteArray().crc32() + assertEquals(actualChecksum, expectedChecksum) + } } } diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 37ec353c9f6..aec1de82d16 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -390,7 +390,7 @@ class S3BucketOpsIntegrationTest { } // generate sequence of "chunks" where each range defines the inclusive start and end bytes -private fun File.chunk(partSize: Int): Sequence = +internal fun File.chunk(partSize: Int): Sequence = (0 until length() step partSize.toLong()).asSequence().map { it until minOf(it + partSize, length()) } diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 4139b4d9e33..e089beb6614 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -10,6 +10,7 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.model.ExpirationStatus import aws.sdk.kotlin.services.s3.model.LifecycleRule +import aws.sdk.kotlin.services.s3.paginators.listBucketsPaginated import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketNotExists @@ -103,26 +104,42 @@ object S3TestUtils { suspend fun getBucketByName( client: S3Client, - bucket: String, + targetBucket: String, region: String? = null, accountId: String? = null, ): String = withTimeout(60.seconds) { - val buckets = client.listBuckets() - .buckets - ?.mapNotNull { it.name } + val bucketNames = mutableListOf() - var testBucket = buckets?.firstOrNull { bucketName -> - bucketName == bucket && - region?.let { + client.listBucketsPaginated() + .collect { response -> + response.buckets?.forEach { bucket -> + bucket.name?.let { bucketNames.add(it) } + } + } + + var testBucket = bucketNames.firstOrNull { bucketName -> + if (bucketName == targetBucket) { + val isInCorrectLocation = region?.let { client.getBucketLocation { - this.bucket = bucketName + bucket = bucketName expectedBucketOwner = accountId }.locationConstraint?.value == region } ?: true + + if (isInCorrectLocation) { + true + } else { + throw RuntimeException( + "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", + ) + } + } else { + false + } } if (testBucket == null) { - testBucket = bucket + testBucket = targetBucket println("Creating S3 bucket: $testBucket") client.createBucket { diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index dd0a9537e6b..696c3c3104a 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -16,7 +16,6 @@ subprojects { apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) val optinAnnotations = listOf( - "kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi", "aws.sdk.kotlin.runtime.InternalSdkApi", ) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index e26afa7908c..e7fe9933394 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -265,6 +265,10 @@ class UserProvidedChecksumHeader { assertTrue( headerReader.containsExpectedHeaders, ) + + assertFalse( + headerReader.containsForbiddenHeaders, + ) } } @@ -370,35 +374,4 @@ class ResponseChecksumValidation { } } } - - @Test - fun compositeChecksumsAreNotValidated(): Unit = runBlocking { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append( - "x-amz-checksum-crc32", - "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1", - ) - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } - } - } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index a8f1f634a1f..c88263bfe70 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -20,8 +20,8 @@ internal class HeaderReader( private val expectedHeaders: Map, private val forbiddenHeaders: Map = emptyMap(), ) : HttpInterceptor { - var containsExpectedHeaders = true - var containsForbiddenHeaders = false + var containsExpectedHeaders = false + var containsForbiddenHeaders = true override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { expectedHeaders.forEach { header -> @@ -31,17 +31,18 @@ internal class HeaderReader( } ?: true if (!containsHeader || !headerValueMatches) { - containsExpectedHeaders = false return } } forbiddenHeaders.forEach { header -> if (context.protocolRequest.headers.contains(header.key)) { - containsForbiddenHeaders = true return } } + + containsExpectedHeaders = true + containsForbiddenHeaders = false } } @@ -66,7 +67,7 @@ internal class HeaderSetter( internal class BusinessMetricsReader( private val expectedBusinessMetrics: Set, ) : HttpInterceptor { - var containsExpectedBusinessMetrics = true + var containsExpectedBusinessMetrics = false override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) From 1aefe666b3e450962428e01c2b3fef5e7a373b48 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 11 Dec 2024 17:44:10 -0500 Subject: [PATCH 49/64] Use head bucket --- services/s3/e2eTest/src/S3TestUtils.kt | 56 ++++++++------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index e089beb6614..6ace834f00f 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -10,7 +10,6 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.model.ExpirationStatus import aws.sdk.kotlin.services.s3.model.LifecycleRule -import aws.sdk.kotlin.services.s3.paginators.listBucketsPaginated import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketNotExists @@ -107,54 +106,31 @@ object S3TestUtils { targetBucket: String, region: String? = null, accountId: String? = null, - ): String = withTimeout(60.seconds) { - val bucketNames = mutableListOf() - - client.listBucketsPaginated() - .collect { response -> - response.buckets?.forEach { bucket -> - bucket.name?.let { bucketNames.add(it) } - } - } - - var testBucket = bucketNames.firstOrNull { bucketName -> - if (bucketName == targetBucket) { - val isInCorrectLocation = region?.let { - client.getBucketLocation { - bucket = bucketName - expectedBucketOwner = accountId - }.locationConstraint?.value == region - } ?: true - - if (isInCorrectLocation) { - true - } else { - throw RuntimeException( - "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", - ) - } - } else { - false + ): Unit = withTimeout(60.seconds) { + try { + val targetBucketRegion = client + .headBucket { + this.bucket = targetBucket + expectedBucketOwner = accountId + }.bucketRegion + + if (targetBucketRegion != region) { + throw RuntimeException( + "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", + ) } - } - - if (testBucket == null) { - testBucket = targetBucket - println("Creating S3 bucket: $testBucket") + } catch (e: Throwable) { + println("Creating S3 bucket: $targetBucket") client.createBucket { - this.bucket = testBucket + bucket = targetBucket createBucketConfiguration { locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) } } - client.waitUntilBucketExists { this.bucket = testBucket } - } else { - println("Using existing S3 bucket: $testBucket") + client.waitUntilBucketExists { bucket = targetBucket } } - - testBucket } suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { From 55bdcb87583698e678ba098e7a52cfd1e7ae3002 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 01:18:30 -0500 Subject: [PATCH 50/64] Presigned URL checksums --- .../sdk/kotlin/codegen/PresignerGenerator.kt | 77 ++++++++++++++++++- services/s3/e2eTest/src/S3ChecksumTest.kt | 39 +++++++++- 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index 34dbab1293f..a912e904579 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,22 +5,32 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable +import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.IllegalStateException +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.lowercase +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinCoroutines.coroutineContext +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Observability.TelemetryApi.warn import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionId import software.amazon.smithy.kotlin.codegen.integration.SectionKey import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes -import software.amazon.smithy.kotlin.codegen.model.buildSymbol -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.getTrait +import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.model.knowledge.AwsSignatureVersion4 import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointResolverAdapterGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingResolver import software.amazon.smithy.kotlin.codegen.rendering.serde.serializerName +import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.knowledge.TopDownIndex import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -149,6 +159,7 @@ class PresignerGenerator : KotlinIntegration { requestSymbol, serializerSymbol, contextMap, + op, ) } } @@ -193,6 +204,7 @@ class PresignerGenerator : KotlinIntegration { requestSymbol: Symbol, serializerSymbol: Symbol, contextMap: Map, Any>, + op: OperationShape, ) = writer.apply { dokka { write("Presign a [#T] using the configuration of this [#T].", requestSymbol, serviceSymbol) @@ -265,6 +277,45 @@ class PresignerGenerator : KotlinIntegration { ) } + checksumAlgorithmMember(op, ctx)?.let { checksumAlgorithmMember -> + withBlock("input.#L?.value?.let { checksumAlgorithmString ->", "}", checksumAlgorithmMember) { + withBlock("when (unsignedRequest.body) {", "}") { + withBlock("is #1T.Bytes, is #1T.Empty -> {", "}", HttpBody) { + write("val checksumAlgorithm = checksumAlgorithmString.#T()", toHashFunctionOrThrow) + withInlineBlock( + "if (checksumAlgorithm.#T) {", + "}", + isSupportedForFlexibleChecksums, + ) { + withBlock("#T {", "}", runBlocking) { + withBlock("checksumAlgorithm.update(", ")") { + write("unsignedRequest.body.#T() ?: byteArrayOf()", readAll) + } + } + write( + "checksum = #S.#T() to checksumAlgorithm.digest().#T()", + "x-amz-checksum-\${checksumAlgorithmString}", + lowercase, + encodeBase64String, + ) + } + withBlock(" else {", "}") { + withBlock("#T {", "}", runBlocking) { + write("class Presigner") + write( + "#T.#T { #S }", + coroutineContext, + warn, + "The requested checksum algorithm is not supported for pre-signed URL checksums, sending request without checksum.", + ) + } + } + } + write("else -> throw #T(#S)", IllegalStateException, "HTTP body type unsupported for pre-signed URL checksums.") + } + } + } + declareSection(SigningConfigCustomizationSection) write("configBlock()") @@ -287,4 +338,24 @@ class PresignerGenerator : KotlinIntegration { * > "my-object/example/photo.user". This is an incorrect path for that object. */ private fun normalizeUriPath(service: ServiceShape) = service.sdkId != "S3" + + /** + * Gets the checksum algorithm member if a user can configure request checksums otherwise null + */ + private fun checksumAlgorithmMember( + operationShape: OperationShape, + ctx: CodegenContext, + ): String? { + operationShape.getTrait()?.let { httpChecksumTrait -> + httpChecksumTrait.requestAlgorithmMember.getOrNull()?.let { requestAlgorithmMember -> + val memberShape = ctx.model + .expectShape(operationShape.input.get()) + .members() + .first { it.memberName == requestAlgorithmMember } + + return ctx.symbolProvider.toMemberName(memberShape) + } + } + return null + } } diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 4b99ae22e8f..aae62d170b9 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -4,10 +4,10 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketByName +import aws.sdk.kotlin.e2etest.S3TestUtils.responseCodeFromPut import aws.sdk.kotlin.services.s3.* -import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload -import aws.sdk.kotlin.services.s3.model.CompletedPart -import aws.sdk.kotlin.services.s3.model.GetObjectRequest +import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.presigners.presignPutObject import aws.smithy.kotlin.runtime.content.* import aws.smithy.kotlin.runtime.hashing.crc32 import aws.smithy.kotlin.runtime.testing.RandomTempFile @@ -16,7 +16,11 @@ import org.junit.jupiter.api.* import java.io.File import java.io.FileInputStream import java.util.* +import kotlin.test.Ignore import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.seconds @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3ChecksumTest { @@ -159,4 +163,33 @@ class S3ChecksumTest { assertEquals(actualChecksum, expectedChecksum) } } + + @Test + fun testPresignedUrlNoDefault() = runBlocking { + val unsignedPutRequest = PutObjectRequest { + bucket = testBucket + key = testKey() + } + val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) + val contents = "presign-test" + + assertFalse(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) + assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) + } + + @Test + @Ignore + // FIXME: Sending checksum via query params should work + fun testPresignedUrlContainsChecksum() = runBlocking { + val unsignedPutRequest = PutObjectRequest { + bucket = testBucket + key = testKey() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) + val contents = "presign-test" + + assertTrue(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) + assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) + } } From 44894e67875e7bb23d63331cf7c24c4ee1e28205 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 01:19:43 -0500 Subject: [PATCH 51/64] Ktlint --- services/s3/e2eTest/src/S3ChecksumTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index aae62d170b9..c3149ba4b82 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -177,9 +177,9 @@ class S3ChecksumTest { assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) } + // FIXME: Sending checksum via query params should work @Test @Ignore - // FIXME: Sending checksum via query params should work fun testPresignedUrlContainsChecksum() = runBlocking { val unsignedPutRequest = PutObjectRequest { bucket = testBucket From ad8d1adfe6e97058299b873b7c7e079c39987b91 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 10:02:50 -0500 Subject: [PATCH 52/64] Trigger CI From 3c259eff149cab843f61994bb696f53b70fa961c Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 15:02:36 -0500 Subject: [PATCH 53/64] PR feeback checkpoint --- .../S3CompositeChecksumsInterceptor.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 18 ++++++++---------- .../FlexibleChecksumsRequest.kt | 2 +- .../S3ExpressDisableChecksumInterceptor.kt | 2 +- services/s3/e2eTest/src/PaginatorTest.kt | 3 +-- services/s3/e2eTest/src/S3ChecksumTest.kt | 3 ++- services/s3/e2eTest/src/S3IntegrationTest.kt | 3 +-- services/s3/e2eTest/src/S3TestUtils.kt | 5 ++--- 8 files changed, 17 insertions(+), 21 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt index 16adc9aeef5..71ee8107ccd 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt @@ -32,7 +32,7 @@ public class S3CompositeChecksumsInterceptor : HttpInterceptor { internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = names().forEach { header -> if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { - logger.warn { "S3 returned a composite checksum, composite checksums are not supported, removing checksum" } + logger.warn { "S3 returned a composite checksum ($header), composite checksums are not supported, removing checksum" } remove(header) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index a912e904579..134b11bb3c9 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -12,9 +12,7 @@ import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.IllegalStateException import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.lowercase import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll @@ -278,10 +276,10 @@ class PresignerGenerator : KotlinIntegration { } checksumAlgorithmMember(op, ctx)?.let { checksumAlgorithmMember -> - withBlock("input.#L?.value?.let { checksumAlgorithmString ->", "}", checksumAlgorithmMember) { + withBlock("input.#L?.value?.let { checksumAlgorithmName ->", "}", checksumAlgorithmMember) { withBlock("when (unsignedRequest.body) {", "}") { withBlock("is #1T.Bytes, is #1T.Empty -> {", "}", HttpBody) { - write("val checksumAlgorithm = checksumAlgorithmString.#T()", toHashFunctionOrThrow) + write("val checksumAlgorithm = checksumAlgorithmName.#T()", toHashFunctionOrThrow) withInlineBlock( "if (checksumAlgorithm.#T) {", "}", @@ -293,9 +291,8 @@ class PresignerGenerator : KotlinIntegration { } } write( - "checksum = #S.#T() to checksumAlgorithm.digest().#T()", - "x-amz-checksum-\${checksumAlgorithmString}", - lowercase, + "checksum = #S.lowercase() to checksumAlgorithm.digest().#T()", + "x-amz-checksum-\${checksumAlgorithmName}", encodeBase64String, ) } @@ -306,12 +303,13 @@ class PresignerGenerator : KotlinIntegration { "#T.#T { #S }", coroutineContext, warn, - "The requested checksum algorithm is not supported for pre-signed URL checksums, sending request without checksum.", + "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + + "Supported checksums for pre-signed URLs include the following: ", ) } } } - write("else -> throw #T(#S)", IllegalStateException, "HTTP body type unsupported for pre-signed URL checksums.") + write("else -> throw IllegalStateException(#S)", "HTTP body type unsupported for pre-signed URL checksums.") } } } @@ -340,7 +338,7 @@ class PresignerGenerator : KotlinIntegration { private fun normalizeUriPath(service: ServiceShape) = service.sdkId != "S3" /** - * Gets the checksum algorithm member if a user can configure request checksums otherwise null + * Gets the checksum algorithm member if configured on a request, otherwise null */ private fun checksumAlgorithmMember( operationShape: OperationShape, diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index d79824212ff..959598100ca 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -69,7 +69,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { } private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { - override val name: String = "FlexibleChecksumsRequest" + override val name: String = "flexibleChecksumsRequestMiddleware" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { val httpChecksumTrait = op.getTrait() diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 1dd06c25406..cd84c2bfc76 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -19,7 +19,7 @@ private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** - * Disables checksums for s3:UploadPart requests that use S3 express. + * Disables checksums for s3:UploadPart requests that use S3 Express. */ internal class S3ExpressDisableChecksumInterceptor( private val userConfiguredChecksum: Boolean, diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index 3d1ed2f4919..f35ea41fff9 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -41,9 +41,8 @@ class PaginatorTest { S3TestUtils.deleteBucketAndAllContents(client, testBucket) } - // FIXME: Enable test when motorcade is ready + // FIXME: Enable test // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 - // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" // ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly. @Ignore @Test diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index c3149ba4b82..0c271178582 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -31,7 +31,7 @@ class S3ChecksumTest { @BeforeAll private fun setUp(): Unit = runBlocking { val accountId = getAccountId() - // FIXME: Use randomly generated bucket instead of hardcoded one when motorcade is ready + // FIXME: Use randomly generated bucket instead of hardcoded one getBucketByName(client, testBucket, "us-west-2", accountId) } @@ -169,6 +169,7 @@ class S3ChecksumTest { val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() + checksumCrc32 = "`" } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) val contents = "presign-test" diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index aec1de82d16..7b9c22fa456 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -188,9 +188,8 @@ class S3BucketOpsIntegrationTest { } } - // FIXME: Enable test when motorcade is ready + // FIXME: Enable test // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 - // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" @Ignore @Test fun testMultipartUpload(): Unit = runBlocking { diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 6ace834f00f..21b8f311c47 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -34,9 +34,8 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-" - // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". - // Non-checksum E2E tests will use it and delete it if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` - // TODO: Change back to "s3-test-bucket-" after motorcade is released. + // Non-checksum E2E tests will use and delete hardcoded bucket required for checksum tests if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` + // TODO: Change back to "s3-test-bucket-" private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html From 0029768dacf39ff1378333d71d4790219ec71623 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 16:39:48 -0500 Subject: [PATCH 54/64] Trigger CI From 427daf35892bdc67381ea976d2ead5c2d43e1b33 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 18:34:23 -0500 Subject: [PATCH 55/64] Refactor checksum interceptors --- .../FlexibleChecksumsRequest.kt | 140 +++++++++++------- .../FlexibleChecksumsResponse.kt | 103 +++++++------ .../s3/express/S3ExpressIntegration.kt | 37 ++--- .../S3ExpressDefaultChecksumAlgorithm.kt | 37 +++++ .../S3ExpressDisableChecksumInterceptor.kt | 77 ---------- 5 files changed, 190 insertions(+), 204 deletions(-) create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt delete mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 959598100ca..ec3acddf0a9 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -22,15 +22,15 @@ import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape /** - * Adds a middleware that enables sending flexible checksums during an HTTP request + * Handles flexible checksum requests */ class FlexibleChecksumsRequest : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings) = model - .shapes() - .any { it.hasTrait() } + override fun enabledForService(model: Model, settings: KotlinSettings) = + model.isTraitApplied(HttpChecksumTrait::class.java) override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf( + // Allows flexible checksum request configuration ConfigProperty { name = "requestChecksumCalculation" symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption @@ -42,66 +42,96 @@ class FlexibleChecksumsRequest : KotlinIntegration { ) override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics - - private val configBusinessMetrics = object : ProtocolMiddleware { - override val name: String = "requestChecksumCalculationBusinessMetric" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.hasTrait() - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.withBlock("when(config.requestChecksumCalculation) {", "}") { - writer.write( - "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - writer.write( - "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - } - } - } + resolved + requestChecksumCalculationBusinessMetric + httpChecksumDefaultAlgorithmMiddleware + flexibleChecksumsRequestMiddleware +} - private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { - override val name: String = "flexibleChecksumsRequestMiddleware" +/** + * Emits business metric based on `requestChecksumCalculation` client config + */ +private val requestChecksumCalculationBusinessMetric = object : ProtocolMiddleware { + override val name: String = "requestChecksumCalculationBusinessMetric" - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val input = op.input.getOrNull()?.let { ctx.model.expectShape(it) } + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() - return (httpChecksumTrait != null) && - (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && - (input?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.requestChecksumCalculation) {", "}") { + // Supported + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + // Required + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) } + } +} - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) +/** + * Adds default checksum algorithm to the execution context + */ +private val httpChecksumDefaultAlgorithmMiddleware = object : ProtocolMiddleware { + override val name: String = "httpChecksumDefaultAlgorithmMiddleware" + override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0) - val httpChecksumTrait = op.getTrait()!! + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + requestChecksumsConfigured(ctx, op) - val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.write( + "op.context[#T.DefaultChecksumAlgorithm] = #S", + RuntimeTypes.HttpClient.Operation.HttpOperationContext, + "CRC32", + ) + } +} + +/** + * Adds interceptor to handle flexible checksum request calculation + */ +private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { + override val name: String = "flexibleChecksumsRequestMiddleware" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + requestChecksumsConfigured(ctx, op) - val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) - val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val httpChecksumTrait = op.getTrait()!! + val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) - writer.withBlock( - "op.interceptors.add(#T<#T>(", - "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, - inputSymbol, - ) { - writer.write("#L,", requestChecksumRequired) - writer.write("config.requestChecksumCalculation,") - writer.write("input.#L?.value,", requestAlgorithmMemberName) - } + writer.withBlock( + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + ) { + writer.write("#L,", requestChecksumRequired) + writer.write("config.requestChecksumCalculation,") + writer.write("input.#L?.value,", requestAlgorithmMemberName) } } } + +/** + * Determines if an operation is set up to send flexible request checksums + */ +private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { + val httpChecksumTrait = op.getTrait() + val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } + + return ( + (httpChecksumTrait != null) && + (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + ) +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 9767d2a9718..c5164d40652 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -19,15 +19,15 @@ import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape /** - * Adds a middleware which validates checksums returned in responses if the user has opted-in. + * Handles flexible checksum responses */ class FlexibleChecksumsResponse : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings) = model - .shapes() - .any { it.hasTrait() } + override fun enabledForService(model: Model, settings: KotlinSettings) = + model.isTraitApplied(HttpChecksumTrait::class.java) override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf( + // Allows flexible checksum response configuration ConfigProperty { name = "responseChecksumValidation" symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption @@ -39,59 +39,66 @@ class FlexibleChecksumsResponse : KotlinIntegration { ) override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsResponseMiddleware + configBusinessMetrics + resolved + flexibleChecksumsResponseMiddleware + responseChecksumValidationBusinessMetric +} - private val configBusinessMetrics = object : ProtocolMiddleware { - override val name: String = "responseChecksumValidationBusinessMetric" +/** + * Emits business metric based on `responseChecksumValidation` client config + */ +private val responseChecksumValidationBusinessMetric = object : ProtocolMiddleware { + override val name: String = "responseChecksumValidationBusinessMetric" - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.withBlock("when(config.responseChecksumValidation) {", "}") { - writer.write( - "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - writer.write( - "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - } + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.responseChecksumValidation) {", "}") { + // Supported + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + // Required + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) } } +} - private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { - override val name: String = "FlexibleChecksumsResponse" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val input = op.input.getOrNull()?.let { ctx.model.expectShape(it) } +/** + * Adds interceptor to handle flexible checksum response validation + */ +private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { + override val name: String = "flexibleChecksumsResponseMiddleware" - return (httpChecksumTrait != null) && - (httpChecksumTrait.requestValidationModeMember?.getOrNull() != null) && - (input?.memberNames?.any { it == httpChecksumTrait.requestValidationModeMember.get() } == true) - } + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { + val httpChecksumTrait = op.getTrait() + val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + return (httpChecksumTrait != null) && + (httpChecksumTrait.requestValidationModeMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestValidationModeMember.get() } == true) + } - val httpChecksumTrait = op.getTrait()!! - val requestValidationModeMember = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } - val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! + val requestValidationModeMember = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } + val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) - writer.withBlock( - "op.interceptors.add(#T<#T>(", - "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, - inputSymbol, - ) { - writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) - writer.write("config.responseChecksumValidation,") - } + writer.withBlock( // TODO: ADD DIFFERENT INTERCEPTORS DEPENDING ON IF THE SERVICE HAS COMPOSITE CHECKSUMS ! + "op.interceptors.add(#T<#T>(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, + inputSymbol, + ) { + writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("config.responseChecksumValidation,") } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 067da2da40f..086534f3777 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,7 +16,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -26,7 +25,7 @@ import software.amazon.smithy.model.transform.ModelTransformer * An integration which handles codegen for S3 Express, such as: * 1. Configure auth scheme by applying a synthetic shape and trait * 2. Add ExpressClient and Bucket to execution context - * 3. Disable all checksums for s3:UploadPart + * 3. Configuring the default checksum algorithm */ class S3ExpressIntegration : KotlinIntegration { companion object { @@ -97,7 +96,7 @@ class S3ExpressIntegration : KotlinIntegration { resolved + listOf( addClientToExecutionContext, addBucketToExecutionContext, - uploadPartDisableChecksum, + s3ExpressDefaultChecksumAlgorithm, ) private val s3AttributesSymbol = buildSymbol { @@ -130,33 +129,23 @@ class S3ExpressIntegration : KotlinIntegration { } /** - * Disable all checksums for s3:UploadPart + * Re-configures the default checksum algorithm for S3 Express. */ - private val uploadPartDisableChecksum = object : ProtocolMiddleware { - override val name: String = "UploadPartDisableChecksum" + private val s3ExpressDefaultChecksumAlgorithm = object : ProtocolMiddleware { + override val name: String = "s3ExpressDefaultChecksumAlgorithm" + override val order: Byte = -1 // After setting the modeled default (-2) and before calculating the checksum (0) override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.isS3UploadPart && op.hasTrait() + op.hasTrait() || op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val httpChecksumTrait = op.getTrait()!! - - val requestAlgorithmMemberName = httpChecksumTrait.requestAlgorithmMember?.getOrNull()?.let { - val requestAlgorithmMemberShape = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - ctx.symbolProvider.toMemberName(requestAlgorithmMemberShape) - } - - val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3.express" - name = "S3ExpressDisableChecksumInterceptor" - } - writer.addImport(interceptorSymbol) writer.write( - "op.interceptors.add(#T(input.#L?.value != null))", - interceptorSymbol, - requestAlgorithmMemberName, + "op.interceptors.add(#T(#L))", + buildSymbol { + namespace = "aws.sdk.kotlin.services.s3.express" + name = "S3ExpressDefaultChecksumAlgorithm" + }, + op.isS3UploadPart ) } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt new file mode 100644 index 00000000000..862e96c1bf9 --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.express + +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.operation.ExecutionContext + +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" + +/** + * Re-configures the default checksum algorithm for S3 Express + * NOTE: Default checksums are disabled for s3:UploadPart. + */ +internal class S3ExpressDefaultChecksumAlgorithm( + private val isS3UploadPart: Boolean, +) : HttpInterceptor { + override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { + if (usingS3Express(context.executionContext)) { + if (isS3UploadPart) { + context.executionContext.remove(HttpOperationContext.DefaultChecksumAlgorithm) + } else { + context.executionContext[HttpOperationContext.DefaultChecksumAlgorithm] = "CRC32" + } + } + return super.modifyBeforeSigning(context) + } +} + +private fun usingS3Express(executionContext: ExecutionContext): Boolean = + executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt deleted file mode 100644 index cd84c2bfc76..00000000000 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.AttributeKey -import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.warn -import kotlin.coroutines.coroutineContext - -private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" -private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" -private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" - -/** - * Disables checksums for s3:UploadPart requests that use S3 Express. - */ -internal class S3ExpressDisableChecksumInterceptor( - private val userConfiguredChecksum: Boolean, -) : HttpInterceptor { - override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { - return context.protocolRequest - } - - if (userConfiguredChecksum) { - coroutineContext.warn { - "Checksums must not be sent with S3 Express UploadPart operation, removing checksum(s)" - } - } - - val request = context.protocolRequest.toBuilder() - - request.headers.removeChecksumHeaders() - request.trailingHeaders.removeChecksumTrailingHeaders() - request.headers.removeChecksumTrailingHeadersFromXAmzTrailer() - - return request.build() - } -} - -/** - * Removes any checksums sent in the request's headers - */ -internal fun HeadersBuilder.removeChecksumHeaders(): Unit = - names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - remove(name) - } - } - -/** - * Removes any checksums sent in the request's trailing headers - */ -internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = - names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - remove(name) - } - } - -/** - * Removes any checksums sent in the request's trailing headers from `x-amz-trailer` - */ -internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { - this.getAll("x-amz-trailer")?.forEach { trailingHeader -> - if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - this.remove("x-amz-trailer", trailingHeader) - } - } -} From 41ad742e63456bc0f0f6a148cc46c535ac758280 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 21:38:29 -0500 Subject: [PATCH 56/64] Fix composite checksums --- .../S3CompositeChecksumsInterceptor.kt | 47 ------------------- .../S3FlexibleChecksumResponseInterceptor.kt | 27 +++++++++++ codegen/aws-sdk-codegen/build.gradle.kts | 1 + .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 3 +- .../FlexibleChecksumsResponse.kt | 17 +++++-- .../s3/S3CompositeChecksumsIntegration.kt | 44 ----------------- ...tlin.codegen.integration.KotlinIntegration | 1 - services/s3/e2eTest/src/S3ChecksumTest.kt | 1 - .../event-stream-model-template.smithy | 4 +- 10 files changed, 45 insertions(+), 102 deletions(-) delete mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt deleted file mode 100644 index 71ee8107ccd..00000000000 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt +++ /dev/null @@ -1,47 +0,0 @@ -package aws.sdk.kotlin.runtime.http.interceptors - -import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.http.response.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.Logger -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.coroutines.coroutineContext - -private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" - -/** - * Removes any checksum headers that contain composite checksums from an S3 response. - */ -public class S3CompositeChecksumsInterceptor : HttpInterceptor { - override suspend fun modifyBeforeDeserialization(context: ProtocolResponseInterceptorContext): HttpResponse { - val response = context.protocolResponse.toBuilder() - val logger = coroutineContext.logger() - - response.headers.removeCompositeChecksums(logger) - - return response.build() - } -} - -/** - * Removes any checksum headers that contain composite checksums. - */ -internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = - names().forEach { header -> - if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { - logger.warn { "S3 returned a composite checksum ($header), composite checksums are not supported, removing checksum" } - remove(header) - } - } - -/** - * Verifies if a checksum is composite. - */ -private fun String.isCompositeChecksum(): Boolean { - // Ends with "-#" where "#" is a number - val regex = Regex("-(\\d)+$") - return regex.containsMatchIn(this) -} diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt new file mode 100644 index 00000000000..7845f2f7f61 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt @@ -0,0 +1,27 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor + +/** + * S3 variant of the flexible checksum interceptor where composite checksums are not validated + */ +public class S3FlexibleChecksumResponseInterceptor( + responseValidationRequired: Boolean, + responseChecksumValidation: HttpChecksumConfigOption?, +) : FlexibleChecksumsResponseInterceptor( + responseValidationRequired, + responseChecksumValidation, +) { + override fun ignoreChecksum(checksum: String): Boolean = + checksum.isCompositeChecksum() +} + +/** + * Verifies if a checksum is composite. + */ +private fun String.isCompositeChecksum(): Boolean { + // Ends with "-#" where "#" is a number + val regex = Regex("-(\\d)+$") + return regex.containsMatchIn(this) +} diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index aa404d7a4cb..0348cd5a7f1 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { api(libs.smithy.protocol.test.traits) implementation(libs.smithy.aws.endpoints) implementation(libs.smithy.smoke.test.traits) + implementation(libs.smithy.kotlin.runtime.core) testImplementation(libs.junit.jupiter) testImplementation(libs.junit.jupiter.params) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 70f3a4950ef..0099e436bb1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,7 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") - val S3CompositeChecksumsInterceptor = symbol("S3CompositeChecksumsInterceptor") + val S3FlexibleChecksumResponseInterceptor = symbol("S3FlexibleChecksumResponseInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index 134b11bb3c9..d9b19377c47 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable +import aws.smithy.kotlin.runtime.hashing.algorithmsSupportedForFlexibleChecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait @@ -304,7 +305,7 @@ class PresignerGenerator : KotlinIntegration { coroutineContext, warn, "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + - "Supported checksums for pre-signed URLs include the following: ", + "Supported checksums for pre-signed URLs: $algorithmsSupportedForFlexibleChecksums", ) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index c5164d40652..3691cc82a26 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -4,6 +4,8 @@ */ package aws.sdk.kotlin.codegen.customization.flexiblechecksums +import aws.sdk.kotlin.codegen.AwsRuntimeTypes +import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* @@ -16,6 +18,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.StructureShape /** @@ -84,18 +87,22 @@ private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) - writer.withBlock( // TODO: ADD DIFFERENT INTERCEPTORS DEPENDING ON IF THE SERVICE HAS COMPOSITE CHECKSUMS ! - "op.interceptors.add(#T<#T>(", + val interceptor = if (ctx.model.expectShape(ctx.settings.service).isS3) { + AwsRuntimeTypes.Http.Interceptors.S3FlexibleChecksumResponseInterceptor + } else { + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor + } + + writer.withBlock( + "op.interceptors.add(#T(", "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, - inputSymbol, + interceptor, ) { writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("config.responseChecksumValidation,") diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt deleted file mode 100644 index cef54247f72..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt +++ /dev/null @@ -1,44 +0,0 @@ -package aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3 - -import aws.sdk.kotlin.codegen.AwsRuntimeTypes.Http.Interceptors.S3CompositeChecksumsInterceptor -import aws.sdk.kotlin.codegen.customization.s3.isS3 -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.KotlinWriter -import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.hasTrait -import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator -import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware -import software.amazon.smithy.model.Model -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ServiceShape - -/** - * Removes composite checksums returned by S3 so that flexible checksums won't validate them. - * Composite checksums are used for multipart uploads and end with "-#" where "#" is a number. - */ -class S3CompositeChecksumsIntegration : KotlinIntegration { - override val order: Byte - get() = -128 - - override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = - model.expectShape(settings.service).isS3 - - override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + s3CompositeChecksumsMiddleware -} - -/** - * Adds [S3CompositeChecksumsInterceptor] to interceptors when operation has flexible checksums. - */ -private val s3CompositeChecksumsMiddleware = object : ProtocolMiddleware { - override val name: String = "S3CompositeChecksumsMiddleware" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.hasTrait() - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.write("op.interceptors.add(#T())", S3CompositeChecksumsInterceptor) - } -} diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 39e5630c930..60ccb728f63 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -12,7 +12,6 @@ aws.sdk.kotlin.codegen.endpoints.AddAwsEndpointFunctions aws.sdk.kotlin.codegen.PresignerGenerator aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsRequest aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsResponse -aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3.S3CompositeChecksumsIntegration aws.sdk.kotlin.codegen.customization.AddAwsSpanAttributes aws.sdk.kotlin.codegen.customization.s3.S3OperationErrorHandler aws.sdk.kotlin.codegen.customization.s3.S3SigningConfig diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 0c271178582..043ae7baae9 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -169,7 +169,6 @@ class S3ChecksumTest { val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() - checksumCrc32 = "`" } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) val contents = "presign-test" diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..dc87f60a32f 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -4,12 +4,12 @@ use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 5c6c0bf02e36bf00d56e3e60796a426785209e62 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:19:52 -0500 Subject: [PATCH 57/64] Make it compile --- aws-runtime/aws-http/api/aws-http.api | 24 ++----------- .../S3CompositeChecksumsInterceptorTest.kt | 35 ------------------- .../FlexibleChecksumsRequest.kt | 6 ++-- .../s3/express/S3ExpressIntegration.kt | 2 +- .../event-stream-model-template.smithy | 4 +-- 5 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 0b475eb05fa..e401ba307dd 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,27 +196,9 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { - public fun ()V - public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +public final class aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { + public fun (ZLaws/smithy/kotlin/runtime/client/config/HttpChecksumConfigOption;)V + public fun ignoreChecksum (Ljava/lang/String;)Z } public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt deleted file mode 100644 index 042dcb5b99e..00000000000 --- a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package aws.sdk.kotlin.runtime.http.interceptors - -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlinx.coroutines.test.runTest -import kotlin.coroutines.coroutineContext -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class S3CompositeChecksumsInterceptorTest { - @Test - fun compositeChecksumRemoval() = runTest { - val headers = HeadersBuilder() - - headers.append("x-amz-checksum-crc32", "foo-1") - headers.append("x-amz-checksum-sha256", "bar") - - assertTrue( - headers.contains("x-amz-checksum-crc32"), - ) - assertTrue( - headers.contains("x-amz-checksum-sha256"), - ) - - headers.removeCompositeChecksums(coroutineContext.logger()) - - assertFalse( - headers.contains("x-amz-checksum-crc32"), - ) - assertTrue( - headers.contains("x-amz-checksum-sha256"), - ) - } -} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index ec3acddf0a9..767e1d00898 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -131,7 +131,7 @@ private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, return ( (httpChecksumTrait != null) && - (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && - (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) - ) + (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + ) } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 086534f3777..d0a76374bcd 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -145,7 +145,7 @@ class S3ExpressIntegration : KotlinIntegration { namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressDefaultChecksumAlgorithm" }, - op.isS3UploadPart + op.isS3UploadPart, ) } } diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dc87f60a32f..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -4,12 +4,12 @@ use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 4d915c65c855afcd0701152ee2096685186c5364 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:25:23 -0500 Subject: [PATCH 58/64] Fix smithy model template --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..8150dcec2db 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#restJson1 +use aws.protocols#{protocol-name} use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 609226c8be776b5f09fda0314a581f7306bc1869 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:29:40 -0500 Subject: [PATCH 59/64] Fix model smithy template again? --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 80ba07013bc186bdc6f21c06f03d4333d96b7cb1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:31:02 -0500 Subject: [PATCH 60/64] Fix model again --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..8150dcec2db 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#restJson1 +use aws.protocols#{protocol-name} use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 77f6ea5be9e899d19a0ca7fefce803be8c7556b1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 19 Dec 2024 09:42:30 -0500 Subject: [PATCH 61/64] Refactor rules engine codegen tests --- tests/codegen/rules-engine/build.gradle.kts | 135 ++---------------- .../operation-context-params.smithy | 0 2 files changed, 13 insertions(+), 122 deletions(-) rename tests/codegen/rules-engine/{ => src/commonTest/resources}/operation-context-params.smithy (100%) diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index ef35526b380..b6242e9f17a 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -3,64 +3,37 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} - -description = "Smithy rules engine codegen integration test suite" - -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, -) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile -} +description = "AWS SDK for Kotlin's rules engine codegen integration test suite" val tests = listOf( - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), + CodegenTest( + "operationContextParams", + Model("operation-context-params.smithy"), + "aws.sdk.kotlin.test#TestService", + ), ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) + projections.register(test.name) { + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) transforms = listOf( """ { "name": "includeServices", "args": { - "services": ["$testServiceShapeId"] + "services": ["${test.serviceShapeId}"] } } """, ) - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -74,85 +47,3 @@ smithyBuild { } } } - -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy similarity index 100% rename from tests/codegen/rules-engine/operation-context-params.smithy rename to tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy From 4b61347212cd695091342f5f848f22dc9837683e Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 19 Dec 2024 12:43:32 -0500 Subject: [PATCH 62/64] Run CI (empty commit) From 756f66d6b1e0ad8a3625c14a5ef3500317b0b821 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 23 Dec 2024 18:57:32 -0500 Subject: [PATCH 63/64] PR feedback --- .../config/AbstractAwsSdkClientFactory.kt | 4 +- .../ResolveFlexibleChecksumsConfig.kt | 37 +- aws-runtime/aws-http/api/aws-http.api | 4 +- ...iteFlexibleChecksumResponseInterceptor.kt} | 8 +- .../aws/sdk/kotlin/shared/CodegenTest.kt | 1 - .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 6 +- .../FlexibleChecksumsRequest.kt | 20 +- .../FlexibleChecksumsResponse.kt | 13 +- .../S3ExpressDefaultChecksumAlgorithm.kt | 6 +- .../s3/express/ChecksumRemovalTest.kt | 87 ---- tests/codegen/build.gradle.kts | 1 + tests/codegen/checksums/build.gradle.kts | 13 +- .../checksums/ChecksumBusinessMetricsTest.kt | 158 ++----- .../codegen/checksums/ChecksumConfigTest.kt | 389 ++++-------------- .../codegen/checksums/ChecksumRequestTest.kt | 118 ++---- .../codegen/checksums/ChecksumResponseTest.kt | 285 ++----------- .../checksums/utils/ChecksumTestUtils.kt | 89 +++- ...-response-test.smithy => checksums.smithy} | 61 +-- .../commonTest/resources/config-test.smithy | 87 ---- tests/codegen/event-stream/build.gradle.kts | 32 -- .../event-stream-model-template.smithy | 6 +- 22 files changed, 388 insertions(+), 1039 deletions(-) rename aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/{S3FlexibleChecksumResponseInterceptor.kt => IgnoreCompositeFlexibleChecksumResponseInterceptor.kt} (67%) delete mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt rename tests/codegen/checksums/src/commonTest/resources/{request-response-test.smithy => checksums.smithy} (72%) delete mode 100644 tests/codegen/checksums/src/commonTest/resources/config-test.smithy diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 190f41a0b12..1ce6f4dc98b 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -24,7 +24,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.SigV4aClientConfig import aws.smithy.kotlin.runtime.client.* import aws.smithy.kotlin.runtime.client.config.ClientSettings import aws.smithy.kotlin.runtime.client.config.CompressionClientConfig -import aws.smithy.kotlin.runtime.client.config.HttpChecksumClientConfig +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.telemetry.TelemetryConfig import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider @@ -97,7 +97,7 @@ public abstract class AbstractAwsSdkClientFactory< config.sigV4aSigningRegionSet ?: resolveSigV4aSigningRegionSet(platform, profile) } - if (config is HttpChecksumClientConfig.Builder) { + if (config is HttpChecksumConfig.Builder) { config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt index bf7049ef2f3..bbbbd241e28 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt @@ -6,7 +6,8 @@ import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider @@ -16,27 +17,43 @@ import aws.smithy.kotlin.runtime.util.PlatformProvider * @return requestChecksumCalculation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { +public suspend fun resolveRequestChecksumCalculation( + platform: PlatformProvider = PlatformProvider.System, + profile: LazyAsyncValue, +): RequestHttpChecksumConfig { val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation - return parseHttpChecksumConfigOption(unparsedString, "requestChecksumCalculation") + return parseRequestHttpChecksumConfig(unparsedString) } +private fun parseRequestHttpChecksumConfig(unparsedString: String?): RequestHttpChecksumConfig = + when (unparsedString?.uppercase()) { + null -> RequestHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> RequestHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> RequestHttpChecksumConfig.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$unparsedString' is not a valid value for 'requestChecksumCalculation'. Valid values are: ${RequestHttpChecksumConfig.entries}", + ) + } + /** * Attempts to resolve responseChecksumValidation from the specified sources. * @return responseChecksumValidation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { +public suspend fun resolveResponseChecksumValidation( + platform: PlatformProvider = PlatformProvider.System, + profile: LazyAsyncValue, +): ResponseHttpChecksumConfig { val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation - return parseHttpChecksumConfigOption(unparsedString, "responseChecksumValidation") + return parseResponseHttpChecksumConfig(unparsedString) } -private fun parseHttpChecksumConfigOption(unparsedString: String?, configOption: String): HttpChecksumConfigOption = +private fun parseResponseHttpChecksumConfig(unparsedString: String?): ResponseHttpChecksumConfig = when (unparsedString?.uppercase()) { - null -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED + null -> ResponseHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> ResponseHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ResponseHttpChecksumConfig.WHEN_REQUIRED else -> throw ConfigurationException( - "'$unparsedString' is not a valid value for $configOption. Valid values are: ${HttpChecksumConfigOption.entries}", + "'$unparsedString' is not a valid value for 'responseChecksumValidation'. Valid values are: ${ResponseHttpChecksumConfig.entries}", ) } diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index e401ba307dd..c9aadd34375 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,8 +196,8 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { - public fun (ZLaws/smithy/kotlin/runtime/client/config/HttpChecksumConfigOption;)V +public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { + public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V public fun ignoreChecksum (Ljava/lang/String;)Z } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt similarity index 67% rename from aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt rename to aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt index 7845f2f7f61..834924d5081 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt @@ -1,14 +1,14 @@ package aws.sdk.kotlin.runtime.http.interceptors -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor /** - * S3 variant of the flexible checksum interceptor where composite checksums are not validated + * Variant of the [FlexibleChecksumsResponseInterceptor] where composite checksums are not validated */ -public class S3FlexibleChecksumResponseInterceptor( +public class IgnoreCompositeFlexibleChecksumResponseInterceptor( responseValidationRequired: Boolean, - responseChecksumValidation: HttpChecksumConfigOption?, + responseChecksumValidation: ResponseHttpChecksumConfig?, ) : FlexibleChecksumsResponseInterceptor( responseValidationRequired, responseChecksumValidation, diff --git a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt index 4ee9580629c..6b72149fcff 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt @@ -7,7 +7,6 @@ data class CodegenTest( val name: String, val model: Model, val serviceShapeId: String, - val protocolName: String? = null, ) /** diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 0099e436bb1..a0b3f834c17 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,7 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") - val S3FlexibleChecksumResponseInterceptor = symbol("S3FlexibleChecksumResponseInterceptor") + val IgnoreCompositeFlexibleChecksumResponseInterceptor = symbol("IgnoreCompositeFlexibleChecksumResponseInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index d9b19377c47..fdbd102c406 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,7 +5,6 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable -import aws.smithy.kotlin.runtime.hashing.algorithmsSupportedForFlexibleChecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait @@ -14,10 +13,10 @@ import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinCoroutines.coroutineContext +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinxCoroutines.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Observability.TelemetryApi.warn import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionId @@ -304,8 +303,7 @@ class PresignerGenerator : KotlinIntegration { "#T.#T { #S }", coroutineContext, warn, - "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + - "Supported checksums for pre-signed URLs: $algorithmsSupportedForFlexibleChecksums", + "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum.", ) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 767e1d00898..51b5eb72c74 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsRequest : KotlinIntegration { // Allows flexible checksum request configuration ConfigProperty { name = "requestChecksumCalculation" - symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption - baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + symbol = RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumConfig useNestedBuilderBaseClass() documentation = "Configures request checksum calculation" - propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") + propertyType = ConfigPropertyType.RequiredWithDefault("RequestHttpChecksumConfig.WHEN_SUPPORTED") }, ) @@ -59,14 +59,14 @@ private val requestChecksumCalculationBusinessMetric = object : ProtocolMiddlewa // Supported writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) // Required writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -82,7 +82,7 @@ private val httpChecksumDefaultAlgorithmMiddleware = object : ProtocolMiddleware override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0) override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - requestChecksumsConfigured(ctx, op) + op.hasRequestAlgorithmMember(ctx) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { writer.write( @@ -100,7 +100,7 @@ private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { override val name: String = "flexibleChecksumsRequestMiddleware" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - requestChecksumsConfigured(ctx, op) + op.hasRequestAlgorithmMember(ctx) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val httpChecksumTrait = op.getTrait()!! @@ -125,9 +125,9 @@ private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { /** * Determines if an operation is set up to send flexible request checksums */ -private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } +private fun OperationShape.hasRequestAlgorithmMember(ctx: ProtocolGenerator.GenerationContext): Boolean { + val httpChecksumTrait = this.getTrait() + val inputShape = this.input.getOrNull()?.let { ctx.model.expectShape(it) } return ( (httpChecksumTrait != null) && diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 3691cc82a26..ae625d3991b 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsResponse : KotlinIntegration { // Allows flexible checksum response configuration ConfigProperty { name = "responseChecksumValidation" - symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption - baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + symbol = RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumConfig useNestedBuilderBaseClass() documentation = "Configures response checksum validation" - propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") + propertyType = ConfigPropertyType.RequiredWithDefault("ResponseHttpChecksumConfig.WHEN_SUPPORTED") }, ) @@ -56,14 +56,14 @@ private val responseChecksumValidationBusinessMetric = object : ProtocolMiddlewa // Supported writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) // Required writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -94,7 +94,8 @@ private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) val interceptor = if (ctx.model.expectShape(ctx.settings.service).isS3) { - AwsRuntimeTypes.Http.Interceptors.S3FlexibleChecksumResponseInterceptor + // S3 needs a custom interceptor because it can send composite checksums, which should be ignored + AwsRuntimeTypes.Http.Interceptors.IgnoreCompositeFlexibleChecksumResponseInterceptor } else { RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt index 862e96c1bf9..075524fb6f2 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt @@ -22,7 +22,7 @@ internal class S3ExpressDefaultChecksumAlgorithm( private val isS3UploadPart: Boolean, ) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (usingS3Express(context.executionContext)) { + if (context.executionContext.usingS3Express()) { if (isS3UploadPart) { context.executionContext.remove(HttpOperationContext.DefaultChecksumAlgorithm) } else { @@ -33,5 +33,5 @@ internal class S3ExpressDefaultChecksumAlgorithm( } } -private fun usingS3Express(executionContext: ExecutionContext): Boolean = - executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE +private fun ExecutionContext.usingS3Express(): Boolean = + this.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt deleted file mode 100644 index c00f78e821f..00000000000 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt +++ /dev/null @@ -1,87 +0,0 @@ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ChecksumRemovalTest { - // Header capitalization shouldn't matter - private val crc32Header = "x-aMz-cHeCkSum-cRC32" - private val sha256Header = "X-Amz-ChEcKsum-sHa256" - - @Test - fun removeChecksumHeaders() { - val headers = HeadersBuilder() - - headers.append(crc32Header, "foo") - headers.append(sha256Header, "bar") - - assertTrue( - headers.contains(crc32Header), - ) - assertTrue( - headers.contains(sha256Header), - ) - - headers.removeChecksumHeaders() - - assertFalse( - headers.contains(crc32Header), - ) - assertFalse( - headers.contains(sha256Header), - ) - } - - @Test - fun removeChecksumTrailingHeaders() { - val trailingHeaders = DeferredHeadersBuilder() - - trailingHeaders.add(crc32Header, "foo") - trailingHeaders.add(sha256Header, "bar") - - assertTrue( - trailingHeaders.contains(crc32Header), - ) - assertTrue( - trailingHeaders.contains(sha256Header), - ) - - trailingHeaders.removeChecksumTrailingHeaders() - - assertFalse( - trailingHeaders.contains(crc32Header), - ) - assertFalse( - trailingHeaders.contains(sha256Header), - ) - } - - @Test - fun removeChecksumTrailingHeadersFromXAmzTrailer() { - val headers = HeadersBuilder() - - headers.append("x-amz-trailer", crc32Header) - headers.append("x-amz-trailer", "x-amz-trailing-header") - - val xAmzTrailer = headers.getAll("x-amz-trailer") - - assertTrue( - xAmzTrailer?.contains(crc32Header) ?: false, - ) - assertTrue( - xAmzTrailer?.contains("x-amz-trailing-header") ?: false, - ) - - headers.removeChecksumTrailingHeadersFromXAmzTrailer() - - assertFalse( - xAmzTrailer?.contains(crc32Header) ?: false, - ) - assertTrue( - xAmzTrailer?.contains("x-amz-trailing-header") ?: false, - ) - } -} diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 696c3c3104a..f6288dac03a 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -76,6 +76,7 @@ subprojects { implementation(libraries.smithy.kotlin.smithy.test) implementation(libraries.smithy.kotlin.aws.signing.default) implementation(libraries.smithy.kotlin.telemetry.api) + implementation(libraries.smithy.kotlin.http.test) } } jvmTest { diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 6867921e4a2..866844297b8 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -6,19 +6,8 @@ import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" -kotlin { - sourceSets { - commonTest { - dependencies { - implementation(libs.smithy.kotlin.http.test) - } - } - } -} - val tests = listOf( - CodegenTest("checksums", Model("request-response-test.smithy"), "aws.sdk.kotlin.test#TestService"), - CodegenTest("clientConfig", Model("config-test.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), + CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt index 065d5f83b76..8c743763011 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt @@ -5,166 +5,86 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.test.checksums.TestClient -import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.sdk.kotlin.tests.codegen.checksums.utils.BusinessMetricsReader +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import kotlin.test.Test -import kotlin.test.assertTrue class ChecksumBusinessMetricsTest { @Test - fun defaultConfigBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun defaultConfigBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + ) @Test - fun whenSupportedBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun whenSupportedBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun whenRequiredBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun whenRequiredBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + ) @Test - fun crc32(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun crc32() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32, + ) @Test - fun crc32c(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun crc32c() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32C, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32C, + ) @Test - fun sha1(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun sha1() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA1, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha1, + ) @Test - fun sha256(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun sha256() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA256, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index e7fe9933394..c734d1f8d18 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -5,373 +5,166 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.clientconfig.* -import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm -import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.checksums.model.ValidationMode import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderSetter -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.Headers -import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.io.SdkSource -import aws.smithy.kotlin.runtime.io.source -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith -import kotlin.test.assertFalse -import kotlin.test.assertTrue /** - * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` param when set to **true**. */ class RequestChecksumRequired { @Test - fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenSupported() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenRequired() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + ) } /** - * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **false**. + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` param when set to **false**. */ class RequestChecksumNotRequired { @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenSupported() = runChecksumTest( + requestChecksumRequired = false, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertFalse( - headerReader.containsExpectedHeaders, - ) - } + fun requestChecksumCalculationWhenRequired() = runChecksumTest( + requestChecksumRequired = false, + headerReader = HeaderReader( + forbiddenHeaders = mapOf("x-amz-checksum-crc32" to null), + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + ) } /** - * Tests the `aws.protocols#httpChecksum` trait's `requestAlgorithmMember`. + * Tests user selected checksum **algorithm** */ class UserSelectedChecksumAlgorithm { @Test - fun userSelectedChecksumAlgorithmIsUsed(): Unit = runBlocking { - val headerReader = HeaderReader( + fun userSelectedChecksumAlgorithmIsUsed() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-sha256" to null), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } /** - * Tests user provided checksum calculations. + * Tests user provided checksum **calculation** */ class UserProvidedChecksumHeader { @Test - fun userProvidedChecksumIsUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun userProvidedChecksumIsUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-crc64nvme" to "foo"), - ) - val headerReader = HeaderReader( + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc64nvme" to "foo"), - forbiddenHeaders = mapOf("x-amz-checksum-sha256" to "foo"), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - - assertFalse( - headerReader.containsForbiddenHeaders, - ) - } + forbiddenHeaders = mapOf( + "x-amz-checksum-sha256" to "foo", // Should be ignored since header checksum has priority + "x-amz-checksum-crc32" to "foo", // Default checksum shouldn't be used + ), + ), + ) @Test - fun unmodeledChecksumIsUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun newChecksumAlgorithmIsUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-some-future-algorithm" to "foo"), - ) - val headerReader = HeaderReader( + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-some-future-algorithm" to "foo"), forbiddenHeaders = mapOf( - "x-amz-checksum-crc32" to "foo", "x-amz-checksum-sha256" to "foo", + "x-amz-checksum-crc32" to "foo", ), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + ) @Test - fun userProvidedMd5IsNotUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun md5IsNotUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-md5" to "foo"), - ) - val headerReader = HeaderReader( + ), + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - forbiddenHeaders = mapOf( - "x-amz-checksum-md5" to "foo", - ), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - - assertFalse( - headerReader.containsForbiddenHeaders, - ) - } + forbiddenHeaders = mapOf("x-amz-checksum-md5" to "foo"), // MD5 is not supported for flexible checksums + ), + ) } /** * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. */ class ResponseChecksumValidation { - private val responseBody = "Hello world" + private val incorrectChecksumValue = "Kaboom!" @Test - fun responseChecksumValidationWhenSupported(): Unit = runBlocking { + fun whenRequiredAndNotEnabled() = runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) + + @Test + fun whenSupportedAndNotEnabled() { assertFailsWith { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } - } + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun responseChecksumValidationWhenRequired(): Unit = runBlocking { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), + fun whenRequiredAndEnabled() { + assertFailsWith { + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + validationModeValue = ValidationMode.Enabled, ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } } } @Test - fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { + fun whenSupportedAndEnabled() { assertFailsWith { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - validationMode = ValidationMode.Enabled - } - } + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + validationModeValue = ValidationMode.Enabled, + ) } } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt index d64794a136d..619402eba65 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt @@ -5,126 +5,56 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import kotlin.test.Test -import kotlin.test.assertTrue +/** + * Tests headers match the configured checksum algorithm + */ class ChecksumRequestTest { @Test - fun crc32(): Unit = runBlocking { - val headerReader = HeaderReader( + fun crc32() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "CRC32", "x-amz-checksum-crc32" to "i9aeUg==", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32, + ) @Test - fun crc32c(): Unit = runBlocking { - val headerReader = HeaderReader( + fun crc32c() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "CRC32C", "x-amz-checksum-crc32c" to "crUfeA==", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32C, + ) @Test - fun sha1(): Unit = runBlocking { - val headerReader = HeaderReader( + fun sha1() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "SHA1", "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha1, + ) @Test - fun sha256(): Unit = runBlocking { - val headerReader = HeaderReader( + fun sha256() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "SHA256", "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index f477a3f1129..5bc5e96a999 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -5,278 +5,83 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.http.* +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.io.SdkSource -import aws.smithy.kotlin.runtime.io.source -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith -private val responseBody = "Hello world" - +/** + * Test the SDK validates correct checksum values + */ class SuccessfulChecksumResponseTest { @Test - fun crc32(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "i9aeUg==") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun crc32() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = "i9aeUg==", + ) @Test - fun crc32c(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32c", "crUfeA==") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun crc32c() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32c", + responseChecksumValue = "crUfeA==", + ) @Test - fun sha1(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun sha1() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha1", + responseChecksumValue = "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ) @Test - fun sha256(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun sha256() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha256", + responseChecksumValue = "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ) } +/** + * Test the SDK throws exception on incorrect checksum values + */ class FailedChecksumResponseTest { + private val incorrectChecksumValue = "Kaboom!" + @Test - fun crc32(): Unit = runBlocking { + fun crc32() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun crc32c(): Unit = runBlocking { + fun crc32c() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32c", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun sha1(): Unit = runBlocking { + fun sha1() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha1", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun sha256(): Unit = runBlocking { + fun sha256() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha256", + responseChecksumValue = incorrectChecksumValue, + ) } } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index c88263bfe70..41fd4faa527 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -5,19 +5,38 @@ package aws.sdk.kotlin.tests.codegen.checksums.utils +import aws.sdk.kotlin.test.checksums.TestClient +import aws.sdk.kotlin.test.checksums.httpChecksumOperation +import aws.sdk.kotlin.test.checksums.httpChecksumRequestChecksumsNotRequiredOperation +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.checksums.model.ValidationMode import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpCall +import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import kotlin.test.assertFalse +import kotlin.test.assertTrue /** * Checks if the specified headers are set in an HTTP request. */ internal class HeaderReader( - private val expectedHeaders: Map, + private val expectedHeaders: Map = emptyMap(), private val forbiddenHeaders: Map = emptyMap(), ) : HttpInterceptor { var containsExpectedHeaders = false @@ -73,3 +92,71 @@ internal class BusinessMetricsReader( containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) } } + +/** + * Runs a checksum test + */ +internal fun runChecksumTest( + // Interceptors + headerReader: HeaderReader? = null, + headerSetter: HeaderSetter? = null, + businessMetricsReader: BusinessMetricsReader? = null, + // Config + requestChecksumCalculationValue: RequestHttpChecksumConfig? = null, + responseChecksumValidationValue: ResponseHttpChecksumConfig? = null, + checksumAlgorithmValue: ChecksumAlgorithm? = null, + validationModeValue: ValidationMode? = null, + // Request/Response + responseChecksumHeader: String? = null, + responseChecksumValue: String? = null, + requestBody: String = "Hello world", + responseBody: String = "Hello world", + // Test type + requestChecksumRequired: Boolean = true, +): Unit = runBlocking { + val interceptorsList = listOfNotNull(headerReader, headerSetter, businessMetricsReader) + + TestClient { + httpClient = TestEngine() + interceptors = interceptorsList.toMutableList() + requestChecksumCalculation = requestChecksumCalculationValue ?: requestChecksumCalculation + responseChecksumValidation = responseChecksumValidationValue ?: responseChecksumValidation + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append(responseChecksumHeader ?: "", responseChecksumValue ?: "") + }, + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + }.use { client -> + if (requestChecksumRequired) { + client.httpChecksumOperation { + body = requestBody.encodeToByteArray() + checksumAlgorithm = checksumAlgorithmValue ?: checksumAlgorithm + validationMode = validationModeValue ?: validationMode + } + } else { + client.httpChecksumRequestChecksumsNotRequiredOperation { + body = requestBody.encodeToByteArray() + checksumAlgorithm = checksumAlgorithmValue ?: checksumAlgorithm + validationMode = validationModeValue ?: validationMode + } + } + } + + businessMetricsReader?.let { assertTrue(it.containsExpectedBusinessMetrics) } + headerReader?.let { + assertTrue(it.containsExpectedHeaders) + assertFalse(it.containsForbiddenHeaders) + } +} diff --git a/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy similarity index 72% rename from tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy rename to tests/codegen/checksums/src/commonTest/resources/checksums.smithy index aa5dc5bc9af..a8b7a89008d 100644 --- a/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy @@ -20,7 +20,7 @@ use smithy.rules#endpointRuleSet }) service TestService { version: "2023-01-01", - operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] + operations: [HttpChecksumOperation, HttpChecksumRequestChecksumsNotRequiredOperation] } @http(uri: "/HttpChecksumOperation", method: "POST") @@ -36,6 +36,19 @@ operation HttpChecksumOperation { output: SomeOutput } +@http(uri: "/HttpChecksumRequestChecksumsNotRequiredOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: false, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumRequestChecksumsNotRequiredOperation { + input: SomeOtherInput, + output: SomeOtherOutput +} + @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -67,40 +80,42 @@ structure SomeInput { body: Blob } -@output -structure SomeOutput {} - -@http(uri: "/HttpChecksumStreamingOperation", method: "POST") -@auth([sigv4]) -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", - requestValidationModeMember: "validationMode", - responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] -) -operation HttpChecksumStreamingOperation { - input: SomeStreamingInput, - output: SomeStreamingOutput -} - -@streaming -blob StreamingBlob - @input -structure SomeStreamingInput { +structure SomeOtherInput { @httpHeader("x-amz-request-algorithm") checksumAlgorithm: ChecksumAlgorithm @httpHeader("x-amz-response-validation-mode") validationMode: ValidationMode + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + @httpPayload @required - body: StreamingBlob + body: Blob } @output -structure SomeStreamingOutput {} +structure SomeOutput {} + +@output +structure SomeOtherOutput {} enum ChecksumAlgorithm { CRC32 diff --git a/tests/codegen/checksums/src/commonTest/resources/config-test.smithy b/tests/codegen/checksums/src/commonTest/resources/config-test.smithy deleted file mode 100644 index e1d1f24e21d..00000000000 --- a/tests/codegen/checksums/src/commonTest/resources/config-test.smithy +++ /dev/null @@ -1,87 +0,0 @@ -$version: "2" -namespace aws.sdk.kotlin.test - -use aws.api#service -use aws.auth#sigv4 -use aws.protocols#httpChecksum -use aws.protocols#restJson1 -use smithy.rules#endpointRuleSet - -@service(sdkId: "dontcare") -@restJson1 -@sigv4(name: "dontcare") -@auth([sigv4]) -@endpointRuleSet({ - "version": "1.0", - "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], - "parameters": { - "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, - } -}) -service ClientConfigTestService { - version: "2023-01-01", - operations: [ChecksumsNotRequiredOperation, ChecksumsRequiredOperation] -} - -@httpChecksum( - requestChecksumRequired: false, - requestAlgorithmMember: "checksumAlgorithm", -) -@http(method: "POST", uri: "/test-checksums", code: 200) -operation ChecksumsNotRequiredOperation { - input: SomeInput, - output: SomeOutput -} - -@input -structure SomeInput { - @httpHeader("x-amz-request-algorithm") - checksumAlgorithm: ChecksumAlgorithm - - @httpPayload - @required - body: String -} - -@output -structure SomeOutput {} - -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", - requestValidationModeMember: "validationMode", - responseAlgorithms: ["CRC32"] -) -@http(method: "POST", uri: "/test-checksums-2", code: 200) -operation ChecksumsRequiredOperation { - input: AnotherInput, - output: AnotherOutput -} - -@input -structure AnotherInput { - @httpHeader("x-amz-request-algorithm") - checksumAlgorithm: ChecksumAlgorithm - - @httpHeader("x-amz-response-validation-mode") - validationMode: ValidationMode - - @httpPayload - @required - body: String -} - -@output -structure AnotherOutput {} - -enum ChecksumAlgorithm { - CRC32 - CRC32C - CRC64NVME - SHA1 - SHA256 -} - -enum ValidationMode { - ENABLED -} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 0de5288432e..ed102cf3e47 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -15,13 +15,11 @@ val tests = listOf( "restJson1", Model("event-stream-model-template.smithy"), "aws.sdk.kotlin.test#TestService", - "restJson1", ), CodegenTest( "awsJson11", Model("event-stream-initial-request-response.smithy"), "aws.sdk.kotlin.test#TestService", - "awsJson1_1", ), ) @@ -65,33 +63,3 @@ kotlin { } } } - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test) } - } -} - -fun fillInModel(test: CodegenTest) { - val modelFile = layout.projectDirectory.file(test.model.path + test.model.fileName).asFile - val model = modelFile.readText() - - val opTraits = - when (test.protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - - val replaced = model - .replace( - "{protocol-name}", - test.protocolName ?: throw IllegalStateException("Please specify a protocol name for the codegen test"), - ) - .replace( - "{op-traits}", - opTraits, - ) - - modelFile.parentFile.mkdirs() - modelFile.writeText(replaced) -} diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 3ed4d917535a480afbde1cbac5353ddf1bdc4156 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 24 Dec 2024 10:47:54 -0500 Subject: [PATCH 64/64] Change JVM version --- codegen/aws-sdk-codegen/build.gradle.kts | 6 +++--- services/s3/e2eTest/src/S3ExpressTest.kt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index 0348cd5a7f1..9a67ee36c09 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -59,14 +59,14 @@ val generateSdkRuntimeVersion by tasks.registering { tasks.withType { compilerOptions { - jvmTarget.set(JvmTarget.JVM_17) + jvmTarget.set(JvmTarget.JVM_1_8) } dependsOn(generateSdkRuntimeVersion) } tasks.withType { - sourceCompatibility = JavaVersion.VERSION_17.toString() - targetCompatibility = JavaVersion.VERSION_17.toString() + sourceCompatibility = JavaVersion.VERSION_1_8.toString() + targetCompatibility = JavaVersion.VERSION_1_8.toString() } // Reusable license copySpec diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index c97c58fc2ce..f831583b38c 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -136,7 +136,7 @@ class S3ExpressTest { } @Test - fun testUploadPartContainsNoChecksums() = runTest { + fun testUploadPartContainsNoDefaultChecksum() = runTest { val testBucket = testBuckets.first() val testObject = "I-will-be-uploaded-in-parts-!"