From a33e0ad3a4bc32b90cc6addd0a5a30d2e0edf6b2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 24 Jan 2024 12:56:12 -0500 Subject: [PATCH 01/20] First test added --- services/build.gradle.kts | 6 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 291 ++++++++++++++++++ 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 services/s3/e2eTest/src/MutliRegionAccessPointTest.kt diff --git a/services/build.gradle.kts b/services/build.gradle.kts index e5912af66ea..ee01f4553c3 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import aws.sdk.kotlin.gradle.dsl.configurePublishing -import aws.sdk.kotlin.gradle.kmp.* +import aws.sdk.kotlin.gradle.kmp.kotlin import aws.sdk.kotlin.gradle.util.typedProp import org.jetbrains.kotlin.gradle.dsl.JvmTarget import java.time.LocalDateTime @@ -54,6 +54,8 @@ subprojects { dependencies { implementation(libraries.kotlinx.coroutines.test) implementation(libraries.smithy.kotlin.http.test) + implementation("aws.sdk.kotlin:s3control:+") // TODO: Duplicates + implementation("aws.smithy.kotlin:aws-signing-crt:+") } } } @@ -73,6 +75,8 @@ subprojects { implementation(libraries.kotlin.test.junit5) implementation(project(":tests:e2e-test-util")) implementation(libraries.slf4j.simple) + implementation("aws.sdk.kotlin:s3control:+") // TODO: Duplicates + implementation("aws.smithy.kotlin:aws-signing-crt:+") } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt new file mode 100644 index 00000000000..c534acd62a9 --- /dev/null +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -0,0 +1,291 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint +import aws.sdk.kotlin.services.s3.model.CreateBucketConfiguration +import aws.sdk.kotlin.services.s3.model.CreateBucketRequest +import aws.sdk.kotlin.services.s3.model.DeleteBucketRequest +import aws.sdk.kotlin.services.s3.withConfig +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.sdk.kotlin.services.s3control.model.* +import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +import kotlinx.coroutines.runBlocking +import java.util.concurrent.TimeUnit +import kotlin.test.Test + +private const val usWestBucket = "aws-sdk-for-kotlin-test-bucket-west" +private const val usEastBucket = "aws-sdk-for-kotlin-test-bucket-east" +private const val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" +private const val testAccountId = "393681621311" +private const val testKeyForObject = "text.txt" + +class MutliRegionAccessPointTest { + @Test + fun testMultRegionAccessPoints(): Unit = runBlocking { + try { + S3Client { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove + }.use { s3West -> + + createS3Bucket( + s3West, + usWestBucket, + BucketLocationConstraint.UsWest2 + ) + + s3West.withConfig { + region = "us-east-2" + }.use { s3East -> + + createS3Bucket( + s3East, + usEastBucket, + BucketLocationConstraint.UsEast2 + ) + + S3ControlClient { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove + }.use { s3Control -> + + createMultiRegionAccessPoint(s3Control) + val multiRegionAccessPointArn = getMultiRegionAccessPointArn(s3Control) ?: throw Exception("Unable to get multi region access point arn") + + s3West.withConfig { + authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) + }.use { s3SigV4a -> + + createObject( + s3SigV4a, + multiRegionAccessPointArn, + ) + + deleteObject( + s3SigV4a, + multiRegionAccessPointArn, + ) + } + + deleteMultiRegionAccessPoint(s3Control) + + deleteS3Bucket( + s3West, + usWestBucket, + ) + + deleteS3Bucket( + s3East, + usEastBucket, + ) + } + } + } + } catch (exception: Throwable) { + cleanUpTest() + throw exception + } + } +} + +private suspend fun createS3Bucket( + s3Client: S3Client, + bucketName: String, + location: BucketLocationConstraint, +) { + s3Client.createBucket( + CreateBucketRequest { + bucket = bucketName + createBucketConfiguration = + CreateBucketConfiguration { + locationConstraint = location + } + } + ) +} + +private suspend fun createMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, +) { + val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( + CreateMultiRegionAccessPointRequest { + accountId = testAccountId + details = + CreateMultiRegionAccessPointInput { + name = multiRegionAccessPoint + regions = + listOf( + Region { + bucket = usWestBucket + }, + Region { + bucket = usEastBucket + } + ) + } + } + ) + + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), + 1000 * 60 * 10, // 10 minutes + ) +} + +private suspend fun getMultiRegionAccessPointArn( + s3ControlClient: S3ControlClient, +): String? = + s3ControlClient.listMultiRegionAccessPoints( + ListMultiRegionAccessPointsRequest { + accountId = testAccountId + } + ).accessPoints?.find { it.name == multiRegionAccessPoint }?.alias?.let { alias -> + return "arn:aws:s3::$testAccountId:accesspoint/$alias" + } + +private suspend fun createObject( + s3Client: S3Client, + bucketName: String, +) { + s3Client.putObject( + PutObjectRequest { + bucket = bucketName + key = testKeyForObject + } + ) +} + +private suspend fun deleteObject( + s3Client: S3Client, + bucketName: String, +) { + s3Client.deleteObject( + DeleteObjectRequest { + bucket = bucketName + key = testKeyForObject + } + ) +} + +private suspend fun deleteMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, +) { + val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( + DeleteMultiRegionAccessPointRequest { + accountId = testAccountId + details = + DeleteMultiRegionAccessPointInput { + name = multiRegionAccessPoint + } + } + ) + + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), + 1000 * 60 * 5, // 5 minutes + ) +} + +private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient: S3ControlClient, + request: String, + timeLimit: Int, +) { + val startTime = System.currentTimeMillis() + + while (System.currentTimeMillis() - startTime < timeLimit) { + val status = s3ControlClient.describeMultiRegionAccessPointOperation( + DescribeMultiRegionAccessPointOperationRequest { + accountId = testAccountId + requestTokenArn = request + } + ).asyncOperation?.requestStatus + + println(status) // TODO: Remove + if (status == "SUCCEEDED") return + TimeUnit.SECONDS.sleep(10L) + } + + throw Exception("The multi-region-access-point operation exceeded the time limit set ($timeLimit ms)") +} + +private suspend fun deleteS3Bucket( + s3Client: S3Client, + bucketName: String, +) { + s3Client.deleteBucket( + DeleteBucketRequest { + bucket = bucketName + } + ) +} + +private suspend fun cleanUpTest() { + S3ControlClient { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove + }. use { s3Control -> + if (multiRegionAccessPointWasCreated(s3Control)) deleteMultiRegionAccessPoint(s3Control) + } + + S3Client { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove + }.use { s3West -> + if (s3BucketWasCreated(s3West, usWestBucket)) deleteS3Bucket(s3West, usWestBucket) + + s3West.withConfig { + region = "us-east-2" + }.use { s3East -> + if (s3BucketWasCreated(s3East, usEastBucket)) { + if (objectWasCreated(s3East, usEastBucket)) deleteObject(s3East, usEastBucket) + deleteS3Bucket(s3East, usEastBucket) + } + } + } +} + +private suspend fun objectWasCreated( + s3: S3Client, + bucketName: String, +) : Boolean { + val search = s3.listObjectsV2( + ListObjectsV2Request { + bucket = bucketName + } + ).contents + + return search?.find { it.key == testKeyForObject } != null +} + +private suspend fun s3BucketWasCreated( + s3: S3Client, + bucketName: String, +): Boolean { + val search = s3.listBuckets( + ListBucketsRequest {} + ).buckets + + return search?.find { it.name == bucketName } != null +} + +private suspend fun multiRegionAccessPointWasCreated( + s3Control: S3ControlClient, +): Boolean { + val search = s3Control.listMultiRegionAccessPoints( + ListMultiRegionAccessPointsRequest { + accountId = testAccountId + } + ).accessPoints?.find { it.name == multiRegionAccessPoint } + + return search != null +} From 906ee38434c2c2f0f00aaaf20f2a2ddd76b0b09a Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 24 Jan 2024 14:30:58 -0500 Subject: [PATCH 02/20] Add unsupported signing algorithm test --- services/build.gradle.kts | 5 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 80 +++++++++---------- .../src/UnsupportedSigningAlgorithmTest.kt | 76 ++++++++++++++++++ 3 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt diff --git a/services/build.gradle.kts b/services/build.gradle.kts index ee01f4553c3..ee13dd69b43 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -54,8 +54,6 @@ subprojects { dependencies { implementation(libraries.kotlinx.coroutines.test) implementation(libraries.smithy.kotlin.http.test) - implementation("aws.sdk.kotlin:s3control:+") // TODO: Duplicates - implementation("aws.smithy.kotlin:aws-signing-crt:+") } } } @@ -75,8 +73,9 @@ subprojects { implementation(libraries.kotlin.test.junit5) implementation(project(":tests:e2e-test-util")) implementation(libraries.slf4j.simple) - implementation("aws.sdk.kotlin:s3control:+") // TODO: Duplicates + implementation("aws.sdk.kotlin:s3control:+") implementation("aws.smithy.kotlin:aws-signing-crt:+") + implementation("io.kotest:kotest-assertions-core:5.8.0") } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index c534acd62a9..53436133333 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.e2etest -import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint @@ -20,25 +19,24 @@ import kotlinx.coroutines.runBlocking import java.util.concurrent.TimeUnit import kotlin.test.Test -private const val usWestBucket = "aws-sdk-for-kotlin-test-bucket-west" -private const val usEastBucket = "aws-sdk-for-kotlin-test-bucket-east" -private const val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" -private const val testAccountId = "393681621311" -private const val testKeyForObject = "text.txt" +internal const val usWestBucket = "aws-sdk-for-kotlin-test-bucket-west" +internal const val usEastBucket = "aws-sdk-for-kotlin-test-bucket-east" +internal const val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" +internal const val testAccountId = "393681621311" +internal const val testKeyForObject = "text.txt" class MutliRegionAccessPointTest { - @Test - fun testMultRegionAccessPoints(): Unit = runBlocking { +// @Test + fun testMultRegionAccessPointOperation(): Unit = runBlocking { try { S3Client { region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove }.use { s3West -> createS3Bucket( s3West, usWestBucket, - BucketLocationConstraint.UsWest2 + BucketLocationConstraint.UsWest2, ) s3West.withConfig { @@ -48,12 +46,11 @@ class MutliRegionAccessPointTest { createS3Bucket( s3East, usEastBucket, - BucketLocationConstraint.UsEast2 + BucketLocationConstraint.UsEast2, ) S3ControlClient { region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove }.use { s3Control -> createMultiRegionAccessPoint(s3Control) @@ -95,7 +92,7 @@ class MutliRegionAccessPointTest { } } -private suspend fun createS3Bucket( +internal suspend fun createS3Bucket( s3Client: S3Client, bucketName: String, location: BucketLocationConstraint, @@ -107,11 +104,11 @@ private suspend fun createS3Bucket( CreateBucketConfiguration { locationConstraint = location } - } + }, ) } -private suspend fun createMultiRegionAccessPoint( +internal suspend fun createMultiRegionAccessPoint( s3ControlClient: S3ControlClient, ) { val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( @@ -127,31 +124,32 @@ private suspend fun createMultiRegionAccessPoint( }, Region { bucket = usEastBucket - } + }, ) } - } + }, ) waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient, createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), 1000 * 60 * 10, // 10 minutes + "createMultiRegionAccessPoint" ) } -private suspend fun getMultiRegionAccessPointArn( +internal suspend fun getMultiRegionAccessPointArn( s3ControlClient: S3ControlClient, ): String? = s3ControlClient.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId - } + }, ).accessPoints?.find { it.name == multiRegionAccessPoint }?.alias?.let { alias -> return "arn:aws:s3::$testAccountId:accesspoint/$alias" } -private suspend fun createObject( +internal suspend fun createObject( s3Client: S3Client, bucketName: String, ) { @@ -159,11 +157,11 @@ private suspend fun createObject( PutObjectRequest { bucket = bucketName key = testKeyForObject - } + }, ) } -private suspend fun deleteObject( +internal suspend fun deleteObject( s3Client: S3Client, bucketName: String, ) { @@ -171,11 +169,11 @@ private suspend fun deleteObject( DeleteObjectRequest { bucket = bucketName key = testKeyForObject - } + }, ) } -private suspend fun deleteMultiRegionAccessPoint( +internal suspend fun deleteMultiRegionAccessPoint( s3ControlClient: S3ControlClient, ) { val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( @@ -185,20 +183,22 @@ private suspend fun deleteMultiRegionAccessPoint( DeleteMultiRegionAccessPointInput { name = multiRegionAccessPoint } - } + }, ) waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient, deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), 1000 * 60 * 5, // 5 minutes + "deleteMultiRegionAccessPoint" ) } -private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( +internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient: S3ControlClient, request: String, timeLimit: Int, + operation: String ) { val startTime = System.currentTimeMillis() @@ -207,10 +207,10 @@ private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( DescribeMultiRegionAccessPointOperationRequest { accountId = testAccountId requestTokenArn = request - } + }, ).asyncOperation?.requestStatus - println(status) // TODO: Remove + println("Waiting on $operation operation. Status: $status") if (status == "SUCCEEDED") return TimeUnit.SECONDS.sleep(10L) } @@ -218,28 +218,26 @@ private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( throw Exception("The multi-region-access-point operation exceeded the time limit set ($timeLimit ms)") } -private suspend fun deleteS3Bucket( +internal suspend fun deleteS3Bucket( s3Client: S3Client, bucketName: String, ) { s3Client.deleteBucket( DeleteBucketRequest { bucket = bucketName - } + }, ) } -private suspend fun cleanUpTest() { +internal suspend fun cleanUpTest() { S3ControlClient { region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove - }. use { s3Control -> + }.use { s3Control -> if (multiRegionAccessPointWasCreated(s3Control)) deleteMultiRegionAccessPoint(s3Control) } S3Client { region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aoperez@amazon.com --role Admin") // TODO: Remove }.use { s3West -> if (s3BucketWasCreated(s3West, usWestBucket)) deleteS3Bucket(s3West, usWestBucket) @@ -254,37 +252,37 @@ private suspend fun cleanUpTest() { } } -private suspend fun objectWasCreated( +internal suspend fun objectWasCreated( s3: S3Client, bucketName: String, -) : Boolean { +): Boolean { val search = s3.listObjectsV2( ListObjectsV2Request { bucket = bucketName - } + }, ).contents return search?.find { it.key == testKeyForObject } != null } -private suspend fun s3BucketWasCreated( +internal suspend fun s3BucketWasCreated( s3: S3Client, bucketName: String, ): Boolean { val search = s3.listBuckets( - ListBucketsRequest {} + ListBucketsRequest {}, ).buckets return search?.find { it.name == bucketName } != null } -private suspend fun multiRegionAccessPointWasCreated( +internal suspend fun multiRegionAccessPointWasCreated( s3Control: S3ControlClient, ): Boolean { val search = s3Control.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId - } + }, ).accessPoints?.find { it.name == multiRegionAccessPoint } return search != null diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt new file mode 100644 index 00000000000..457c1431941 --- /dev/null +++ b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint +import aws.sdk.kotlin.services.s3.withConfig +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.sdk.kotlin.services.s3control.model.* +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +import kotlin.test.assertFailsWith +import io.kotest.matchers.string.shouldContain + +class UnsupportedSigningAlgorithmTest { +// @Test + fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { + try { + S3Client { + region = "us-west-2" + }.use { s3West -> + + createS3Bucket( + s3West, + usWestBucket, + BucketLocationConstraint.UsWest2, + ) + + s3West.withConfig { + region = "us-east-2" + }.use { s3East -> + + createS3Bucket( + s3East, + usEastBucket, + BucketLocationConstraint.UsEast2, + ) + + S3ControlClient { + region = "us-west-2" + }.use { s3Control -> + + createMultiRegionAccessPoint(s3Control) + val multiRegionAccessPointArn = getMultiRegionAccessPointArn(s3Control) ?: throw Exception("Unable to get multi region access point arn") + + assertFailsWith { + createObject( + s3West, + multiRegionAccessPointArn, + ) + }.message.shouldContain("SIGV4A support is not yet implemented for the default signer.") + + deleteMultiRegionAccessPoint(s3Control) + + deleteS3Bucket( + s3West, + usWestBucket, + ) + + deleteS3Bucket( + s3East, + usEastBucket, + ) + } + } + } + } catch (exception: Throwable) { + cleanUpTest() + throw exception + } + } +} From 5e1bca6e42bc013d507445f6a8fd4e99346d6b55 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 24 Jan 2024 17:33:02 -0500 Subject: [PATCH 03/20] Added account id & clean up --- services/build.gradle.kts | 1 + .../e2eTest/src/MutliRegionAccessPointTest.kt | 246 ++++++++++++------ .../src/UnsupportedSigningAlgorithmTest.kt | 122 +++++---- 3 files changed, 241 insertions(+), 128 deletions(-) diff --git a/services/build.gradle.kts b/services/build.gradle.kts index ee13dd69b43..971310ef9ca 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -74,6 +74,7 @@ subprojects { implementation(project(":tests:e2e-test-util")) implementation(libraries.slf4j.simple) implementation("aws.sdk.kotlin:s3control:+") + implementation("aws.sdk.kotlin:sts:+") implementation("aws.smithy.kotlin:aws-signing-crt:+") implementation("io.kotest:kotest-assertions-core:5.8.0") } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 53436133333..98626f1ad27 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -13,85 +13,131 @@ import aws.sdk.kotlin.services.s3.model.DeleteBucketRequest import aws.sdk.kotlin.services.s3.withConfig import aws.sdk.kotlin.services.s3control.S3ControlClient import aws.sdk.kotlin.services.s3control.model.* +import aws.sdk.kotlin.services.sts.StsClient +import aws.sdk.kotlin.services.sts.model.GetCallerIdentityRequest import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme import kotlinx.coroutines.runBlocking import java.util.concurrent.TimeUnit import kotlin.test.Test -internal const val usWestBucket = "aws-sdk-for-kotlin-test-bucket-west" -internal const val usEastBucket = "aws-sdk-for-kotlin-test-bucket-east" -internal const val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" -internal const val testAccountId = "393681621311" -internal const val testKeyForObject = "text.txt" - class MutliRegionAccessPointTest { -// @Test + @Test fun testMultRegionAccessPointOperation(): Unit = runBlocking { - try { - S3Client { - region = "us-west-2" - }.use { s3West -> - - createS3Bucket( - s3West, - usWestBucket, - BucketLocationConstraint.UsWest2, - ) - - s3West.withConfig { - region = "us-east-2" - }.use { s3East -> - - createS3Bucket( - s3East, - usEastBucket, - BucketLocationConstraint.UsEast2, - ) - - S3ControlClient { - region = "us-west-2" - }.use { s3Control -> - - createMultiRegionAccessPoint(s3Control) - val multiRegionAccessPointArn = getMultiRegionAccessPointArn(s3Control) ?: throw Exception("Unable to get multi region access point arn") - - s3West.withConfig { - authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) - }.use { s3SigV4a -> - - createObject( - s3SigV4a, - multiRegionAccessPointArn, - ) - - deleteObject( - s3SigV4a, - multiRegionAccessPointArn, - ) - } + val accountId = getAccountId() + val s3West = S3Client { + region = "us-west-2" + } + val s3East = s3West.withConfig { + region = "us-east-2" + } + val s3SigV4a = s3West.withConfig { + authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) + } + val s3Control = S3ControlClient { + region = "us-west-2" + } - deleteMultiRegionAccessPoint(s3Control) + val usWestBucket = "aws-sdk-for-kotlin-test-bucket-us-west-2" + val usEastBucket = "aws-sdk-for-kotlin-test-bucket-us-east-2" + val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + val keyForObject = "test.txt" - deleteS3Bucket( - s3West, - usWestBucket, - ) + try { + createS3Bucket( + s3West, + usWestBucket, + BucketLocationConstraint.UsWest2, + ) + + createS3Bucket( + s3East, + usEastBucket, + BucketLocationConstraint.UsEast2, + ) + + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) + + val multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) - deleteS3Bucket( - s3East, - usEastBucket, - ) - } - } - } + createObject( + s3SigV4a, + multiRegionAccessPointArn, + keyForObject, + ) + + deleteObject( + s3SigV4a, + multiRegionAccessPointArn, + keyForObject, + ) + + deleteMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + accountId, + ) + + deleteS3Bucket( + s3West, + usWestBucket, + ) + + deleteS3Bucket( + s3East, + usEastBucket, + ) } catch (exception: Throwable) { - cleanUpTest() + closeClients( + s3West, + s3East, + s3SigV4a, + s3Control, + ) + cleanUpTest( + accountId, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + keyForObject, + ) throw exception } + closeClients( + s3West, + s3East, + s3SigV4a, + s3Control, + ) } } +internal suspend fun getAccountId(): String { + val sts = StsClient { + region = "us-west-2" + } + + val accountId = sts.getCallerIdentity( + GetCallerIdentityRequest { }, + ).account + + sts.close() + + return accountId ?: throw Exception("Unable to get AWS account ID") +} + internal suspend fun createS3Bucket( s3Client: S3Client, bucketName: String, @@ -110,20 +156,24 @@ internal suspend fun createS3Bucket( internal suspend fun createMultiRegionAccessPoint( s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + regionOneBucket: String, + regionTwoBucket: String, + testAccountId: String, ) { val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( CreateMultiRegionAccessPointRequest { accountId = testAccountId details = CreateMultiRegionAccessPointInput { - name = multiRegionAccessPoint + name = multiRegionAccessPointName regions = listOf( Region { - bucket = usWestBucket + bucket = regionOneBucket }, Region { - bucket = usEastBucket + bucket = regionTwoBucket }, ) } @@ -134,29 +184,35 @@ internal suspend fun createMultiRegionAccessPoint( s3ControlClient, createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), 1000 * 60 * 10, // 10 minutes - "createMultiRegionAccessPoint" + testAccountId, + "createMultiRegionAccessPoint", ) } internal suspend fun getMultiRegionAccessPointArn( s3ControlClient: S3ControlClient, -): String? = + multiRegionAccessPointName: String, + testAccountId: String, +): String { s3ControlClient.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId }, - ).accessPoints?.find { it.name == multiRegionAccessPoint }?.alias?.let { alias -> + ).accessPoints?.find { it.name == multiRegionAccessPointName }?.alias?.let { alias -> return "arn:aws:s3::$testAccountId:accesspoint/$alias" } + throw Exception("Unable to get multi region access point arn") +} internal suspend fun createObject( s3Client: S3Client, bucketName: String, + keyName: String, ) { s3Client.putObject( PutObjectRequest { bucket = bucketName - key = testKeyForObject + key = keyName }, ) } @@ -164,24 +220,27 @@ internal suspend fun createObject( internal suspend fun deleteObject( s3Client: S3Client, bucketName: String, + keyName: String, ) { s3Client.deleteObject( DeleteObjectRequest { bucket = bucketName - key = testKeyForObject + key = keyName }, ) } internal suspend fun deleteMultiRegionAccessPoint( s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, ) { val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( DeleteMultiRegionAccessPointRequest { accountId = testAccountId details = DeleteMultiRegionAccessPointInput { - name = multiRegionAccessPoint + name = multiRegionAccessPointName } }, ) @@ -190,15 +249,17 @@ internal suspend fun deleteMultiRegionAccessPoint( s3ControlClient, deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), 1000 * 60 * 5, // 5 minutes - "deleteMultiRegionAccessPoint" + testAccountId, + "deleteMultiRegionAccessPoint", ) } -internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( +private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient: S3ControlClient, request: String, timeLimit: Int, - operation: String + testAccountId: String, + operation: String, ) { val startTime = System.currentTimeMillis() @@ -212,10 +273,10 @@ internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( println("Waiting on $operation operation. Status: $status") if (status == "SUCCEEDED") return - TimeUnit.SECONDS.sleep(10L) + TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks } - throw Exception("The multi-region-access-point operation exceeded the time limit set ($timeLimit ms)") + throw Exception("The multi-region-access-point $operation operation exceeded the time limit set ($timeLimit ms)") } internal suspend fun deleteS3Bucket( @@ -229,11 +290,17 @@ internal suspend fun deleteS3Bucket( ) } -internal suspend fun cleanUpTest() { +internal suspend fun cleanUpTest( + testAccountId: String, + multiRegionAccessPointName: String, + usWestBucket: String, + usEastBucket: String, + keyName: String, +) { S3ControlClient { region = "us-west-2" }.use { s3Control -> - if (multiRegionAccessPointWasCreated(s3Control)) deleteMultiRegionAccessPoint(s3Control) + if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPointName, testAccountId)) deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPointName, testAccountId) } S3Client { @@ -245,16 +312,17 @@ internal suspend fun cleanUpTest() { region = "us-east-2" }.use { s3East -> if (s3BucketWasCreated(s3East, usEastBucket)) { - if (objectWasCreated(s3East, usEastBucket)) deleteObject(s3East, usEastBucket) + if (objectWasCreated(s3East, usEastBucket, keyName)) deleteObject(s3East, usEastBucket, keyName) deleteS3Bucket(s3East, usEastBucket) } } } } -internal suspend fun objectWasCreated( +private suspend fun objectWasCreated( s3: S3Client, bucketName: String, + keyName: String, ): Boolean { val search = s3.listObjectsV2( ListObjectsV2Request { @@ -262,10 +330,10 @@ internal suspend fun objectWasCreated( }, ).contents - return search?.find { it.key == testKeyForObject } != null + return search?.find { it.key == keyName } != null } -internal suspend fun s3BucketWasCreated( +private suspend fun s3BucketWasCreated( s3: S3Client, bucketName: String, ): Boolean { @@ -276,14 +344,24 @@ internal suspend fun s3BucketWasCreated( return search?.find { it.name == bucketName } != null } -internal suspend fun multiRegionAccessPointWasCreated( +private suspend fun multiRegionAccessPointWasCreated( s3Control: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, ): Boolean { val search = s3Control.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId }, - ).accessPoints?.find { it.name == multiRegionAccessPoint } + ).accessPoints?.find { it.name == multiRegionAccessPointName } return search != null } + +internal fun closeClients( + vararg clients: SdkClient, +) { + clients.forEach { client -> + client.close() + } +} diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt index 457c1431941..b4cbdb8b479 100644 --- a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt +++ b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt @@ -10,67 +10,101 @@ import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.withConfig import aws.sdk.kotlin.services.s3control.S3ControlClient import aws.sdk.kotlin.services.s3control.model.* +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +import io.kotest.matchers.string.shouldContain import kotlinx.coroutines.runBlocking import kotlin.test.Test -import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException import kotlin.test.assertFailsWith -import io.kotest.matchers.string.shouldContain class UnsupportedSigningAlgorithmTest { -// @Test + @Test fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { - try { - S3Client { - region = "us-west-2" - }.use { s3West -> + val accountId = getAccountId() + val s3West = S3Client { + region = "us-west-2" + } + val s3East = s3West.withConfig { + region = "us-east-2" + } + val s3Control = S3ControlClient { + region = "us-west-2" + } - createS3Bucket( - s3West, - usWestBucket, - BucketLocationConstraint.UsWest2, - ) + val usWestBucket = "aws-sdk-for-kotlin-test-bucket-us-west-2" + val usEastBucket = "aws-sdk-for-kotlin-test-bucket-us-east-2" + val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + val keyForObject = "test.txt" - s3West.withConfig { - region = "us-east-2" - }.use { s3East -> + try { + createS3Bucket( + s3West, + usWestBucket, + BucketLocationConstraint.UsWest2, + ) - createS3Bucket( - s3East, - usEastBucket, - BucketLocationConstraint.UsEast2, - ) + createS3Bucket( + s3East, + usEastBucket, + BucketLocationConstraint.UsEast2, + ) - S3ControlClient { - region = "us-west-2" - }.use { s3Control -> + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) - createMultiRegionAccessPoint(s3Control) - val multiRegionAccessPointArn = getMultiRegionAccessPointArn(s3Control) ?: throw Exception("Unable to get multi region access point arn") + val multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) - assertFailsWith { - createObject( - s3West, - multiRegionAccessPointArn, - ) - }.message.shouldContain("SIGV4A support is not yet implemented for the default signer.") + assertFailsWith { + createObject( + s3West, + multiRegionAccessPointArn, + keyForObject, + ) + }.message.shouldContain("SIGV4A support is not yet implemented for the default signer.") - deleteMultiRegionAccessPoint(s3Control) + deleteMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + accountId, + ) - deleteS3Bucket( - s3West, - usWestBucket, - ) + deleteS3Bucket( + s3West, + usWestBucket, + ) - deleteS3Bucket( - s3East, - usEastBucket, - ) - } - } - } + deleteS3Bucket( + s3East, + usEastBucket, + ) } catch (exception: Throwable) { - cleanUpTest() + closeClients( + s3West, + s3East, + s3Control, + ) + cleanUpTest( + accountId, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + keyForObject, + ) throw exception } + closeClients( + s3West, + s3East, + s3Control, + ) } } From 49d102bb6bdb7549b7781ed9262e0424fe9a96c8 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 24 Jan 2024 17:58:47 -0500 Subject: [PATCH 04/20] Refactoring --- .../e2eTest/src/MutliRegionAccessPointTest.kt | 224 +---------------- services/s3/e2eTest/src/S3TestUtils.kt | 227 ++++++++++++++++++ .../src/UnsupportedSigningAlgorithmTest.kt | 2 +- 3 files changed, 230 insertions(+), 223 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 98626f1ad27..e09690e0089 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -7,19 +7,12 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint -import aws.sdk.kotlin.services.s3.model.CreateBucketConfiguration -import aws.sdk.kotlin.services.s3.model.CreateBucketRequest -import aws.sdk.kotlin.services.s3.model.DeleteBucketRequest import aws.sdk.kotlin.services.s3.withConfig import aws.sdk.kotlin.services.s3control.S3ControlClient import aws.sdk.kotlin.services.s3control.model.* -import aws.sdk.kotlin.services.sts.StsClient -import aws.sdk.kotlin.services.sts.model.GetCallerIdentityRequest import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme import kotlinx.coroutines.runBlocking -import java.util.concurrent.TimeUnit import kotlin.test.Test class MutliRegionAccessPointTest { @@ -106,7 +99,7 @@ class MutliRegionAccessPointTest { s3SigV4a, s3Control, ) - cleanUpTest( + cleanUpMrapTest( accountId, multiRegionAccessPoint, usWestBucket, @@ -124,173 +117,7 @@ class MutliRegionAccessPointTest { } } -internal suspend fun getAccountId(): String { - val sts = StsClient { - region = "us-west-2" - } - - val accountId = sts.getCallerIdentity( - GetCallerIdentityRequest { }, - ).account - - sts.close() - - return accountId ?: throw Exception("Unable to get AWS account ID") -} - -internal suspend fun createS3Bucket( - s3Client: S3Client, - bucketName: String, - location: BucketLocationConstraint, -) { - s3Client.createBucket( - CreateBucketRequest { - bucket = bucketName - createBucketConfiguration = - CreateBucketConfiguration { - locationConstraint = location - } - }, - ) -} - -internal suspend fun createMultiRegionAccessPoint( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - regionOneBucket: String, - regionTwoBucket: String, - testAccountId: String, -) { - val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( - CreateMultiRegionAccessPointRequest { - accountId = testAccountId - details = - CreateMultiRegionAccessPointInput { - name = multiRegionAccessPointName - regions = - listOf( - Region { - bucket = regionOneBucket - }, - Region { - bucket = regionTwoBucket - }, - ) - } - }, - ) - - waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient, - createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), - 1000 * 60 * 10, // 10 minutes - testAccountId, - "createMultiRegionAccessPoint", - ) -} - -internal suspend fun getMultiRegionAccessPointArn( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -): String { - s3ControlClient.listMultiRegionAccessPoints( - ListMultiRegionAccessPointsRequest { - accountId = testAccountId - }, - ).accessPoints?.find { it.name == multiRegionAccessPointName }?.alias?.let { alias -> - return "arn:aws:s3::$testAccountId:accesspoint/$alias" - } - throw Exception("Unable to get multi region access point arn") -} - -internal suspend fun createObject( - s3Client: S3Client, - bucketName: String, - keyName: String, -) { - s3Client.putObject( - PutObjectRequest { - bucket = bucketName - key = keyName - }, - ) -} - -internal suspend fun deleteObject( - s3Client: S3Client, - bucketName: String, - keyName: String, -) { - s3Client.deleteObject( - DeleteObjectRequest { - bucket = bucketName - key = keyName - }, - ) -} - -internal suspend fun deleteMultiRegionAccessPoint( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -) { - val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( - DeleteMultiRegionAccessPointRequest { - accountId = testAccountId - details = - DeleteMultiRegionAccessPointInput { - name = multiRegionAccessPointName - } - }, - ) - - waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient, - deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), - 1000 * 60 * 5, // 5 minutes - testAccountId, - "deleteMultiRegionAccessPoint", - ) -} - -private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient: S3ControlClient, - request: String, - timeLimit: Int, - testAccountId: String, - operation: String, -) { - val startTime = System.currentTimeMillis() - - while (System.currentTimeMillis() - startTime < timeLimit) { - val status = s3ControlClient.describeMultiRegionAccessPointOperation( - DescribeMultiRegionAccessPointOperationRequest { - accountId = testAccountId - requestTokenArn = request - }, - ).asyncOperation?.requestStatus - - println("Waiting on $operation operation. Status: $status") - if (status == "SUCCEEDED") return - TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks - } - - throw Exception("The multi-region-access-point $operation operation exceeded the time limit set ($timeLimit ms)") -} - -internal suspend fun deleteS3Bucket( - s3Client: S3Client, - bucketName: String, -) { - s3Client.deleteBucket( - DeleteBucketRequest { - bucket = bucketName - }, - ) -} - -internal suspend fun cleanUpTest( +internal suspend fun cleanUpMrapTest( testAccountId: String, multiRegionAccessPointName: String, usWestBucket: String, @@ -318,50 +145,3 @@ internal suspend fun cleanUpTest( } } } - -private suspend fun objectWasCreated( - s3: S3Client, - bucketName: String, - keyName: String, -): Boolean { - val search = s3.listObjectsV2( - ListObjectsV2Request { - bucket = bucketName - }, - ).contents - - return search?.find { it.key == keyName } != null -} - -private suspend fun s3BucketWasCreated( - s3: S3Client, - bucketName: String, -): Boolean { - val search = s3.listBuckets( - ListBucketsRequest {}, - ).buckets - - return search?.find { it.name == bucketName } != null -} - -private suspend fun multiRegionAccessPointWasCreated( - s3Control: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -): Boolean { - val search = s3Control.listMultiRegionAccessPoints( - ListMultiRegionAccessPointsRequest { - accountId = testAccountId - }, - ).accessPoints?.find { it.name == multiRegionAccessPointName } - - return search != null -} - -internal fun closeClients( - vararg clients: SdkClient, -) { - clients.forEach { client -> - client.close() - } -} diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 7ceb34708a4..072ddbaded0 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -5,15 +5,29 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint +import aws.sdk.kotlin.services.s3.model.CreateBucketConfiguration +import aws.sdk.kotlin.services.s3.model.CreateBucketRequest +import aws.sdk.kotlin.services.s3.model.DeleteBucketRequest +import aws.sdk.kotlin.services.s3.model.ExpirationStatus +import aws.sdk.kotlin.services.s3.model.LifecycleRule +import aws.sdk.kotlin.services.s3.model.LifecycleRuleFilter import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.sdk.kotlin.services.s3control.model.* +import aws.sdk.kotlin.services.sts.StsClient +import aws.sdk.kotlin.services.sts.model.GetCallerIdentityRequest +import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.http.request.HttpRequest import kotlinx.coroutines.* import kotlinx.coroutines.flow.* import java.io.OutputStreamWriter import java.net.URL import java.util.* +import java.util.concurrent.TimeUnit import javax.net.ssl.HttpsURLConnection import kotlin.time.Duration.Companion.seconds @@ -116,3 +130,216 @@ object S3TestUtils { return connection.responseCode } } + +internal suspend fun getAccountId(): String { + val sts = StsClient { + region = "us-west-2" + } + + val accountId = sts.getCallerIdentity( + GetCallerIdentityRequest { }, + ).account + + sts.close() + + return accountId ?: throw Exception("Unable to get AWS account ID") +} + +internal suspend fun createS3Bucket( + s3Client: S3Client, + bucketName: String, + location: BucketLocationConstraint, +) { + s3Client.createBucket( + CreateBucketRequest { + bucket = bucketName + createBucketConfiguration = + CreateBucketConfiguration { + locationConstraint = location + } + }, + ) +} + +internal suspend fun createMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + regionOneBucket: String, + regionTwoBucket: String, + testAccountId: String, +) { + val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( + CreateMultiRegionAccessPointRequest { + accountId = testAccountId + details = + CreateMultiRegionAccessPointInput { + name = multiRegionAccessPointName + regions = + listOf( + Region { + bucket = regionOneBucket + }, + Region { + bucket = regionTwoBucket + }, + ) + } + }, + ) + + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), + 1000 * 60 * 10, // 10 minutes + testAccountId, + "createMultiRegionAccessPoint", + ) +} + +internal suspend fun getMultiRegionAccessPointArn( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, +): String { + s3ControlClient.listMultiRegionAccessPoints( + ListMultiRegionAccessPointsRequest { + accountId = testAccountId + }, + ).accessPoints?.find { it.name == multiRegionAccessPointName }?.alias?.let { alias -> + return "arn:aws:s3::$testAccountId:accesspoint/$alias" + } + throw Exception("Unable to get multi region access point arn") +} + +internal suspend fun createObject( + s3Client: S3Client, + bucketName: String, + keyName: String, +) { + s3Client.putObject( + PutObjectRequest { + bucket = bucketName + key = keyName + }, + ) +} + +internal suspend fun deleteObject( + s3Client: S3Client, + bucketName: String, + keyName: String, +) { + s3Client.deleteObject( + DeleteObjectRequest { + bucket = bucketName + key = keyName + }, + ) +} + +internal suspend fun deleteMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, +) { + val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( + DeleteMultiRegionAccessPointRequest { + accountId = testAccountId + details = + DeleteMultiRegionAccessPointInput { + name = multiRegionAccessPointName + } + }, + ) + + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), + 1000 * 60 * 5, // 5 minutes + testAccountId, + "deleteMultiRegionAccessPoint", + ) +} + +internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient: S3ControlClient, + request: String, + timeLimit: Int, + testAccountId: String, + operation: String, +) { + val startTime = System.currentTimeMillis() + + while (System.currentTimeMillis() - startTime < timeLimit) { + val status = s3ControlClient.describeMultiRegionAccessPointOperation( + DescribeMultiRegionAccessPointOperationRequest { + accountId = testAccountId + requestTokenArn = request + }, + ).asyncOperation?.requestStatus + + println("Waiting on $operation operation. Status: $status") + if (status == "SUCCEEDED") return + TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks + } + + throw Exception("The multi-region-access-point $operation operation exceeded the time limit set ($timeLimit ms)") +} + +internal suspend fun deleteS3Bucket( + s3Client: S3Client, + bucketName: String, +) { + s3Client.deleteBucket( + DeleteBucketRequest { + bucket = bucketName + }, + ) +} + +internal suspend fun objectWasCreated( + s3: S3Client, + bucketName: String, + keyName: String, +): Boolean { + val search = s3.listObjectsV2( + ListObjectsV2Request { + bucket = bucketName + }, + ).contents + + return search?.find { it.key == keyName } != null +} + +internal suspend fun s3BucketWasCreated( + s3: S3Client, + bucketName: String, +): Boolean { + val search = s3.listBuckets( + ListBucketsRequest {}, + ).buckets + + return search?.find { it.name == bucketName } != null +} + +internal suspend fun multiRegionAccessPointWasCreated( + s3Control: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, +): Boolean { + val search = s3Control.listMultiRegionAccessPoints( + ListMultiRegionAccessPointsRequest { + accountId = testAccountId + }, + ).accessPoints?.find { it.name == multiRegionAccessPointName } + + return search != null +} + +internal fun closeClients( + vararg clients: SdkClient, +) { + clients.forEach { client -> + client.close() + } +} diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt index b4cbdb8b479..13f58018a14 100644 --- a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt +++ b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt @@ -92,7 +92,7 @@ class UnsupportedSigningAlgorithmTest { s3East, s3Control, ) - cleanUpTest( + cleanUpMrapTest( accountId, multiRegionAccessPoint, usWestBucket, From 44f5efac04a7c219631fda51209b5a2b6c8e353a Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 25 Jan 2024 15:33:18 -0500 Subject: [PATCH 05/20] Empty commit --- services/s3/e2eTest/src/S3TestUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 072ddbaded0..8e60feb991c 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -278,7 +278,7 @@ internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( }, ).asyncOperation?.requestStatus - println("Waiting on $operation operation. Status: $status") + println("Waiting on $operation operation. Status: $status ") if (status == "SUCCEEDED") return TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks } From 80a1bb0b5866de267eab4ea242000ad2786506a1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 26 Jan 2024 14:18:56 -0500 Subject: [PATCH 06/20] Improve logging & add prefix to S3 buckets --- .../e2eTest/src/MutliRegionAccessPointTest.kt | 4 +-- services/s3/e2eTest/src/S3TestUtils.kt | 29 ++++++++++++++++++- .../src/UnsupportedSigningAlgorithmTest.kt | 4 +-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index e09690e0089..869d0a21453 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -32,8 +32,8 @@ class MutliRegionAccessPointTest { region = "us-west-2" } - val usWestBucket = "aws-sdk-for-kotlin-test-bucket-us-west-2" - val usEastBucket = "aws-sdk-for-kotlin-test-bucket-us-east-2" + val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-west-2" + val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-east-2" val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" val keyForObject = "test.txt" diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 8e60feb991c..b04bde15077 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -132,6 +132,8 @@ object S3TestUtils { } internal suspend fun getAccountId(): String { + println("Getting account ID") + val sts = StsClient { region = "us-west-2" } @@ -150,6 +152,8 @@ internal suspend fun createS3Bucket( bucketName: String, location: BucketLocationConstraint, ) { + println("Creating S3 bucket: $bucketName") + s3Client.createBucket( CreateBucketRequest { bucket = bucketName @@ -168,6 +172,8 @@ internal suspend fun createMultiRegionAccessPoint( regionTwoBucket: String, testAccountId: String, ) { + println("Creating multi region access point: $multiRegionAccessPointName") + val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( CreateMultiRegionAccessPointRequest { accountId = testAccountId @@ -201,6 +207,8 @@ internal suspend fun getMultiRegionAccessPointArn( multiRegionAccessPointName: String, testAccountId: String, ): String { + println("Getting multi region access point arn for: $multiRegionAccessPointName") + s3ControlClient.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId @@ -216,6 +224,8 @@ internal suspend fun createObject( bucketName: String, keyName: String, ) { + println("Creating object '$keyName' in S3 bucket: $bucketName") + s3Client.putObject( PutObjectRequest { bucket = bucketName @@ -229,6 +239,8 @@ internal suspend fun deleteObject( bucketName: String, keyName: String, ) { + println("Deleting object '$keyName' in S3 bucket: $bucketName") + s3Client.deleteObject( DeleteObjectRequest { bucket = bucketName @@ -242,6 +254,8 @@ internal suspend fun deleteMultiRegionAccessPoint( multiRegionAccessPointName: String, testAccountId: String, ) { + println("Deleting multi region access point: $multiRegionAccessPointName") + val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( DeleteMultiRegionAccessPointRequest { accountId = testAccountId @@ -279,7 +293,12 @@ internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( ).asyncOperation?.requestStatus println("Waiting on $operation operation. Status: $status ") - if (status == "SUCCEEDED") return + + if (status == "SUCCEEDED") { + println("$operation operation succeeded.") + return + } + TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks } @@ -290,6 +309,8 @@ internal suspend fun deleteS3Bucket( s3Client: S3Client, bucketName: String, ) { + println("Deleting S3 bucket: $bucketName") + s3Client.deleteBucket( DeleteBucketRequest { bucket = bucketName @@ -302,6 +323,8 @@ internal suspend fun objectWasCreated( bucketName: String, keyName: String, ): Boolean { + println("Checking if object '$keyName' was created in S3 bucket: $bucketName") + val search = s3.listObjectsV2( ListObjectsV2Request { bucket = bucketName @@ -315,6 +338,8 @@ internal suspend fun s3BucketWasCreated( s3: S3Client, bucketName: String, ): Boolean { + println("Checking if S3 bucket was created: $bucketName") + val search = s3.listBuckets( ListBucketsRequest {}, ).buckets @@ -327,6 +352,8 @@ internal suspend fun multiRegionAccessPointWasCreated( multiRegionAccessPointName: String, testAccountId: String, ): Boolean { + println("Checking if multi region access point was created: $multiRegionAccessPointName") + val search = s3Control.listMultiRegionAccessPoints( ListMultiRegionAccessPointsRequest { accountId = testAccountId diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt index 13f58018a14..61042d56cce 100644 --- a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt +++ b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt @@ -30,8 +30,8 @@ class UnsupportedSigningAlgorithmTest { region = "us-west-2" } - val usWestBucket = "aws-sdk-for-kotlin-test-bucket-us-west-2" - val usEastBucket = "aws-sdk-for-kotlin-test-bucket-us-east-2" + val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-west-2" + val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-east-2" val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" val keyForObject = "test.txt" From 78a5370f4babcd692f0290e729e746c7397df3c0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 26 Jan 2024 14:20:28 -0500 Subject: [PATCH 07/20] Change bucket names --- services/s3/e2eTest/src/MutliRegionAccessPointTest.kt | 4 ++-- services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 869d0a21453..26baf993184 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -32,8 +32,8 @@ class MutliRegionAccessPointTest { region = "us-west-2" } - val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-west-2" - val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-east-2" + val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" + val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" val keyForObject = "test.txt" diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt index 61042d56cce..e1786ecee84 100644 --- a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt +++ b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt @@ -30,8 +30,8 @@ class UnsupportedSigningAlgorithmTest { region = "us-west-2" } - val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-west-2" - val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}us-east-2" + val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" + val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" val keyForObject = "test.txt" From bb14a9fe67f11d5abf99025271103e06cb043ae0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 26 Jan 2024 14:37:45 -0500 Subject: [PATCH 08/20] Comment and small fix --- services/s3/e2eTest/src/S3TestUtils.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index b04bde15077..8ef1fbb17e0 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -35,7 +35,8 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" - private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + // The E2E test account only has permission to operate on buckets with the prefix + internal const val TEST_BUCKET_PREFIX = "s3-test-bucket-" suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) From ae169336635497ffdb8cb8b3284f504fec2511d1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 29 Jan 2024 14:21:12 -0800 Subject: [PATCH 09/20] Refactoring --- .../e2eTest/src/MutliRegionAccessPointTest.kt | 184 +++++++++--------- .../src/UnsupportedSigningAlgorithmTest.kt | 110 ----------- 2 files changed, 90 insertions(+), 204 deletions(-) delete mode 100644 services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 26baf993184..b911d1346bb 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -10,53 +10,87 @@ import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.withConfig import aws.sdk.kotlin.services.s3control.S3ControlClient import aws.sdk.kotlin.services.s3control.model.* +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +import io.kotest.matchers.string.shouldContain import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll import kotlin.test.Test +import kotlin.test.assertFailsWith class MutliRegionAccessPointTest { - @Test - fun testMultRegionAccessPointOperation(): Unit = runBlocking { + private val s3West = S3Client { region = "us-west-2" } + private val s3East = s3West.withConfig { region = "us-east-2" } + private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } + private val s3Control = S3ControlClient { region = "us-west-2" } + + private val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" + private val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" + private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + private val keyForObject = "test.txt" + + @BeforeAll + private suspend fun setUpMrapTest() { + println("Setting up MutliRegionAccessPointTest tests") + val accountId = getAccountId() - val s3West = S3Client { - region = "us-west-2" - } - val s3East = s3West.withConfig { - region = "us-east-2" + + createS3Bucket( // TODO: Use function already there + s3West, + usWestBucket, + BucketLocationConstraint.UsWest2, + ) + + createS3Bucket( + s3East, + usEastBucket, + BucketLocationConstraint.UsEast2, + ) + + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) + } + + @AfterAll + private suspend fun cleanUpMrapTest() { + println("Cleaning up MutliRegionAccessPointTest tests") + + val accountId = getAccountId() + + if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { + deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) } - val s3SigV4a = s3West.withConfig { - authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) + + if (s3BucketWasCreated(s3West, usWestBucket)) { + deleteS3Bucket(s3West, usWestBucket) // TODO: Use function already there } - val s3Control = S3ControlClient { - region = "us-west-2" + + if (s3BucketWasCreated(s3East, usEastBucket)) { + if (objectWasCreated(s3East, usEastBucket, keyForObject)) { // TODO: Change to MRAP > How ? + deleteObject(s3East, usEastBucket, keyForObject) + } + deleteS3Bucket(s3East, usEastBucket) } - val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" - val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" - val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" - val keyForObject = "test.txt" + closeClients( + s3West, + s3East, + s3SigV4a, + s3Control, + ) + } + @Test + fun testMultRegionAccessPointOperation(): Unit = runBlocking { try { - createS3Bucket( - s3West, - usWestBucket, - BucketLocationConstraint.UsWest2, - ) - - createS3Bucket( - s3East, - usEastBucket, - BucketLocationConstraint.UsEast2, - ) - - createMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - accountId, - ) + val accountId = getAccountId() val multiRegionAccessPointArn = getMultiRegionAccessPointArn( @@ -76,72 +110,34 @@ class MutliRegionAccessPointTest { multiRegionAccessPointArn, keyForObject, ) - - deleteMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - accountId, - ) - - deleteS3Bucket( - s3West, - usWestBucket, - ) - - deleteS3Bucket( - s3East, - usEastBucket, - ) } catch (exception: Throwable) { - closeClients( - s3West, - s3East, - s3SigV4a, - s3Control, - ) - cleanUpMrapTest( - accountId, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - keyForObject, - ) + println("Test failed with exception: ${exception.cause}") throw exception } - closeClients( - s3West, - s3East, - s3SigV4a, - s3Control, - ) } -} -internal suspend fun cleanUpMrapTest( - testAccountId: String, - multiRegionAccessPointName: String, - usWestBucket: String, - usEastBucket: String, - keyName: String, -) { - S3ControlClient { - region = "us-west-2" - }.use { s3Control -> - if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPointName, testAccountId)) deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPointName, testAccountId) - } + @Test + fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { + try { + val accountId = getAccountId() - S3Client { - region = "us-west-2" - }.use { s3West -> - if (s3BucketWasCreated(s3West, usWestBucket)) deleteS3Bucket(s3West, usWestBucket) - - s3West.withConfig { - region = "us-east-2" - }.use { s3East -> - if (s3BucketWasCreated(s3East, usEastBucket)) { - if (objectWasCreated(s3East, usEastBucket, keyName)) deleteObject(s3East, usEastBucket, keyName) - deleteS3Bucket(s3East, usEastBucket) - } + val multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) + + assertFailsWith { + createObject( + s3West, + multiRegionAccessPointArn, + keyForObject, + ) + }.message.shouldContain("SIGV4_ASYMMETRIC support is not yet implemented for the default signer.") + } catch (exception: Throwable) { + println("Test failed with exception: ${exception.cause}") + throw exception } } } diff --git a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt b/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt deleted file mode 100644 index e1786ecee84..00000000000 --- a/services/s3/e2eTest/src/UnsupportedSigningAlgorithmTest.kt +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.e2etest - -import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.model.* -import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint -import aws.sdk.kotlin.services.s3.withConfig -import aws.sdk.kotlin.services.s3control.S3ControlClient -import aws.sdk.kotlin.services.s3control.model.* -import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -import io.kotest.matchers.string.shouldContain -import kotlinx.coroutines.runBlocking -import kotlin.test.Test -import kotlin.test.assertFailsWith - -class UnsupportedSigningAlgorithmTest { - @Test - fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { - val accountId = getAccountId() - val s3West = S3Client { - region = "us-west-2" - } - val s3East = s3West.withConfig { - region = "us-east-2" - } - val s3Control = S3ControlClient { - region = "us-west-2" - } - - val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" - val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" - val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" - val keyForObject = "test.txt" - - try { - createS3Bucket( - s3West, - usWestBucket, - BucketLocationConstraint.UsWest2, - ) - - createS3Bucket( - s3East, - usEastBucket, - BucketLocationConstraint.UsEast2, - ) - - createMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - accountId, - ) - - val multiRegionAccessPointArn = - getMultiRegionAccessPointArn( - s3Control, - multiRegionAccessPoint, - accountId, - ) - - assertFailsWith { - createObject( - s3West, - multiRegionAccessPointArn, - keyForObject, - ) - }.message.shouldContain("SIGV4A support is not yet implemented for the default signer.") - - deleteMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - accountId, - ) - - deleteS3Bucket( - s3West, - usWestBucket, - ) - - deleteS3Bucket( - s3East, - usEastBucket, - ) - } catch (exception: Throwable) { - closeClients( - s3West, - s3East, - s3Control, - ) - cleanUpMrapTest( - accountId, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - keyForObject, - ) - throw exception - } - closeClients( - s3West, - s3East, - s3Control, - ) - } -} From fc59631e7389de5ed0e4f58376ca0a1fa774251c Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 29 Jan 2024 14:21:49 -0800 Subject: [PATCH 10/20] Refactoring --- services/s3/e2eTest/src/MutliRegionAccessPointTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index b911d1346bb..e9ca211a8ad 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -73,7 +73,7 @@ class MutliRegionAccessPointTest { } if (s3BucketWasCreated(s3East, usEastBucket)) { - if (objectWasCreated(s3East, usEastBucket, keyForObject)) { // TODO: Change to MRAP > How ? + if (objectWasCreated(s3East, usEastBucket, keyForObject)) { // TODO: Change to MRAP ARN - How ? deleteObject(s3East, usEastBucket, keyForObject) } deleteS3Bucket(s3East, usEastBucket) From 4da19ee95b4f97d167ec507b84f71cba0f2c9c35 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 8 Feb 2024 19:49:08 -0500 Subject: [PATCH 11/20] refactoring --- services/s3/e2eTest/src/MutliRegionAccessPointTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index e9ca211a8ad..602a33b6ef3 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -37,7 +37,7 @@ class MutliRegionAccessPointTest { val accountId = getAccountId() - createS3Bucket( // TODO: Use function already there + createS3Bucket( s3West, usWestBucket, BucketLocationConstraint.UsWest2, @@ -69,11 +69,11 @@ class MutliRegionAccessPointTest { } if (s3BucketWasCreated(s3West, usWestBucket)) { - deleteS3Bucket(s3West, usWestBucket) // TODO: Use function already there + deleteS3Bucket(s3West, usWestBucket) } if (s3BucketWasCreated(s3East, usEastBucket)) { - if (objectWasCreated(s3East, usEastBucket, keyForObject)) { // TODO: Change to MRAP ARN - How ? + if (objectWasCreated(s3East, usEastBucket, keyForObject)) { deleteObject(s3East, usEastBucket, keyForObject) } deleteS3Bucket(s3East, usEastBucket) From ec59a8c798b913fe370a37dd874bfa47b447b264 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 9 Feb 2024 11:00:30 -0500 Subject: [PATCH 12/20] Fixed before all and after all --- services/s3/e2eTest/src/MutliRegionAccessPointTest.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 602a33b6ef3..9810189442e 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -17,9 +17,11 @@ import io.kotest.matchers.string.shouldContain import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance import kotlin.test.Test import kotlin.test.assertFailsWith +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class MutliRegionAccessPointTest { private val s3West = S3Client { region = "us-west-2" } private val s3East = s3West.withConfig { region = "us-east-2" } @@ -32,7 +34,7 @@ class MutliRegionAccessPointTest { private val keyForObject = "test.txt" @BeforeAll - private suspend fun setUpMrapTest() { + private fun setUpMrapTest(): Unit = runBlocking { println("Setting up MutliRegionAccessPointTest tests") val accountId = getAccountId() @@ -59,7 +61,7 @@ class MutliRegionAccessPointTest { } @AfterAll - private suspend fun cleanUpMrapTest() { + private fun cleanUpMrapTest(): Unit = runBlocking { println("Cleaning up MutliRegionAccessPointTest tests") val accountId = getAccountId() From 190f4eb23f78e6d46975af44b5a5acf4d7a1ebdc Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 14 Feb 2024 16:00:22 -0500 Subject: [PATCH 13/20] Use util functions already there --- .../s3/e2eTest/src/MutliRegionAccessPointTest.kt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 9810189442e..16d7c1155bd 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -4,6 +4,7 @@ */ package aws.sdk.kotlin.e2etest +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint @@ -70,16 +71,8 @@ class MutliRegionAccessPointTest { deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) } - if (s3BucketWasCreated(s3West, usWestBucket)) { - deleteS3Bucket(s3West, usWestBucket) - } - - if (s3BucketWasCreated(s3East, usEastBucket)) { - if (objectWasCreated(s3East, usEastBucket, keyForObject)) { - deleteObject(s3East, usEastBucket, keyForObject) - } - deleteS3Bucket(s3East, usEastBucket) - } + deleteBucketAndAllContents(s3West, usWestBucket) + deleteBucketAndAllContents(s3East, usEastBucket) closeClients( s3West, From 47ae5a497fbd44689399020cffe843ff81a350c8 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 21 Feb 2024 12:42:29 -0500 Subject: [PATCH 14/20] Checkpoint --- services/build.gradle.kts | 19 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 262 +++++++------- services/s3/e2eTest/src/S3TestUtils.kt | 331 ++++++------------ 3 files changed, 251 insertions(+), 361 deletions(-) diff --git a/services/build.gradle.kts b/services/build.gradle.kts index 5659dcab072..62c355e6e54 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import aws.sdk.kotlin.gradle.dsl.configurePublishing -import aws.sdk.kotlin.gradle.kmp.kotlin +import aws.sdk.kotlin.gradle.kmp.* import aws.sdk.kotlin.gradle.util.typedProp import org.jetbrains.kotlin.gradle.dsl.JvmTarget import java.time.LocalDateTime @@ -75,10 +75,6 @@ subprojects { implementation(libraries.kotlin.test.junit5) implementation(project(":tests:e2e-test-util")) implementation(libraries.slf4j.simple) - implementation("aws.sdk.kotlin:s3control:+") - implementation("aws.sdk.kotlin:sts:+") - implementation("aws.smithy.kotlin:aws-signing-crt:+") - implementation("io.kotest:kotest-assertions-core:5.8.0") } } @@ -89,6 +85,19 @@ subprojects { } tasks.register("e2eTest") { + if (project.name == "s3") { + dependencies { + project.parent?.subprojects?.forEach { + println("\n\n\n\n\n") + println(it.name) + println("\n\n\n\n\n") + } + implementation("aws.sdk.kotlin:s3control:+") // services:s3control (if present) + implementation("aws.sdk.kotlin:sts:+") // services:sts (if present) + implementation("aws.smithy.kotlin:aws-signing-crt:+") // :runtime:auth:aws-signing-crt (in smithy) + } + } + description = "Run e2e service tests" group = "verification" diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 16d7c1155bd..a8175dee1bb 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,138 +1,124 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.e2etest - -import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.model.* -import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint -import aws.sdk.kotlin.services.s3.withConfig -import aws.sdk.kotlin.services.s3control.S3ControlClient -import aws.sdk.kotlin.services.s3control.model.* -import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -import io.kotest.matchers.string.shouldContain -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.TestInstance -import kotlin.test.Test -import kotlin.test.assertFailsWith - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -class MutliRegionAccessPointTest { - private val s3West = S3Client { region = "us-west-2" } - private val s3East = s3West.withConfig { region = "us-east-2" } - private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } - private val s3Control = S3ControlClient { region = "us-west-2" } - - private val usWestBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-west-2" - private val usEastBucket = "${S3TestUtils.TEST_BUCKET_PREFIX}for-aws-kotlin-sdk-us-east-2" - private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" - private val keyForObject = "test.txt" - - @BeforeAll - private fun setUpMrapTest(): Unit = runBlocking { - println("Setting up MutliRegionAccessPointTest tests") - - val accountId = getAccountId() - - createS3Bucket( - s3West, - usWestBucket, - BucketLocationConstraint.UsWest2, - ) - - createS3Bucket( - s3East, - usEastBucket, - BucketLocationConstraint.UsEast2, - ) - - createMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - accountId, - ) - } - - @AfterAll - private fun cleanUpMrapTest(): Unit = runBlocking { - println("Cleaning up MutliRegionAccessPointTest tests") - - val accountId = getAccountId() - - if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { - deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) - } - - deleteBucketAndAllContents(s3West, usWestBucket) - deleteBucketAndAllContents(s3East, usEastBucket) - - closeClients( - s3West, - s3East, - s3SigV4a, - s3Control, - ) - } - - @Test - fun testMultRegionAccessPointOperation(): Unit = runBlocking { - try { - val accountId = getAccountId() - - val multiRegionAccessPointArn = - getMultiRegionAccessPointArn( - s3Control, - multiRegionAccessPoint, - accountId, - ) - - createObject( - s3SigV4a, - multiRegionAccessPointArn, - keyForObject, - ) - - deleteObject( - s3SigV4a, - multiRegionAccessPointArn, - keyForObject, - ) - } catch (exception: Throwable) { - println("Test failed with exception: ${exception.cause}") - throw exception - } - } - - @Test - fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { - try { - val accountId = getAccountId() - - val multiRegionAccessPointArn = - getMultiRegionAccessPointArn( - s3Control, - multiRegionAccessPoint, - accountId, - ) - - assertFailsWith { - createObject( - s3West, - multiRegionAccessPointArn, - keyForObject, - ) - }.message.shouldContain("SIGV4_ASYMMETRIC support is not yet implemented for the default signer.") - } catch (exception: Throwable) { - println("Test failed with exception: ${exception.cause}") - throw exception - } - } -} +///* +// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// * SPDX-License-Identifier: Apache-2.0 +// */ +//package aws.sdk.kotlin.e2etest +// +//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +//import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket +//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +//import aws.sdk.kotlin.services.s3.S3Client +//import aws.sdk.kotlin.services.s3.deleteObject +//import aws.sdk.kotlin.services.s3.model.* +//import aws.sdk.kotlin.services.s3.putObject +//import aws.sdk.kotlin.services.s3.withConfig +//import aws.sdk.kotlin.services.s3control.S3ControlClient +//import aws.sdk.kotlin.services.s3control.model.* +//import aws.sdk.kotlin.services.sts.StsClient +//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +//import kotlinx.coroutines.invoke +//import kotlinx.coroutines.runBlocking +//import org.junit.jupiter.api.AfterAll +//import org.junit.jupiter.api.BeforeAll +//import org.junit.jupiter.api.TestInstance +//import kotlin.test.Test +//import kotlin.test.assertEquals +//import kotlin.test.assertFailsWith +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//class MutliRegionAccessPointTest { +// private val s3West = S3Client { region = "us-west-2" } +// private val s3East = s3West.withConfig { region = "us-east-2" } +// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } +// private val s3Control = S3ControlClient { region = "us-west-2" } +// +// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" +// private val objectKey = "test.txt" +// +// private lateinit var accountId: String +// private lateinit var multiRegionAccessPointArn: String +// private lateinit var usWestBucket: String +// private lateinit var usEastBucket: String +// +// @BeforeAll +// private fun setUpMrapTest(): Unit = runBlocking { +// accountId = getAccountId() +// usWestBucket = getTestBucket(s3West, "us-west-2", accountId) +// usEastBucket = getTestBucket(s3East, "us-east-2", accountId) +// +// createMultiRegionAccessPoint( +// s3Control, +// multiRegionAccessPoint, +// usWestBucket, +// usEastBucket, +// accountId, +// ) +// +// multiRegionAccessPointArn = +// getMultiRegionAccessPointArn( +// s3Control, +// multiRegionAccessPoint, +// accountId, +// ) +// } +// +// @AfterAll +// private fun cleanUpMrapTest(): Unit = runBlocking { +// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { +// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) +// } +// +// deleteBucketAndAllContents(s3West, usWestBucket) +// deleteBucketAndAllContents(s3East, usEastBucket) +// +// s3West.close() +// s3East.close() +// s3SigV4a.close() +// s3Control.close() +// } +// +// @Test +// fun testMultRegionAccessPointOperation(): Unit = runBlocking { +// s3SigV4a.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// +// s3SigV4a.deleteObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// @Test +// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { +// val ex = assertFailsWith { +// s3West.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// assertEquals( +// ex.message, +// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533" +// ) +// } +// +// private suspend fun getAccountId(): String { +// println("Getting account ID") +// +// val accountId = StsClient { +// region = "us-west-2" +// }.use { +// it.getCallerIdentity().account +// } +// +// return checkNotNull(accountId) { "Unable to get AWS account ID"} +// } +//} diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 8ef1fbb17e0..9e0ef48e3fb 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -8,28 +8,23 @@ import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint -import aws.sdk.kotlin.services.s3.model.CreateBucketConfiguration -import aws.sdk.kotlin.services.s3.model.CreateBucketRequest -import aws.sdk.kotlin.services.s3.model.DeleteBucketRequest import aws.sdk.kotlin.services.s3.model.ExpirationStatus import aws.sdk.kotlin.services.s3.model.LifecycleRule import aws.sdk.kotlin.services.s3.model.LifecycleRuleFilter import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists -import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.sdk.kotlin.services.s3control.* import aws.sdk.kotlin.services.s3control.model.* -import aws.sdk.kotlin.services.sts.StsClient -import aws.sdk.kotlin.services.sts.model.GetCallerIdentityRequest -import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.http.request.HttpRequest import kotlinx.coroutines.* import kotlinx.coroutines.flow.* import java.io.OutputStreamWriter import java.net.URL import java.util.* -import java.util.concurrent.TimeUnit import javax.net.ssl.HttpsURLConnection import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.withTimeout +import kotlin.time.Duration.Companion.milliseconds object S3TestUtils { @@ -38,13 +33,32 @@ object S3TestUtils { // The E2E test account only has permission to operate on buckets with the prefix internal const val TEST_BUCKET_PREFIX = "s3-test-bucket-" - suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) - - private suspend fun getBucketWithPrefix(client: S3Client, prefix: String): String = withTimeout(60.seconds) { - var testBucket = client.listBuckets() + suspend fun getTestBucket( + client: S3Client, + region: String? = null, + accountId: String? = null, + ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId) + + private suspend fun getBucketWithPrefix( + client: S3Client, + prefix: String, + region: String?, + accountId: String?, + ): String = withTimeout(60.seconds) { + val buckets = client.listBuckets() .buckets ?.mapNotNull { it.name } - ?.firstOrNull { it.startsWith(prefix) } + + var testBucket = if (region != null) { + buckets?.firstOrNull { + client.getBucketLocation { + bucket = it + expectedBucketOwner = accountId + }.locationConstraint?.value == region && it.startsWith(prefix) + } + } else { + buckets?.firstOrNull { it.startsWith(prefix) } + } if (testBucket == null) { testBucket = prefix + UUID.randomUUID() @@ -53,7 +67,7 @@ object S3TestUtils { client.createBucket { bucket = testBucket createBucketConfiguration { - locationConstraint = BucketLocationConstraint.fromValue(client.config.region!!) + locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) } } @@ -130,53 +144,17 @@ object S3TestUtils { return connection.responseCode } -} - -internal suspend fun getAccountId(): String { - println("Getting account ID") - - val sts = StsClient { - region = "us-west-2" - } - - val accountId = sts.getCallerIdentity( - GetCallerIdentityRequest { }, - ).account - - sts.close() - - return accountId ?: throw Exception("Unable to get AWS account ID") -} -internal suspend fun createS3Bucket( - s3Client: S3Client, - bucketName: String, - location: BucketLocationConstraint, -) { - println("Creating S3 bucket: $bucketName") - - s3Client.createBucket( - CreateBucketRequest { - bucket = bucketName - createBucketConfiguration = - CreateBucketConfiguration { - locationConstraint = location - } - }, - ) -} + internal suspend fun createMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + regionOneBucket: String, + regionTwoBucket: String, + testAccountId: String, + ) { + println("Creating multi region access point: $multiRegionAccessPointName") -internal suspend fun createMultiRegionAccessPoint( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - regionOneBucket: String, - regionTwoBucket: String, - testAccountId: String, -) { - println("Creating multi region access point: $multiRegionAccessPointName") - - val createRequestToken = s3ControlClient.createMultiRegionAccessPoint( - CreateMultiRegionAccessPointRequest { + val createRequestToken = s3ControlClient.createMultiRegionAccessPoint { accountId = testAccountId details = CreateMultiRegionAccessPointInput { @@ -191,183 +169,100 @@ internal suspend fun createMultiRegionAccessPoint( }, ) } - }, - ) - - waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient, - createRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), - 1000 * 60 * 10, // 10 minutes - testAccountId, - "createMultiRegionAccessPoint", - ) -} - -internal suspend fun getMultiRegionAccessPointArn( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -): String { - println("Getting multi region access point arn for: $multiRegionAccessPointName") + } - s3ControlClient.listMultiRegionAccessPoints( - ListMultiRegionAccessPointsRequest { - accountId = testAccountId - }, - ).accessPoints?.find { it.name == multiRegionAccessPointName }?.alias?.let { alias -> - return "arn:aws:s3::$testAccountId:accesspoint/$alias" + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + checkNotNull(createRequestToken.requestTokenArn) { "Unable to get request token ARN" }, + 1000 * 60 * 10, // 10 minutes + testAccountId, + "createMultiRegionAccessPoint", + ) } - throw Exception("Unable to get multi region access point arn") -} -internal suspend fun createObject( - s3Client: S3Client, - bucketName: String, - keyName: String, -) { - println("Creating object '$keyName' in S3 bucket: $bucketName") - - s3Client.putObject( - PutObjectRequest { - bucket = bucketName - key = keyName - }, - ) -} + internal suspend fun getMultiRegionAccessPointArn( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, + ): String { + println("Getting multi region access point arn for: $multiRegionAccessPointName") -internal suspend fun deleteObject( - s3Client: S3Client, - bucketName: String, - keyName: String, -) { - println("Deleting object '$keyName' in S3 bucket: $bucketName") - - s3Client.deleteObject( - DeleteObjectRequest { - bucket = bucketName - key = keyName - }, - ) -} + s3ControlClient.getMultiRegionAccessPoint { + accountId = testAccountId + name = multiRegionAccessPointName + }.accessPoint?.alias?.let { alias -> + return "arn:aws:s3::$testAccountId:accesspoint/$alias" + } + throw Exception("Unable to get multi region access point arn") + } -internal suspend fun deleteMultiRegionAccessPoint( - s3ControlClient: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -) { - println("Deleting multi region access point: $multiRegionAccessPointName") + internal suspend fun deleteMultiRegionAccessPoint( + s3ControlClient: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, + ) { + println("Deleting multi region access point: $multiRegionAccessPointName") - val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint( - DeleteMultiRegionAccessPointRequest { + val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint { accountId = testAccountId details = DeleteMultiRegionAccessPointInput { name = multiRegionAccessPointName } - }, - ) - - waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient, - deleteRequestToken.requestTokenArn ?: throw Exception("Unable to get request token ARN"), - 1000 * 60 * 5, // 5 minutes - testAccountId, - "deleteMultiRegionAccessPoint", - ) -} - -internal suspend fun waitUntilMultiRegionAccessPointOperationCompletes( - s3ControlClient: S3ControlClient, - request: String, - timeLimit: Int, - testAccountId: String, - operation: String, -) { - val startTime = System.currentTimeMillis() - - while (System.currentTimeMillis() - startTime < timeLimit) { - val status = s3ControlClient.describeMultiRegionAccessPointOperation( - DescribeMultiRegionAccessPointOperationRequest { - accountId = testAccountId - requestTokenArn = request - }, - ).asyncOperation?.requestStatus - - println("Waiting on $operation operation. Status: $status ") - - if (status == "SUCCEEDED") { - println("$operation operation succeeded.") - return } - TimeUnit.SECONDS.sleep(10L) // Avoid constant status checks + waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient, + checkNotNull(deleteRequestToken.requestTokenArn) { "Unable to get request token ARN" }, + 1000 * 60 * 5, // 5 minutes + testAccountId, + "deleteMultiRegionAccessPoint", + ) } - throw Exception("The multi-region-access-point $operation operation exceeded the time limit set ($timeLimit ms)") -} - -internal suspend fun deleteS3Bucket( - s3Client: S3Client, - bucketName: String, -) { - println("Deleting S3 bucket: $bucketName") - - s3Client.deleteBucket( - DeleteBucketRequest { - bucket = bucketName - }, - ) -} - -internal suspend fun objectWasCreated( - s3: S3Client, - bucketName: String, - keyName: String, -): Boolean { - println("Checking if object '$keyName' was created in S3 bucket: $bucketName") - - val search = s3.listObjectsV2( - ListObjectsV2Request { - bucket = bucketName - }, - ).contents - - return search?.find { it.key == keyName } != null -} - -internal suspend fun s3BucketWasCreated( - s3: S3Client, - bucketName: String, -): Boolean { - println("Checking if S3 bucket was created: $bucketName") - - val search = s3.listBuckets( - ListBucketsRequest {}, - ).buckets - - return search?.find { it.name == bucketName } != null -} + private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( + s3ControlClient: S3ControlClient, + request: String, + duration: Int, + testAccountId: String, + operation: String, + ) { + withTimeout(duration.milliseconds) { + while (true) { + val status = s3ControlClient.describeMultiRegionAccessPointOperation { + accountId = testAccountId + requestTokenArn = request + }.asyncOperation?.requestStatus + + println("Waiting on $operation operation. Status: $status ") + + if (status == "SUCCEEDED") { + println("$operation operation succeeded.") + return@withTimeout + } -internal suspend fun multiRegionAccessPointWasCreated( - s3Control: S3ControlClient, - multiRegionAccessPointName: String, - testAccountId: String, -): Boolean { - println("Checking if multi region access point was created: $multiRegionAccessPointName") + if (status == "FAILED") throw Exception("$operation operation failed.") - val search = s3Control.listMultiRegionAccessPoints( - ListMultiRegionAccessPointsRequest { - accountId = testAccountId - }, - ).accessPoints?.find { it.name == multiRegionAccessPointName } + delay(10.seconds) // Avoid constant status checks + } + } + } - return search != null -} + internal suspend fun multiRegionAccessPointWasCreated( + s3Control: S3ControlClient, + multiRegionAccessPointName: String, + testAccountId: String, + ): Boolean { + println("Checking if multi region access point was created: $multiRegionAccessPointName") -internal fun closeClients( - vararg clients: SdkClient, -) { - clients.forEach { client -> - client.close() + try { + s3Control.getMultiRegionAccessPoint { + accountId = testAccountId + name = multiRegionAccessPointName + } + } catch (ex: Throwable) { + return false + } + return true } } From 9a8c055b41b829883df012e5831bd3fc9bd0e076 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 21 Feb 2024 23:49:52 -0500 Subject: [PATCH 15/20] PR feedback --- services/build.gradle.kts | 27 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 245 +++++++++--------- services/s3/e2eTest/src/S3TestUtils.kt | 14 +- 3 files changed, 143 insertions(+), 143 deletions(-) diff --git a/services/build.gradle.kts b/services/build.gradle.kts index 62c355e6e54..befae733af5 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -85,22 +85,29 @@ subprojects { } tasks.register("e2eTest") { + description = "Run e2e service tests" + group = "verification" + if (project.name == "s3") { dependencies { - project.parent?.subprojects?.forEach { - println("\n\n\n\n\n") - println(it.name) - println("\n\n\n\n\n") + val services = project.parent?.subprojects + + if (services?.firstOrNull { it.name == "s3control" } == null) { + implementation("aws.sdk.kotlin:s3control:+") + } else { + implementation(project(":services:s3control")) + } + + if (services?.firstOrNull { it.name == "sts" } == null) { + implementation("aws.sdk.kotlin:sts:+") + } else { + implementation(project(":services:sts")) } - implementation("aws.sdk.kotlin:s3control:+") // services:s3control (if present) - implementation("aws.sdk.kotlin:sts:+") // services:sts (if present) - implementation("aws.smithy.kotlin:aws-signing-crt:+") // :runtime:auth:aws-signing-crt (in smithy) + + implementation(libs.smithy.kotlin.aws.signing.crt) } } - description = "Run e2e service tests" - group = "verification" - // Run the tests with the classpath containing the compile dependencies (including 'main'), // runtime dependencies, and the outputs of this compilation: classpath = compileDependencyFiles + runtimeDependencyFiles + output.allOutputs diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index a8175dee1bb..1315be6ca08 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,124 +1,121 @@ -///* -// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// * SPDX-License-Identifier: Apache-2.0 -// */ -//package aws.sdk.kotlin.e2etest -// -//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -//import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket -//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -//import aws.sdk.kotlin.services.s3.S3Client -//import aws.sdk.kotlin.services.s3.deleteObject -//import aws.sdk.kotlin.services.s3.model.* -//import aws.sdk.kotlin.services.s3.putObject -//import aws.sdk.kotlin.services.s3.withConfig -//import aws.sdk.kotlin.services.s3control.S3ControlClient -//import aws.sdk.kotlin.services.s3control.model.* -//import aws.sdk.kotlin.services.sts.StsClient -//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -//import kotlinx.coroutines.invoke -//import kotlinx.coroutines.runBlocking -//import org.junit.jupiter.api.AfterAll -//import org.junit.jupiter.api.BeforeAll -//import org.junit.jupiter.api.TestInstance -//import kotlin.test.Test -//import kotlin.test.assertEquals -//import kotlin.test.assertFailsWith -// -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//class MutliRegionAccessPointTest { -// private val s3West = S3Client { region = "us-west-2" } -// private val s3East = s3West.withConfig { region = "us-east-2" } -// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } -// private val s3Control = S3ControlClient { region = "us-west-2" } -// -// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" -// private val objectKey = "test.txt" -// -// private lateinit var accountId: String -// private lateinit var multiRegionAccessPointArn: String -// private lateinit var usWestBucket: String -// private lateinit var usEastBucket: String -// -// @BeforeAll -// private fun setUpMrapTest(): Unit = runBlocking { -// accountId = getAccountId() -// usWestBucket = getTestBucket(s3West, "us-west-2", accountId) -// usEastBucket = getTestBucket(s3East, "us-east-2", accountId) -// -// createMultiRegionAccessPoint( -// s3Control, -// multiRegionAccessPoint, -// usWestBucket, -// usEastBucket, -// accountId, -// ) -// -// multiRegionAccessPointArn = -// getMultiRegionAccessPointArn( -// s3Control, -// multiRegionAccessPoint, -// accountId, -// ) -// } -// -// @AfterAll -// private fun cleanUpMrapTest(): Unit = runBlocking { -// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { -// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) -// } -// -// deleteBucketAndAllContents(s3West, usWestBucket) -// deleteBucketAndAllContents(s3East, usEastBucket) -// -// s3West.close() -// s3East.close() -// s3SigV4a.close() -// s3Control.close() -// } -// -// @Test -// fun testMultRegionAccessPointOperation(): Unit = runBlocking { -// s3SigV4a.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// -// s3SigV4a.deleteObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// @Test -// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { -// val ex = assertFailsWith { -// s3West.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// assertEquals( -// ex.message, -// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533" -// ) -// } -// -// private suspend fun getAccountId(): String { -// println("Getting account ID") -// -// val accountId = StsClient { -// region = "us-west-2" -// }.use { -// it.getCallerIdentity().account -// } -// -// return checkNotNull(accountId) { "Unable to get AWS account ID"} -// } -//} +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket +import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.deleteObject +import aws.sdk.kotlin.services.s3.putObject +import aws.sdk.kotlin.services.s3.withConfig +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.sdk.kotlin.services.sts.StsClient +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MutliRegionAccessPointTest { + private val s3West = S3Client { region = "us-west-2" } + private val s3East = s3West.withConfig { region = "us-east-2" } + private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } + private val s3Control = S3ControlClient { region = "us-west-2" } + + private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + private val objectKey = "test.txt" + + private lateinit var accountId: String + private lateinit var multiRegionAccessPointArn: String + private lateinit var usWestBucket: String + private lateinit var usEastBucket: String + + @BeforeAll + private fun setUpMrapTest(): Unit = runBlocking { + accountId = getAccountId() + usWestBucket = getTestBucket(s3West, "us-west-2", accountId) + usEastBucket = getTestBucket(s3East, "us-east-2", accountId) + + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) + + multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) + } + + @AfterAll + private fun cleanUpMrapTest(): Unit = runBlocking { + if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { + deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) + } + + deleteBucketAndAllContents(s3West, usWestBucket) + deleteBucketAndAllContents(s3East, usEastBucket) + + s3West.close() + s3East.close() + s3SigV4a.close() + s3Control.close() + } + + @Test + fun testMultiRegionAccessPointOperation(): Unit = runBlocking { + s3SigV4a.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + + s3SigV4a.deleteObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + @Test + fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { + val ex = assertFailsWith { + s3West.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + assertEquals( + ex.message, + "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533" + ) + } + + private suspend fun getAccountId(): String { + println("Getting account ID") + + val accountId = StsClient { + region = "us-west-2" + }.use { + it.getCallerIdentity().account + } + + return checkNotNull(accountId) { "Unable to get AWS account ID"} + } +} diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 9e0ef48e3fb..31101365069 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -255,14 +255,10 @@ object S3TestUtils { ): Boolean { println("Checking if multi region access point was created: $multiRegionAccessPointName") - try { - s3Control.getMultiRegionAccessPoint { - accountId = testAccountId - name = multiRegionAccessPointName - } - } catch (ex: Throwable) { - return false - } - return true + val search = s3Control.listMultiRegionAccessPoints { + accountId = testAccountId + }.accessPoints?.find { it.name == multiRegionAccessPointName } + + return search != null } } From af8ec0225a789807095c4e1ef2bc2f9f5e66b715 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 22 Feb 2024 00:36:08 -0500 Subject: [PATCH 16/20] Refactor --- .../s3/e2eTest/src/MutliRegionAccessPointTest.kt | 14 +------------- services/s3/e2eTest/src/S3TestUtils.kt | 13 +++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 1315be6ca08..8c76066cb25 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -7,6 +7,7 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated @@ -15,7 +16,6 @@ import aws.sdk.kotlin.services.s3.deleteObject import aws.sdk.kotlin.services.s3.putObject import aws.sdk.kotlin.services.s3.withConfig import aws.sdk.kotlin.services.s3control.S3ControlClient -import aws.sdk.kotlin.services.sts.StsClient import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme @@ -106,16 +106,4 @@ class MutliRegionAccessPointTest { "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533" ) } - - private suspend fun getAccountId(): String { - println("Getting account ID") - - val accountId = StsClient { - region = "us-west-2" - }.use { - it.getCallerIdentity().account - } - - return checkNotNull(accountId) { "Unable to get AWS account ID"} - } } diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 31101365069..4f2d180dc6d 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -15,6 +15,7 @@ import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.sdk.kotlin.services.s3control.* import aws.sdk.kotlin.services.s3control.model.* +import aws.sdk.kotlin.services.sts.StsClient import aws.smithy.kotlin.runtime.http.request.HttpRequest import kotlinx.coroutines.* import kotlinx.coroutines.flow.* @@ -145,6 +146,18 @@ object S3TestUtils { return connection.responseCode } + internal suspend fun getAccountId(): String { + println("Getting account ID") + + val accountId = StsClient { + region = "us-west-2" + }.use { + it.getCallerIdentity().account + } + + return checkNotNull(accountId) { "Unable to get AWS account ID"} + } + internal suspend fun createMultiRegionAccessPoint( s3ControlClient: S3ControlClient, multiRegionAccessPointName: String, From bb979b1d340e20f74c8ab395afbb6f179a09ec17 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 22 Feb 2024 09:20:27 -0500 Subject: [PATCH 17/20] Lint --- .../e2eTest/src/MutliRegionAccessPointTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 35 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 8c76066cb25..c44405a947e 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -103,7 +103,7 @@ class MutliRegionAccessPointTest { assertEquals( ex.message, - "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533" + "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", ) } } diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 4f2d180dc6d..ec242e0b326 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -19,20 +19,20 @@ import aws.sdk.kotlin.services.sts.StsClient import aws.smithy.kotlin.runtime.http.request.HttpRequest import kotlinx.coroutines.* import kotlinx.coroutines.flow.* +import kotlinx.coroutines.withTimeout import java.io.OutputStreamWriter import java.net.URL import java.util.* import javax.net.ssl.HttpsURLConnection -import kotlin.time.Duration.Companion.seconds -import kotlinx.coroutines.withTimeout import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds object S3TestUtils { const val DEFAULT_REGION = "us-west-2" // The E2E test account only has permission to operate on buckets with the prefix - internal const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" suspend fun getTestBucket( client: S3Client, @@ -155,7 +155,7 @@ object S3TestUtils { it.getCallerIdentity().account } - return checkNotNull(accountId) { "Unable to get AWS account ID"} + return checkNotNull(accountId) { "Unable to get AWS account ID" } } internal suspend fun createMultiRegionAccessPoint( @@ -169,19 +169,13 @@ object S3TestUtils { val createRequestToken = s3ControlClient.createMultiRegionAccessPoint { accountId = testAccountId - details = - CreateMultiRegionAccessPointInput { - name = multiRegionAccessPointName - regions = - listOf( - Region { - bucket = regionOneBucket - }, - Region { - bucket = regionTwoBucket - }, - ) - } + details { + name = multiRegionAccessPointName + regions = listOf( + Region { bucket = regionOneBucket }, + Region { bucket = regionTwoBucket }, + ) + } } waitUntilMultiRegionAccessPointOperationCompletes( @@ -218,10 +212,9 @@ object S3TestUtils { val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint { accountId = testAccountId - details = - DeleteMultiRegionAccessPointInput { - name = multiRegionAccessPointName - } + details { + name = multiRegionAccessPointName + } } waitUntilMultiRegionAccessPointOperationCompletes( From 943c4c3caf27ac2ca1c7842f095fa550eb050937 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 22 Feb 2024 09:46:00 -0500 Subject: [PATCH 18/20] Small refactor --- services/build.gradle.kts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/build.gradle.kts b/services/build.gradle.kts index befae733af5..f1fb421cd62 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -92,16 +92,16 @@ subprojects { dependencies { val services = project.parent?.subprojects - if (services?.firstOrNull { it.name == "s3control" } == null) { - implementation("aws.sdk.kotlin:s3control:+") - } else { + if (services?.any { it.name == "s3control" } == true) { implementation(project(":services:s3control")) + } else { + implementation("aws.sdk.kotlin:s3control:+") } - if (services?.firstOrNull { it.name == "sts" } == null) { - implementation("aws.sdk.kotlin:sts:+") - } else { + if (services?.any { it.name == "sts" } == true) { implementation(project(":services:sts")) + } else { + implementation("aws.sdk.kotlin:sts:+") } implementation(libs.smithy.kotlin.aws.signing.crt) From 83d9ef9de97a1c517469af59e82e57bbe1a49551 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 22 Feb 2024 15:07:13 -0500 Subject: [PATCH 19/20] Minor Feedback --- services/s3/e2eTest/src/S3TestUtils.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index ec242e0b326..7eec74b9fcf 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -24,7 +24,8 @@ import java.io.OutputStreamWriter import java.net.URL import java.util.* import javax.net.ssl.HttpsURLConnection -import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds object S3TestUtils { @@ -181,7 +182,7 @@ object S3TestUtils { waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient, checkNotNull(createRequestToken.requestTokenArn) { "Unable to get request token ARN" }, - 1000 * 60 * 10, // 10 minutes + 10.minutes, testAccountId, "createMultiRegionAccessPoint", ) @@ -220,7 +221,7 @@ object S3TestUtils { waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient, checkNotNull(deleteRequestToken.requestTokenArn) { "Unable to get request token ARN" }, - 1000 * 60 * 5, // 5 minutes + 5.minutes, testAccountId, "deleteMultiRegionAccessPoint", ) @@ -229,11 +230,11 @@ object S3TestUtils { private suspend fun waitUntilMultiRegionAccessPointOperationCompletes( s3ControlClient: S3ControlClient, request: String, - duration: Int, + timeoutAfter: Duration, testAccountId: String, operation: String, ) { - withTimeout(duration.milliseconds) { + withTimeout(timeoutAfter) { while (true) { val status = s3ControlClient.describeMultiRegionAccessPointOperation { accountId = testAccountId @@ -247,7 +248,7 @@ object S3TestUtils { return@withTimeout } - if (status == "FAILED") throw Exception("$operation operation failed.") + check(status != "FAILED") { "$operation operation failed" } delay(10.seconds) // Avoid constant status checks } From c2aa248c4aa52909844dde3a25290299da720a55 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 22 Feb 2024 15:40:07 -0500 Subject: [PATCH 20/20] Minor Feedback --- .../e2eTest/src/MutliRegionAccessPointTest.kt | 4 ++-- services/s3/e2eTest/src/S3TestUtils.kt | 23 ++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index c44405a947e..9c3cb087fa6 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -43,7 +43,7 @@ class MutliRegionAccessPointTest { private lateinit var usEastBucket: String @BeforeAll - private fun setUpMrapTest(): Unit = runBlocking { + private fun setUp(): Unit = runBlocking { accountId = getAccountId() usWestBucket = getTestBucket(s3West, "us-west-2", accountId) usEastBucket = getTestBucket(s3East, "us-east-2", accountId) @@ -65,7 +65,7 @@ class MutliRegionAccessPointTest { } @AfterAll - private fun cleanUpMrapTest(): Unit = runBlocking { + private fun cleanUp(): Unit = runBlocking { if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) } diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 7eec74b9fcf..c6497cc74f2 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -51,15 +51,14 @@ object S3TestUtils { .buckets ?.mapNotNull { it.name } - var testBucket = if (region != null) { - buckets?.firstOrNull { - client.getBucketLocation { - bucket = it - expectedBucketOwner = accountId - }.locationConstraint?.value == region && it.startsWith(prefix) - } - } else { - buckets?.firstOrNull { it.startsWith(prefix) } + var testBucket = buckets?.firstOrNull { bucketName -> + bucketName.startsWith(prefix) && + region?.let { + client.getBucketLocation { + bucket = bucketName + expectedBucketOwner = accountId + }.locationConstraint?.value == region + } ?: true } if (testBucket == null) { @@ -262,10 +261,8 @@ object S3TestUtils { ): Boolean { println("Checking if multi region access point was created: $multiRegionAccessPointName") - val search = s3Control.listMultiRegionAccessPoints { + return s3Control.listMultiRegionAccessPoints { accountId = testAccountId - }.accessPoints?.find { it.name == multiRegionAccessPointName } - - return search != null + }.accessPoints?.any { it.name == multiRegionAccessPointName } ?: false } }