misc: e2e tests for s3 mrap (#1193)

services/build.gradle.kts

@@ -88,6 +88,26 @@ subprojects {
                         description = "Run e2e service tests"
                         group = "verification"
 
+                        if (project.name == "s3") {
+                            dependencies {
+                                val services = project.parent?.subprojects
+
+                                if (services?.any { it.name == "s3control" } == true) {
+                                    implementation(project(":services:s3control"))
+                                } else {
+                                    implementation("aws.sdk.kotlin:s3control:+")
+                                }
+
+                                if (services?.any { it.name == "sts" } == true) {
+                                    implementation(project(":services:sts"))
+                                } else {
+                                    implementation("aws.sdk.kotlin:sts:+")
+                                }
+
+                                implementation(libs.smithy.kotlin.aws.signing.crt)
+                            }
+                        }
+
                         // Run the tests with the classpath containing the compile dependencies (including 'main'),
                         // runtime dependencies, and the outputs of this compilation:
                         classpath = compileDependencyFiles + runtimeDependencyFiles + output.allOutputs

services/s3/e2eTest/src/MutliRegionAccessPointTest.kt

@@ -0,0 +1,109 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package aws.sdk.kotlin.e2etest
+
+import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint
+import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents
+import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint
+import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId
+import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn
+import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket
+import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.deleteObject
+import aws.sdk.kotlin.services.s3.putObject
+import aws.sdk.kotlin.services.s3.withConfig
+import aws.sdk.kotlin.services.s3control.S3ControlClient
+import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException
+import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner
+import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.AfterAll
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.TestInstance
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class MutliRegionAccessPointTest {
+    private val s3West = S3Client { region = "us-west-2" }
+    private val s3East = s3West.withConfig { region = "us-east-2" }
+    private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) }
+    private val s3Control = S3ControlClient { region = "us-west-2" }
+
+    private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point"
+    private val objectKey = "test.txt"
+
+    private lateinit var accountId: String
+    private lateinit var multiRegionAccessPointArn: String
+    private lateinit var usWestBucket: String
+    private lateinit var usEastBucket: String
+
+    @BeforeAll
+    private fun setUp(): Unit = runBlocking {
+        accountId = getAccountId()
+        usWestBucket = getTestBucket(s3West, "us-west-2", accountId)
+        usEastBucket = getTestBucket(s3East, "us-east-2", accountId)
+
+        createMultiRegionAccessPoint(
+            s3Control,
+            multiRegionAccessPoint,
+            usWestBucket,
+            usEastBucket,
+            accountId,
+        )
+
+        multiRegionAccessPointArn =
+            getMultiRegionAccessPointArn(
+                s3Control,
+                multiRegionAccessPoint,
+                accountId,
+            )
+    }
+
+    @AfterAll
+    private fun cleanUp(): Unit = runBlocking {
+        if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) {
+            deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId)
+        }
+
+        deleteBucketAndAllContents(s3West, usWestBucket)
+        deleteBucketAndAllContents(s3East, usEastBucket)
+
+        s3West.close()
+        s3East.close()
+        s3SigV4a.close()
+        s3Control.close()
+    }
+
+    @Test
+    fun testMultiRegionAccessPointOperation(): Unit = runBlocking {
+        s3SigV4a.putObject {
+            bucket = multiRegionAccessPointArn
+            key = objectKey
+        }
+
+        s3SigV4a.deleteObject {
+            bucket = multiRegionAccessPointArn
+            key = objectKey
+        }
+    }
+
+    @Test
+    fun testUnsupportedSigningAlgorithm(): Unit = runBlocking {
+        val ex = assertFailsWith<UnsupportedSigningAlgorithmException> {
+            s3West.putObject {
+                bucket = multiRegionAccessPointArn
+                key = objectKey
+            }
+        }
+
+        assertEquals(
+            ex.message,
+            "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533",
+        )
+    }
+}

services/s3/e2eTest/src/S3TestUtils.kt

@@ -5,31 +5,61 @@
 package aws.sdk.kotlin.e2etest
 
 import aws.sdk.kotlin.services.s3.*
+import aws.sdk.kotlin.services.s3.S3Client
 import aws.sdk.kotlin.services.s3.model.*
+import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint
+import aws.sdk.kotlin.services.s3.model.ExpirationStatus
+import aws.sdk.kotlin.services.s3.model.LifecycleRule
+import aws.sdk.kotlin.services.s3.model.LifecycleRuleFilter
 import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated
 import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists
+import aws.sdk.kotlin.services.s3control.*
+import aws.sdk.kotlin.services.s3control.model.*
+import aws.sdk.kotlin.services.sts.StsClient
 import aws.smithy.kotlin.runtime.http.request.HttpRequest
 import kotlinx.coroutines.*
 import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.withTimeout
 import java.io.OutputStreamWriter
 import java.net.URL
 import java.util.*
 import javax.net.ssl.HttpsURLConnection
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.minutes
 import kotlin.time.Duration.Companion.seconds
 
 object S3TestUtils {
 
     const val DEFAULT_REGION = "us-west-2"
 
+    // The E2E test account only has permission to operate on buckets with the prefix
     private const val TEST_BUCKET_PREFIX = "s3-test-bucket-"
 
-    suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX)
+    suspend fun getTestBucket(
+        client: S3Client,
+        region: String? = null,
+        accountId: String? = null,
+    ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId)
 
-    private suspend fun getBucketWithPrefix(client: S3Client, prefix: String): String = withTimeout(60.seconds) {
-        var testBucket = client.listBuckets()
+    private suspend fun getBucketWithPrefix(
+        client: S3Client,
+        prefix: String,
+        region: String?,
+        accountId: String?,
+    ): String = withTimeout(60.seconds) {
+        val buckets = client.listBuckets()
             .buckets
             ?.mapNotNull { it.name }
-            ?.firstOrNull { it.startsWith(prefix) }
+
+        var testBucket = buckets?.firstOrNull { bucketName ->
+            bucketName.startsWith(prefix) &&
+                region?.let {
+                    client.getBucketLocation {
+                        bucket = bucketName
+                        expectedBucketOwner = accountId
+                    }.locationConstraint?.value == region
+                } ?: true
+        }
 
         if (testBucket == null) {
             testBucket = prefix + UUID.randomUUID()
@@ -38,7 +68,7 @@ object S3TestUtils {
             client.createBucket {
                 bucket = testBucket
                 createBucketConfiguration {
-                    locationConstraint = BucketLocationConstraint.fromValue(client.config.region!!)
+                    locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!)
                 }
             }
 
@@ -115,4 +145,124 @@ object S3TestUtils {
 
         return connection.responseCode
     }
+
+    internal suspend fun getAccountId(): String {
+        println("Getting account ID")
+
+        val accountId = StsClient {
+            region = "us-west-2"
+        }.use {
+            it.getCallerIdentity().account
+        }
+
+        return checkNotNull(accountId) { "Unable to get AWS account ID" }
+    }
+
+    internal suspend fun createMultiRegionAccessPoint(
+        s3ControlClient: S3ControlClient,
+        multiRegionAccessPointName: String,
+        regionOneBucket: String,
+        regionTwoBucket: String,
+        testAccountId: String,
+    ) {
+        println("Creating multi region access point: $multiRegionAccessPointName")
+
+        val createRequestToken = s3ControlClient.createMultiRegionAccessPoint {
+            accountId = testAccountId
+            details {
+                name = multiRegionAccessPointName
+                regions = listOf(
+                    Region { bucket = regionOneBucket },
+                    Region { bucket = regionTwoBucket },
+                )
+            }
+        }
+
+        waitUntilMultiRegionAccessPointOperationCompletes(
+            s3ControlClient,
+            checkNotNull(createRequestToken.requestTokenArn) { "Unable to get request token ARN" },
+            10.minutes,
+            testAccountId,
+            "createMultiRegionAccessPoint",
+        )
+    }
+
+    internal suspend fun getMultiRegionAccessPointArn(
+        s3ControlClient: S3ControlClient,
+        multiRegionAccessPointName: String,
+        testAccountId: String,
+    ): String {
+        println("Getting multi region access point arn for: $multiRegionAccessPointName")
+
+        s3ControlClient.getMultiRegionAccessPoint {
+            accountId = testAccountId
+            name = multiRegionAccessPointName
+        }.accessPoint?.alias?.let { alias ->
+            return "arn:aws:s3::$testAccountId:accesspoint/$alias"
+        }
+        throw Exception("Unable to get multi region access point arn")
+    }
+
+    internal suspend fun deleteMultiRegionAccessPoint(
+        s3ControlClient: S3ControlClient,
+        multiRegionAccessPointName: String,
+        testAccountId: String,
+    ) {
+        println("Deleting multi region access point: $multiRegionAccessPointName")
+
+        val deleteRequestToken = s3ControlClient.deleteMultiRegionAccessPoint {
+            accountId = testAccountId
+            details {
+                name = multiRegionAccessPointName
+            }
+        }
+
+        waitUntilMultiRegionAccessPointOperationCompletes(
+            s3ControlClient,
+            checkNotNull(deleteRequestToken.requestTokenArn) { "Unable to get request token ARN" },
+            5.minutes,
+            testAccountId,
+            "deleteMultiRegionAccessPoint",
+        )
+    }
+
+    private suspend fun waitUntilMultiRegionAccessPointOperationCompletes(
+        s3ControlClient: S3ControlClient,
+        request: String,
+        timeoutAfter: Duration,
+        testAccountId: String,
+        operation: String,
+    ) {
+        withTimeout(timeoutAfter) {
+            while (true) {
+                val status = s3ControlClient.describeMultiRegionAccessPointOperation {
+                    accountId = testAccountId
+                    requestTokenArn = request
+                }.asyncOperation?.requestStatus
+
+                println("Waiting on $operation operation. Status: $status ")
+
+                if (status == "SUCCEEDED") {
+                    println("$operation operation succeeded.")
+                    return@withTimeout
+                }
+
+                check(status != "FAILED") { "$operation operation failed" }
+
+                delay(10.seconds) // Avoid constant status checks
+            }
+        }
+    }
+
+    internal suspend fun multiRegionAccessPointWasCreated(
+        s3Control: S3ControlClient,
+        multiRegionAccessPointName: String,
+        testAccountId: String,
+    ): Boolean {
+        println("Checking if multi region access point was created: $multiRegionAccessPointName")
+
+        return s3Control.listMultiRegionAccessPoints {
+            accountId = testAccountId
+        }.accessPoints?.any { it.name == multiRegionAccessPointName } ?: false
+    }
 }