Skip to content

Commit

Permalink
Use service exceptions to determine skew
Browse files Browse the repository at this point in the history
  • Loading branch information
@lauzadis
lauzadis committed Oct 11, 2023
1 parent 928d58f commit 34c1b3b
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 18 deletions.
44 changes: 36 additions & 8 deletions ...ttp-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/ClockSkewInterceptor.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
*/
package aws.smithy.kotlin.runtime.http.interceptors

import aws.smithy.kotlin.runtime.ErrorMetadata
import aws.smithy.kotlin.runtime.SdkBaseException
import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext
import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext
import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext
import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext
import aws.smithy.kotlin.runtime.http.request.HttpRequest
import aws.smithy.kotlin.runtime.http.response.HttpResponse
Expand All @@ -31,9 +33,30 @@ public class ClockSkewInterceptor : HttpInterceptor {

/**
* Determine whether the client's clock is skewed relative to the server.
* @return true if the service's response represents a definite clock skew error
* OR a *possible* clock skew error AND the skew exists. false otherwise.
* @param responseCodeDescription the server's response code description
* @param serverTime the server's time
*/
internal fun Instant.isSkewed(serverTime: Instant): Boolean = until(serverTime).absoluteValue >= CLOCK_SKEW_THRESHOLD
internal fun Instant.isSkewed(serverTime: Instant, responseCodeDescription: String?): Boolean =
responseCodeDescription?.let {
CLOCK_SKEW_ERROR_CODES.contains(it) || (POSSIBLE_CLOCK_SKEW_ERROR_CODES.contains(it) && until(serverTime).absoluteValue >= CLOCK_SKEW_THRESHOLD)
} ?: false

// Errors definitely caused by clock skew
private val CLOCK_SKEW_ERROR_CODES = listOf(
"RequestTimeTooSkewed",
"RequestExpired",
"RequestInTheFuture",
)

// Errors possibly caused by clock skew
private val POSSIBLE_CLOCK_SKEW_ERROR_CODES = listOf(
"PriorRequestNotComplete",
"RequestTimeout",
"RequestTimeoutException",
"InternalError"
)
}

// Clock skew to be applied to all requests
Expand All @@ -59,31 +82,36 @@ public class ClockSkewInterceptor : HttpInterceptor {
/**
* After receiving a response, check if the client clock is skewed and apply a correction if necessary.
*/
public override suspend fun modifyBeforeDeserialization(context: ProtocolResponseInterceptorContext<Any, HttpRequest, HttpResponse>): HttpResponse {
public override suspend fun modifyBeforeAttemptCompletion(context: ResponseInterceptorContext<Any, Any, HttpRequest, HttpResponse?>): Result<Any> {
val logger = coroutineContext.logger<ClockSkewInterceptor>()

val serverTime = context.protocolResponse.header("Date")?.let {
val serverTime = context.protocolResponse?.header("Date")?.let {
Instant.fromRfc5322(it)
} ?: run {
logger.debug { "service did not return \"Date\" header, skipping skew calculation" }
return context.protocolResponse
return context.response
}

val clientTime = context.protocolRequest.headers["Date"]?.let {
Instant.fromRfc5322(it)
} ?: context.protocolRequest.headers["x-amz-date"]?.let {
Instant.fromIso8601(it)
} ?: context.executionContext.get(HttpOperationContext.ClockSkewApproximateSigningTime)
} ?: context.executionContext[HttpOperationContext.ClockSkewApproximateSigningTime]

if (clientTime.isSkewed(serverTime)) {
if (clientTime.isSkewed(serverTime, context.protocolResponse?.status?.description)) {
val skew = clientTime.until(serverTime)
logger.warn { "client clock ($clientTime) is skewed $skew from the server ($serverTime), applying correction" }
_currentSkew.getAndSet(skew)
context.executionContext[HttpOperationContext.ClockSkew] = skew

// Mark the exception as retryable
val ex = (context.response.exceptionOrNull() as? SdkBaseException)
ex?.sdkErrorMetadata?.attributes?.set(ErrorMetadata.Retryable, true)
ex?.let { return Result.failure(it) }
} else {
logger.trace { "client clock ($clientTime) is not skewed from the server ($serverTime)" }
}

return context.protocolResponse
return context.response
}
}
35 changes: 25 additions & 10 deletions ...lient/common/test/aws/smithy/kotlin/runtime/http/interceptors/ClockSkewInterceptorTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,27 +23,40 @@ import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.seconds

class ClockSkewInterceptorTest {
val SKEWED_RESPONSE_CODE_DESCRIPTION = "RequestTimeTooSkewed"
val POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION = "InternalError"
val NOT_SKEWED_RESPONSE_CODE_DESCRIPTION = "RequestThrottled"

@Test
fun testNotSkewed() {
val clientTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
val serverTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
assertEquals(clientTime, serverTime)
assertFalse(clientTime.isSkewed(serverTime))
assertFalse(clientTime.isSkewed(serverTime, NOT_SKEWED_RESPONSE_CODE_DESCRIPTION))
}

@Test
fun testSkewedByResponseCode() {
// clocks are exactly the same, but service returned skew error
val clientTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
val serverTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
assertTrue(clientTime.isSkewed(serverTime, SKEWED_RESPONSE_CODE_DESCRIPTION))
assertEquals(0.days, clientTime.until(serverTime))
}

@Test
fun testSkewed() {
fun testSkewedByTime() {
val clientTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
val serverTime = Instant.fromRfc5322("Wed, 7 Oct 2023 16:20:50 -0400")
assertTrue(clientTime.isSkewed(serverTime))
assertTrue(clientTime.isSkewed(serverTime, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION))
assertEquals(1.days, clientTime.until(serverTime))
}

@Test
fun testNegativeSkewed() {
fun testNegativeSkewedByTime() {
val clientTime = Instant.fromRfc5322("Wed, 7 Oct 2023 16:20:50 -0400")
val serverTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:20:50 -0400")
assertTrue(clientTime.isSkewed(serverTime))
assertTrue(clientTime.isSkewed(serverTime, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION))
assertEquals(-1.days, clientTime.until(serverTime))
}

Expand All @@ -52,12 +65,12 @@ class ClockSkewInterceptorTest {
val minute = 20
var clientTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:${minute - CLOCK_SKEW_THRESHOLD.inWholeMinutes}:50 -0400")
val serverTime = Instant.fromRfc5322("Wed, 6 Oct 2023 16:$minute:50 -0400")
assertTrue(clientTime.isSkewed(serverTime))
assertTrue(clientTime.isSkewed(serverTime, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION))
assertEquals(CLOCK_SKEW_THRESHOLD, clientTime.until(serverTime))

// shrink the skew by one second, crossing the threshold
clientTime += 1.seconds
assertFalse(clientTime.isSkewed(serverTime))
assertFalse(clientTime.isSkewed(serverTime, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION))
}

@Test
Expand All @@ -73,6 +86,7 @@ class ClockSkewInterceptorTest {
Headers {
append("Date", serverTimeString)
},
HttpStatusCode(403, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION)
)

val req = HttpRequestBuilder().apply {
Expand All @@ -87,7 +101,7 @@ class ClockSkewInterceptorTest {

// Validate the skew got stored in execution context
val expectedSkew = clientTime.until(serverTime)
assertEquals(op.context.getOrNull(HttpOperationContext.ClockSkew), expectedSkew)
assertEquals(expectedSkew, op.context.getOrNull(HttpOperationContext.ClockSkew))
}

@Test
Expand All @@ -101,6 +115,7 @@ class ClockSkewInterceptorTest {
Headers {
append("Date", serverTimeString)
},
HttpStatusCode(403, POSSIBLE_SKEWED_RESPONSE_CODE_DESCRIPTION)
)

val req = HttpRequestBuilder().apply {
Expand All @@ -117,14 +132,14 @@ class ClockSkewInterceptorTest {
assertNull(op.context.getOrNull(HttpOperationContext.ClockSkew))
}

private fun getMockClient(response: ByteArray, responseHeaders: Headers = Headers.Empty): SdkHttpClient {
private fun getMockClient(response: ByteArray, responseHeaders: Headers = Headers.Empty, httpStatusCode: HttpStatusCode = HttpStatusCode.OK): SdkHttpClient {
val mockEngine = TestEngine { _, request ->
val body = object : HttpBody.SourceContent() {
override val contentLength: Long = response.size.toLong()
override fun readFrom(): SdkSource = response.source()
override val isOneShot: Boolean get() = false
}
val resp = HttpResponse(HttpStatusCode.OK, responseHeaders, body)
val resp = HttpResponse(httpStatusCode, responseHeaders, body)
HttpCall(request, resp, Instant.now(), Instant.now())
}
return SdkHttpClient(mockEngine)
Expand Down

0 comments on commit 34c1b3b

Please sign in to comment.