From 4b97717886e894be3cf66bc27fa7edd5bfa156ca Mon Sep 17 00:00:00 2001 From: Simon Pilkington Date: Fri, 23 Apr 2021 17:17:36 -0700 Subject: [PATCH 1/4] Add initial async/await support for AWSClientProtocol and AWSQueryClientProtocol. --- .travis.yml | 36 ++++-- Package.resolved | 4 +- Package.swift | 10 +- Package@swift-5.1.swift | 8 +- .../AWSClientProtocol.swift | 107 ++++++++++++++++++ .../AWSQueryClientProtocol.swift | 85 ++++++++++++++ 6 files changed, 235 insertions(+), 15 deletions(-) create mode 100644 Sources/_SmokeAWSHttpConcurrency/AWSClientProtocol.swift create mode 100644 Sources/_SmokeAWSHttpConcurrency/AWSQueryClientProtocol.swift diff --git a/.travis.yml b/.travis.yml index b5def4fa..ece2489b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ matrix: dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.3.1-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + env: DOCKER_IMAGE_TAG=swift:5.3.3-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes - os: linux dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.3.1-xenial ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + env: DOCKER_IMAGE_TAG=swift:5.3.3-xenial ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes - os: linux dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.3.1-focal ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + env: DOCKER_IMAGE_TAG=swift:5.3.3-focal ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes - os: linux dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.3.1-amazonlinux2 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no + env: DOCKER_IMAGE_TAG=swift:5.3.3-amazonlinux2 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no - os: linux dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.3.1-centos8 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no + env: DOCKER_IMAGE_TAG=swift:5.3.3-centos8 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no - os: linux dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.2.4-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + env: DOCKER_IMAGE_TAG=swift:5.2.5-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes - os: linux dist: xenial @@ -39,12 +39,26 @@ matrix: services: docker env: DOCKER_IMAGE_TAG=swift:5.1-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + # Verify against nightly of upcoming Swift 5.4 + - os: linux + dist: xenial + sudo: required + services: docker + env: DOCKER_IMAGE_TAG=swiftlang/swift:nightly-5.4-amazonlinux2 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no + + # Verify against nightly of Swift mainline + - os: linux + dist: xenial + sudo: required + services: docker + env: DOCKER_IMAGE_TAG=swiftlang/swift:nightly-amazonlinux2 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no + # Use a docker image that contains the SwiftLint executable the verify the code against the linter. -# - os: linux -# dist: xenial -# sudo: required -# services: docker -# env: DOCKER_IMAGE_TAG=bytesguy/swiftlint:0.39.2 ONLY_RUN_SWIFT_LINT=yes USE_APT_GET=no + - os: linux + dist: xenial + sudo: required + services: docker + env: DOCKER_IMAGE_TAG=bytesguy/swiftlint:0.39.2 ONLY_RUN_SWIFT_LINT=yes USE_APT_GET=no before_install: - docker pull $DOCKER_IMAGE_TAG diff --git a/Package.resolved b/Package.resolved index f52b09d1..b94e2d4b 100644 --- a/Package.resolved +++ b/Package.resolved @@ -15,8 +15,8 @@ "repositoryURL": "https://github.com/amzn/smoke-http.git", "state": { "branch": null, - "revision": "6619fe6f8e56a85d9a7d9481c27afb2dc5688459", - "version": "2.7.0" + "revision": "ab695d0c4593c39ac4494ee318703ac329ca2ae8", + "version": "2.8.0" } }, { diff --git a/Package.swift b/Package.swift index 516850c2..8fb9fd63 100644 --- a/Package.swift +++ b/Package.swift @@ -112,6 +112,9 @@ let package = Package( .library( name: "SmokeAWSHttp", targets: ["SmokeAWSHttp"]), + .library( + name: "_SmokeAWSHttpConcurrency", + targets: ["_SmokeAWSHttpConcurrency"]), .library( name: "SmokeAWSMetrics", targets: ["SmokeAWSMetrics"]), @@ -122,7 +125,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-log", from: "1.0.0"), .package(url: "https://github.com/apple/swift-metrics.git", "1.0.0"..<"3.0.0"), .package(url: "https://github.com/LiveUI/XMLCoding.git", from: "0.4.1"), - .package(url: "https://github.com/amzn/smoke-http.git", from: "2.7.0"), + .package(url: "https://github.com/amzn/smoke-http.git", from: "2.8.0"), .package(url: "https://github.com/apple/swift-crypto.git", from: "1.0.0"), ], targets: [ @@ -271,6 +274,11 @@ let package = Package( .product(name: "HTTPHeadersCoding", package: "smoke-http"), .product(name: "Crypto", package: "swift-crypto"), ]), + .target( + name: "_SmokeAWSHttpConcurrency", dependencies: [ + .target(name: "SmokeAWSHttp"), + .product(name: "_SmokeHTTPClientConcurrency", package: "smoke-http"), + ]), .target( name: "SmokeAWSMetrics", dependencies: [ .product(name: "Logging", package: "swift-log"), diff --git a/Package@swift-5.1.swift b/Package@swift-5.1.swift index 50368b0d..4f7fcb94 100644 --- a/Package@swift-5.1.swift +++ b/Package@swift-5.1.swift @@ -112,6 +112,9 @@ let package = Package( .library( name: "SmokeAWSHttp", targets: ["SmokeAWSHttp"]), + .library( + name: "_SmokeAWSHttpConcurrency", + targets: ["_SmokeAWSHttpConcurrency"]), .library( name: "SmokeAWSMetrics", targets: ["SmokeAWSMetrics"]), @@ -122,7 +125,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-log", from: "1.0.0"), .package(url: "https://github.com/apple/swift-metrics.git", "1.0.0"..<"3.0.0"), .package(url: "https://github.com/LiveUI/XMLCoding.git", from: "0.4.1"), - .package(url: "https://github.com/amzn/smoke-http.git", from: "2.7.0"), + .package(url: "https://github.com/amzn/smoke-http.git", from: "2.8.0"), .package(url: "https://github.com/IBM-Swift/BlueCryptor.git", from: "1.0.0"), ], targets: [ @@ -218,6 +221,9 @@ let package = Package( dependencies: ["Logging", "NIO", "NIOHTTP1", "SmokeAWSCore", "SmokeHTTPClient", "QueryCoding", "HTTPPathCoding", "HTTPHeadersCoding", "Cryptor"]), + .target( + name: "_SmokeAWSHttpConcurrency", + dependencies: ["SmokeAWSHttp", "_SmokeHTTPClientConcurrency"]), .target( name: "SmokeAWSMetrics", dependencies: ["Logging", "Metrics", "CloudWatchClient"]), diff --git a/Sources/_SmokeAWSHttpConcurrency/AWSClientProtocol.swift b/Sources/_SmokeAWSHttpConcurrency/AWSClientProtocol.swift new file mode 100644 index 00000000..3de4e865 --- /dev/null +++ b/Sources/_SmokeAWSHttpConcurrency/AWSClientProtocol.swift @@ -0,0 +1,107 @@ +// +// AWSClientProtocol.swift +// + +#if compiler(>=5.5) && $AsyncAwait + +import SmokeHTTPClient +import SmokeAWSCore +import NIO +import NIOHTTP1 +import NIOTransportServices +import AsyncHTTPClient +import SmokeAWSHttp +import _SmokeHTTPClientConcurrency + +public extension AWSClientProtocol { + func executeWithoutOutput( + httpClient: HTTPOperationsClient, + endpointPath: String = "/", + httpMethod: HTTPMethod = .POST, + requestInput: InputType, + operation: String, + reporting: InvocationReportingType, + signAllHeaders: Bool = false, + errorType: ErrorType.Type) async throws { + let handlerDelegate = AWSClientInvocationDelegate( + credentialsProvider: credentialsProvider, + awsRegion: awsRegion, + service: service, + operation: operation, + target: target, + signAllHeaders: signAllHeaders) + + let invocationContext = HTTPClientInvocationContext(reporting: reporting, + handlerDelegate: handlerDelegate) + + do { + return try await httpClient.executeRetriableWithoutOutput( + endpointPath: endpointPath, + httpMethod: httpMethod, + input: requestInput, + invocationContext: invocationContext, + retryConfiguration: retryConfiguration, + retryOnError: retryOnErrorProvider) + } catch { + let typedError: ErrorType = error.asTypedError() + throw typedError + } + } + + func executeWithOutput( + httpClient: HTTPOperationsClient, + endpointPath: String = "/", + httpMethod: HTTPMethod = .POST, + requestInput: InputType, + operation: String, + reporting: InvocationReportingType, + signAllHeaders: Bool = false, + errorType: ErrorType.Type) async throws -> OutputType { + let handlerDelegate = AWSClientInvocationDelegate( + credentialsProvider: credentialsProvider, + awsRegion: awsRegion, + service: service, + operation: operation, + target: target, + signAllHeaders: signAllHeaders) + + let invocationContext = HTTPClientInvocationContext(reporting: reporting, + handlerDelegate: handlerDelegate) + + do { + return try await httpClient.executeRetriableWithOutput( + endpointPath: endpointPath, + httpMethod: httpMethod, + input: requestInput, + invocationContext: invocationContext, + retryConfiguration: retryConfiguration, + retryOnError: retryOnErrorProvider) + } catch { + let typedError: ErrorType = error.asTypedError() + throw typedError + } + } +} + +public struct AWSClientHelper { + public static func getEventLoop(eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider) -> EventLoopGroup { + switch eventLoopGroupProvider { + case .shared(let group): + return group + case .createNew: + #if canImport(Network) + if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) { + return NIOTSEventLoopGroup() + } else { + return MultiThreadedEventLoopGroup(numberOfThreads: 1) + } + #else + return MultiThreadedEventLoopGroup(numberOfThreads: 1) + #endif + } + } +} + +#endif diff --git a/Sources/_SmokeAWSHttpConcurrency/AWSQueryClientProtocol.swift b/Sources/_SmokeAWSHttpConcurrency/AWSQueryClientProtocol.swift new file mode 100644 index 00000000..79443833 --- /dev/null +++ b/Sources/_SmokeAWSHttpConcurrency/AWSQueryClientProtocol.swift @@ -0,0 +1,85 @@ +// +// AWSClientProtocol.swift +// + +#if compiler(>=5.5) && $AsyncAwait + +import SmokeHTTPClient +import SmokeAWSCore +import NIO +import SmokeAWSHttp +import _SmokeHTTPClientConcurrency + +public extension AWSQueryClientProtocol { + func executeWithoutOutput( + httpClient: HTTPOperationsClient, + wrappedInput: WrappedInputType, + action: String, + reporting: InvocationReportingType, + errorType: ErrorType.Type) async throws { + let handlerDelegate = AWSClientInvocationDelegate( + credentialsProvider: credentialsProvider, + awsRegion: awsRegion, + service: service, + target: target) + + let invocationContext = HTTPClientInvocationContext(reporting: reporting, + handlerDelegate: handlerDelegate) + + let requestInput = QueryWrapperHTTPRequestInput( + wrappedInput: wrappedInput, + action: action, + version: apiVersion) + + do { + return try await httpClient.executeRetriableWithoutOutput( + endpointPath: "/", + httpMethod: .POST, + input: requestInput, + invocationContext: invocationContext, + retryConfiguration: retryConfiguration, + retryOnError: retryOnErrorProvider) + } catch { + let typedError: ErrorType = error.asTypedError() + throw typedError + } + } + + func executeWithOutput( + httpClient: HTTPOperationsClient, + wrappedInput: WrappedInputType, + action: String, + reporting: InvocationReportingType, + errorType: ErrorType.Type) async throws -> OutputType { + let handlerDelegate = AWSClientInvocationDelegate( + credentialsProvider: credentialsProvider, + awsRegion: awsRegion, + service: service, + target: target) + + let invocationContext = HTTPClientInvocationContext(reporting: reporting, + handlerDelegate: handlerDelegate) + + let requestInput = QueryWrapperHTTPRequestInput( + wrappedInput: wrappedInput, + action: action, + version: apiVersion) + + do { + return try await httpClient.executeRetriableWithOutput( + endpointPath: "/", + httpMethod: .POST, + input: requestInput, + invocationContext: invocationContext, + retryConfiguration: retryConfiguration, + retryOnError: retryOnErrorProvider) + } catch { + let typedError: ErrorType = error.asTypedError() + throw typedError + } + } +} + +#endif From 487dd5f0662cbe34503bd86159f5a3fa24c675f4 Mon Sep 17 00:00:00 2001 From: Simon Pilkington Date: Mon, 26 Apr 2021 10:12:03 -0700 Subject: [PATCH 2/4] Fix compilation for Swift 5.1 and re-disable swift-lint. --- .travis.yml | 10 +++++----- Package@swift-5.1.swift | 8 +------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index ece2489b..56e142c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,11 +54,11 @@ matrix: env: DOCKER_IMAGE_TAG=swiftlang/swift:nightly-amazonlinux2 ONLY_RUN_SWIFT_LINT=no USE_APT_GET=no # Use a docker image that contains the SwiftLint executable the verify the code against the linter. - - os: linux - dist: xenial - sudo: required - services: docker - env: DOCKER_IMAGE_TAG=bytesguy/swiftlint:0.39.2 ONLY_RUN_SWIFT_LINT=yes USE_APT_GET=no +# - os: linux +# dist: xenial +# sudo: required +# services: docker +# env: DOCKER_IMAGE_TAG=bytesguy/swiftlint:0.39.2 ONLY_RUN_SWIFT_LINT=yes USE_APT_GET=no before_install: - docker pull $DOCKER_IMAGE_TAG diff --git a/Package@swift-5.1.swift b/Package@swift-5.1.swift index 4f7fcb94..50368b0d 100644 --- a/Package@swift-5.1.swift +++ b/Package@swift-5.1.swift @@ -112,9 +112,6 @@ let package = Package( .library( name: "SmokeAWSHttp", targets: ["SmokeAWSHttp"]), - .library( - name: "_SmokeAWSHttpConcurrency", - targets: ["_SmokeAWSHttpConcurrency"]), .library( name: "SmokeAWSMetrics", targets: ["SmokeAWSMetrics"]), @@ -125,7 +122,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-log", from: "1.0.0"), .package(url: "https://github.com/apple/swift-metrics.git", "1.0.0"..<"3.0.0"), .package(url: "https://github.com/LiveUI/XMLCoding.git", from: "0.4.1"), - .package(url: "https://github.com/amzn/smoke-http.git", from: "2.8.0"), + .package(url: "https://github.com/amzn/smoke-http.git", from: "2.7.0"), .package(url: "https://github.com/IBM-Swift/BlueCryptor.git", from: "1.0.0"), ], targets: [ @@ -221,9 +218,6 @@ let package = Package( dependencies: ["Logging", "NIO", "NIOHTTP1", "SmokeAWSCore", "SmokeHTTPClient", "QueryCoding", "HTTPPathCoding", "HTTPHeadersCoding", "Cryptor"]), - .target( - name: "_SmokeAWSHttpConcurrency", - dependencies: ["SmokeAWSHttp", "_SmokeHTTPClientConcurrency"]), .target( name: "SmokeAWSMetrics", dependencies: ["Logging", "Metrics", "CloudWatchClient"]), From 634fd06fa82ee1fbf9f811240eeddefde408a851 Mon Sep 17 00:00:00 2001 From: Simon Pilkington Date: Mon, 26 Apr 2021 13:37:27 -0700 Subject: [PATCH 3/4] Add details to README detailing how to add metrics configuration. --- README.md | 12 +++++++++++- .../SmokeAWSClientReportingConfiguration.swift | 10 ++++++++++ docs/Support_Policy.md | 9 +++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1818fe23..d3aaa1d7 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Build - Master Branch -Swift 5.1, 5.2 and 5.3 Tested +Swift 5.1, 5.2 and 5.3 Tested Ubuntu 16.04, 18.04 and 20.04 Tested CentOS 8 Tested @@ -136,6 +136,16 @@ import SmokeAWSCredentials guard let credentialsProvider = AwsContainerRotatingCredentials.getCredentials(fromEnvironment: environment) else { return Log.error("Unable to obtain credentials from the container environment.") } + + // for the EC2 clients, only emit the retry count metric + // only report 5XX error counts for DescribeInstances (even if additional operations are added in the future) + // only report 4XX error counts for operations other than DescribeInstances (including if they are added in the future) + let reportingConfiguration = SmokeAWSClientReportingConfiguration( + successCounterMatchingOperations: .none, + failure5XXCounterMatchingRequests: .onlyForOperations([.describeInstances]), + failure4XXCounterMatchingRequests: .exceptForOperations([.describeInstances]), + retryCountRecorderMatchingOperations: .all, + latencyTimerMatchingOperations: .none) self.ec2ClientGenerator = AWSElasticComputeCloudClientGenerator( credentialsProvider: credentialsProvider, diff --git a/Sources/SmokeAWSCore/SmokeAWSClientReportingConfiguration.swift b/Sources/SmokeAWSCore/SmokeAWSClientReportingConfiguration.swift index 6e00788c..32198ca7 100644 --- a/Sources/SmokeAWSCore/SmokeAWSClientReportingConfiguration.swift +++ b/Sources/SmokeAWSCore/SmokeAWSClientReportingConfiguration.swift @@ -19,6 +19,8 @@ import Foundation public struct SmokeAWSClientReportingConfiguration { + // TODO: Remove non-inclusive language + // https://github.com/amzn/smoke-aws/issues/84 public enum MatchingOperations { case all case whitelist(Set) @@ -60,6 +62,14 @@ public struct SmokeAWSClientReportingConfiguration return .init(matchingOperations: .all) } + public static func onlyForOperations(_ operations: Set) -> Self { + return .init(matchingOperations: .whitelist(operations)) + } + + public static func exceptForOperations(_ operations: Set) -> Self { + return .init(matchingOperations: .blacklist(operations)) + } + public func reportSuccessForOperation(_ operation: OperationIdentifer) -> Bool { return isMatchingOperation(operation, matchingOperations: successCounterMatchingOperations) } diff --git a/docs/Support_Policy.md b/docs/Support_Policy.md index 4d302946..e281bfe9 100644 --- a/docs/Support_Policy.md +++ b/docs/Support_Policy.md @@ -1,5 +1,5 @@ --- -date: 2020-12-11 13:00 +date: 2021-04-26 15:00 description: Support Policy for SmokeAWS. tags: support policy --- @@ -25,9 +25,10 @@ Once support has been removed, testing via continuous integration may be removed Following these rules, the support level for different Swift Toolchain versions are- 1. **Swift 5.0 and earlier**: No support. -5. **Swift 5.1**: Supported for SmokeAWS 2.x. Support to be removed after 12th of May, 2021. -6. **Swift 5.2**: Supported for SmokeAWS 2.x. -7. **Swift 5.3**: Supported for SmokeAWS 2.x. +2. **Swift 5.1**: Supported for SmokeAWS 2.x. Support to be removed after 12th of May, 2021. +3. **Swift 5.2**: Supported for SmokeAWS 2.x. Support to be removed after 26th of October, 2021. +4. **Swift 5.3**: Supported for SmokeAWS 2.x. +5. **Swift 5.4**: Supported for SmokeAWS 2.x. # Runtime Support From afa6e5a998ab7d5e4becee4f3c88ad71190fb0cc Mon Sep 17 00:00:00 2001 From: Simon Pilkington Date: Tue, 27 Apr 2021 09:17:34 -0700 Subject: [PATCH 4/4] Make it clearer what is an optional input for the generator. --- .travis.yml | 2 +- README.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56e142c5..15a34f5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: dist: xenial sudo: required services: docker - env: DOCKER_IMAGE_TAG=swift:5.2.5-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes + env: DOCKER_IMAGE_TAG=swift:5.2.4-bionic ONLY_RUN_SWIFT_LINT=no USE_APT_GET=yes - os: linux dist: xenial diff --git a/README.md b/README.md index d3aaa1d7..1a5dd6d2 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ import SmokeAWSCredentials return Log.error("Unable to obtain credentials from the container environment.") } - // for the EC2 clients, only emit the retry count metric + // optional: for the EC2 clients, only emit the retry count metric // only report 5XX error counts for DescribeInstances (even if additional operations are added in the future) // only report 4XX error counts for operations other than DescribeInstances (including if they are added in the future) let reportingConfiguration = SmokeAWSClientReportingConfiguration( @@ -151,10 +151,10 @@ import SmokeAWSCredentials credentialsProvider: credentialsProvider, awsRegion: region, endpointHostName: ec2EndpointHostName, - connectionTimeoutSeconds: connectionTimeoutSeconds, - retryConfiguration: retryConfiguration, - eventLoopProvider: .createNew, - reportingConfiguration: reportingConfiguration) + connectionTimeoutSeconds: connectionTimeoutSeconds, // optional + retryConfiguration: retryConfiguration, // optional + eventLoopProvider: .createNew, // optional + reportingConfiguration: reportingConfiguration) // optional ``` The inputs to this constructor are-