feat!: Streaming + Event Streams (#946)

* refactor: streams (#889)

smithy-lang/smithy-swift#530

* feat: event stream runtime support (#910)

To support event streaming, SDK needs implementations for MessageSigner, MessageEncoder, MessageDecoder, MessageDecoderStream and MessageEncoderStream.

- Add AWSMessageEncoder which is a MessageEncoder implementation along with its tests
- Add AWSMessageDecoder which is a MessageDecoder implementation along with its tests
- Add AWSMessageDecoderStream which is a MessageDecoderStream implementation along with its tests
- Add AWSMessageEncoderStream which is a MessageEncoderStream implementation along with its tests
- Integrate Payload signer in the SDK which is used in MessageEncoderStream implementation

All tests pass.

* feat: codegen and integration (#925)

* wip

* revert to explict header types

* comments and fixes for Int/Long

* fix test cases

* fix test case input

* mark

* Update Sources/Core/AWSClientRuntime/EventStream/AWSEventStream.swift

Co-authored-by: Ed Paulosky <[email protected]>

* docs

* lint

* remove messageDecoder from protocol instead use as local variable

* rename method

* docs

* Update Sources/Core/AWSClientRuntime/HttpContextBuilder+Extension.swift

Co-authored-by: Ed Paulosky <[email protected]>

* use same pattern

* update other places too

* remove local deps check

* update branch

* ALPN change

* remove env var

* remove ALPN from setup

---------

Co-authored-by: Ganesh Jangir <[email protected]>
Co-authored-by: Ed Paulosky <[email protected]>

* chore: regen models

* feat: Forces http2 for bidirectional event streams (#950)

* Fixes tests

* Fixes codegen test

* Some tweaks to tests

* Fixes encoder stream tests some times failing

* fixes linter warnings

* regens models

* Post rebase fixes

* Adds command to CLI to generate package.swift with integration tests (#965)

---------

Co-authored-by: Ganesh Jangir <[email protected]>
Co-authored-by: Ganesh Jangir <[email protected]>

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Commands/GeneratePackageManifest.swift

@@ -31,14 +31,18 @@ struct GeneratePackageManifestCommand: ParsableCommand {
 
     @Option(help: "The names of the services to include in the package manifest. This defaults to all services located in aws-sdk-swift/Sources/Services")
     var services: [String] = []
+
+    @Flag(help: "If the package manifest should include the integration tests.")
+    var includesIntegrationTests: Bool = false
 
     func run() throws {
         let generatePackageManifest = GeneratePackageManifest.standard(
             repoPath: repoPath,
             packageFileName: packageFileName,
             clientRuntimeVersion: clientRuntimeVersion,
             crtVersion: crtVersion,
-            services: services.isEmpty ? nil : services
+            services: services.isEmpty ? nil : services,
+            includesIntegrationTests: includesIntegrationTests
         )
         try generatePackageManifest.run()
     }
@@ -61,11 +65,13 @@ struct GeneratePackageManifest {
     /// The list of services to include as products
     /// If `nil` then the list is populated with the names of all items within the `Sources/Services` directory
     let services: [String]?
+    /// If the package manifest should include the integration tests.
+    let includesIntegrationTests: Bool
 
     typealias BuildPackageManifest = (
         _ clientRuntimeVersion: Version,
         _ crtVersion: Version,
-        _ services: [String]
+        _ services: [PackageManifestBuilder.Service]
     ) throws -> String
     /// Returns the contents of the package manifest file given the versions of dependencies and the list of services.
     let buildPackageManifest: BuildPackageManifest
@@ -85,7 +91,13 @@ struct GeneratePackageManifest {
     /// - Returns: The contents of the generated package manifest.
     func generatePackageManifestContents() throws -> String {
         let versions = try resolveVersions()
-        let services = try resolveServices()
+        let servicesWithIntegrationTests = try includesIntegrationTests ? resolveServicesWithIntegrationTests() : []
+        let services = try resolveServices().map {
+            PackageManifestBuilder.Service(
+                name: $0,
+                includeIntegrationTests: servicesWithIntegrationTests.contains($0)
+            )
+        }
         log("Creating package manifest contents...")
         let contents = try buildPackageManifest(versions.clientRuntime, versions.crt, services)
         log("Successfully created package manifest contents")
@@ -170,6 +182,21 @@ struct GeneratePackageManifest {
         log("Resolved list of services: \(resolvedServices.count)")
         return resolvedServices
     }
+
+    /// Returns the list of services to include in the package manifest.
+    /// If an explicit list of services was provided by the command, then this returns the specified services.
+    /// Otherwise, this returns the list of services that exist within `Sources/Services`
+    ///
+    /// - Returns: The list of services to include in the package manifest
+    func resolveServicesWithIntegrationTests() throws -> [String] {
+        log("Resolving services with integration tests...")
+        let resolvedServices = try FileManager
+            .default
+            .integrationTests()
+            .map { $0.replacingOccurrences(of: "IntegrationTests", with: "") }
+        log("List of services with integration tests: \(resolvedServices.count)")
+        return resolvedServices
+    }
 }
 
 // MARK: - Factory
@@ -191,14 +218,16 @@ extension GeneratePackageManifest {
         packageFileName: String,
         clientRuntimeVersion: Version? = nil,
         crtVersion: Version? = nil,
-        services: [String]? = nil
+        services: [String]? = nil,
+        includesIntegrationTests: Bool = false
     ) -> Self {
         GeneratePackageManifest(
             repoPath: repoPath,
             packageFileName: packageFileName,
             clientRuntimeVersion: clientRuntimeVersion,
             crtVersion: crtVersion,
-            services: services
+            services: services,
+            includesIntegrationTests: includesIntegrationTests
         ) { _clientRuntimeVersion, _crtVersion, _services in
             let builder = PackageManifestBuilder(
                 clientRuntimeVersion: _clientRuntimeVersion,

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Models/PackageManifestBuilder.swift

@@ -10,15 +10,20 @@ import PackageDescription
 
 /// Builds the contents of the package manifest file.
 struct PackageManifestBuilder {
+    struct Service {
+        let name: String
+        let includeIntegrationTests: Bool
+    }
+
     let clientRuntimeVersion: Version
     let crtVersion: Version
-    let services: [String]
+    let services: [Service]
     let basePackageContents: () throws -> String
 
     init(
         clientRuntimeVersion: Version,
         crtVersion: Version,
-        services: [String],
+        services: [Service],
         basePackageContents: @escaping () throws -> String
     ) {
         self.clientRuntimeVersion = clientRuntimeVersion
@@ -30,7 +35,7 @@ struct PackageManifestBuilder {
     init(
         clientRuntimeVersion: Version,
         crtVersion: Version,
-        services: [String]
+        services: [Service]
     ) {
         self.init(clientRuntimeVersion: clientRuntimeVersion, crtVersion: crtVersion, services: services) {
             // Returns the contents of the base package manifest stored in the bundle at `Resources/Package.Base.swift`
@@ -75,7 +80,10 @@ struct PackageManifestBuilder {
             buildDependencies(),
             "",
             // Add the generated content that defines the list of services to include
-            buildServiceTargets()
+            buildServiceTargets(),
+            "",
+            // Add the generated content that defines the list of services with integration tests to include
+            buildIntegrationTestsTargets()
         ]
         return contents.joined(separator: .newline)
     }
@@ -125,11 +133,38 @@ struct PackageManifestBuilder {
 
         var lines: [String] = []
         lines += ["let \(propertyName): [String] = ["]
-        lines += services.map { "    \($0.wrappedInQuotes())," }
+        lines += services.map { "    \($0.name.wrappedInQuotes())," }
         lines += ["]"]
         lines += [""]
         lines += ["\(propertyName).forEach(addServiceTarget)"]
 
         return lines.joined(separator: .newline)
     }
+
+    /// Builds the list of services to add integration test targets for..
+    /// This generates an array of strings, where the each item is a name of a service
+    /// and calls the `addIntegrationTestTarget` for each item.
+    ///
+    ///```
+    ///let servicesWithIntegrationTests: [String] = [
+    ///    "Service Name 1",
+    ///    "Service Name 2",
+    ///    "Service Name 3"
+    ///    // etc...
+    ///]
+    ///serviceTagets.forEach(addIntegrationTestTarget)
+    ///```
+    ///
+    private func buildIntegrationTestsTargets() -> String {
+        let propertyName = "servicesWithIntegrationTests"
+
+        var lines: [String] = []
+        lines += ["let \(propertyName): [String] = ["]
+        lines += services.filter(\.includeIntegrationTests).map { "    \($0.name.wrappedInQuotes())," }
+        lines += ["]"]
+        lines += [""]
+        lines += ["\(propertyName).forEach(addIntegrationTestTarget)"]
+
+        return lines.joined(separator: .newline)
+    }
 }

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Resources/Package.Base.swift

@@ -103,3 +103,15 @@ func addServiceTarget(_ name: String) {
         )
     ]
 }
+
+func addIntegrationTestTarget(_ name: String) {
+    let integrationTestName = "\(name)IntegrationTests"
+    package.targets += [
+        .testTarget(
+            name: integrationTestName,
+            dependencies: [.crt, .clientRuntime, .awsClientRuntime, .byName(name: name), .smithyTestUtils],
+            path: "./IntegrationTests/Services/\(integrationTestName)",
+            resources: [.process("Resources")]
+        )
+    ]
+}

AWSSDKSwiftCLI/Sources/AWSSDKSwiftCLI/Utils/FileManager+Utils.swift

@@ -35,4 +35,14 @@ extension FileManager {
             .sorted()
             .filter { !$0.hasPrefix(".") }
     }
+
+    /// Returns the list of integration tests.
+    ///
+    /// - Returns: The list of integration tests.
+    func integrationTests() throws -> [String] {
+        try FileManager.default
+            .contentsOfDirectory(atPath: "IntegrationTests/Services")
+            .sorted()
+            .filter { !$0.hasPrefix(".") }
+    }
 }

AWSSDKSwiftCLI/Tests/AWSSDKSwiftCLITests/Commands/GeneratePackageManifestTests.swift

@@ -48,7 +48,7 @@ class GeneratePackageManifestTests: CLITestCase {
         createServiceFolders(services)
 
         let subject = GeneratePackageManifest.mock(buildPackageManifest: { _clientRuntimeVersion, _crtVersion, services in
-            "\(_clientRuntimeVersion)-\(_crtVersion)-\(services.joined(separator: "-"))"
+            "\(_clientRuntimeVersion)-\(_crtVersion)-\(services.map(\.name).joined(separator: "-"))"
         })
         try! subject.run()
         let result = try! String(contentsOfFile: "Package.swift", encoding: .utf8)
@@ -110,6 +110,7 @@ extension GeneratePackageManifest {
         clientRuntimeVersion: Version? = nil,
         crtVersion: Version? = nil,
         services: [String]? = nil,
+        includesIntegrationTests: Bool = false,
         buildPackageManifest: @escaping BuildPackageManifest = { (_,_,_) throws -> String in "" }
     ) -> GeneratePackageManifest {
         GeneratePackageManifest(
@@ -118,6 +119,7 @@ extension GeneratePackageManifest {
             clientRuntimeVersion: clientRuntimeVersion,
             crtVersion: crtVersion,
             services: services,
+            includesIntegrationTests: includesIntegrationTests,
             buildPackageManifest: buildPackageManifest
         )
     }

AWSSDKSwiftCLI/Tests/AWSSDKSwiftCLITests/Commands/PrepareReleaseTests.swift

@@ -104,7 +104,7 @@ class PrepareReleaseTests: CLITestCase {
         ProcessRunner.testRunner = runner
         let subject = PrepareRelease.mock(repoType: .awsSdkSwift)
         try! subject.stageFiles()
-        XCTAssertTrue(command.hasSuffix("git add Package.swift Package.version Sources/Services Tests/Services"))
+        XCTAssertTrue(command.hasSuffix("git add Package.swift Package.version packageDependencies.plist Sources/Services Tests/Services"))
     }
 
     func testStageFilesForSmithySwift() {

AWSSDKSwiftCLI/Tests/AWSSDKSwiftCLITests/Models/PackageManifestBuilderTests.swift

@@ -29,13 +29,23 @@ class PackageManifestBuilderTests: XCTestCase {
     ]
     
     serviceTargets.forEach(addServiceTarget)
+
+    let servicesWithIntegrationTests: [String] = [
+        "A",
+        "B",
+        "C",
+        "D",
+        "E",
+    ]
+
+    servicesWithIntegrationTests.forEach(addIntegrationTestTarget)
     """
 
     func testBuild() {
         let subjeect = PackageManifestBuilder(
             clientRuntimeVersion: .init("1.2.3"),
             crtVersion: .init("4.5.6"),
-            services: ["A","B","C","D","E"],
+            services: ["A","B","C","D","E"].map { PackageManifestBuilder.Service(name: $0, includeIntegrationTests: true) },
             basePackageContents: { "<contents of base package>" }
         )
         let result = try! subjeect.build()

IntegrationTests/Services/AWSS3IntegrationTests/Resources/hello-world

@@ -0,0 +1 @@
+Hello, world!

IntegrationTests/Services/AWSS3IntegrationTests/S3Tests.swift

@@ -0,0 +1,112 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import XCTest
+import AWSS3
+
+final class S3Tests: XCTestCase {
+    var sut: S3Client!
+    let bucketName = "aws-sdk-swift-test-\(UUID().uuidString.split(separator: "-").first!.lowercased())"
+    let objectName = "hello-world"
+    let expected = "Hello, world!"
+
+    override func setUp() async throws {
+        sut = try S3Client(region: "us-west-2")
+        try await createBucket()
+    }
+
+    override func tearDown() async throws {
+        try await emptyBucket()
+        try await deleteBucket()
+    }
+
+    func testGetObject() async throws {
+        try await putObject()
+        let input = GetObjectInput(bucket: bucketName, key: objectName)
+        let output = try await sut.getObject(input: input)
+        XCTAssertNotNil(output)
+        XCTAssertNotNil(output.body)
+
+        switch output.body! {
+        case .data(let data):
+            let actual = String(data: data!, encoding: .utf8)
+            XCTAssertEqual(actual, expected)
+        case .stream(let stream):
+            let actual = String(data: try stream.readToEnd()!, encoding: .utf8)
+            print(actual)
+            XCTAssertEqual(actual, expected)
+        }
+    }
+
+    func testPutObject_givenDataBody() async throws {
+        let input = PutObjectInput(body: .data(expected.data(using: .utf8)), bucket: bucketName, key: objectName)
+        let output = try await sut.putObject(input: input)
+        XCTAssertNotNil(output)
+
+        let actual = try await getObject()
+        XCTAssertEqual(expected, actual)
+    }
+
+    func testPutObject_givenStreamBody() async throws {
+        let audioURL = Bundle.module.url(forResource: "hello-world", withExtension: nil)!
+        let fileHandle = FileHandle(forReadingAtPath: audioURL.relativePath)!
+        let input = PutObjectInput(body: .from(fileHandle: fileHandle), bucket: bucketName, key: objectName)
+        let output = try await sut.putObject(input: input)
+        XCTAssertNotNil(output)
+
+        let actual = try await getObject()
+        XCTAssertEqual(expected, actual)
+    }
+
+    // MARK: Helpers
+
+    private func createBucket() async throws {
+        let input = CreateBucketInput(bucket: bucketName, createBucketConfiguration: S3ClientTypes.CreateBucketConfiguration(locationConstraint: S3ClientTypes.BucketLocationConstraint.usWest2))
+        let output = try await sut.createBucket(input: input)
+        print(output)
+    }
+
+    private func getObject() async throws -> String? {
+        let input = GetObjectInput(bucket: bucketName, key: objectName)
+        let output = try await sut.getObject(input: input)
+        XCTAssertNotNil(output)
+        XCTAssertNotNil(output.body)
+
+        switch output.body! {
+        case .data(let data):
+            return String(data: data!, encoding: .utf8)
+        case .stream(let stream):
+            return String(data: try stream.readToEnd()!, encoding: .utf8)
+        }
+    }
+
+    private func deleteBucket() async throws {
+        let input = DeleteBucketInput(bucket: bucketName)
+        let output = try await sut.deleteBucket(input: input)
+        print(output)
+    }
+
+    private func emptyBucket() async throws {
+        let input = ListObjectsV2Input(bucket: bucketName)
+        let output = try await sut.listObjectsV2(input: input)
+        let objects = output.contents ?? []
+        print("Deleting \(objects.count) objects in \(bucketName)")
+        for object in objects {
+            print("Deleting \(String(describing: object.key)) in \(bucketName)")
+            let deleteInput = DeleteObjectInput(bucket: bucketName, key: object.key)
+            let deleteOutput = try await sut.deleteObject(input: deleteInput)
+            print(deleteOutput)
+        }
+    }
+
+    private func putObject() async throws {
+        let input = PutObjectInput(body: .data(expected.data(using: .utf8)), bucket: bucketName, key: objectName)
+        let output = try await sut.putObject(input: input)
+        print(output)
+    }
+
+}