chore: update epic branch with new changes in main (#617)

* chore: Require Swift 5.7, fix deprecation warnings (#600)

* feat: support initial-response in RPC based event streams (#597)

* chore: Updates version to 0.32.0

* chore: Add newline to README.md (#602)

* feat: add limited support in smithy-swift for visionOS (#606)

* feat: add support for requiresLength trait and Transfer-Encoding: Chunked (#604)

* chore: Update to aws-crt-swift 0.15.0 (#607)

* fix: content-length middleware should not error on event streams (#608)

* chore: Updates version to 0.33.0

* chore: Improved downstream task (#568)

* chore: Convert idempotency token middleware from closure to reusable type (#610)

* fix: Update aws-crt-swift dependency to 0.17.0 (#612)

* chore: Updates version to 0.34.0

* fix: Endpoint url should be nil if host or scheme is missing (#614)

* fix: Pool HTTP connections based on scheme, host, and port (#615)

* add default log level to initialize method (#616)

* feat: add utility method for converting SdkHttpRequest to URLRequest. (#613)

* Add extension constructor to URLRequest to convert SDKHttpRequest

* Add preprocessor conditional import functionality to SwiftWriter.

---------

Co-authored-by: Sichan Yoo <[email protected]>

* chore: Updates version to 0.35.0

* Update test cases added from main to reflect sra identity & auth fields in middleware context & sra identity & auth middlewares in operation stack.

---------

Co-authored-by: Josh Elkins <[email protected]>
Co-authored-by: David Yaffe <[email protected]>
Co-authored-by: AWS SDK Swift Automation <[email protected]>
Co-authored-by: Cyprien Ricque <[email protected]>
Co-authored-by: Sichan Yoo <[email protected]>

.github/workflows/continuous-integration.yml

@@ -7,43 +7,109 @@ on:
   workflow_dispatch:
 
 env:
-  BUILDER_VERSION: v0.9.11
-  BUILDER_SOURCE: releases
-  # host owned by CRT team to host aws-crt-builder releases. Contact their on-call with any issues
-  BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
-  PACKAGE_NAME: smithy-swift
-  LINUX_BASE_IMAGE: ubuntu-16-x64
-  RUN: ${{ github.run_id }}-${{ github.run_number }}
-  AWS_SDK_SWIFT_CI_DIR: /Users/runner/work/smithy-swift/smithy-swift/target/build/deps/aws-sdk-swift
-  AWS_CRT_SWIFT_CI_DIR: /Users/runner/work/smithy-swift/smithy-swift/target/build/deps/aws-crt-swift
-  SMITHY_SWIFT_CI_DIR: /Users/runner/work/smithy-swift/smithy-swift
+  AWS_SWIFT_SDK_USE_LOCAL_DEPS: 1
 
 jobs:
   downstream:
-    runs-on: macos-13
+    runs-on: ${{ matrix.runner }}
     env:
-      DEVELOPER_DIR: /Applications/Xcode_14.3.app/Contents/Developer
+      DEVELOPER_DIR: /Applications/${{ matrix.xcode }}.app/Contents/Developer
+    strategy:
+      fail-fast: false
+      matrix:
+        # This matrix runs tests on iOS sim & Mac, on oldest & newest supported Xcodes
+        runner:
+          - macos-12
+          - macos-13
+        xcode:
+          - Xcode_14.0.1
+          - Xcode_15.0
+        destination:
+          - 'platform=iOS Simulator,OS=16.0,name=iPhone 13'
+          - 'platform=iOS Simulator,OS=17.0,name=iPhone 15'
+          - 'platform=OS X'
+        exclude:
+          # Don't run old macOS with new Xcode
+          - runner: macos-12
+            xcode: Xcode_15.0
+          # Don't run new macOS with old Xcode
+          - runner: macos-13
+            xcode: Xcode_14.0.1
+          # Don't run old iOS simulator with new Xcode
+          - destination: 'platform=iOS Simulator,OS=16.0,name=iPhone 13'
+            xcode: Xcode_15.0
+          # Don't run new iOS simulator with old Xcode
+          - destination: 'platform=iOS Simulator,OS=17.0,name=iPhone 15'
+            xcode: Xcode_14.0.1
     steps:
-      - name: Checkout sources
+      - name: Checkout smithy-swift
         uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - name: Select aws-sdk-swift branch
+        run: |
+          ORIGINAL_REPO_HEAD_REF="$GITHUB_HEAD_REF" \
+          DEPENDENCY_REPO_URL="https://github.com/awslabs/aws-sdk-swift.git" \
+          ./scripts/ci_steps/select_dependency_branch.sh
+      - name: Checkout aws-sdk-swift
+        uses: actions/checkout@v3
+        with:
+          repository: awslabs/aws-sdk-swift
+          ref: ${{ env.DEPENDENCY_REPO_SHA }}
+          path: aws-sdk-swift
+      - name: Move aws-sdk-swift into place
+        run: mv aws-sdk-swift ..
+      - name: Cache Gradle
+        uses: actions/cache@v3
         with:
           path: |
             ~/.gradle/caches
             ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+          key: 1-${{ runner.os }}-gradle-${{ hashFiles('settings.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }}
+          restore-keys: |
+            1-${{ runner.os }}-gradle-${{ hashFiles('settings.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }}
+            1-${{ runner.os }}-gradle-
+      - name: Cache Swift
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/org.swift.swiftpm
+            ~/.cache/org.swift.swiftpm
+          key: 1-${{ runner.os }}-${{ matrix.xcode }}-spm-${{ hashFiles('Package.swift') }}
           restore-keys: |
-            ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
-            ${{ runner.os }}-gradle-
-      - uses: actions/setup-java@v3
+            1-${{ runner.os }}-${{ matrix.xcode }}-spm-${{ hashFiles('Package.swift') }}
+            1-${{ runner.os }}-${{ matrix.xcode }}-spm-
+      - name: Setup Java
+        uses: actions/setup-java@v3
         with:
           distribution: corretto
           java-version: 17
-      - name: Build and Test ${{ env.PACKAGE_NAME }} Downstream Consumers
+      - name: Tools Versions
+        run: |
+          cd ../aws-sdk-swift
+          ./scripts/ci_steps/log_tool_versions.sh
+      - name: Build & Run smithy-swift Kotlin Unit Tests
+        run: ./gradlew build
+      - name: Build & Run smithy-swift Swift Unit Tests
+        run: |
+          set -o pipefail && \
+          NSUnbufferedIO=YES xcodebuild \
+            -scheme smithy-swift-Package \
+            -destination '${{ matrix.destination }}' \
+            test 2>&1 \
+            | xcpretty
+      - name: Prepare aws-sdk-swift Protocol & Unit Tests
         run: |
-          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
-          chmod a+x builder
-          AWS_CRT_SWIFT_CI_DIR="${{ env.AWS_CRT_SWIFT_CI_DIR }}" AWS_SDK_SWIFT_CI_DIR="${{ env.AWS_SDK_SWIFT_CI_DIR }}" SMITHY_SWIFT_CI_DIR="${{ env.SMITHY_SWIFT_CI_DIR }}" AWS_SWIFT_SDK_USE_LOCAL_DEPS=1 ./builder build -p ${{ env.PACKAGE_NAME }} --spec downstream
+          cd ../aws-sdk-swift
+          ./scripts/ci_steps/prepare_protocol_and_unit_tests.sh
+      - name: Build and Run aws-sdk-swift Protocol & Unit Tests
+        run: |
+          cd ../aws-sdk-swift
+          set -o pipefail && \
+          NSUnbufferedIO=YES xcodebuild \
+            -scheme aws-sdk-swift \
+            -destination '${{ matrix.destination }}' \
+            test 2>&1 \
+            | xcpretty
+
   linux:
     runs-on: ubuntu-latest
     strategy:
@@ -57,14 +123,41 @@ jobs:
     container:
       image: swift:${{ matrix.swift }}
     steps:
+      - name: Checkout Sources
+        uses: actions/checkout@v3
       - name: Install openssl
         run: |
           if [ -x "$(command -v apt)" ]; then
             apt-get update && apt-get install -y libssl-dev
           else
-            yum install -y openssl-devel
+            yum install -y openssl-devel which
           fi
-      - name: Checkout Sources
-        uses: actions/checkout@v3
-      - name: Build and Test
+      - name: Cache Gradle
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: 1-${{ runner.os }}-gradle-${{ hashFiles('settings.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }}
+          restore-keys: |
+            1-${{ runner.os }}-gradle-${{ hashFiles('settings.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }}
+            1-${{ runner.os }}-gradle-
+      - name: Cache Swift
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/org.swift.swiftpm
+            ~/.cache/org.swift.swiftpm
+          key: 1-${{ runner.os }}-${{ matrix.xcode }}-spm-${{ hashFiles('Package.swift') }}
+          restore-keys: |
+            1-${{ runner.os }}-${{ matrix.xcode }}-spm-${{ hashFiles('Package.swift') }}
+            1-${{ runner.os }}-${{ matrix.xcode }}-spm-
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: corretto
+          java-version: 17
+      - name: Build & Run Kotlin Unit Tests
+        run: ./gradlew build
+      - name: Build & Run Swift Unit Tests
         run: swift test

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version:5.5
+// swift-tools-version:5.7
 
 import PackageDescription
 
@@ -13,9 +13,9 @@ let package = Package(
         .library(name: "SmithyTestUtil", targets: ["SmithyTestUtil"])
     ],
     dependencies: [
-        .package(url: "https://github.com/awslabs/aws-crt-swift.git", .exact("0.13.0")),
+        .package(url: "https://github.com/awslabs/aws-crt-swift.git", exact: "0.17.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
-        .package(url: "https://github.com/MaxDesiatov/XMLCoder.git", .exact("0.17.0"))
+        .package(url: "https://github.com/MaxDesiatov/XMLCoder.git", exact: "0.17.0")
     ],
     targets: [
         .target(

Package.version

@@ -1 +1 @@
-0.31.0
+0.35.0

README.md

@@ -28,3 +28,4 @@ This project is licensed under the Apache-2.0 License.
 
 See [CONTRIBUTING](CONTRIBUTING.md) for more information.
 
+

Sources/ClientRuntime/Idempotency/IdempotencyTokenMiddleware.swift

@@ -0,0 +1,33 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+public struct IdempotencyTokenMiddleware<OperationStackInput,
+                                         OperationStackOutput: HttpResponseBinding,
+                                         OperationStackError: HttpResponseErrorBinding>: ClientRuntime.Middleware {
+    public let id: Swift.String = "IdempotencyTokenMiddleware"
+    private let keyPath: WritableKeyPath<OperationStackInput, String?>
+
+    public init(keyPath: WritableKeyPath<OperationStackInput, String?>) {
+        self.keyPath = keyPath
+    }
+
+    public func handle<H>(context: Context,
+                          input: MInput,
+                          next: H) async throws -> MOutput
+    where H: Handler, Self.MInput == H.Input, Self.MOutput == H.Output, Self.Context == H.Context {
+        var copiedInput = input
+        if input[keyPath: keyPath] == nil {
+            let idempotencyTokenGenerator = context.getIdempotencyTokenGenerator()
+            copiedInput[keyPath: keyPath] = idempotencyTokenGenerator.generateToken()
+        }
+        return try await next.handle(context: context, input: copiedInput)
+    }
+
+    public typealias MInput = OperationStackInput
+    public typealias MOutput = OperationOutput<OperationStackOutput>
+    public typealias Context = HttpContext
+}

Sources/ClientRuntime/Logging/SDKLoggingSystem.swift

@@ -15,13 +15,13 @@ public class SDKLoggingSystem {
         factories[label] = logHandlerFactory
     }
 
-    public class func initialize() {
+    public class func initialize(defaultLogLevel: SDKLogLevel = .info) {
         LoggingSystem.bootstrap { label in
             if let factory = factories[label] {
                 return factory.construct(label: label)
             }
             var handler = StreamLogHandler.standardOutput(label: label)
-            handler.logLevel = .info
+            handler.logLevel = defaultLogLevel.toLoggerType()
             return handler
         }
     }

Sources/ClientRuntime/Networking/Endpoint.swift

@@ -63,10 +63,10 @@ extension Endpoint {
     public var url: URL? {
         var components = URLComponents()
         components.scheme = protocolType?.rawValue
-        components.host = host
+        components.host = host.isEmpty ? nil : host // If host is empty, URL is invalid
         components.percentEncodedPath = path
         components.percentEncodedQuery = queryItemString
-        return components.url
+        return (components.host == nil || components.scheme == nil) ? nil : components.url
     }
 
     var queryItemString: String? {

Sources/ClientRuntime/Networking/Http/CRT/CRTClientEngine.swift

@@ -12,12 +12,31 @@ import Darwin
 
 public class CRTClientEngine: HttpClientEngine {
     actor SerialExecutor {
+
+        /// Stores the common properties of requests that should share a HTTP connection, such that requests
+        /// with equal `ConnectionID` values should be pooled together.
+        ///
+        /// Used as a dictionary key for storing CRT connection managers once they have been created.
+        /// When a new request is made, a connection manager is reused if it matches the request's scheme,
+        /// host, and port.
+        private struct ConnectionPoolID: Hashable {
+            private let protocolType: ProtocolType?
+            private let host: String
+            private let port: Int16
+
+            init(endpoint: Endpoint) {
+                self.protocolType = endpoint.protocolType
+                self.host = endpoint.host
+                self.port = endpoint.port
+            }
+        }
+
         private var logger: LogAgent
 
         private let windowSize: Int
         private let maxConnectionsPerEndpoint: Int
-        private var connectionPools: [Endpoint: HTTPClientConnectionManager] = [:]
-        private var http2ConnectionPools: [Endpoint: HTTP2StreamManager] = [:]
+        private var connectionPools: [ConnectionPoolID: HTTPClientConnectionManager] = [:]
+        private var http2ConnectionPools: [ConnectionPoolID: HTTP2StreamManager] = [:]
         private let sharedDefaultIO = SDKDefaultIO.shared
         private let connectTimeoutMs: UInt32?
 
@@ -29,22 +48,22 @@ public class CRTClientEngine: HttpClientEngine {
         }
 
         func getOrCreateConnectionPool(endpoint: Endpoint) throws -> HTTPClientConnectionManager {
-            guard let connectionPool = connectionPools[endpoint] else {
+            let poolID = ConnectionPoolID(endpoint: endpoint)
+            guard let connectionPool = connectionPools[poolID] else {
                 let newConnectionPool = try createConnectionPool(endpoint: endpoint)
-                connectionPools[endpoint] = newConnectionPool // save in dictionary
+                connectionPools[poolID] = newConnectionPool // save in dictionary
                 return newConnectionPool
             }
-
             return connectionPool
         }
 
         func getOrCreateHTTP2ConnectionPool(endpoint: Endpoint) throws -> HTTP2StreamManager {
-            guard let connectionPool = http2ConnectionPools[endpoint] else {
+            let poolID = ConnectionPoolID(endpoint: endpoint)
+            guard let connectionPool = http2ConnectionPools[poolID] else {
                 let newConnectionPool = try createHTTP2ConnectionPool(endpoint: endpoint)
-                http2ConnectionPools[endpoint] = newConnectionPool // save in dictionary
+                http2ConnectionPools[poolID] = newConnectionPool // save in dictionary
                 return newConnectionPool
             }
-
             return connectionPool
         }
 

Sources/ClientRuntime/Networking/Http/Middlewares/ContentLengthMiddleware.swift

@@ -6,7 +6,20 @@ public struct ContentLengthMiddleware<OperationStackOutput: HttpResponseBinding>
 
     private let contentLengthHeaderName = "Content-Length"
 
-    public init() {}
+    private var requiresLength: Bool?
+
+    private var unsignedPayload: Bool?
+
+    /// Creates a new `ContentLengthMiddleware` with the supplied parameters
+    /// - Parameters:
+    ///   - requiresLength: Trait requires the length of a blob stream to be known. 
+    ///     When the request body is not a streaming blob, `nil` should be passed. Defaults to `nil`.
+    ///   - unsignedPayload: Trait signifies that the length of a stream in payload does not need to be known.
+    ///     When the request body is not a streaming blob, `nil` should be passed. Defaults to `nil`.
+    public init(requiresLength: Bool? = nil, unsignedPayload: Bool? = nil) {
+        self.requiresLength = requiresLength
+        self.unsignedPayload = unsignedPayload
+    }
 
     public func handle<H>(context: Context,
                           input: MInput,
@@ -22,8 +35,19 @@ public struct ContentLengthMiddleware<OperationStackOutput: HttpResponseBinding>
         case .stream(let stream):
             if let length = stream.length {
                 input.headers.update(name: "Content-Length", value: String(length))
+            } else if (requiresLength == false && unsignedPayload == true) ||
+                        (requiresLength == nil && unsignedPayload == nil) {
+                // Transfer-Encoding can be sent on all Event Streams where length cannot be determined
+                // or on blob Data Streams where requiresLength is true and unsignedPayload is false
+                // Only for HTTP/1.1 requests, will be removed in all HTTP/2 requests
+                input.headers.update(name: "Transfer-Encoding", value: "Chunked")
             } else {
-                input.headers.update(name: "Transfer-Encoded", value: "Chunked")
+                let operation = context.attributes.get(key: AttributeKey<String>(name: "Operation"))
+                             ?? "Error getting operation name"
+                let errorMessage = (unsignedPayload ?? false) ?
+                    "Missing content-length for operation: \(operation)" :
+                    "Missing content-length for SigV4 signing on operation: \(operation)"
+                throw StreamError.notSupported(errorMessage)
             }
         default:
             input.headers.update(name: "Content-Length", value: "0")