Merge branch 'release-1.33.0'

* release-1.33.0:
  Bumping version to 1.33.0
  Add changelog entries from botocore
  Add changelog and version bump information
  Address feedback
  Run tests with CRT
  Add CRT tests
  Initial implementation of AWS CRT S3 transfers

.changes/1.33.0.json

@@ -0,0 +1,72 @@
+[
+  {
+    "category": "Versioning",
+    "description": "Bump boto3 from 1.29.7 to 1.33.0 to match Botocore versioning scheme.",
+    "type": "enhancement"
+  },
+  {
+    "category": "``s3``",
+    "description": "Boto3 will now opt into using the awscrt on select EC2 instance types for s3 transfers.",
+    "type": "feature"
+  },
+  {
+    "category": "Versioning",
+    "description": "[``botocore``] With the release of Botocore 1.33.0, Boto3 and Botocore will share the same version number.",
+    "type": "feature"
+  },
+  {
+    "category": "``appsync``",
+    "description": "[``botocore``] This update enables introspection of Aurora cluster databases using the RDS Data API",
+    "type": "api-change"
+  },
+  {
+    "category": "``b2bi``",
+    "description": "[``botocore``] This is the initial SDK release for AWS B2B Data Interchange.",
+    "type": "api-change"
+  },
+  {
+    "category": "``backup``",
+    "description": "[``botocore``] AWS Backup now supports restore testing, a new feature that allows customers to automate restore testing and validating their backups. Additionally, this release adds support for EBS Snapshots Archive tier.",
+    "type": "api-change"
+  },
+  {
+    "category": "``controltower``",
+    "description": "[``botocore``] This release adds the following support: 1. The EnableControl API can configure controls that are configurable.  2. The GetEnabledControl API shows the configured parameters on an enabled control. 3. The new UpdateEnabledControl API can change parameters on an enabled control.",
+    "type": "api-change"
+  },
+  {
+    "category": "``efs``",
+    "description": "[``botocore``] Update efs client to latest version",
+    "type": "api-change"
+  },
+  {
+    "category": "``fis``",
+    "description": "[``botocore``] AWS FIS adds support for multi-account experiments & empty target resolution. This release also introduces the CreateTargetAccountConfiguration API that allows experiments across multiple AWS accounts, and the ListExperimentResolvedTargets API to list target details.",
+    "type": "api-change"
+  },
+  {
+    "category": "``glue``",
+    "description": "[``botocore``] add observations support to DQ CodeGen config model + update document for connectiontypes supported by ConnectorData entities",
+    "type": "api-change"
+  },
+  {
+    "category": "``rds``",
+    "description": "[``botocore``] Updates Amazon RDS documentation for support for RDS for Db2.",
+    "type": "api-change"
+  },
+  {
+    "category": "``securityhub``",
+    "description": "[``botocore``] Adds and updates APIs to support central configuration. This feature allows the Security Hub delegated administrator to configure Security Hub for their entire AWS Org across multiple regions from a home Region. With this release, findings also include account name and application metadata.",
+    "type": "api-change"
+  },
+  {
+    "category": "``transcribe``",
+    "description": "[``botocore``] This release adds support for AWS HealthScribe APIs within Amazon Transcribe",
+    "type": "api-change"
+  },
+  {
+    "category": "``endpoint-rules``",
+    "description": "[``botocore``] Update endpoint-rules client to latest version",
+    "type": "api-change"
+  }
+]

.github/workflows/run-crt-test.yml

@@ -0,0 +1,31 @@
+name: Run CRT tests
+
+on:
+  push:
+  pull_request:
+    branches-ignore: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: '${{ matrix.os }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
+      - name: 'Set up Python ${{ matrix.python-version }}'
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
+        with:
+          python-version: '${{ matrix.python-version }}'
+      - name: Install dependencies and CRT
+        run: |
+          python scripts/ci/install --extras crt
+      - name: Run tests
+        run: |
+          python scripts/ci/run-crt-tests --with-cov --with-xdist

CHANGELOG.rst

@@ -2,6 +2,25 @@
 CHANGELOG
 =========
 
+1.33.0
+======
+
+* enhancement:Versioning: Bump boto3 from 1.29.7 to 1.33.0 to match Botocore versioning scheme.
+* feature:``s3``: Boto3 will now opt into using the awscrt on select EC2 instance types for s3 transfers.
+* feature:Versioning: [``botocore``] With the release of Botocore 1.33.0, Boto3 and Botocore will share the same version number.
+* api-change:``appsync``: [``botocore``] This update enables introspection of Aurora cluster databases using the RDS Data API
+* api-change:``b2bi``: [``botocore``] This is the initial SDK release for AWS B2B Data Interchange.
+* api-change:``backup``: [``botocore``] AWS Backup now supports restore testing, a new feature that allows customers to automate restore testing and validating their backups. Additionally, this release adds support for EBS Snapshots Archive tier.
+* api-change:``controltower``: [``botocore``] This release adds the following support: 1. The EnableControl API can configure controls that are configurable.  2. The GetEnabledControl API shows the configured parameters on an enabled control. 3. The new UpdateEnabledControl API can change parameters on an enabled control.
+* api-change:``efs``: [``botocore``] Update efs client to latest version
+* api-change:``fis``: [``botocore``] AWS FIS adds support for multi-account experiments & empty target resolution. This release also introduces the CreateTargetAccountConfiguration API that allows experiments across multiple AWS accounts, and the ListExperimentResolvedTargets API to list target details.
+* api-change:``glue``: [``botocore``] add observations support to DQ CodeGen config model + update document for connectiontypes supported by ConnectorData entities
+* api-change:``rds``: [``botocore``] Updates Amazon RDS documentation for support for RDS for Db2.
+* api-change:``securityhub``: [``botocore``] Adds and updates APIs to support central configuration. This feature allows the Security Hub delegated administrator to configure Security Hub for their entire AWS Org across multiple regions from a home Region. With this release, findings also include account name and application metadata.
+* api-change:``transcribe``: [``botocore``] This release adds support for AWS HealthScribe APIs within Amazon Transcribe
+* api-change:``endpoint-rules``: [``botocore``] Update endpoint-rules client to latest version
+
+
 1.29.7
 ======
 

boto3/__init__.py

@@ -17,7 +17,7 @@
 from boto3.session import Session
 
 __author__ = 'Amazon Web Services'
-__version__ = '1.29.7'
+__version__ = '1.33.0'
 
 
 # The default Boto3 session; autoloaded when needed.

boto3/crt.py

@@ -0,0 +1,167 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# https://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""
+This file contains private functionality for interacting with the AWS
+Common Runtime library (awscrt) in boto3.
+
+All code contained within this file is for internal usage within this
+project and is not intended for external consumption. All interfaces
+contained within are subject to abrupt breaking changes.
+"""
+
+import threading
+
+import botocore.exceptions
+from botocore.session import Session
+from s3transfer.crt import (
+    BotocoreCRTCredentialsWrapper,
+    BotocoreCRTRequestSerializer,
+    CRTTransferManager,
+    acquire_crt_s3_process_lock,
+    create_s3_crt_client,
+)
+
+# Singletons for CRT-backed transfers
+CRT_S3_CLIENT = None
+BOTOCORE_CRT_SERIALIZER = None
+
+CLIENT_CREATION_LOCK = threading.Lock()
+PROCESS_LOCK_NAME = 'boto3'
+
+
+def _create_crt_client(session, config, region_name, cred_provider):
+    """Create a CRT S3 Client for file transfer.
+
+    Instantiating many of these may lead to degraded performance or
+    system resource exhaustion.
+    """
+    create_crt_client_kwargs = {
+        'region': region_name,
+        'use_ssl': True,
+        'crt_credentials_provider': cred_provider,
+    }
+    return create_s3_crt_client(**create_crt_client_kwargs)
+
+
+def _create_crt_request_serializer(session, region_name):
+    return BotocoreCRTRequestSerializer(
+        session, {'region_name': region_name, 'endpoint_url': None}
+    )
+
+
+def _create_crt_s3_client(
+    session, config, region_name, credentials, lock, **kwargs
+):
+    """Create boto3 wrapper class to manage crt lock reference and S3 client."""
+    cred_wrapper = BotocoreCRTCredentialsWrapper(credentials)
+    cred_provider = cred_wrapper.to_crt_credentials_provider()
+    return CRTS3Client(
+        _create_crt_client(session, config, region_name, cred_provider),
+        lock,
+        region_name,
+        cred_wrapper,
+    )
+
+
+def _initialize_crt_transfer_primatives(client, config):
+    lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME)
+    if lock is None:
+        # If we're unable to acquire the lock, we cannot
+        # use the CRT in this process and should default to
+        # the classic s3transfer manager.
+        return None, None
+
+    session = Session()
+    region_name = client.meta.region_name
+    credentials = client._get_credentials()
+
+    serializer = _create_crt_request_serializer(session, region_name)
+    s3_client = _create_crt_s3_client(
+        session, config, region_name, credentials, lock
+    )
+    return serializer, s3_client
+
+
+def get_crt_s3_client(client, config):
+    global CRT_S3_CLIENT
+    global BOTOCORE_CRT_SERIALIZER
+
+    with CLIENT_CREATION_LOCK:
+        if CRT_S3_CLIENT is None:
+            serializer, s3_client = _initialize_crt_transfer_primatives(
+                client, config
+            )
+            BOTOCORE_CRT_SERIALIZER = serializer
+            CRT_S3_CLIENT = s3_client
+
+    return CRT_S3_CLIENT
+
+
+class CRTS3Client:
+    """
+    This wrapper keeps track of our underlying CRT client, the lock used to
+    acquire it and the region we've used to instantiate the client.
+
+    Due to limitations in the existing CRT interfaces, we can only make calls
+    in a single region and does not support redirects. We track the region to
+    ensure we don't use the CRT client when a successful request cannot be made.
+    """
+
+    def __init__(self, crt_client, process_lock, region, cred_provider):
+        self.crt_client = crt_client
+        self.process_lock = process_lock
+        self.region = region
+        self.cred_provider = cred_provider
+
+
+def is_crt_compatible_request(client, crt_s3_client):
+    """
+    Boto3 client must use same signing region and credentials
+    as the CRT_S3_CLIENT singleton. Otherwise fallback to classic.
+    """
+    if crt_s3_client is None:
+        return False
+
+    boto3_creds = client._get_credentials()
+    if boto3_creds is None:
+        return False
+
+    is_same_identity = compare_identity(
+        boto3_creds.get_frozen_credentials(), crt_s3_client.cred_provider
+    )
+    is_same_region = client.meta.region_name == crt_s3_client.region
+    return is_same_region and is_same_identity
+
+
+def compare_identity(boto3_creds, crt_s3_creds):
+    try:
+        crt_creds = crt_s3_creds()
+    except botocore.exceptions.NoCredentialsError:
+        return False
+
+    is_matching_identity = (
+        boto3_creds.access_key == crt_creds.access_key_id
+        and boto3_creds.secret_key == crt_creds.secret_access_key
+        and boto3_creds.token == crt_creds.session_token
+    )
+    return is_matching_identity
+
+
+def create_crt_transfer_manager(client, config):
+    """Create a CRTTransferManager for optimized data transfer."""
+    crt_s3_client = get_crt_s3_client(client, config)
+    if is_crt_compatible_request(client, crt_s3_client):
+        return CRTTransferManager(
+            crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER
+        )
+    return None

boto3/s3/constants.py

@@ -0,0 +1,17 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# https://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+# TransferConfig preferred_transfer_client settings
+CLASSIC_TRANSFER_CLIENT = "classic"
+AUTO_RESOLVE_TRANSFER_CLIENT = "auto"

boto3/s3/inject.py

@@ -10,6 +10,8 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import copy as python_copy
+
 from botocore.exceptions import ClientError
 
 from boto3 import utils
@@ -432,7 +434,11 @@ def copy(
     if config is None:
         config = TransferConfig()
 
-    with create_transfer_manager(self, config) as manager:
+    # copy is not supported in the CRT
+    new_config = python_copy.copy(config)
+    new_config.preferred_transfer_client = "classic"
+
+    with create_transfer_manager(self, new_config) as manager:
         future = manager.copy(
             copy_source=CopySource,
             bucket=Bucket,