Merge pull request #175 from hnousiainen/htn_s3_dualstack_endpoints

Use S3 dualstack endpoint by default

#175

rohmu/object_storage/config.py

@@ -151,6 +151,7 @@ class S3ObjectStorageConfig(StorageModel):
     connect_timeout: Optional[str] = None
     read_timeout: Optional[str] = None
     aws_session_token: Optional[str] = Field(None, repr=False)
+    use_dualstack_endpoint: Optional[bool] = True
     storage_type: Literal[StorageDriver.s3] = StorageDriver.s3
 
     @root_validator(skip_on_failure=True)

rohmu/object_storage/s3.py

@@ -120,6 +120,7 @@ def __init__(
         read_timeout: Optional[float] = None,
         notifier: Optional[Notifier] = None,
         aws_session_token: Optional[str] = None,
+        use_dualstack_endpoint: Optional[bool] = True,
         statsd_info: Optional[StatsdConfig] = None,
     ) -> None:
         super().__init__(prefix=prefix, notifier=notifier, statsd_info=statsd_info)
@@ -137,6 +138,8 @@ def __init__(
             if proxy_info:
                 proxy_url = get_proxy_url(proxy_info)
                 custom_config["proxies"] = {"https": proxy_url}
+            if use_dualstack_endpoint is True:
+                custom_config["use_dualstack_endpoint"] = True
             self.s3_client = create_s3_client(
                 session=session,
                 config=botocore.config.Config(**custom_config),

test/test_factory.py

@@ -42,7 +42,10 @@ def test_get_transfer_s3(
 ) -> None:
     expected_config_arg = dict(config)
     expected_config_arg.pop("notifier")
-    expected_botocore_config = {"proxies": {"https": "socks5://bob:[email protected]:16666"}}
+    expected_botocore_config = {
+        "proxies": {"https": "socks5://bob:[email protected]:16666"},
+        "use_dualstack_endpoint": True,
+    }
     mock_config_model.return_value = S3ObjectStorageConfig(**expected_config_arg)
 
     transfer_object = get_transfer(config)