Merge pull request #2344 from aws/feat-request-compression-2

Add Gzip request compression feature

.changelog/a96149e35456411aaca3224016feff3c.json

@@ -0,0 +1,16 @@
+{
+    "id": "a96149e3-5456-411a-aca3-224016feff3c",
+    "type": "feature",
+    "collapse": true,
+    "description": "Support modeled request compression. The only algorithm supported at this time is `gzip`.",
+    "modules": [
+        ".",
+        "config",
+        "internal/protocoltest/awsrestjson",
+        "internal/protocoltest/ec2query",
+        "internal/protocoltest/jsonrpc",
+        "internal/protocoltest/jsonrpc10",
+        "internal/protocoltest/query",
+        "internal/protocoltest/restxml"
+    ]
+}

aws/config.go

@@ -150,6 +150,18 @@ type Config struct {
 	// BaseEndpoint is an intermediary transfer location to a service specific
 	// BaseEndpoint on a service's Options.
 	BaseEndpoint *string
+
+	// DisableRequestCompression toggles if an operation request could be
+	// compressed or not. Will be set to false by default. This variable is sourced from
+	// environment variable AWS_DISABLE_REQUEST_COMPRESSION or the shared config profile attribute
+	// disable_request_compression
+	DisableRequestCompression bool
+
+	// RequestMinCompressSizeBytes sets the inclusive min bytes of a request body that could be
+	// compressed. Will be set to 10240 by default and must be within 0 and 10485760 bytes inclusively.
+	// This variable is sourced from environment variable AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES or
+	// the shared config profile attribute request_min_compression_size_bytes
+	RequestMinCompressSizeBytes int64
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java

@@ -37,6 +37,7 @@
 import software.amazon.smithy.go.codegen.integration.ConfigFieldResolver;
 import software.amazon.smithy.go.codegen.integration.GoIntegration;
 import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
+import software.amazon.smithy.go.codegen.requestcompression.RequestCompression;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.shapes.ServiceShape;
 import software.amazon.smithy.model.traits.HttpBearerAuthTrait;
@@ -76,6 +77,10 @@ public class AddAwsConfigFields implements GoIntegration {
 
     private static final String SDK_APP_ID = "AppID";
 
+    private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression";
+
+    private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes";
+
     private static final List<AwsConfigField> AWS_CONFIG_FIELDS = ListUtils.of(
             AwsConfigField.builder()
                     .name(REGION_CONFIG_NAME)
@@ -214,6 +219,20 @@ public class AddAwsConfigFields implements GoIntegration {
                     .type(getUniversalSymbol("string"))
                     .documentation("The optional application specific identifier appended to the User-Agent header.")
                     .generatedOnClient(false)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(DISABLE_REQUEST_COMPRESSION)
+                    .type(getUniversalSymbol("bool"))
+                    .documentation("Configure whether or not a operation request could be compressed.")
+                    .servicePredicate(RequestCompression::isRequestCompressionService)
+                    .generatedOnClient(false)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES)
+                    .type(getUniversalSymbol("int64"))
+                    .documentation("The inclusive min request body size to be compressed.")
+                    .servicePredicate(RequestCompression::isRequestCompressionService)
+                    .generatedOnClient(false)
                     .build()
     );
 

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsProtocolUtils.java

@@ -107,42 +107,6 @@ static void generateHttpProtocolTests(GenerationContext context) {
 
         // skip request compression tests, not yet implemented in the SDK
         Set<HttpProtocolUnitTestGenerator.SkipTest> inputSkipTests = new TreeSet<>(SetUtils.of(
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.json10#JsonRpc10"))
-                        .operation(ShapeId.from("aws.protocoltests.json10#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_awsJson1_0")
-                        .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_0")
-                        .build(),
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.json#JsonProtocol"))
-                        .operation(ShapeId.from("aws.protocoltests.json#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_awsJson1_1")
-                        .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_1")
-                        .build(),
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.query#AwsQuery"))
-                        .operation(ShapeId.from("aws.protocoltests.query#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_awsQuery")
-                        .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsQuery")
-                        .build(),
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.ec2#AwsEc2"))
-                        .operation(ShapeId.from("aws.protocoltests.ec2#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_ec2Query")
-                        .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_ec2Query")
-                        .build(),
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.restjson#RestJson"))
-                        .operation(ShapeId.from("aws.protocoltests.restjson#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_restJson1")
-                        .addTestName("SDKAppendedGzipAfterProvidedEncoding_restJson1")
-                        .build(),
-                HttpProtocolUnitTestGenerator.SkipTest.builder()
-                        .service(ShapeId.from("aws.protocoltests.restxml#RestXml"))
-                        .operation(ShapeId.from("aws.protocoltests.restxml#PutWithContentEncoding"))
-                        .addTestName("SDKAppliedContentEncoding_restXml")
-                        .addTestName("SDKAppendedGzipAfterProvidedEncoding_restXml")
-                        .build(),
                 HttpProtocolUnitTestGenerator.SkipTest.builder()
                         .service(ShapeId.from("aws.protocoltests.restxml#RestXml"))
                         .operation(ShapeId.from("aws.protocoltests.restxml#HttpPayloadWithUnion"))

config/config.go

@@ -70,10 +70,16 @@ var defaultAWSConfigResolvers = []awsConfigResolver{
 	// httpBearerAuth authentication scheme.
 	resolveBearerAuthToken,
 
-	// Sets the sdk app ID if present in shared config profile
+	// Sets the sdk app ID if present in env var or shared config profile
 	resolveAppID,
 
 	resolveBaseEndpoint,
+
+	// Sets the DisableRequestCompression if present in env var or shared config profile
+	resolveDisableRequestCompression,
+
+	// Sets the RequestMinCompressSizeBytes if present in env var or shared config profile
+	resolveRequestMinCompressSizeBytes,
 }
 
 // A Config represents a generic configuration value or set of values. This type

config/env_config.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+	smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression"
 )
 
 // CredentialsSourceName provides a name of the provider when config is
@@ -75,6 +76,9 @@ const (
 	awsIgnoreConfiguredEndpoints = "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS"
 	awsEndpointURL               = "AWS_ENDPOINT_URL"
 
+	awsDisableRequestCompression      = "AWS_DISABLE_REQUEST_COMPRESSION"
+	awsRequestMinCompressionSizeBytes = "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES"
+
 	awsS3DisableExpressSessionAuthEnv = "AWS_S3_DISABLE_EXPRESS_SESSION_AUTH"
 )
 
@@ -271,6 +275,15 @@ type EnvConfig struct {
 	// corresponding endpoint resolution field.
 	BaseEndpoint string
 
+	// determine if request compression is allowed, default to false
+	// retrieved from env var AWS_DISABLE_REQUEST_COMPRESSION
+	DisableRequestCompression *bool
+
+	// inclusive threshold request body size to trigger compression,
+	// default to 10240 and must be within 0 and 10485760 bytes inclusive
+	// retrieved from env var AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES
+	RequestMinCompressSizeBytes *int64
+
 	// Whether S3Express auth is disabled.
 	//
 	// This will NOT prevent requests from being made to S3Express buckets, it
@@ -319,6 +332,13 @@ func NewEnvConfig() (EnvConfig, error) {
 
 	cfg.AppID = os.Getenv(awsSdkAppID)
 
+	if err := setBoolPtrFromEnvVal(&cfg.DisableRequestCompression, []string{awsDisableRequestCompression}); err != nil {
+		return cfg, err
+	}
+	if err := setInt64PtrFromEnvVal(&cfg.RequestMinCompressSizeBytes, []string{awsRequestMinCompressionSizeBytes}, smithyrequestcompression.MaxRequestMinCompressSizeBytes); err != nil {
+		return cfg, err
+	}
+
 	if err := setEndpointDiscoveryTypeFromEnvVal(&cfg.EnableEndpointDiscovery, []string{awsEnableEndpointDiscoveryEnvVar}); err != nil {
 		return cfg, err
 	}
@@ -383,6 +403,20 @@ func (c EnvConfig) getAppID(context.Context) (string, bool, error) {
 	return c.AppID, len(c.AppID) > 0, nil
 }
 
+func (c EnvConfig) getDisableRequestCompression(context.Context) (bool, bool, error) {
+	if c.DisableRequestCompression == nil {
+		return false, false, nil
+	}
+	return *c.DisableRequestCompression, true, nil
+}
+
+func (c EnvConfig) getRequestMinCompressSizeBytes(context.Context) (int64, bool, error) {
+	if c.RequestMinCompressSizeBytes == nil {
+		return 0, false, nil
+	}
+	return *c.RequestMinCompressSizeBytes, true, nil
+}
+
 // GetRetryMaxAttempts returns the value of AWS_MAX_ATTEMPTS if was specified,
 // and not 0.
 func (c EnvConfig) GetRetryMaxAttempts(ctx context.Context) (int, bool, error) {
@@ -639,6 +673,30 @@ func setBoolPtrFromEnvVal(dst **bool, keys []string) error {
 	return nil
 }
 
+func setInt64PtrFromEnvVal(dst **int64, keys []string, max int64) error {
+	for _, k := range keys {
+		value := os.Getenv(k)
+		if len(value) == 0 {
+			continue
+		}
+
+		v, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			return fmt.Errorf("invalid value for env var, %s=%s, need int64", k, value)
+		} else if v < 0 || v > max {
+			return fmt.Errorf("invalid range for env var min request compression size bytes %q, must be within 0 and 10485760 inclusively", v)
+		}
+		if *dst == nil {
+			*dst = new(int64)
+		}
+
+		**dst = v
+		break
+	}
+
+	return nil
+}
+
 func setEndpointDiscoveryTypeFromEnvVal(dst *aws.EndpointDiscoveryEnableState, keys []string) error {
 	for _, k := range keys {
 		value := os.Getenv(k)

config/env_config_test.go

@@ -456,6 +456,47 @@ func TestNewEnvConfig(t *testing.T) {
 			},
 			WantErr: true,
 		},
+		42: {
+			Env: map[string]string{
+				"AWS_DISABLE_REQUEST_COMPRESSION":        "true",
+				"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "12345",
+			},
+			Config: EnvConfig{
+				DisableRequestCompression:   aws.Bool(true),
+				RequestMinCompressSizeBytes: aws.Int64(12345),
+			},
+		},
+		43: {
+			Env: map[string]string{
+				"AWS_DISABLE_REQUEST_COMPRESSION":        "blabla",
+				"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "12345",
+			},
+			Config: EnvConfig{
+				DisableRequestCompression: aws.Bool(false),
+			},
+			WantErr: true,
+		},
+		44: {
+			Env: map[string]string{
+				"AWS_DISABLE_REQUEST_COMPRESSION":        "true",
+				"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "1.1",
+			},
+			Config: EnvConfig{
+				DisableRequestCompression: aws.Bool(true),
+			},
+			WantErr: true,
+		},
+		// expect err detected due to AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES exceeding max 10485760
+		45: {
+			Env: map[string]string{
+				"AWS_DISABLE_REQUEST_COMPRESSION":        "false",
+				"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "10485761",
+			},
+			Config: EnvConfig{
+				DisableRequestCompression: aws.Bool(false),
+			},
+			WantErr: true,
+		},
 	}
 
 	for i, c := range cases {

config/load_options.go

@@ -207,6 +207,12 @@ type LoadOptions struct {
 	// The sdk app ID retrieved from env var or shared config to be added to request user agent header
 	AppID string
 
+	// Specifies whether an operation request could be compressed
+	DisableRequestCompression *bool
+
+	// The inclusive min bytes of a request body that could be compressed
+	RequestMinCompressSizeBytes *int64
+
 	// Whether S3 Express auth is disabled.
 	S3DisableExpressAuth *bool
 }
@@ -256,6 +262,22 @@ func (o LoadOptions) getAppID(ctx context.Context) (string, bool, error) {
 	return o.AppID, len(o.AppID) > 0, nil
 }
 
+// getDisableRequestCompression returns DisableRequestCompression from config's LoadOptions
+func (o LoadOptions) getDisableRequestCompression(ctx context.Context) (bool, bool, error) {
+	if o.DisableRequestCompression == nil {
+		return false, false, nil
+	}
+	return *o.DisableRequestCompression, true, nil
+}
+
+// getRequestMinCompressSizeBytes returns RequestMinCompressSizeBytes from config's LoadOptions
+func (o LoadOptions) getRequestMinCompressSizeBytes(ctx context.Context) (int64, bool, error) {
+	if o.RequestMinCompressSizeBytes == nil {
+		return 0, false, nil
+	}
+	return *o.RequestMinCompressSizeBytes, true, nil
+}
+
 // WithRegion is a helper function to construct functional options
 // that sets Region on config's LoadOptions. Setting the region to
 // an empty string, will result in the region value being ignored.
@@ -277,6 +299,30 @@ func WithAppID(ID string) LoadOptionsFunc {
 	}
 }
 
+// WithDisableRequestCompression is a helper function to construct functional options
+// that sets DisableRequestCompression on config's LoadOptions.
+func WithDisableRequestCompression(DisableRequestCompression *bool) LoadOptionsFunc {
+	return func(o *LoadOptions) error {
+		if DisableRequestCompression == nil {
+			return nil
+		}
+		o.DisableRequestCompression = DisableRequestCompression
+		return nil
+	}
+}
+
+// WithRequestMinCompressSizeBytes is a helper function to construct functional options
+// that sets RequestMinCompressSizeBytes on config's LoadOptions.
+func WithRequestMinCompressSizeBytes(RequestMinCompressSizeBytes *int64) LoadOptionsFunc {
+	return func(o *LoadOptions) error {
+		if RequestMinCompressSizeBytes == nil {
+			return nil
+		}
+		o.RequestMinCompressSizeBytes = RequestMinCompressSizeBytes
+		return nil
+	}
+}
+
 // getDefaultRegion returns DefaultRegion from config's LoadOptions
 func (o LoadOptions) getDefaultRegion(ctx context.Context) (string, bool, error) {
 	if len(o.DefaultRegion) == 0 {

config/provider.go

@@ -191,6 +191,40 @@ func getAppID(ctx context.Context, configs configs) (value string, found bool, e
 	return
 }
 
+// disableRequestCompressionProvider provides access to the DisableRequestCompression
+type disableRequestCompressionProvider interface {
+	getDisableRequestCompression(context.Context) (bool, bool, error)
+}
+
+func getDisableRequestCompression(ctx context.Context, configs configs) (value bool, found bool, err error) {
+	for _, cfg := range configs {
+		if p, ok := cfg.(disableRequestCompressionProvider); ok {
+			value, found, err = p.getDisableRequestCompression(ctx)
+			if err != nil || found {
+				break
+			}
+		}
+	}
+	return
+}
+
+// requestMinCompressSizeBytesProvider provides access to the MinCompressSizeBytes
+type requestMinCompressSizeBytesProvider interface {
+	getRequestMinCompressSizeBytes(context.Context) (int64, bool, error)
+}
+
+func getRequestMinCompressSizeBytes(ctx context.Context, configs configs) (value int64, found bool, err error) {
+	for _, cfg := range configs {
+		if p, ok := cfg.(requestMinCompressSizeBytesProvider); ok {
+			value, found, err = p.getRequestMinCompressSizeBytes(ctx)
+			if err != nil || found {
+				break
+			}
+		}
+	}
+	return
+}
+
 // ec2IMDSRegionProvider provides access to the ec2 imds region
 // configuration value
 type ec2IMDSRegionProvider interface {