From 791088de7892866adfbf15592b1e2ce1cf32c5de Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Thu, 2 Nov 2023 18:15:42 -0400 Subject: [PATCH 01/14] Add and Merge request compression feature --- .../smithy/go/codegen/SmithyGoDependency.java | 2 + .../RequestCompression.java | 118 ++++++++++++ ...mithy.go.codegen.integration.GoIntegration | 1 + .../requestcompression/request_compression.go | 105 +++++++++++ .../request_compression_test.go | 169 ++++++++++++++++++ 5 files changed, 395 insertions(+) create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java create mode 100644 private/requestcompression/request_compression.go create mode 100644 private/requestcompression/request_compression_test.go diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java index 3df52d363..071c66cc1 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java @@ -53,6 +53,8 @@ public final class SmithyGoDependency { public static final GoDependency SMITHY_HTTP_TRANSPORT = smithy("transport/http", "smithyhttp"); public static final GoDependency SMITHY_MIDDLEWARE = smithy("middleware"); public static final GoDependency SMITHY_PRIVATE_PROTOCOL = smithy("private/protocol", "smithyprivateprotocol"); + public static final GoDependency SMITHY_REQUEST_COMPRESSION = + smithy("private/requestcompression", "smithyrequestcompression"); public static final GoDependency SMITHY_TIME = smithy("time", "smithytime"); public static final GoDependency SMITHY_HTTP_BINDING = smithy("encoding/httpbinding"); public static final GoDependency SMITHY_JSON = smithy("encoding/json", "smithyjson"); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java new file mode 100644 index 000000000..ec72e72bb --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -0,0 +1,118 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.requestcompression; + +import java.util.List; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.SmithyGoDependency; +import software.amazon.smithy.go.codegen.SymbolUtils; +import software.amazon.smithy.go.codegen.integration.ConfigField; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ToShapeId; +import software.amazon.smithy.model.traits.RequestCompressionTrait; +import software.amazon.smithy.utils.ListUtils; + +public final class RequestCompression implements GoIntegration { + private static final String ADD_REQUEST_COMPRESSION = "addRequestCompression"; + + private static final String ADD_REQUEST_COMPRESSION_INTERNAL = "AddRequestCompression"; + + private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression"; + + private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes"; + + @Override + public void writeAdditionalFiles( + GoSettings settings, + Model model, + SymbolProvider symbolProvider, + GoDelegator goDelegator + ) { + ServiceShape service = settings.getService(model); + if (!isRequestCompressionService(model, service)) { + return; + } + + goDelegator.useShapeWriter(service, this::writeMiddlewareHelper); + } + + + public static boolean isRequestCompressionService(Model model, ServiceShape service) { + TopDownIndex topDownIndex = TopDownIndex.of(model); + for (ToShapeId operation : topDownIndex.getContainedOperations(service)) { + OperationShape operationShape = model.expectShape(operation.toShapeId(), OperationShape.class); + if (operationShape.hasTrait(RequestCompressionTrait.class)) { + return true; + } + } + return false; + } + + private void writeMiddlewareHelper(GoWriter writer) { + writer.openBlock("func $L(stack *middleware.Stack, options Options) error {", "}", + ADD_REQUEST_COMPRESSION, () -> { + writer.write( + "return $T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes)", + SymbolUtils.createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, + SmithyGoDependency.SMITHY_REQUEST_COMPRESSION).build() + ); + }); + writer.insertTrailingNewline(); + } + + @Override + public List getClientPlugins() { + return ListUtils.of( + RuntimeClientPlugin.builder() + .operationPredicate((model, service, operation) -> + operation.hasTrait(RequestCompressionTrait.class)) + .registerMiddleware(MiddlewareRegistrar.builder() + .resolvedFunction(SymbolUtils.createValueSymbolBuilder(ADD_REQUEST_COMPRESSION).build()) + .useClientOptions() + .build()) + .build(), + RuntimeClientPlugin.builder() + .servicePredicate(RequestCompression::isRequestCompressionService) + .configFields(ListUtils.of( + ConfigField.builder() + .name(DISABLE_REQUEST_COMPRESSION) + .type(SymbolUtils.createValueSymbolBuilder("bool") + .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) + .build()) + .documentation("Determine if request compression is allowed, default to false") + .build(), + ConfigField.builder() + .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES) + .type(SymbolUtils.createValueSymbolBuilder("int64") + .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) + .build()) + .documentation("Inclusive threshold request body size to trigger compression, " + + "default to 10240 and must be within 0 and 10485760 bytes inclusively") + .build() + )) + .build() + ); + } +} diff --git a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 908c87260..b65413a5f 100644 --- a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -9,3 +9,4 @@ software.amazon.smithy.go.codegen.integration.Paginators software.amazon.smithy.go.codegen.integration.Waiters software.amazon.smithy.go.codegen.integration.ClientLogger software.amazon.smithy.go.codegen.endpoints.EndpointClientPluginsGenerator +software.amazon.smithy.go.codegen.requestcompression.RequestCompression \ No newline at end of file diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go new file mode 100644 index 000000000..a40342b2e --- /dev/null +++ b/private/requestcompression/request_compression.go @@ -0,0 +1,105 @@ +package requestcompression + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/transport/http" + "io" +) + +// AddRequestCompression add requestCompression middleware to op stack +func AddRequestCompression(stack *middleware.Stack, DisableRequestCompression bool, RequestMinCompressSizeBytes int64) error { + return stack.Build.Add(&requestCompression{ + disableRequestCompression: DisableRequestCompression, + requestMinCompressSizeBytes: RequestMinCompressSizeBytes, + }, middleware.After) +} + +type requestCompression struct { + disableRequestCompression bool + requestMinCompressSizeBytes int64 +} + +// ID returns the ID of the middleware +func (m requestCompression) ID() string { + return "RequestCompression" +} + +// HandleBuild gzip compress the request's stream/body if enabled by config fields +func (m requestCompression) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + if m.disableRequestCompression { + return next.HandleBuild(ctx, in) + } + // still need to check requestMinCompressSizeBytes in case it is out of range after service client config + if m.requestMinCompressSizeBytes < 0 || m.requestMinCompressSizeBytes > 10485760 { + return out, metadata, fmt.Errorf("invalid range for min request compression size bytes %d, must be within 0 and 10485760 inclusively", m.requestMinCompressSizeBytes) + } + + req, ok := in.Request.(*http.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + var isCompressed bool + if stream := req.GetStream(); stream != nil { + compressedBytes, err := compress(stream) + if err != nil { + return out, metadata, fmt.Errorf("failed to compress request stream, %v", err) + } + + var newReq *http.Request + if newReq, err = req.SetStream(bytes.NewReader(compressedBytes)); err != nil { + return out, metadata, fmt.Errorf("failed to set request stream, %v", err) + } + *req = *newReq + isCompressed = true + } else if req.ContentLength >= m.requestMinCompressSizeBytes { + compressedBytes, err := compress(req.Body) + if err != nil { + return out, metadata, fmt.Errorf("failed to compress request body, %v", err) + } + + isCompressed = true + req.Body = io.NopCloser(bytes.NewReader(compressedBytes)) + } + + if isCompressed { + // Either append to the header if it already exists, else set it + if len(req.Header["Content-Encoding"]) != 0 { + req.Header["Content-Encoding"][0] += ", gzip" + } else { + req.Header.Set("Content-Encoding", "gzip") + } + } + + return next.HandleBuild(ctx, in) +} + +func compress(input io.Reader) ([]byte, error) { + var b bytes.Buffer + w, err := gzip.NewWriterLevel(&b, gzip.BestCompression) + if err != nil { + return nil, fmt.Errorf("failed to create gzip writer, %v", err) + } + + inBytes, err := io.ReadAll(input) + if err != nil { + return nil, fmt.Errorf("failed read payload to compress, %v", err) + } + + if _, err = w.Write(inBytes); err != nil { + return nil, fmt.Errorf("failed to write payload to be compressed, %v", err) + } + if err = w.Close(); err != nil { + return nil, fmt.Errorf("failed to flush payload being compressed, %v", err) + } + + return b.Bytes(), nil +} diff --git a/private/requestcompression/request_compression_test.go b/private/requestcompression/request_compression_test.go new file mode 100644 index 000000000..474617a8e --- /dev/null +++ b/private/requestcompression/request_compression_test.go @@ -0,0 +1,169 @@ +package requestcompression + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/transport/http" + "io" + "reflect" + "strings" + "testing" +) + +func TestRequestCompression(t *testing.T) { + cases := map[string]struct { + DisableRequestCompression bool + RequestMinCompressSizeBytes int64 + Body io.ReadCloser + ContentLength int64 + Header map[string][]string + Stream io.Reader + ExpectedBody []byte + ExpectedStream []byte + ExpectedHeader map[string][]string + }{ + "GZip request stream": { + Stream: strings.NewReader("Hi, world!"), + ExpectedStream: []byte("Hi, world!"), + ExpectedHeader: map[string][]string{ + "Content-Encoding": {"gzip"}, + }, + }, + "GZip request body": { + RequestMinCompressSizeBytes: 0, + Body: io.NopCloser(strings.NewReader("Hello, world!")), + ContentLength: 13, + ExpectedBody: []byte("Hello, world!"), + ExpectedHeader: map[string][]string{ + "Content-Encoding": {"gzip"}, + }, + }, + "GZip request body with existing encoding header": { + RequestMinCompressSizeBytes: 0, + Body: io.NopCloser(strings.NewReader("Hello, world!")), + ContentLength: 13, + Header: map[string][]string{ + "Content-Encoding": {"custom"}, + }, + ExpectedBody: []byte("Hello, world!"), + ExpectedHeader: map[string][]string{ + "Content-Encoding": {"custom, gzip"}, + }, + }, + "GZip request stream ignoring min compress request size": { + RequestMinCompressSizeBytes: 100, + Stream: strings.NewReader("Hi, world!"), + ExpectedStream: []byte("Hi, world!"), + ExpectedHeader: map[string][]string{ + "Content-Encoding": {"gzip"}, + }, + }, + "Disable GZip request stream": { + DisableRequestCompression: true, + Stream: strings.NewReader("Hi, world!"), + ExpectedStream: []byte("Hi, world!"), + ExpectedHeader: map[string][]string{}, + }, + "Disable GZip request body": { + DisableRequestCompression: true, + RequestMinCompressSizeBytes: 0, + Body: io.NopCloser(strings.NewReader("Hello, world!")), + ContentLength: 13, + ExpectedBody: []byte("Hello, world!"), + ExpectedHeader: map[string][]string{}, + }, + "Disable Gzip request body due to size threshold": { + RequestMinCompressSizeBytes: 14, + Body: io.NopCloser(strings.NewReader("Hello, world!")), + ContentLength: 13, + ExpectedBody: []byte("Hello, world!"), + ExpectedHeader: map[string][]string{}, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var err error + req := http.NewStackRequest().(*http.Request) + req.ContentLength = c.ContentLength + req.Body = c.Body + req, _ = req.SetStream(c.Stream) + if c.Header != nil { + req.Header = c.Header + } + var updatedRequest *http.Request + + m := requestCompression{ + disableRequestCompression: c.DisableRequestCompression, + requestMinCompressSizeBytes: c.RequestMinCompressSizeBytes, + } + _, _, err = m.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, input middleware.BuildInput) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error) { + updatedRequest = input.Request.(*http.Request) + return out, metadata, nil + }), + ) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + if stream := updatedRequest.GetStream(); stream != nil { + if err := testUnzipContent(stream, c.ExpectedStream, c.DisableRequestCompression); err != nil { + t.Errorf("error while checking request stream: %q", err) + } + } + + if body := updatedRequest.Body; body != nil { + if c.RequestMinCompressSizeBytes > c.ContentLength { + bodyBytes, err := io.ReadAll(body) + if err != nil { + t.Errorf("error while reading request body") + } + if e, a := c.ExpectedBody, bodyBytes; !bytes.Equal(e, a) { + t.Errorf("expect body to be %s, got %s", e, a) + } + } else if err := testUnzipContent(body, c.ExpectedBody, c.DisableRequestCompression); err != nil { + t.Errorf("error while checking request body: %q", err) + } + } + + if e, a := c.ExpectedHeader, map[string][]string(updatedRequest.Header); !reflect.DeepEqual(e, a) { + t.Errorf("expect request header to be %q, got %q", e, a) + } + }) + } +} + +func testUnzipContent(content io.Reader, expect []byte, disableRequestCompression bool) error { + if disableRequestCompression { + b, err := io.ReadAll(content) + if err != nil { + return fmt.Errorf("error while reading request") + } + if e, a := expect, b; !bytes.Equal(e, a) { + return fmt.Errorf("expect content to be %s, got %s", e, a) + } + } else { + r, err := gzip.NewReader(content) + if err != nil { + return fmt.Errorf("error while reading request") + } + + var actualBytes bytes.Buffer + _, err = actualBytes.ReadFrom(r) + if err != nil { + return fmt.Errorf("error while unzipping request payload") + } + + if e, a := expect, actualBytes.Bytes(); !bytes.Equal(e, a) { + return fmt.Errorf("expect unzipped content to be %s, got %s", e, a) + } + } + + return nil +} From f65e28ca3b0ab1be480487c7e4f4c105d2bb4d8f Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 7 Nov 2023 22:47:56 -0500 Subject: [PATCH 02/14] Modify and Merge request compression codegen --- .../RequestCompression.java | 50 +++++---- .../requestcompression/request_compression.go | 104 ++++++++++++------ .../request_compression_test.go | 11 +- 3 files changed, 102 insertions(+), 63 deletions(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index ec72e72bb..35b2ff519 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -15,6 +15,8 @@ package software.amazon.smithy.go.codegen.requestcompression; +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; + import java.util.List; import software.amazon.smithy.codegen.core.SymbolProvider; import software.amazon.smithy.go.codegen.GoDelegator; @@ -28,11 +30,11 @@ import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.knowledge.TopDownIndex; -import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; -import software.amazon.smithy.model.shapes.ToShapeId; import software.amazon.smithy.model.traits.RequestCompressionTrait; import software.amazon.smithy.utils.ListUtils; +import software.amazon.smithy.utils.MapUtils; + public final class RequestCompression implements GoIntegration { private static final String ADD_REQUEST_COMPRESSION = "addRequestCompression"; @@ -55,31 +57,34 @@ public void writeAdditionalFiles( return; } - goDelegator.useShapeWriter(service, this::writeMiddlewareHelper); + goDelegator.useShapeWriter(service, writeMiddlewareHelper()); } public static boolean isRequestCompressionService(Model model, ServiceShape service) { - TopDownIndex topDownIndex = TopDownIndex.of(model); - for (ToShapeId operation : topDownIndex.getContainedOperations(service)) { - OperationShape operationShape = model.expectShape(operation.toShapeId(), OperationShape.class); - if (operationShape.hasTrait(RequestCompressionTrait.class)) { - return true; - } - } - return false; + return TopDownIndex.of(model) + .getContainedOperations(service).stream() + .anyMatch(it -> it.hasTrait(RequestCompressionTrait.class)); } - private void writeMiddlewareHelper(GoWriter writer) { - writer.openBlock("func $L(stack *middleware.Stack, options Options) error {", "}", - ADD_REQUEST_COMPRESSION, () -> { - writer.write( - "return $T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes)", - SymbolUtils.createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, - SmithyGoDependency.SMITHY_REQUEST_COMPRESSION).build() - ); - }); - writer.insertTrailingNewline(); + private GoWriter.Writable writeMiddlewareHelper() { + var stackSymbol = SymbolUtils + .createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE) + .build(); + var addInternalSymbol = SymbolUtils + .createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, + SmithyGoDependency.SMITHY_REQUEST_COMPRESSION) + .build(); + return goTemplate(""" + func $add:L(stack $stack:P, options Options) error { + return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes) + } + """, + MapUtils.of( + "add", ADD_REQUEST_COMPRESSION, + "stack", stackSymbol, + "addInternal", addInternalSymbol + )); } @Override @@ -101,7 +106,8 @@ public List getClientPlugins() { .type(SymbolUtils.createValueSymbolBuilder("bool") .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) .build()) - .documentation("Determine if request compression is allowed, default to false") + .documentation( + "Whether to disable automatic request compression for supported operations.") .build(), ConfigField.builder() .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES) diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go index a40342b2e..7fc2613e2 100644 --- a/private/requestcompression/request_compression.go +++ b/private/requestcompression/request_compression.go @@ -1,3 +1,9 @@ +// Package requestcompression implements runtime support for smithy-modeled +// request compression. +// +// This package is designated as private and is intended for use only by the +// smithy client runtime. The exported API therein is not considered stable and +// is subject to breaking changes without notice. package requestcompression import ( @@ -10,17 +16,35 @@ import ( "io" ) +// Algorithm represents the request compression algorithms supported +type Algorithm string + +const maxRequestMinCompressSizeBytes = 10485760 + +// Enumeration values for supported compress Algorithms. +const ( + GZIP Algorithm = "gzip" +) + +type compressFunc func(io.Reader) ([]byte, error) + +var allowedAlgorithms = map[Algorithm]compressFunc{ + GZIP: gzipCompress, +} + // AddRequestCompression add requestCompression middleware to op stack func AddRequestCompression(stack *middleware.Stack, DisableRequestCompression bool, RequestMinCompressSizeBytes int64) error { - return stack.Build.Add(&requestCompression{ + return stack.Serialize.Add(&requestCompression{ disableRequestCompression: DisableRequestCompression, requestMinCompressSizeBytes: RequestMinCompressSizeBytes, + compressAlgorithms: []Algorithm{GZIP}, }, middleware.After) } type requestCompression struct { disableRequestCompression bool requestMinCompressSizeBytes int64 + compressAlgorithms []Algorithm } // ID returns the ID of the middleware @@ -28,17 +52,17 @@ func (m requestCompression) ID() string { return "RequestCompression" } -// HandleBuild gzip compress the request's stream/body if enabled by config fields -func (m requestCompression) HandleBuild( - ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +// HandleSerialize gzip compress the request's stream/body if enabled by config fields +func (m requestCompression) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, ) ( - out middleware.BuildOutput, metadata middleware.Metadata, err error, + out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { if m.disableRequestCompression { - return next.HandleBuild(ctx, in) + return next.HandleSerialize(ctx, in) } // still need to check requestMinCompressSizeBytes in case it is out of range after service client config - if m.requestMinCompressSizeBytes < 0 || m.requestMinCompressSizeBytes > 10485760 { + if m.requestMinCompressSizeBytes < 0 || m.requestMinCompressSizeBytes > maxRequestMinCompressSizeBytes { return out, metadata, fmt.Errorf("invalid range for min request compression size bytes %d, must be within 0 and 10485760 inclusively", m.requestMinCompressSizeBytes) } @@ -47,44 +71,52 @@ func (m requestCompression) HandleBuild( return out, metadata, fmt.Errorf("unknown request type %T", req) } - var isCompressed bool - if stream := req.GetStream(); stream != nil { - compressedBytes, err := compress(stream) - if err != nil { - return out, metadata, fmt.Errorf("failed to compress request stream, %v", err) - } + for _, algorithm := range m.compressAlgorithms { + compressFunc := allowedAlgorithms[algorithm] + if compressFunc != nil { + if stream := req.GetStream(); stream != nil { + compressedBytes, err := compressFunc(stream) + if err != nil { + return out, metadata, fmt.Errorf("failed to compress request stream, %v", err) + } - var newReq *http.Request - if newReq, err = req.SetStream(bytes.NewReader(compressedBytes)); err != nil { - return out, metadata, fmt.Errorf("failed to set request stream, %v", err) - } - *req = *newReq - isCompressed = true - } else if req.ContentLength >= m.requestMinCompressSizeBytes { - compressedBytes, err := compress(req.Body) - if err != nil { - return out, metadata, fmt.Errorf("failed to compress request body, %v", err) - } + var newReq *http.Request + if newReq, err = req.SetStream(bytes.NewReader(compressedBytes)); err != nil { + return out, metadata, fmt.Errorf("failed to set request stream, %v", err) + } + *req = *newReq + } else { + if req.Body == nil { + break + } + body, err := io.ReadAll(req.Body) + if err != nil { + return out, metadata, fmt.Errorf("failed to read request body") + } + if int64(len(body)) < m.requestMinCompressSizeBytes { + req.Body = io.NopCloser(bytes.NewReader(body)) + break + } + compressedBytes, err := compressFunc(bytes.NewReader(body)) + if err != nil { + return out, metadata, fmt.Errorf("failed to compress request body, %v", err) + } - isCompressed = true - req.Body = io.NopCloser(bytes.NewReader(compressedBytes)) - } + req.Body = io.NopCloser(bytes.NewReader(compressedBytes)) + } + + req.Header.Add("Content-Encoding", "gzip") - if isCompressed { - // Either append to the header if it already exists, else set it - if len(req.Header["Content-Encoding"]) != 0 { - req.Header["Content-Encoding"][0] += ", gzip" - } else { - req.Header.Set("Content-Encoding", "gzip") + break } } - return next.HandleBuild(ctx, in) + return next.HandleSerialize(ctx, in) } -func compress(input io.Reader) ([]byte, error) { +func gzipCompress(input io.Reader) ([]byte, error) { var b bytes.Buffer - w, err := gzip.NewWriterLevel(&b, gzip.BestCompression) + w, err := gzip.NewWriterLevel(&b, gzip.DefaultCompression) if err != nil { return nil, fmt.Errorf("failed to create gzip writer, %v", err) } diff --git a/private/requestcompression/request_compression_test.go b/private/requestcompression/request_compression_test.go index 474617a8e..c50b0163c 100644 --- a/private/requestcompression/request_compression_test.go +++ b/private/requestcompression/request_compression_test.go @@ -50,7 +50,7 @@ func TestRequestCompression(t *testing.T) { }, ExpectedBody: []byte("Hello, world!"), ExpectedHeader: map[string][]string{ - "Content-Encoding": {"custom, gzip"}, + "Content-Encoding": {"custom", "gzip"}, }, }, "GZip request stream ignoring min compress request size": { @@ -99,11 +99,12 @@ func TestRequestCompression(t *testing.T) { m := requestCompression{ disableRequestCompression: c.DisableRequestCompression, requestMinCompressSizeBytes: c.RequestMinCompressSizeBytes, + compressAlgorithms: []Algorithm{GZIP}, } - _, _, err = m.HandleBuild(context.Background(), - middleware.BuildInput{Request: req}, - middleware.BuildHandlerFunc(func(ctx context.Context, input middleware.BuildInput) ( - out middleware.BuildOutput, metadata middleware.Metadata, err error) { + _, _, err = m.HandleSerialize(context.Background(), + middleware.SerializeInput{Request: req}, + middleware.SerializeHandlerFunc(func(ctx context.Context, input middleware.SerializeInput) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error) { updatedRequest = input.Request.(*http.Request) return out, metadata, nil }), From f07d7c16a8de3cf4e8b29e292c54969bd77aefc6 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 7 Nov 2023 22:51:33 -0500 Subject: [PATCH 03/14] Add and Merge changelog for last commit --- .changelog/80ed28327bcd4301a264f318efaf8216.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changelog/80ed28327bcd4301a264f318efaf8216.json diff --git a/.changelog/80ed28327bcd4301a264f318efaf8216.json b/.changelog/80ed28327bcd4301a264f318efaf8216.json new file mode 100644 index 000000000..af1fdbe4a --- /dev/null +++ b/.changelog/80ed28327bcd4301a264f318efaf8216.json @@ -0,0 +1,8 @@ +{ + "id": "80ed2832-7bcd-4301-a264-f318efaf8216", + "type": "feature", + "description": "Modify and Merge request compression codegen", + "modules": [ + "." + ] +} \ No newline at end of file From be821cebf39a73f849e24244c4cbef8070c50b7b Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Fri, 17 Nov 2023 10:36:04 -0500 Subject: [PATCH 04/14] Modify logic of request compression middleware --- .../requestcompression/request_compression.go | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go index 7fc2613e2..ee650c2be 100644 --- a/private/requestcompression/request_compression.go +++ b/private/requestcompression/request_compression.go @@ -85,29 +85,8 @@ func (m requestCompression) HandleSerialize( return out, metadata, fmt.Errorf("failed to set request stream, %v", err) } *req = *newReq - } else { - if req.Body == nil { - break - } - body, err := io.ReadAll(req.Body) - if err != nil { - return out, metadata, fmt.Errorf("failed to read request body") - } - if int64(len(body)) < m.requestMinCompressSizeBytes { - req.Body = io.NopCloser(bytes.NewReader(body)) - break - } - compressedBytes, err := compressFunc(bytes.NewReader(body)) - if err != nil { - return out, metadata, fmt.Errorf("failed to compress request body, %v", err) - } - - req.Body = io.NopCloser(bytes.NewReader(compressedBytes)) + req.Header.Add("Content-Encoding", "gzip") } - - req.Header.Add("Content-Encoding", "gzip") - - break } } From 833fea1d4ffd0b6d6b3138b7ecfaf5eb131a8ea3 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 13:49:56 -0500 Subject: [PATCH 05/14] Add request compression algorithm codegen part --- .../80ed28327bcd4301a264f318efaf8216.json | 2 +- .../RequestCompression.java | 16 +++-- .../requestcompression/request_compression.go | 24 ++++--- .../request_compression_test.go | 63 +++---------------- 4 files changed, 35 insertions(+), 70 deletions(-) diff --git a/.changelog/80ed28327bcd4301a264f318efaf8216.json b/.changelog/80ed28327bcd4301a264f318efaf8216.json index af1fdbe4a..74bf98c30 100644 --- a/.changelog/80ed28327bcd4301a264f318efaf8216.json +++ b/.changelog/80ed28327bcd4301a264f318efaf8216.json @@ -1,7 +1,7 @@ { "id": "80ed2832-7bcd-4301-a264-f318efaf8216", "type": "feature", - "description": "Modify and Merge request compression codegen", + "description": "Support modeled request compression.", "modules": [ "." ] diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index 35b2ff519..85b1ef177 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -18,6 +18,7 @@ import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; import java.util.List; +import java.util.Set; import software.amazon.smithy.codegen.core.SymbolProvider; import software.amazon.smithy.go.codegen.GoDelegator; import software.amazon.smithy.go.codegen.GoSettings; @@ -57,7 +58,8 @@ public void writeAdditionalFiles( return; } - goDelegator.useShapeWriter(service, writeMiddlewareHelper()); + Set algorithms = RequestCompressionTrait.SUPPORTED_COMPRESSION_ALGORITHMS; + goDelegator.useShapeWriter(service, writeMiddlewareHelper(algorithms)); } @@ -67,7 +69,7 @@ public static boolean isRequestCompressionService(Model model, ServiceShape serv .anyMatch(it -> it.hasTrait(RequestCompressionTrait.class)); } - private GoWriter.Writable writeMiddlewareHelper() { + private GoWriter.Writable writeMiddlewareHelper(Set algorithms) { var stackSymbol = SymbolUtils .createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE) .build(); @@ -77,13 +79,15 @@ private GoWriter.Writable writeMiddlewareHelper() { .build(); return goTemplate(""" func $add:L(stack $stack:P, options Options) error { - return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes) + return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + $algorithms:L) } """, MapUtils.of( - "add", ADD_REQUEST_COMPRESSION, - "stack", stackSymbol, - "addInternal", addInternalSymbol + "add", ADD_REQUEST_COMPRESSION, + "stack", stackSymbol, + "addInternal", addInternalSymbol, + "algorithms", String.format("\"%s\"", String.join(",", algorithms)) )); } diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go index ee650c2be..ede4ecea6 100644 --- a/private/requestcompression/request_compression.go +++ b/private/requestcompression/request_compression.go @@ -14,37 +14,35 @@ import ( "github.com/aws/smithy-go/middleware" "github.com/aws/smithy-go/transport/http" "io" + "strings" ) -// Algorithm represents the request compression algorithms supported -type Algorithm string - const maxRequestMinCompressSizeBytes = 10485760 // Enumeration values for supported compress Algorithms. const ( - GZIP Algorithm = "gzip" + GZIP = "gzip" ) type compressFunc func(io.Reader) ([]byte, error) -var allowedAlgorithms = map[Algorithm]compressFunc{ +var allowedAlgorithms = map[string]compressFunc{ GZIP: gzipCompress, } // AddRequestCompression add requestCompression middleware to op stack -func AddRequestCompression(stack *middleware.Stack, DisableRequestCompression bool, RequestMinCompressSizeBytes int64) error { +func AddRequestCompression(stack *middleware.Stack, DisableRequestCompression bool, RequestMinCompressSizeBytes int64, algorithms string) error { return stack.Serialize.Add(&requestCompression{ disableRequestCompression: DisableRequestCompression, requestMinCompressSizeBytes: RequestMinCompressSizeBytes, - compressAlgorithms: []Algorithm{GZIP}, + compressAlgorithms: strings.Split(algorithms, ","), }, middleware.After) } type requestCompression struct { disableRequestCompression bool requestMinCompressSizeBytes int64 - compressAlgorithms []Algorithm + compressAlgorithms []string } // ID returns the ID of the middleware @@ -75,6 +73,13 @@ func (m requestCompression) HandleSerialize( compressFunc := allowedAlgorithms[algorithm] if compressFunc != nil { if stream := req.GetStream(); stream != nil { + size, found, err := req.StreamLength() + if err != nil { + return out, metadata, fmt.Errorf("error while finding request stream length, %v", err) + } else if !found || size < m.requestMinCompressSizeBytes { + return next.HandleSerialize(ctx, in) + } + compressedBytes, err := compressFunc(stream) if err != nil { return out, metadata, fmt.Errorf("failed to compress request stream, %v", err) @@ -85,8 +90,9 @@ func (m requestCompression) HandleSerialize( return out, metadata, fmt.Errorf("failed to set request stream, %v", err) } *req = *newReq - req.Header.Add("Content-Encoding", "gzip") + req.Header.Add("Content-Encoding", algorithm) } + break } } diff --git a/private/requestcompression/request_compression_test.go b/private/requestcompression/request_compression_test.go index c50b0163c..7e57c5c3c 100644 --- a/private/requestcompression/request_compression_test.go +++ b/private/requestcompression/request_compression_test.go @@ -17,11 +17,9 @@ func TestRequestCompression(t *testing.T) { cases := map[string]struct { DisableRequestCompression bool RequestMinCompressSizeBytes int64 - Body io.ReadCloser ContentLength int64 Header map[string][]string Stream io.Reader - ExpectedBody []byte ExpectedStream []byte ExpectedHeader map[string][]string }{ @@ -32,34 +30,21 @@ func TestRequestCompression(t *testing.T) { "Content-Encoding": {"gzip"}, }, }, - "GZip request body": { - RequestMinCompressSizeBytes: 0, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - ContentLength: 13, - ExpectedBody: []byte("Hello, world!"), - ExpectedHeader: map[string][]string{ - "Content-Encoding": {"gzip"}, - }, - }, - "GZip request body with existing encoding header": { - RequestMinCompressSizeBytes: 0, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - ContentLength: 13, + "GZip request stream with existing encoding header": { + Stream: strings.NewReader("Hi, world!"), + ExpectedStream: []byte("Hi, world!"), Header: map[string][]string{ "Content-Encoding": {"custom"}, }, - ExpectedBody: []byte("Hello, world!"), ExpectedHeader: map[string][]string{ "Content-Encoding": {"custom", "gzip"}, }, }, - "GZip request stream ignoring min compress request size": { + "GZip request stream smaller than min compress request size": { RequestMinCompressSizeBytes: 100, Stream: strings.NewReader("Hi, world!"), ExpectedStream: []byte("Hi, world!"), - ExpectedHeader: map[string][]string{ - "Content-Encoding": {"gzip"}, - }, + ExpectedHeader: map[string][]string{}, }, "Disable GZip request stream": { DisableRequestCompression: true, @@ -67,21 +52,6 @@ func TestRequestCompression(t *testing.T) { ExpectedStream: []byte("Hi, world!"), ExpectedHeader: map[string][]string{}, }, - "Disable GZip request body": { - DisableRequestCompression: true, - RequestMinCompressSizeBytes: 0, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - ContentLength: 13, - ExpectedBody: []byte("Hello, world!"), - ExpectedHeader: map[string][]string{}, - }, - "Disable Gzip request body due to size threshold": { - RequestMinCompressSizeBytes: 14, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - ContentLength: 13, - ExpectedBody: []byte("Hello, world!"), - ExpectedHeader: map[string][]string{}, - }, } for name, c := range cases { @@ -89,7 +59,6 @@ func TestRequestCompression(t *testing.T) { var err error req := http.NewStackRequest().(*http.Request) req.ContentLength = c.ContentLength - req.Body = c.Body req, _ = req.SetStream(c.Stream) if c.Header != nil { req.Header = c.Header @@ -99,7 +68,7 @@ func TestRequestCompression(t *testing.T) { m := requestCompression{ disableRequestCompression: c.DisableRequestCompression, requestMinCompressSizeBytes: c.RequestMinCompressSizeBytes, - compressAlgorithms: []Algorithm{GZIP}, + compressAlgorithms: []string{GZIP}, } _, _, err = m.HandleSerialize(context.Background(), middleware.SerializeInput{Request: req}, @@ -114,25 +83,11 @@ func TestRequestCompression(t *testing.T) { } if stream := updatedRequest.GetStream(); stream != nil { - if err := testUnzipContent(stream, c.ExpectedStream, c.DisableRequestCompression); err != nil { + if err := testUnzipContent(stream, c.ExpectedStream, c.DisableRequestCompression, c.RequestMinCompressSizeBytes); err != nil { t.Errorf("error while checking request stream: %q", err) } } - if body := updatedRequest.Body; body != nil { - if c.RequestMinCompressSizeBytes > c.ContentLength { - bodyBytes, err := io.ReadAll(body) - if err != nil { - t.Errorf("error while reading request body") - } - if e, a := c.ExpectedBody, bodyBytes; !bytes.Equal(e, a) { - t.Errorf("expect body to be %s, got %s", e, a) - } - } else if err := testUnzipContent(body, c.ExpectedBody, c.DisableRequestCompression); err != nil { - t.Errorf("error while checking request body: %q", err) - } - } - if e, a := c.ExpectedHeader, map[string][]string(updatedRequest.Header); !reflect.DeepEqual(e, a) { t.Errorf("expect request header to be %q, got %q", e, a) } @@ -140,8 +95,8 @@ func TestRequestCompression(t *testing.T) { } } -func testUnzipContent(content io.Reader, expect []byte, disableRequestCompression bool) error { - if disableRequestCompression { +func testUnzipContent(content io.Reader, expect []byte, disableRequestCompression bool, requestMinCompressionSizeBytes int64) error { + if disableRequestCompression || int64(len(expect)) < requestMinCompressionSizeBytes { b, err := io.ReadAll(content) if err != nil { return fmt.Errorf("error while reading request") From e23df69a39f1a03b37a8418f05500ec99173f02f Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 14:04:35 -0500 Subject: [PATCH 06/14] resolve METAINFO conflict --- ...tware.amazon.smithy.go.codegen.integration.GoIntegration | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index b65413a5f..6e2ac6a93 100644 --- a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -9,4 +9,8 @@ software.amazon.smithy.go.codegen.integration.Paginators software.amazon.smithy.go.codegen.integration.Waiters software.amazon.smithy.go.codegen.integration.ClientLogger software.amazon.smithy.go.codegen.endpoints.EndpointClientPluginsGenerator -software.amazon.smithy.go.codegen.requestcompression.RequestCompression \ No newline at end of file +software.amazon.smithy.go.codegen.requestcompression.RequestCompression + +# modeled auth schemes +software.amazon.smithy.go.codegen.integration.auth.SigV4AuthScheme +software.amazon.smithy.go.codegen.integration.auth.AnonymousAuthScheme From fc068fafb66a719dd2e5984926292ec4280a1ed2 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 14:40:39 -0500 Subject: [PATCH 07/14] Change dependency format --- .../software/amazon/smithy/go/codegen/SmithyGoDependency.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java index 0d8c7e602..0c08db5e7 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java @@ -53,8 +53,7 @@ public final class SmithyGoDependency { public static final GoDependency SMITHY_HTTP_TRANSPORT = smithy("transport/http", "smithyhttp"); public static final GoDependency SMITHY_MIDDLEWARE = smithy("middleware"); public static final GoDependency SMITHY_PRIVATE_PROTOCOL = smithy("private/protocol", "smithyprivateprotocol"); - public static final GoDependency SMITHY_REQUEST_COMPRESSION = - smithy("private/requestcompression", "smithyrequestcompression"); + public static final GoDependency SMITHY_REQUEST_COMPRESSION = smithy("private/requestcompression", "smithyrequestcompression"); public static final GoDependency SMITHY_TIME = smithy("time", "smithytime"); public static final GoDependency SMITHY_HTTP_BINDING = smithy("encoding/httpbinding"); public static final GoDependency SMITHY_JSON = smithy("encoding/json", "smithyjson"); From 293356e67698521d32adddd970d1ce26c587785b Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 14:43:06 -0500 Subject: [PATCH 08/14] Revert dependency format --- .../software/amazon/smithy/go/codegen/SmithyGoDependency.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java index 0c08db5e7..0d8c7e602 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java @@ -53,7 +53,8 @@ public final class SmithyGoDependency { public static final GoDependency SMITHY_HTTP_TRANSPORT = smithy("transport/http", "smithyhttp"); public static final GoDependency SMITHY_MIDDLEWARE = smithy("middleware"); public static final GoDependency SMITHY_PRIVATE_PROTOCOL = smithy("private/protocol", "smithyprivateprotocol"); - public static final GoDependency SMITHY_REQUEST_COMPRESSION = smithy("private/requestcompression", "smithyrequestcompression"); + public static final GoDependency SMITHY_REQUEST_COMPRESSION = + smithy("private/requestcompression", "smithyrequestcompression"); public static final GoDependency SMITHY_TIME = smithy("time", "smithytime"); public static final GoDependency SMITHY_HTTP_BINDING = smithy("encoding/httpbinding"); public static final GoDependency SMITHY_JSON = smithy("encoding/json", "smithyjson"); From 129647422d3bdbfafb5b17408d7733128013fb19 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 28 Nov 2023 13:44:44 -0500 Subject: [PATCH 09/14] Change request compression middleware to operation level --- .../RequestCompression.java | 79 ++++++++++++++----- .../requestcompression/request_compression.go | 16 ++-- .../request_compression_test.go | 2 +- 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index 85b1ef177..c3a307edd 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -17,9 +17,10 @@ import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import java.util.ArrayList; import java.util.List; -import java.util.Set; import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoCodegenPlugin; import software.amazon.smithy.go.codegen.GoDelegator; import software.amazon.smithy.go.codegen.GoSettings; import software.amazon.smithy.go.codegen.GoWriter; @@ -31,21 +32,52 @@ import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.traits.RequestCompressionTrait; import software.amazon.smithy.utils.ListUtils; import software.amazon.smithy.utils.MapUtils; public final class RequestCompression implements GoIntegration { - private static final String ADD_REQUEST_COMPRESSION = "addRequestCompression"; - private static final String ADD_REQUEST_COMPRESSION_INTERNAL = "AddRequestCompression"; private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression"; private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes"; + private final List runtimeClientPlugins = new ArrayList<>(); + + private static String getAddRequestCompressionMiddlewareFuncName(String operationName) { + return String.format("addOperation%sRequestCompressionMiddleware", operationName); + } + + // Write operation plugin for request compression middleware + @Override + public void processFinalizedModel(GoSettings settings, Model model) { + ServiceShape service = settings.getService(model); + TopDownIndex.of(model) + .getContainedOperations(service).forEach(operation -> { + if (!operation.hasTrait(RequestCompressionTrait.class)) { + return; + } + SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings); + String funcName = getAddRequestCompressionMiddlewareFuncName( + symbolProvider.toSymbol(operation).getName() + ); + runtimeClientPlugins.add(RuntimeClientPlugin.builder().operationPredicate((m, s, o) -> { + if (!o.hasTrait(RequestCompressionTrait.class)) { + return false; + } + return o.equals(operation); + }).registerMiddleware(MiddlewareRegistrar.builder() + .resolvedFunction(SymbolUtils.createValueSymbolBuilder(funcName).build()) + .useClientOptions().build()) + .build()); + }); + } + @Override public void writeAdditionalFiles( GoSettings settings, @@ -54,12 +86,13 @@ public void writeAdditionalFiles( GoDelegator goDelegator ) { ServiceShape service = settings.getService(model); - if (!isRequestCompressionService(model, service)) { - return; + for (ShapeId operationID : service.getAllOperations()) { + OperationShape operation = model.expectShape(operationID, OperationShape.class); + if (!operation.hasTrait(RequestCompressionTrait.class)) { + continue; + } + goDelegator.useShapeWriter(operation, writeMiddlewareHelper(symbolProvider, operation)); } - - Set algorithms = RequestCompressionTrait.SUPPORTED_COMPRESSION_ALGORITHMS; - goDelegator.useShapeWriter(service, writeMiddlewareHelper(algorithms)); } @@ -69,7 +102,7 @@ public static boolean isRequestCompressionService(Model model, ServiceShape serv .anyMatch(it -> it.hasTrait(RequestCompressionTrait.class)); } - private GoWriter.Writable writeMiddlewareHelper(Set algorithms) { + private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { var stackSymbol = SymbolUtils .createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE) .build(); @@ -77,6 +110,16 @@ private GoWriter.Writable writeMiddlewareHelper(Set algorithms) { .createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, SmithyGoDependency.SMITHY_REQUEST_COMPRESSION) .build(); + String operationName = symbolProvider.toSymbol(operation).getName(); + RequestCompressionTrait trait = operation.expectTrait(RequestCompressionTrait.class); + + // build encoding list symbol + StringBuilder algorithmList = new StringBuilder("[]string{"); + for (String algo : trait.getEncodings()) { + algorithmList.append(String.format("\"%s\", ", algo)); + } + String algorithms = algorithmList.substring(0, algorithmList.length() - 2) + "}"; + return goTemplate(""" func $add:L(stack $stack:P, options Options) error { return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, @@ -84,24 +127,16 @@ private GoWriter.Writable writeMiddlewareHelper(Set algorithms) { } """, MapUtils.of( - "add", ADD_REQUEST_COMPRESSION, + "add", getAddRequestCompressionMiddlewareFuncName(operationName), "stack", stackSymbol, "addInternal", addInternalSymbol, - "algorithms", String.format("\"%s\"", String.join(",", algorithms)) + "algorithms", algorithms )); } @Override public List getClientPlugins() { - return ListUtils.of( - RuntimeClientPlugin.builder() - .operationPredicate((model, service, operation) -> - operation.hasTrait(RequestCompressionTrait.class)) - .registerMiddleware(MiddlewareRegistrar.builder() - .resolvedFunction(SymbolUtils.createValueSymbolBuilder(ADD_REQUEST_COMPRESSION).build()) - .useClientOptions() - .build()) - .build(), + runtimeClientPlugins.add( RuntimeClientPlugin.builder() .servicePredicate(RequestCompression::isRequestCompressionService) .configFields(ListUtils.of( @@ -123,6 +158,8 @@ public List getClientPlugins() { .build() )) .build() - ); + ); + + return runtimeClientPlugins; } } diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go index ede4ecea6..be5d545dd 100644 --- a/private/requestcompression/request_compression.go +++ b/private/requestcompression/request_compression.go @@ -14,7 +14,6 @@ import ( "github.com/aws/smithy-go/middleware" "github.com/aws/smithy-go/transport/http" "io" - "strings" ) const maxRequestMinCompressSizeBytes = 10485760 @@ -31,11 +30,11 @@ var allowedAlgorithms = map[string]compressFunc{ } // AddRequestCompression add requestCompression middleware to op stack -func AddRequestCompression(stack *middleware.Stack, DisableRequestCompression bool, RequestMinCompressSizeBytes int64, algorithms string) error { +func AddRequestCompression(stack *middleware.Stack, disabled bool, minBytes int64, algorithms []string) error { return stack.Serialize.Add(&requestCompression{ - disableRequestCompression: DisableRequestCompression, - requestMinCompressSizeBytes: RequestMinCompressSizeBytes, - compressAlgorithms: strings.Split(algorithms, ","), + disableRequestCompression: disabled, + requestMinCompressSizeBytes: minBytes, + compressAlgorithms: algorithms, }, middleware.After) } @@ -90,7 +89,12 @@ func (m requestCompression) HandleSerialize( return out, metadata, fmt.Errorf("failed to set request stream, %v", err) } *req = *newReq - req.Header.Add("Content-Encoding", algorithm) + + if val := req.Header.Get("Content-Encoding"); val != "" { + req.Header.Set("Content-Encoding", val+", "+algorithm) + } else { + req.Header.Set("Content-Encoding", algorithm) + } } break } diff --git a/private/requestcompression/request_compression_test.go b/private/requestcompression/request_compression_test.go index 7e57c5c3c..b29947dfd 100644 --- a/private/requestcompression/request_compression_test.go +++ b/private/requestcompression/request_compression_test.go @@ -37,7 +37,7 @@ func TestRequestCompression(t *testing.T) { "Content-Encoding": {"custom"}, }, ExpectedHeader: map[string][]string{ - "Content-Encoding": {"custom", "gzip"}, + "Content-Encoding": {"custom, gzip"}, }, }, "GZip request stream smaller than min compress request size": { From 5b5d3bf7ad1b129fb216b55b5f472aa31705991c Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 28 Nov 2023 13:55:53 -0500 Subject: [PATCH 10/14] Change codegen comment --- .../go/codegen/requestcompression/RequestCompression.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index c3a307edd..1298735e8 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -153,8 +153,9 @@ public List getClientPlugins() { .type(SymbolUtils.createValueSymbolBuilder("int64") .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) .build()) - .documentation("Inclusive threshold request body size to trigger compression, " - + "default to 10240 and must be within 0 and 10485760 bytes inclusively") + .documentation("The minimum request body size, in bytes, at which compression " + + "should occur. The default value is 10 KiB. Values must fall within " + + "[0, 1MiB].") .build() )) .build() From 7840dc4235ee65ae73eeef83cc931bbe5e5f0cde Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 28 Nov 2023 14:37:19 -0500 Subject: [PATCH 11/14] Change static middleware import --- .../go/codegen/requestcompression/RequestCompression.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index 1298735e8..b3b0be865 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -25,6 +25,7 @@ import software.amazon.smithy.go.codegen.GoSettings; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.SmithyGoDependency; +import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.SymbolUtils; import software.amazon.smithy.go.codegen.integration.ConfigField; import software.amazon.smithy.go.codegen.integration.GoIntegration; @@ -103,9 +104,6 @@ public static boolean isRequestCompressionService(Model model, ServiceShape serv } private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { - var stackSymbol = SymbolUtils - .createPointableSymbolBuilder("Stack", SmithyGoDependency.SMITHY_MIDDLEWARE) - .build(); var addInternalSymbol = SymbolUtils .createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, SmithyGoDependency.SMITHY_REQUEST_COMPRESSION) @@ -128,7 +126,7 @@ private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, O """, MapUtils.of( "add", getAddRequestCompressionMiddlewareFuncName(operationName), - "stack", stackSymbol, + "stack", SmithyGoTypes.Middleware.Stack, "addInternal", addInternalSymbol, "algorithms", algorithms )); From 6031dadb98bb008506c224418c7e057c39a0b255 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 29 Nov 2023 14:11:07 -0500 Subject: [PATCH 12/14] Change go dependency codegen --- .../smithy/go/codegen/SmithyGoTypes.java | 6 +++ .../RequestCompression.java | 44 +++++++++---------- private/requestcompression/gzip.go | 30 +++++++++++++ .../requestcompression/request_compression.go | 25 +---------- 4 files changed, 57 insertions(+), 48 deletions(-) create mode 100644 private/requestcompression/gzip.go diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java index c4e2b75c6..057bc3a2d 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java @@ -105,4 +105,10 @@ public static final class Bearer { public static final Symbol NewSignHTTPSMessage = SmithyGoDependency.SMITHY_AUTH_BEARER.valueSymbol("NewSignHTTPSMessage"); } } + + public static final class Private { + public static final class RequestCompression { + public static final Symbol AddRequestCompression = SmithyGoDependency.SMITHY_REQUEST_COMPRESSION.valueSymbol("AddRequestCompression"); + } + } } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index b3b0be865..74cc84e2e 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -23,8 +23,8 @@ import software.amazon.smithy.go.codegen.GoCodegenPlugin; import software.amazon.smithy.go.codegen.GoDelegator; import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.GoUniverseTypes; import software.amazon.smithy.go.codegen.GoWriter; -import software.amazon.smithy.go.codegen.SmithyGoDependency; import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.SymbolUtils; import software.amazon.smithy.go.codegen.integration.ConfigField; @@ -42,8 +42,6 @@ public final class RequestCompression implements GoIntegration { - private static final String ADD_REQUEST_COMPRESSION_INTERNAL = "AddRequestCompression"; - private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression"; private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes"; @@ -73,7 +71,7 @@ public void processFinalizedModel(GoSettings settings, Model model) { } return o.equals(operation); }).registerMiddleware(MiddlewareRegistrar.builder() - .resolvedFunction(SymbolUtils.createValueSymbolBuilder(funcName).build()) + .resolvedFunction(SymbolUtils.buildPackageSymbol(funcName)) .useClientOptions().build()) .build()); }); @@ -104,31 +102,20 @@ public static boolean isRequestCompressionService(Model model, ServiceShape serv } private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { - var addInternalSymbol = SymbolUtils - .createValueSymbolBuilder(ADD_REQUEST_COMPRESSION_INTERNAL, - SmithyGoDependency.SMITHY_REQUEST_COMPRESSION) - .build(); String operationName = symbolProvider.toSymbol(operation).getName(); RequestCompressionTrait trait = operation.expectTrait(RequestCompressionTrait.class); - // build encoding list symbol - StringBuilder algorithmList = new StringBuilder("[]string{"); - for (String algo : trait.getEncodings()) { - algorithmList.append(String.format("\"%s\", ", algo)); - } - String algorithms = algorithmList.substring(0, algorithmList.length() - 2) + "}"; - return goTemplate(""" func $add:L(stack $stack:P, options Options) error { return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - $algorithms:L) + $algorithms:W) } """, MapUtils.of( "add", getAddRequestCompressionMiddlewareFuncName(operationName), "stack", SmithyGoTypes.Middleware.Stack, - "addInternal", addInternalSymbol, - "algorithms", algorithms + "addInternal", SmithyGoTypes.Private.RequestCompression.AddRequestCompression, + "algorithms", generateAlgorithmList(trait.getEncodings()) )); } @@ -140,17 +127,13 @@ public List getClientPlugins() { .configFields(ListUtils.of( ConfigField.builder() .name(DISABLE_REQUEST_COMPRESSION) - .type(SymbolUtils.createValueSymbolBuilder("bool") - .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) - .build()) + .type(GoUniverseTypes.Bool) .documentation( "Whether to disable automatic request compression for supported operations.") .build(), ConfigField.builder() .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES) - .type(SymbolUtils.createValueSymbolBuilder("int64") - .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) - .build()) + .type(GoUniverseTypes.Int64) .documentation("The minimum request body size, in bytes, at which compression " + "should occur. The default value is 10 KiB. Values must fall within " + "[0, 1MiB].") @@ -161,4 +144,17 @@ public List getClientPlugins() { return runtimeClientPlugins; } + + private GoWriter.Writable generateAlgorithmList(List algorithms) { + return goTemplate(""" + []string{ + $W + } + """, + GoWriter.ChainWritable.of( + algorithms.stream() + .map(it -> goTemplate("$S,", it)) + .toList() + ).compose(false)); + } } diff --git a/private/requestcompression/gzip.go b/private/requestcompression/gzip.go new file mode 100644 index 000000000..004d78f21 --- /dev/null +++ b/private/requestcompression/gzip.go @@ -0,0 +1,30 @@ +package requestcompression + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" +) + +func gzipCompress(input io.Reader) ([]byte, error) { + var b bytes.Buffer + w, err := gzip.NewWriterLevel(&b, gzip.DefaultCompression) + if err != nil { + return nil, fmt.Errorf("failed to create gzip writer, %v", err) + } + + inBytes, err := io.ReadAll(input) + if err != nil { + return nil, fmt.Errorf("failed read payload to compress, %v", err) + } + + if _, err = w.Write(inBytes); err != nil { + return nil, fmt.Errorf("failed to write payload to be compressed, %v", err) + } + if err = w.Close(); err != nil { + return nil, fmt.Errorf("failed to flush payload being compressed, %v", err) + } + + return b.Bytes(), nil +} diff --git a/private/requestcompression/request_compression.go b/private/requestcompression/request_compression.go index be5d545dd..cc1a7fc13 100644 --- a/private/requestcompression/request_compression.go +++ b/private/requestcompression/request_compression.go @@ -8,7 +8,6 @@ package requestcompression import ( "bytes" - "compress/gzip" "context" "fmt" "github.com/aws/smithy-go/middleware" @@ -91,7 +90,7 @@ func (m requestCompression) HandleSerialize( *req = *newReq if val := req.Header.Get("Content-Encoding"); val != "" { - req.Header.Set("Content-Encoding", val+", "+algorithm) + req.Header.Set("Content-Encoding", fmt.Sprintf("%s, %s", val, algorithm)) } else { req.Header.Set("Content-Encoding", algorithm) } @@ -102,25 +101,3 @@ func (m requestCompression) HandleSerialize( return next.HandleSerialize(ctx, in) } - -func gzipCompress(input io.Reader) ([]byte, error) { - var b bytes.Buffer - w, err := gzip.NewWriterLevel(&b, gzip.DefaultCompression) - if err != nil { - return nil, fmt.Errorf("failed to create gzip writer, %v", err) - } - - inBytes, err := io.ReadAll(input) - if err != nil { - return nil, fmt.Errorf("failed read payload to compress, %v", err) - } - - if _, err = w.Write(inBytes); err != nil { - return nil, fmt.Errorf("failed to write payload to be compressed, %v", err) - } - if err = w.Close(); err != nil { - return nil, fmt.Errorf("failed to flush payload being compressed, %v", err) - } - - return b.Bytes(), nil -} From 2054530f65817fccc5b97ee52754582210b178b0 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Mon, 4 Dec 2023 23:25:37 -0500 Subject: [PATCH 13/14] Add body compare fn to request compress op unit test --- CHANGELOG.md | 10 +++ auth/option_test.go | 31 ++++++++ .../amazon/smithy/go/codegen/GoWriter.java | 17 +++++ .../smithy/go/codegen/ServiceGenerator.java | 16 ++++ .../smithy/go/codegen/SmithyGoDependency.java | 2 +- .../smithy/go/codegen/SmithyGoTypes.java | 1 + .../HttpProtocolUnitTestRequestGenerator.java | 52 ++++++++++++- .../RequestCompression.java | 46 ++++++------ endpoints/private/rulesfn/strings.go | 1 - go_module_metadata.go | 2 +- .../middleware_capture_request_compression.go | 52 +++++++++++++ properties_test.go | 11 ++- testing/bytes.go | 35 +++++++++ testing/gzip.go | 27 +++++++ transport/http/auth_schemes_test.go | 21 ++++++ transport/http/identity_test.go | 18 +++++ transport/http/properties_test.go | 74 +++++++++++++++++++ 17 files changed, 388 insertions(+), 28 deletions(-) create mode 100644 auth/option_test.go create mode 100644 private/requestcompression/middleware_capture_request_compression.go create mode 100644 testing/gzip.go create mode 100644 transport/http/auth_schemes_test.go create mode 100644 transport/http/identity_test.go create mode 100644 transport/http/properties_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e6afaccfc..8c83df00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# Release (2023-11-30) + +* No change notes available for this release. + +# Release (2023-11-29) + +## Module Highlights +* `github.com/aws/smithy-go`: v1.18.0 + * **Feature**: Expose Options() method on generated service clients. + # Release (2023-11-15) ## Module Highlights diff --git a/auth/option_test.go b/auth/option_test.go new file mode 100644 index 000000000..3ff4cf304 --- /dev/null +++ b/auth/option_test.go @@ -0,0 +1,31 @@ +package auth + +import ( + "testing" + "reflect" + smithy "github.com/aws/smithy-go" +) + +func TestAuthOptions(t *testing.T) { + var ip smithy.Properties + ip.Set("foo", "bar") + + var sp smithy.Properties + sp.Set("foo", "bar") + + expected := []*Option{ + &Option{ + SchemeID: "fakeSchemeID", + IdentityProperties: ip, + SignerProperties: sp, + }, + } + + var m smithy.Properties + SetAuthOptions(&m, expected) + actual, _ := GetAuthOptions(&m) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expect AuthOptions to be equivalent %v != %v", expected, actual) + } +} \ No newline at end of file diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java index 195113c2b..e5cd6b83f 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java @@ -152,6 +152,23 @@ public static Writable goDocTemplate(String contents) { return goDocTemplate(contents, new HashMap<>()); } + /** + * Auto-formats a multi-paragraph string as a doc writable (including line wrapping). + * @param contents The docs. + * @return writer for formatted docs. + */ + public static Writable autoDocTemplate(String contents) { + return GoWriter.ChainWritable.of( + Arrays.stream(contents.split("\n\n")) + .map(it -> docParagraphWriter(it.replace("\n", " "))) + .toList() + ).compose(false); + } + + private static GoWriter.Writable docParagraphWriter(String paragraph) { + return writer -> writer.writeDocs(paragraph).writeDocs(""); + } + @SafeVarargs public static Writable goDocTemplate(String contents, Map... args) { validateTemplateArgsNotNull(args); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java index 055329971..da823214d 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java @@ -15,6 +15,7 @@ package software.amazon.smithy.go.codegen; +import static software.amazon.smithy.go.codegen.GoWriter.autoDocTemplate; import static software.amazon.smithy.go.codegen.GoWriter.emptyGoTemplate; import static software.amazon.smithy.go.codegen.GoWriter.goDocTemplate; import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; @@ -95,6 +96,7 @@ private GoWriter.Writable generate() { generateMetadata(), generateClient(), generateNew(), + generateGetOptions(), generateInvokeOperation(), generateInputContextFuncs(), generateAddProtocolFinalizerMiddleware() @@ -223,6 +225,20 @@ func New(options $options:L, optFns ...func(*$options:L)) *$client:L { )); } + private GoWriter.Writable generateGetOptions() { + var docs = autoDocTemplate(""" + Options returns a copy of the client configuration. + + Callers SHOULD NOT perform mutations on any inner structures within client config. Config overrides + should instead be made on a per-operation basis through functional options."""); + return goTemplate(""" + $W + func (c $P) Options() $L { + return c.options.Copy() + } + """, docs, symbolProvider.toSymbol(service), ClientOptions.NAME); + } + private GoWriter.Writable generateConfigFieldResolver(ConfigFieldResolver resolver) { return writer -> { writer.writeInline("$T(&options", resolver.getResolver()); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java index 0d8c7e602..4f613e1ad 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java @@ -54,7 +54,7 @@ public final class SmithyGoDependency { public static final GoDependency SMITHY_MIDDLEWARE = smithy("middleware"); public static final GoDependency SMITHY_PRIVATE_PROTOCOL = smithy("private/protocol", "smithyprivateprotocol"); public static final GoDependency SMITHY_REQUEST_COMPRESSION = - smithy("private/requestcompression", "smithyrequestcompression"); + smithy("private/requestcompression", "smithyrequestcompression"); public static final GoDependency SMITHY_TIME = smithy("time", "smithytime"); public static final GoDependency SMITHY_HTTP_BINDING = smithy("encoding/httpbinding"); public static final GoDependency SMITHY_JSON = smithy("encoding/json", "smithyjson"); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java index 057bc3a2d..50a34ea42 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java @@ -109,6 +109,7 @@ public static final class Bearer { public static final class Private { public static final class RequestCompression { public static final Symbol AddRequestCompression = SmithyGoDependency.SMITHY_REQUEST_COMPRESSION.valueSymbol("AddRequestCompression"); + public static final Symbol AddCaptureUncompressedRequest = SmithyGoDependency.SMITHY_REQUEST_COMPRESSION.valueSymbol("AddCaptureUncompressedRequestMiddleware"); } } } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpProtocolUnitTestRequestGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpProtocolUnitTestRequestGenerator.java index 15d4adf6a..3c4f3cea8 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpProtocolUnitTestRequestGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpProtocolUnitTestRequestGenerator.java @@ -17,13 +17,22 @@ package software.amazon.smithy.go.codegen.integration; +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoTypes.Private.RequestCompression.AddCaptureUncompressedRequest; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.function.Consumer; import java.util.logging.Logger; import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.SmithyGoDependency; +import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.SymbolUtils; +import software.amazon.smithy.model.traits.RequestCompressionTrait; import software.amazon.smithy.protocoltests.traits.HttpRequestTestCase; +import software.amazon.smithy.utils.MapUtils; /** * Generates HTTP protocol unit tests for HTTP request test cases. @@ -31,6 +40,8 @@ public class HttpProtocolUnitTestRequestGenerator extends HttpProtocolUnitTestGenerator { private static final Logger LOGGER = Logger.getLogger(HttpProtocolUnitTestRequestGenerator.class.getName()); + private static final Set ALLOWED_ALGORITHMS = new HashSet<>(Arrays.asList("gzip")); + /** * Initializes the protocol test generator. * @@ -198,6 +209,10 @@ protected void generateTestCaseValues(GoWriter writer, HttpRequestTestCase testC */ protected void generateTestBodySetup(GoWriter writer) { writer.write("actualReq := &http.Request{}"); + if (operation.hasTrait(RequestCompressionTrait.class)) { + writer.addUseImports(SmithyGoDependency.BYTES); + writer.write("rawBodyBuf := &bytes.Buffer{}"); + } } /** @@ -227,8 +242,29 @@ protected void generateTestInvokeClientOperation(GoWriter writer, String clientN writer.write("return $T(stack, actualReq)", SymbolUtils.createValueSymbolBuilder("AddCaptureRequestMiddleware", SmithyGoDependency.SMITHY_PRIVATE_PROTOCOL).build()); - }); + }); + if (operation.hasTrait(RequestCompressionTrait.class)) { + writer.write(goTemplate(""" + options.APIOptions = append(options.APIOptions, func(stack $stack:P) error { + return $captureRequest:T(stack, rawBodyBuf) + }) + """, + MapUtils.of( + "stack", SmithyGoTypes.Middleware.Stack, + "captureRequest", AddCaptureUncompressedRequest + ))); + } }); + + if (operation.hasTrait(RequestCompressionTrait.class)) { + writer.write(goTemplate(""" + disable := $client:L.Options().DisableRequestCompression + min := $client:L.Options().RequestMinCompressSizeBytes + """, + MapUtils.of( + "client", clientName + ))); + } } /** @@ -259,6 +295,20 @@ protected void generateTestAssertions(GoWriter writer) { writer.write("t.Errorf(\"expect body equal, got %v\", err)"); }); }); + + if (operation.hasTrait(RequestCompressionTrait.class)) { + String algorithm = operation.expectTrait(RequestCompressionTrait.class).getEncodings() + .stream().filter(it -> ALLOWED_ALGORITHMS.contains(it)).findFirst().get(); + writer.write(goTemplate(""" + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, $algorithm:S); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } + """, + MapUtils.of( + "algorithm", algorithm + ))); + } } public static class Builder extends HttpProtocolUnitTestGenerator.Builder { diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java index 74cc84e2e..b6f7779c8 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/requestcompression/RequestCompression.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -48,10 +48,6 @@ public final class RequestCompression implements GoIntegration { private final List runtimeClientPlugins = new ArrayList<>(); - private static String getAddRequestCompressionMiddlewareFuncName(String operationName) { - return String.format("addOperation%sRequestCompressionMiddleware", operationName); - } - // Write operation plugin for request compression middleware @Override public void processFinalizedModel(GoSettings settings, Model model) { @@ -101,24 +97,6 @@ public static boolean isRequestCompressionService(Model model, ServiceShape serv .anyMatch(it -> it.hasTrait(RequestCompressionTrait.class)); } - private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { - String operationName = symbolProvider.toSymbol(operation).getName(); - RequestCompressionTrait trait = operation.expectTrait(RequestCompressionTrait.class); - - return goTemplate(""" - func $add:L(stack $stack:P, options Options) error { - return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - $algorithms:W) - } - """, - MapUtils.of( - "add", getAddRequestCompressionMiddlewareFuncName(operationName), - "stack", SmithyGoTypes.Middleware.Stack, - "addInternal", SmithyGoTypes.Private.RequestCompression.AddRequestCompression, - "algorithms", generateAlgorithmList(trait.getEncodings()) - )); - } - @Override public List getClientPlugins() { runtimeClientPlugins.add( @@ -157,4 +135,26 @@ private GoWriter.Writable generateAlgorithmList(List algorithms) { .toList() ).compose(false)); } + + private static String getAddRequestCompressionMiddlewareFuncName(String operationName) { + return String.format("addOperation%sRequestCompressionMiddleware", operationName); + } + + private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { + String operationName = symbolProvider.toSymbol(operation).getName(); + RequestCompressionTrait trait = operation.expectTrait(RequestCompressionTrait.class); + + return goTemplate(""" + func $add:L(stack $stack:P, options Options) error { + return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + $algorithms:W) + } + """, + MapUtils.of( + "add", getAddRequestCompressionMiddlewareFuncName(operationName), + "stack", SmithyGoTypes.Middleware.Stack, + "addInternal", SmithyGoTypes.Private.RequestCompression.AddRequestCompression, + "algorithms", generateAlgorithmList(trait.getEncodings()) + )); + } } diff --git a/endpoints/private/rulesfn/strings.go b/endpoints/private/rulesfn/strings.go index 8e230783e..5cf4a7b02 100644 --- a/endpoints/private/rulesfn/strings.go +++ b/endpoints/private/rulesfn/strings.go @@ -1,6 +1,5 @@ package rulesfn - // Substring returns the substring of the input provided. If the start or stop // indexes are not valid for the input nil will be returned. If errors occur // they will be added to the provided [ErrorCollector]. diff --git a/go_module_metadata.go b/go_module_metadata.go index 184aaae1c..bb04fc9e8 100644 --- a/go_module_metadata.go +++ b/go_module_metadata.go @@ -3,4 +3,4 @@ package smithy // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.17.0" +const goModuleVersion = "1.18.1" diff --git a/private/requestcompression/middleware_capture_request_compression.go b/private/requestcompression/middleware_capture_request_compression.go new file mode 100644 index 000000000..06c16afc1 --- /dev/null +++ b/private/requestcompression/middleware_capture_request_compression.go @@ -0,0 +1,52 @@ +package requestcompression + +import ( + "bytes" + "context" + "fmt" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "io" + "net/http" +) + +const captureUncompressedRequestID = "CaptureUncompressedRequest" + +// AddCaptureUncompressedRequestMiddleware captures http request before compress encoding for check +func AddCaptureUncompressedRequestMiddleware(stack *middleware.Stack, buf *bytes.Buffer) error { + return stack.Serialize.Insert(&captureUncompressedRequestMiddleware{ + buf: buf, + }, "RequestCompression", middleware.Before) +} + +type captureUncompressedRequestMiddleware struct { + req *http.Request + buf *bytes.Buffer + bytes []byte +} + +// ID returns id of the captureUncompressedRequestMiddleware +func (*captureUncompressedRequestMiddleware) ID() string { + return captureUncompressedRequestID +} + +// HandleSerialize captures request payload before it is compressed by request compression middleware +func (m *captureUncompressedRequestMiddleware) HandleSerialize(ctx context.Context, input middleware.SerializeInput, next middleware.SerializeHandler, +) ( + output middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + request, ok := input.Request.(*smithyhttp.Request) + if !ok { + return output, metadata, fmt.Errorf("error when retrieving http request") + } + + _, err = io.Copy(m.buf, request.GetStream()) + if err != nil { + return output, metadata, fmt.Errorf("error when copying http request stream: %q", err) + } + if err = request.RewindStream(); err != nil { + return output, metadata, fmt.Errorf("error when rewinding request stream: %q", err) + } + + return next.HandleSerialize(ctx, input) +} diff --git a/properties_test.go b/properties_test.go index 93637c3b1..8e75130b9 100644 --- a/properties_test.go +++ b/properties_test.go @@ -17,4 +17,13 @@ func TestProperties(t *testing.T) { t.Errorf("expect key / value properties to be equivalent: %v / %v", k, v) } } -} + + var n Properties + n.SetAll(&m) + for k, v := range original { + if n.Get(k) != v { + t.Errorf("expect key / value properties to be equivalent: %v / %v", k, v) + } + } + +} \ No newline at end of file diff --git a/testing/bytes.go b/testing/bytes.go index 8f966846d..955612552 100644 --- a/testing/bytes.go +++ b/testing/bytes.go @@ -8,6 +8,17 @@ import ( "io/ioutil" ) +// Enumeration values for supported compress Algorithms. +const ( + GZIP = "gzip" +) + +type compareCompressFunc func([]byte, io.Reader) error + +var allowedAlgorithms = map[string]compareCompressFunc{ + GZIP: GzipCompareCompressBytes, +} + // CompareReaderEmpty checks if the reader is nil, or contains no bytes. // Returns an error if not empty. func CompareReaderEmpty(r io.Reader) error { @@ -94,3 +105,27 @@ func CompareURLFormReaderBytes(r io.Reader, expect []byte) error { } return nil } + +// CompareCompressedBytes compares the request stream before and after possible request compression +func CompareCompressedBytes(expect *bytes.Buffer, actual io.Reader, disable bool, min int64, algorithm string) error { + expectBytes := expect.Bytes() + if disable || int64(len(expectBytes)) < min { + actualBytes, err := io.ReadAll(actual) + if err != nil { + return fmt.Errorf("error while reading request: %q", err) + } + if e, a := expectBytes, actualBytes; !bytes.Equal(e, a) { + return fmt.Errorf("expect content to be %s, got %s", e, a) + } + } else { + compareFn := allowedAlgorithms[algorithm] + if compareFn == nil { + return fmt.Errorf("compress algorithm %s is not allowed", algorithm) + } + if err := compareFn(expectBytes, actual); err != nil { + return fmt.Errorf("error while comparing unzipped content: %q", err) + } + } + + return nil +} diff --git a/testing/gzip.go b/testing/gzip.go new file mode 100644 index 000000000..e6f4d6524 --- /dev/null +++ b/testing/gzip.go @@ -0,0 +1,27 @@ +package testing + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" +) + +func GzipCompareCompressBytes(expect []byte, actual io.Reader) error { + content, err := gzip.NewReader(actual) + if err != nil { + return fmt.Errorf("error while reading request") + } + + var actualBytes bytes.Buffer + _, err = actualBytes.ReadFrom(content) + if err != nil { + return fmt.Errorf("error while unzipping request payload") + } + + if e, a := expect, actualBytes.Bytes(); !bytes.Equal(e, a) { + return fmt.Errorf("expect unzipped content to be %s, got %s", e, a) + } + + return nil +} diff --git a/transport/http/auth_schemes_test.go b/transport/http/auth_schemes_test.go new file mode 100644 index 000000000..44ae003be --- /dev/null +++ b/transport/http/auth_schemes_test.go @@ -0,0 +1,21 @@ +package http + +import ( + "github.com/aws/smithy-go/auth" + "testing" +) + +func TestAnonymousScheme(t *testing.T) { + expectedID := auth.SchemeIDAnonymous + scheme := NewAnonymousScheme() + actualID := scheme.SchemeID() + if expectedID != actualID { + t.Errorf("AnonymousScheme constructor is not producing the correct scheme ID") + } + + var expectedSigner Signer = &nopSigner{} + actualSigner := scheme.Signer() + if expectedSigner != actualSigner { + t.Errorf("AnonymousScheme constructor is not producing the correct signer") + } +} diff --git a/transport/http/identity_test.go b/transport/http/identity_test.go new file mode 100644 index 000000000..2727ce14b --- /dev/null +++ b/transport/http/identity_test.go @@ -0,0 +1,18 @@ +package http + +import ( + "context" + "testing" + smithy "github.com/aws/smithy-go" + "github.com/aws/smithy-go/auth" +) + +func TestIdentity(t *testing.T) { + var expected auth.Identity = &auth.AnonymousIdentity{} + + resolver := auth.AnonymousIdentityResolver{} + actual, _ := resolver.GetIdentity(context.TODO(), smithy.Properties{}) + if expected != actual { + t.Errorf("Anonymous identity resolver does not produce correct identity") + } +} \ No newline at end of file diff --git a/transport/http/properties_test.go b/transport/http/properties_test.go new file mode 100644 index 000000000..f79cde2f4 --- /dev/null +++ b/transport/http/properties_test.go @@ -0,0 +1,74 @@ +package http + +import ( + "testing" + "reflect" + smithy "github.com/aws/smithy-go" +) + + +func TestSigV4SigningName(t *testing.T) { + expected := "foo" + var m smithy.Properties + SetSigV4SigningName(&m, expected) + actual, _ := GetSigV4SigningName(&m) + + if expected != actual { + t.Errorf("Expect SigV4SigningName to be equivalent %s != %s", expected, actual) + } +} + +func TestSigV4SigningRegion(t *testing.T) { + expected := "foo" + var m smithy.Properties + SetSigV4SigningRegion(&m, expected) + actual, _ := GetSigV4SigningRegion(&m) + + if expected != actual { + t.Errorf("Expect SigV4SigningRegion to be equivalent %s != %s", expected, actual) + } +} + +func TestSigV4ASigningName(t *testing.T) { + expected := "foo" + var m smithy.Properties + SetSigV4ASigningName(&m, expected) + actual, _ := GetSigV4ASigningName(&m) + + if expected != actual { + t.Errorf("Expect SigV4ASigningName to be equivalent %s != %s", expected, actual) + } +} + +func TestSigV4SigningRegions(t *testing.T) { + expected := []string{"foo", "bar"} + var m smithy.Properties + SetSigV4ASigningRegions(&m, expected) + actual, _ := GetSigV4ASigningRegions(&m) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expect SigV4ASigningRegions to be equivalent %v != %v", expected, actual) + } +} + +func TestUnsignedPayload(t *testing.T) { + expected := true + var m smithy.Properties + SetIsUnsignedPayload(&m, expected) + actual, _ := GetIsUnsignedPayload(&m) + + if expected != actual { + t.Errorf("Expect IsUnsignedPayload to be equivalent %v != %v", expected, actual) + } +} + +func TestDisableDoubleEncoding(t *testing.T) { + expected := true + var m smithy.Properties + SetDisableDoubleEncoding(&m, expected) + actual, _ := GetDisableDoubleEncoding(&m) + + if expected != actual { + t.Errorf("Expect DisableDoubleEncoding to be equivalent %v != %v", expected, actual) + } +} \ No newline at end of file From eb29c1c5d83a1fedd79e103eadf3f88b125b8e4d Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Mon, 4 Dec 2023 23:30:58 -0500 Subject: [PATCH 14/14] Solve rebase conflict --- ...software.amazon.smithy.go.codegen.integration.GoIntegration | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 5ccf5c135..09af27932 100644 --- a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -8,8 +8,9 @@ software.amazon.smithy.go.codegen.integration.Paginators software.amazon.smithy.go.codegen.integration.Waiters software.amazon.smithy.go.codegen.integration.ClientLogger software.amazon.smithy.go.codegen.endpoints.EndpointClientPluginsGenerator -software.amazon.smithy.go.codegen.requestcompression.RequestCompression # modeled auth schemes software.amazon.smithy.go.codegen.integration.auth.SigV4AuthScheme software.amazon.smithy.go.codegen.integration.auth.AnonymousAuthScheme + +software.amazon.smithy.go.codegen.requestcompression.RequestCompression