From 26601ee23455880695722c6df31fc3385921ba06 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 11 Aug 2022 21:22:02 -0700 Subject: [PATCH] Bump flyteidl version (#462) * Bump flyteidl version Signed-off-by: Kevin Su * Bump flyteidl version Signed-off-by: Kevin Su * lint Signed-off-by: Kevin Su * make ReadHeaderTimeout configurable Signed-off-by: Kevin Su * update_boilerplate Signed-off-by: Kevin Su * Address comment Signed-off-by: Kevin Su * Address comment Signed-off-by: Kevin Su * fix test Signed-off-by: Kevin Su * update run-tests Signed-off-by: Kevin Su Signed-off-by: Kevin Su --- boilerplate/flyte/end2end/run-tests.py | 4 ++-- go.mod | 2 +- go.sum | 6 ++---- pkg/config/config.go | 4 +++- pkg/config/serverconfig_flags.go | 1 + pkg/config/serverconfig_flags_test.go | 14 ++++++++++++++ pkg/server/service.go | 2 ++ 7 files changed, 25 insertions(+), 8 deletions(-) diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py index d461b0efb..b868be57f 100755 --- a/boilerplate/flyte/end2end/run-tests.py +++ b/boilerplate/flyte/end2end/run-tests.py @@ -15,14 +15,14 @@ WAIT_TIME = 10 -MAX_ATTEMPTS = 60 +MAX_ATTEMPTS = 200 # This dictionary maps the names found in the flytesnacks manifest to a list of workflow names and # inputs. This is so we can progressively cover all priorities in the original flytesnacks manifest, # starting with "core". FLYTESNACKS_WORKFLOW_GROUPS: Mapping[str, List[Tuple[str, dict]]] = { "core": [ - ("core.control_flow.chain_tasks.chain_tasks_wf", {}), + ("core.control_flow.chain_entities.chain_workflows_wf", {}), ("core.control_flow.dynamics.wf", {"s1": "Pear", "s2": "Earth"}), ("core.control_flow.map_task.my_map_workflow", {"a": [1, 2, 3, 4, 5]}), # Workflows that use nested executions cannot be launched via flyteremote. diff --git a/go.mod b/go.mod index 77c84a7dc..2722e33dd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cloudevents/sdk-go/v2 v2.8.0 github.com/coreos/go-oidc v2.2.1+incompatible github.com/evanphx/json-patch v4.9.0+incompatible - github.com/flyteorg/flyteidl v1.1.9 + github.com/flyteorg/flyteidl v1.1.11 github.com/flyteorg/flyteplugins v1.0.0 github.com/flyteorg/flytepropeller v1.1.13 github.com/flyteorg/flytestdlib v1.0.4 diff --git a/go.sum b/go.sum index d1073a3a6..feec1f89e 100644 --- a/go.sum +++ b/go.sum @@ -380,10 +380,8 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flyteorg/flyteidl v1.0.0/go.mod h1:JW0z1ZaHS9zWvDAwSMIyGhsf+V4zrzBBgh5IuqzMFCM= -github.com/flyteorg/flyteidl v1.1.5 h1:awptNJfw2yESkdNOm1Pe9KPILAzVImkiViUFP1K7UPk= -github.com/flyteorg/flyteidl v1.1.5/go.mod h1:f1tvw5CDjqmrzNxKpRYr6BdAhHL8f7Wp1Duxl0ZOV4g= -github.com/flyteorg/flyteidl v1.1.9 h1:ZIjbBqNYvxnT8Tw6HYH5IDs3CJuCmT7ecrJPczty38Y= -github.com/flyteorg/flyteidl v1.1.9/go.mod h1:f1tvw5CDjqmrzNxKpRYr6BdAhHL8f7Wp1Duxl0ZOV4g= +github.com/flyteorg/flyteidl v1.1.11 h1:tHwtRnuO8s0jSLAjTKtqG0nlqDvVhD2T+pT7/wKEgY4= +github.com/flyteorg/flyteidl v1.1.11/go.mod h1:f1tvw5CDjqmrzNxKpRYr6BdAhHL8f7Wp1Duxl0ZOV4g= github.com/flyteorg/flyteplugins v1.0.0 h1:77hUJjiIxBmQ9rd3+cXjSGnzOVAFrSzCd59aIaYFB/8= github.com/flyteorg/flyteplugins v1.0.0/go.mod h1:4Cpn+9RfanIieTTh2XsuL6zPYXtsR5UDe8YaEmXONT4= github.com/flyteorg/flytepropeller v1.1.13 h1:Y23JopdtsMyVykIBXm/zpSxIqWPuSqw/a+9pPePOejY= diff --git a/pkg/config/config.go b/pkg/config/config.go index b2388b9c1..5d5bc3dbc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -25,7 +25,8 @@ type ServerConfig struct { // Deprecated: please use auth.AppAuth.ThirdPartyConfig instead. DeprecatedThirdPartyConfig authConfig.ThirdPartyConfigOptions `json:"thirdPartyConfig" pflag:",Deprecated please use auth.appAuth.thirdPartyConfig instead."` - DataProxy DataProxyConfig `json:"dataProxy" pflag:",Defines data proxy configuration."` + DataProxy DataProxyConfig `json:"dataProxy" pflag:",Defines data proxy configuration."` + ReadHeaderTimeoutSeconds int `json:"readHeaderTimeoutSeconds" pflag:",The amount of time allowed to read request headers."` } type DataProxyConfig struct { @@ -95,6 +96,7 @@ var defaultServerConfig = &ServerConfig{ MaxExpiresIn: config.Duration{Duration: time.Hour}, }, }, + ReadHeaderTimeoutSeconds: 32, // just shy of requestTimeoutUpperBound } var serverConfig = config.MustRegisterSection(SectionKey, defaultServerConfig) diff --git a/pkg/config/serverconfig_flags.go b/pkg/config/serverconfig_flags.go index fa2ee145f..54d374c80 100755 --- a/pkg/config/serverconfig_flags.go +++ b/pkg/config/serverconfig_flags.go @@ -74,5 +74,6 @@ func (cfg ServerConfig) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "dataProxy.upload.defaultFileNameLength"), defaultServerConfig.DataProxy.Upload.DefaultFileNameLength, "Default length for the generated file name if not provided in the request.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "dataProxy.upload.storagePrefix"), defaultServerConfig.DataProxy.Upload.StoragePrefix, "Storage prefix to use for all upload requests.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "dataProxy.download.maxExpiresIn"), defaultServerConfig.DataProxy.Download.MaxExpiresIn.String(), "Maximum allowed expiration duration.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "readHeaderTimeoutSeconds"), defaultServerConfig.ReadHeaderTimeoutSeconds, "The amount of time allowed to read request headers.") return cmdFlags } diff --git a/pkg/config/serverconfig_flags_test.go b/pkg/config/serverconfig_flags_test.go index bd3bb83e9..c2493f110 100755 --- a/pkg/config/serverconfig_flags_test.go +++ b/pkg/config/serverconfig_flags_test.go @@ -435,4 +435,18 @@ func TestServerConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_readHeaderTimeoutSeconds", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("readHeaderTimeoutSeconds", testValue) + if vInt, err := cmdFlags.GetInt("readHeaderTimeoutSeconds"); err == nil { + testDecodeJson_ServerConfig(t, fmt.Sprintf("%v", vInt), &actual.ReadHeaderTimeoutSeconds) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/pkg/server/service.go b/pkg/server/service.go index b998936df..f5273da35 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "strings" + "time" "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/promutils/labeled" @@ -401,6 +402,7 @@ func serveGatewaySecure(ctx context.Context, pluginRegistry *plugins.Registry, c Certificates: []tls.Certificate{*cert}, NextProtos: []string{"h2"}, }, + ReadHeaderTimeout: time.Duration(cfg.ReadHeaderTimeoutSeconds) * time.Second, } err = srv.Serve(tls.NewListener(conn, srv.TLSConfig))