offload the WorkflowClosure to storage for flytepropeller (#458)

* store separate crds parts to json files

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* use single object for static execution

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* make offloading configurable from flyteadmin config

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* empty crd fields when offloaded to storage

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* offload workflowClosure instead of parts of crd

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* use open flytepropeller with new crd fields

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* lint

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* pass config rather than argument and reuse datastore proto write function

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* cleanup

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* fixed tests

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* setting crd closure reference to existing metastore data

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* fixed unit tests

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* added test for stripping static fields on offload

Signed-off-by: Daniel Rammer <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* rename offload config

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>

* fix lint

Signed-off-by: Babis Kiosidis <[email protected]>

* upgrade flytepropeller version

Signed-off-by: Babis Kiosidis <[email protected]>

* goimports pkg/runtime/application_config_provider.go

Signed-off-by: Babis Kiosidis <[email protected]>

Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Babis Kiosidis <[email protected]>
Signed-off-by: Daniel Rammer <[email protected]>
Co-authored-by: Babis Kiosidis <[email protected]>
Co-authored-by: Daniel Rammer <[email protected]>

flyteadmin_config.yaml

@@ -60,6 +60,7 @@ flyteadmin:
   metadataStoragePrefix:
     - "metadata"
     - "admin"
+  useOffloadedWorkflowClosure: false
 database:
   postgres:
     port: 5432

go.mod

@@ -14,9 +14,9 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/evanphx/json-patch v4.12.0+incompatible
 	github.com/flyteorg/flyteidl v1.1.11
-	github.com/flyteorg/flyteplugins v1.0.0
-	github.com/flyteorg/flytepropeller v1.1.13
-	github.com/flyteorg/flytestdlib v1.0.4
+	github.com/flyteorg/flyteplugins v1.0.10
+	github.com/flyteorg/flytepropeller v1.1.28
+	github.com/flyteorg/flytestdlib v1.0.5
 	github.com/flyteorg/stow v0.3.5
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-gormigrate/gormigrate/v2 v2.0.0
@@ -34,7 +34,7 @@ require (
 	github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69
 	github.com/jackc/pgconn v1.10.1
 	github.com/lestrrat-go/jwx v1.1.6
-	github.com/magiconair/properties v1.8.5
+	github.com/magiconair/properties v1.8.6
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/ory/fosite v0.39.0
 	github.com/ory/x v0.0.162
@@ -45,7 +45,7 @@ require (
 	github.com/sendgrid/sendgrid-go v3.10.0+incompatible
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.7.2
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
 	google.golang.org/api v0.76.0
@@ -55,9 +55,9 @@ require (
 	gorm.io/driver/postgres v1.2.3
 	gorm.io/driver/sqlite v1.1.1
 	gorm.io/gorm v1.22.4
-	k8s.io/api v0.24.0
-	k8s.io/apimachinery v0.24.0
-	k8s.io/client-go v0.24.0
+	k8s.io/api v0.24.1
+	k8s.io/apimachinery v0.24.1
+	k8s.io/client-go v0.24.1
 	sigs.k8s.io/controller-runtime v0.12.1
 )
 
@@ -97,7 +97,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
@@ -151,41 +151,42 @@ require (
 	github.com/ory/viper v1.7.5 // indirect
 	github.com/pborman/uuid v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/sendgrid/rest v2.6.8+incompatible // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.10.1 // indirect
+	github.com/spf13/viper v1.11.0 // indirect
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
-	golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
+	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/tools v0.1.11 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
 	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.24.0 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.24.1 // indirect
+	sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
@@ -194,7 +195,7 @@ require (
 	github.com/Shopify/sarama v1.26.4
 	github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 // indirect
 	github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2 v2.8.0
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	go.uber.org/atomic v1.7.0 // indirect