From ace62c7a6ab2b5b5d26642286deb6db406391d8f Mon Sep 17 00:00:00 2001 From: Osama Nabih Date: Tue, 1 Aug 2023 02:54:43 +0300 Subject: [PATCH] feat: add new configuration "sync-interval" which controls the HTTP polling interval (#404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …, in minutes, bet. each HTTP_sync calls EDIT by @toddbaert : I have modified this PR. I've done a couple things differently than @OsamaNabih : - interval is in seconds (I think it's likely some people will want sub-minute behavior, and the existing behavior was seconds - using `sources` for this, as @james-milligan suggested: - for example: `./bin/flagd start --sources='[{"uri":"https://raw.githubusercontent.com/open-feature/playground/main/config/flagd/flags.json","provider":"http","interval":1}]' --debug` will poll every 1s - defaults to 5s to maintain current behavior - added tests --------- Signed-off-by: Todd Baert Co-authored-by: Todd Baert --- core/pkg/runtime/from_config.go | 8 +++++ core/pkg/runtime/from_config_test.go | 6 ++++ core/pkg/sync/http/http_sync.go | 8 +++-- docs/configuration/configuration.md | 52 ++++++++++++++++------------ 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/core/pkg/runtime/from_config.go b/core/pkg/runtime/from_config.go index 299b7a43e..9882773cb 100644 --- a/core/pkg/runtime/from_config.go +++ b/core/pkg/runtime/from_config.go @@ -54,6 +54,7 @@ type SourceConfig struct { TLS bool `json:"tls,omitempty"` ProviderID string `json:"providerID,omitempty"` Selector string `json:"selector,omitempty"` + Interval uint32 `json:"interval,omitempty"` } // Config is the configuration structure derived from startup arguments. @@ -216,6 +217,12 @@ func NewGRPC(config SourceConfig, logger *logger.Logger) *grpc.Sync { } func NewHTTP(config SourceConfig, logger *logger.Logger) *httpSync.Sync { + // Default to 5 seconds + var interval uint32 = 5 + if config.Interval != 0 { + interval = config.Interval + } + return &httpSync.Sync{ URI: config.URI, Client: &http.Client{ @@ -226,6 +233,7 @@ func NewHTTP(config SourceConfig, logger *logger.Logger) *httpSync.Sync { zap.String("sync", "remote"), ), BearerToken: config.BearerToken, + Interval: interval, Cron: cron.New(), } } diff --git a/core/pkg/runtime/from_config_test.go b/core/pkg/runtime/from_config_test.go index 65016c44f..5e41eb18e 100644 --- a/core/pkg/runtime/from_config_test.go +++ b/core/pkg/runtime/from_config_test.go @@ -57,6 +57,7 @@ func TestParseSource(t *testing.T) { in: `[{"uri":"config/samples/example_flags.json","provider":"file"}, {"uri":"http://my-flag-source.json","provider":"http","bearerToken":"bearer-dji34ld2l"}, {"uri":"https://secure-remote","provider":"http","bearerToken":"bearer-dji34ld2l"}, + {"uri":"http://site.com","provider":"http","interval":77 }, {"uri":"default/my-flag-config","provider":"kubernetes"}, {"uri":"grpc-source:8080","provider":"grpc"}, {"uri":"my-flag-source:8080","provider":"grpc", "tls":true, "certPath": "/certs/ca.cert", "providerID": "flagd-weatherapp-sidecar", "selector": "source=database,app=weatherapp"}] @@ -77,6 +78,11 @@ func TestParseSource(t *testing.T) { Provider: syncProviderHTTP, BearerToken: "bearer-dji34ld2l", }, + { + URI: "http://site.com", + Provider: syncProviderHTTP, + Interval: 77, + }, { URI: "default/my-flag-config", Provider: syncProviderKubernetes, diff --git a/core/pkg/sync/http/http_sync.go b/core/pkg/sync/http/http_sync.go index 4bdb4d0b1..4401cb12f 100644 --- a/core/pkg/sync/http/http_sync.go +++ b/core/pkg/sync/http/http_sync.go @@ -21,8 +21,8 @@ type Sync struct { LastBodySHA string Logger *logger.Logger BearerToken string - - ready bool + Interval uint32 + ready bool } // Client defines the behaviour required of a http client @@ -65,7 +65,9 @@ func (hs *Sync) Sync(ctx context.Context, dataSync chan<- sync.DataSync) error { // Set ready state hs.ready = true - _ = hs.Cron.AddFunc("*/5 * * * *", func() { + hs.Logger.Debug(fmt.Sprintf("polling %s every %d seconds", hs.URI, hs.Interval)) + _ = hs.Cron.AddFunc(fmt.Sprintf("*/%d * * * *", hs.Interval), func() { + hs.Logger.Debug(fmt.Sprintf("fetching configuration from %s", hs.URI)) body, err := hs.fetchBodyFromURL(ctx, hs.URI) if err != nil { hs.Logger.Error(err.Error()) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index d35f4bc99..5b1f070ee 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -1,15 +1,15 @@ # Configuration -* [Configuration](#configuration) - * [Sync providers](#sync-providers) - * [Kubernetes provider](#kubernetes-provider) - * [Filepath provider](#filepath-provider) - * [Remote provider](#remote-provider) - * [GRPC provider](#grpc-provider) - * [Sync provider configurations](#sync-provider-configurations) - * [URI patterns](#uri-patterns) - * [Source Configuration](#source-configuration) +- [Configuration](#configuration) + - [Sync providers](#sync-providers) + - [Kubernetes provider](#kubernetes-provider) + - [Filepath provider](#filepath-provider) + - [Remote provider](#remote-provider) + - [GRPC provider](#grpc-provider) + - [Sync provider configurations](#sync-provider-configurations) + - [URI patterns](#uri-patterns) + - [Source Configuration](#source-configuration) `flagd` supports configuration via config file, environment variables and start-up flags. In cases of a conflict, @@ -27,10 +27,15 @@ The config file expects the keys to have the exact naming as startup-flags flags Sync providers are a core part of flagd; they are the abstraction that enables different sources for feature flag configurations. flagd currently support the following sync providers: -* [Kubernetes provider](#kubernetes-provider) -* [Filepath Configuration](#filepath-provider) -* [Remote Configuration](#remote-provider) -* [GRPC Configuration](#grpc-provider) +- [Configuration](#configuration) + - [Sync providers](#sync-providers) + - [Kubernetes provider](#kubernetes-provider) + - [Filepath provider](#filepath-provider) + - [Remote provider](#remote-provider) + - [GRPC provider](#grpc-provider) + - [Sync provider configurations](#sync-provider-configurations) + - [URI patterns](#uri-patterns) + - [Source Configuration](#source-configuration) ### Kubernetes provider @@ -83,8 +88,8 @@ In this example, `grpc-sync-source` is a grpc target implementing flagd protobuf There are two mechanisms to provide configurations of sync providers, -* [URI patterns](#uri-patterns) -* [Source Configuration](#source-configuration) +- [URI patterns](#uri-patterns) +- [Source Configuration](#source-configuration) ## Sync provider configurations @@ -94,7 +99,7 @@ Any URI passed to flagd via the `--uri` flag must follow one of the 4 following it is passed to the correct implementation: | Sync | Prefix | Example | -|------------|------------------------|---------------------------------------| +| ---------- | ---------------------- | ------------------------------------- | | Kubernetes | `core.openfeature.dev` | `core.openfeature.dev/default/my-crd` | | Filepath | `file:` | `file:etc/flagd/my-flags.json` | | Remote | `http(s)://` | `https://my-flags.com/flags` | @@ -110,10 +115,11 @@ The flagd accepts a string argument, which should be a JSON representation of an Alternatively, these configurations can be passed to flagd via config file, specified using the `--config` flag. | Field | Type | Note | -|-------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| ----------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | | uri | required `string` | Flag configuration source of the provider | | provider | required `string` | Provider type - `file`, `kubernetes`, `http` or `grpc` | -| bearerToken | optional `string` | Used for http sync and token get appended to `Authorization` header with [bearer schema](https://www.rfc-editor.org/rfc/rfc6750#section-2.1) | +| bearerToken | optional `string` | Used for http sync; token gets appended to `Authorization` header with [bearer schema](https://www.rfc-editor.org/rfc/rfc6750#section-2.1) | +| interval | optional `uint32` | Used for http sync; requests will be made at this interval. Defaults to 5 seconds. | | tls | optional `boolean` | Enable/Disable secure TLS connectivity. Currently used only by GRPC sync. Default(ex:- if unset) is false, which will use an insecure connection | | providerID | optional `string` | Value binds to grpc connection's providerID field. GRPC server implementations may use this to identify connecting flagd instance | | selector | optional `string` | Value binds to grpc connection's selector field. GRPC server implementations may use this to filter flag configurations | @@ -127,11 +133,11 @@ Given below are example sync providers, startup command and equivalent config fi Sync providers, -* `file` - config/samples/example_flags.json -* `http` - -* `kubernetes` - default/my-flag-config -* `grpc`(insecure) - grpc-source:8080 -* `grpc`(secure) - my-flag-source:8080 +- `file` - config/samples/example_flags.json +- `http` - +- `kubernetes` - default/my-flag-config +- `grpc`(insecure) - grpc-source:8080 +- `grpc`(secure) - my-flag-source:8080 Startup command,