Skip to content

Commit

Permalink
getting pump type from config file when loading env vars
Browse files Browse the repository at this point in the history
  • Loading branch information
@mativm02
mativm02 committed Nov 22, 2023
1 parent ba5151b commit 5e7c75b
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 313 deletions.
17 changes: 9 additions & 8 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,6 @@ func (cfg *TykPumpConfiguration) LoadPumpsByEnv() error {
//first we look for all the pumps names in the env vars from the os
for _, env := range os.Environ() {
if strings.HasPrefix(env, PUMPS_ENV_PREFIX) {

// We trim everything after PUMPS_ENV_PREFIX. For example, if we have TYK_PUMP_PUMPS_CSV_TYPE we would have CSV_TYPE here
envWoPrefix := strings.TrimPrefix(env, PUMPS_ENV_PREFIX+"_")

Expand All @@ -302,7 +301,6 @@ func (cfg *TykPumpConfiguration) LoadPumpsByEnv() error {

//The name of the pump is always going to be the first keyword after the PUMPS_ENV_PREFIX
pmpName := strings.ToUpper(envSplit[0])

osPumpsEnvNames[pmpName] = true
}
}
Expand All @@ -317,18 +315,21 @@ func (cfg *TykPumpConfiguration) LoadPumpsByEnv() error {
pmp = jsonPump
}
//We look if the pmpName is one of our available pumps. If it's not, we look if the env with the TYPE filed exists.
var pmpType string
if _, ok := pumps.AvailablePumps[strings.ToLower(pmpName)]; !ok {
pmpType, found := os.LookupEnv(PUMPS_ENV_PREFIX + "_" + pmpName + "_TYPE")
var found bool
pmpType, found = os.LookupEnv(PUMPS_ENV_PREFIX + "_" + pmpName + "_TYPE")
if !found {
log.Error(fmt.Sprintf("TYPE Env var for pump %s not found", pmpName))
continue
if pmp.Type == "" {
log.Error(fmt.Sprintf("TYPE Env var for pump %s not found", pmpName))
continue
}
pmpType = pmp.Type
}
pmp.Type = pmpType
} else {
pmp.Type = pmpName
pmpType = pmpName
}

pmpType := pmp.Type
//We fetch the env vars for that pump.
overrideErr := envconfig.Process(PUMPS_ENV_PREFIX+"_"+pmpName, &pmp)
if overrideErr != nil {
Expand Down
113 changes: 113 additions & 0 deletions config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,116 @@ func TestIgnoreConfig(t *testing.T) {
assert.Equal(t, 5, initialConfig.PurgeDelay, "Nonexistent config file should not affect the configuration")
})
}

func TestTykPumpConfiguration_LoadPumpsByEnv(t *testing.T) {
tcs := []struct {
cfg *TykPumpConfiguration
wanted map[string]PumpConfig
setup func()
teardown func()
name string
}{
{
name: "no initial pumps",
cfg: &TykPumpConfiguration{},
setup: func() {
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_TYPE", "mongo-pump-aggregate")
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_META_MONGOURL", "mongodb://localhost:27017")
},
teardown: func() {
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_TYPE")
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_META_MONGOURL")
},
wanted: map[string]PumpConfig{
"ENVTEST": {
Type: "mongo-pump-aggregate",
Meta: map[string]interface{}{
"meta_env_prefix": ENV_PREVIX + "_PUMPS_ENVTEST_META",
},
},
},
},
{
name: "with initial pumps",
cfg: &TykPumpConfiguration{
Pumps: map[string]PumpConfig{
"INITIAL": {
Type: "csv",
Meta: map[string]interface{}{
"csv_dir": "/tmp",
},
},
},
},
setup: func() {
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_TYPE", "mongo-pump-aggregate")
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_META_MONGOURL", "mongodb://localhost:27017")
},
teardown: func() {
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_TYPE")
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_META_MONGOURL")
},
wanted: map[string]PumpConfig{
"INITIAL": {
Type: "csv",
Meta: map[string]interface{}{
"csv_dir": "/tmp",
},
},
"ENVTEST": {
Type: "mongo-pump-aggregate",
Meta: map[string]interface{}{
"meta_env_prefix": ENV_PREVIX + "_PUMPS_ENVTEST_META",
},
},
},
},
{
name: "type env var not found and type in cfg is empty",
cfg: &TykPumpConfiguration{},
setup: func() {
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_META_MONGOURL", "mongodb://localhost:27017")
},
teardown: func() {
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_META_MONGOURL")
},
wanted: map[string]PumpConfig{},
},
{
name: "type env var not found but type in cfg is set",
cfg: &TykPumpConfiguration{
Pumps: map[string]PumpConfig{
"ENVTEST": {
Type: "mongo",
},
},
},
setup: func() {
// Deliberately not setting the TYPE env var for ENVTEST
os.Setenv(ENV_PREVIX+"_PUMPS_ENVTEST_META_MONGOURL", "mongodb://localhost:27017")
},
teardown: func() {
os.Unsetenv(ENV_PREVIX + "_PUMPS_ENVTEST_META_MONGOURL")
},
wanted: map[string]PumpConfig{
"ENVTEST": {
Type: "mongo", // Expecting the predefined type to be retained
Meta: map[string]interface{}{
"meta_env_prefix": ENV_PREVIX + "_PUMPS_ENVTEST_META",
},
},
},
},
}

for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
tc.setup()
defer tc.teardown()

err := tc.cfg.LoadPumpsByEnv()
assert.NoError(t, err)
assert.Equal(t, tc.wanted, tc.cfg.Pumps)
})
}
}
34 changes: 5 additions & 29 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@ go 1.19
require (
github.com/DataDog/datadog-go v4.7.0+incompatible
github.com/TykTechnologies/gorpc v0.0.0-20210624160652-fe65bda0ccb9
github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517
github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632
github.com/TykTechnologies/storage v1.0.8
github.com/aws/aws-sdk-go-v2 v1.16.14
github.com/aws/aws-sdk-go-v2/config v1.9.0
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.9.0
github.com/buger/jsonparser v1.1.1
github.com/cenkalti/backoff/v4 v4.0.2
github.com/fatih/structs v1.1.0
github.com/go-redis/redis/v8 v8.3.1
Expand Down Expand Up @@ -51,9 +49,6 @@ require (
)

require (
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
Expand All @@ -76,15 +71,10 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/eclipse/paho.mqtt.golang v1.2.0 // indirect
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/helloeave/json v1.15.3 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.10.0 // indirect
Expand All @@ -95,40 +85,31 @@ require (
github.com/jackc/pgtype v1.8.1 // indirect
github.com/jackc/pgx/v4 v4.13.0 // indirect
github.com/jehiah/go-strftime v0.0.0-20151206194810-2efbe75097a5 // indirect
github.com/jensneuse/abstractlogger v0.0.4 // indirect
github.com/jensneuse/byte-template v0.0.0-20200214152254-4f3cf06e5c68 // indirect
github.com/jensneuse/pipeline v0.0.0-20200117120358-9fb4de085cd6 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.2 // indirect
github.com/joho/godotenv v1.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.10.6 // indirect
github.com/lintianzhi/graylogd v0.0.0-20180503131252-dc68342f04dc // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-sqlite3 v1.14.3 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.1.1 // indirect
github.com/mitchellh/reflectwalk v1.0.1 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/nats-io/nats.go v1.11.1-0.20210623165838-4b75fc59ae30 // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/olivere/elastic v6.2.31+incompatible // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.20.0 // indirect
github.com/pierrec/lz4 v2.6.0+incompatible // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/qri-io/jsonpointer v0.1.1 // indirect
github.com/qri-io/jsonschema v0.2.1 // indirect
github.com/r3labs/sse/v2 v2.8.1 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/segmentio/backo-go v0.0.0-20160424052352-204274ad699c // indirect
github.com/shirou/gopsutil v3.20.11+incompatible // indirect
github.com/syndtr/goleveldb v0.0.0-20190318030020-c3a204f8e965 // indirect
github.com/tidwall/gjson v1.11.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.0.4 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
Expand All @@ -139,19 +120,14 @@ require (
go.mongodb.org/mongo-driver v1.11.2 // indirect
go.opentelemetry.io/otel v0.13.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.18.1 // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
nhooyr.io/websocket v1.8.7 // indirect
)

//replace gorm.io/gorm => ../gorm
Expand Down
Loading

0 comments on commit 5e7c75b

Please sign in to comment.