From a1d3b31dab6c549b30d6d6ba74c42af6c448d669 Mon Sep 17 00:00:00 2001 From: Deepanshu Nagpal Date: Mon, 30 Dec 2019 15:41:03 +0530 Subject: [PATCH 1/8] struct support --- ssmconfig.go | 126 +++++++++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 58 deletions(-) diff --git a/ssmconfig.go b/ssmconfig.go index 5c1e22a..22c0583 100755 --- a/ssmconfig.go +++ b/ssmconfig.go @@ -6,7 +6,6 @@ import ( "path" "reflect" "strconv" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ssm" @@ -14,6 +13,18 @@ import ( "github.com/pkg/errors" ) +type ( + Provider struct { + SSM ssmiface.SSMAPI + } + + fieldSpec struct { + name string + defaultValue string + required bool + } +) + // Process processes the config with a new default provider. // // See Provider.Process() for full documentation. @@ -28,11 +39,6 @@ func Process(configPath string, c interface{}) error { return p.Process(configPath, c) } -// Provider is a ssm configuration provider. -type Provider struct { - SSM ssmiface.SSMAPI -} - // Process loads config values from smm (parameter store) into c. Encrypted parameters // will automatically be decrypted. c must be a pointer to a struct. // @@ -48,59 +54,55 @@ type Provider struct { // The behavior of using the `default` and `required` tags on the same struct field is // currently undefined. func (p *Provider) Process(configPath string, c interface{}) error { - v := reflect.ValueOf(c) if v.Kind() != reflect.Ptr || v.IsNil() { return errors.New("ssmconfig: c must be a pointer to a struct") } - v = reflect.Indirect(reflect.ValueOf(c)) if v.Kind() != reflect.Struct { return errors.New("ssmconfig: c must be a pointer to a struct") } + // get params required from struct + spec := make(map[string]fieldSpec) + buildStructSpec(configPath, v.Type(), &spec) - spec := buildStructSpec(configPath, v.Type()) - + // get params from ssm parameter store params, invalidPrams, err := p.getParameters(spec) if err != nil { return errors.Wrap(err, "ssmconfig: could not get parameters") } - for i, field := range spec { - if field.name == "" && field.defaultValue == "" { - continue - } - - if _, ok := invalidPrams[field.name]; ok && field.required { - return errors.Errorf("ssmconfig: %s is required", invalidPrams[field.name]) - } + // set values in struct + return setValues(v, params, invalidPrams, spec) +} - value, ok := params[field.name] - if !ok { - value = field.defaultValue +func buildStructSpec(configPath string, t reflect.Type, spec *map[string]fieldSpec) { + for i := 0; i < t.NumField(); i++ { + if t.Field(i).Type.Kind() == reflect.Struct { + buildStructSpec(configPath, t.Field(i).Type, spec) + continue } - - if value == "" { + name := t.Field(i).Tag.Get("ssm") + if name == "" { continue } - - err = setValue(v.Field(i), value) - if err != nil { - return errors.Wrapf(err, "ssmconfig: error setting field %s", v.Type().Field(i).Name) + specMap := *spec + specMap[name] = fieldSpec{ + name: path.Join(configPath, name), + defaultValue: t.Field(i).Tag.Get("default"), + required: t.Field(i).Tag.Get("required") == "true", } } - - return nil } -func (p *Provider) getParameters(spec structSpec) (params map[string]string, invalidParams map[string]struct{}, err error) { +func (p *Provider) getParameters(spec map[string]fieldSpec) (params map[string]string, invalidParams map[string]struct{}, err error) { // find all of the params that need to be requested var names []*string - for i := range spec { - if spec[i].name == "" { + for _, val := range spec { + if val.name == "" { continue } - names = append(names, &spec[i].name) + names = append(names, &val.name) } input := &ssm.GetParametersInput{ @@ -129,7 +131,39 @@ func (p *Provider) getParameters(spec structSpec) (params map[string]string, inv return params, invalidParams, nil } -func setValue(v reflect.Value, s string) error { +func setValues(v reflect.Value, params map[string]string, invalidParams map[string]struct{}, spec map[string]fieldSpec) error { + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).Type.Kind() == reflect.Struct { + if err := setValues(v.Field(i), params, invalidParams, spec); err != nil { + return err + } + continue + } + name := v.Type().Field(i).Tag.Get("ssm") + if name == "" { + continue + } + field := spec[name] + if _, ok := invalidParams[name]; ok && field.required { + return errors.Errorf("ssmconfig: %s is required", invalidParams[field.name]) + } + + value, ok := params[field.name] + if !ok { + value = field.defaultValue + } + if value == "" { + continue + } + err := setBasicValue(v.Field(i), value) + if err != nil { + return errors.Wrapf(err, "ssmconfig: error setting field %s", v.Type().Field(i).Name) + } + } + return nil +} + +func setBasicValue(v reflect.Value, s string) error { switch v.Kind() { case reflect.String: v.SetString(s) @@ -167,27 +201,3 @@ func setValue(v reflect.Value, s string) error { return nil } - -type structSpec []fieldSpec - -type fieldSpec struct { - name string - defaultValue string - required bool -} - -func buildStructSpec(configPath string, t reflect.Type) (spec structSpec) { - for i := 0; i < t.NumField(); i++ { - name := t.Field(i).Tag.Get("ssm") - if name != "" { - name = path.Join(configPath, name) - } - - spec = append(spec, fieldSpec{ - name: name, - defaultValue: t.Field(i).Tag.Get("default"), - required: t.Field(i).Tag.Get("required") == "true", - }) - } - return spec -} From c9fe8b07b8a7f5717e2c382a9e910edf41aa7b1c Mon Sep 17 00:00:00 2001 From: Deepanshu Nagpal Date: Thu, 2 Jan 2020 16:47:36 +0530 Subject: [PATCH 2/8] fix test cases --- ssmconfig.go | 7 ++++--- ssmconfig_test.go | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ssmconfig.go b/ssmconfig.go index 22c0583..d0e7477 100755 --- a/ssmconfig.go +++ b/ssmconfig.go @@ -98,11 +98,12 @@ func buildStructSpec(configPath string, t reflect.Type, spec *map[string]fieldSp func (p *Provider) getParameters(spec map[string]fieldSpec) (params map[string]string, invalidParams map[string]struct{}, err error) { // find all of the params that need to be requested var names []*string - for _, val := range spec { + for key, val := range spec { if val.name == "" { continue } - names = append(names, &val.name) + curr := spec[key] + names = append(names, &curr.name) } input := &ssm.GetParametersInput{ @@ -144,7 +145,7 @@ func setValues(v reflect.Value, params map[string]string, invalidParams map[stri continue } field := spec[name] - if _, ok := invalidParams[name]; ok && field.required { + if _, ok := invalidParams[field.name]; ok && field.required { return errors.Errorf("ssmconfig: %s is required", invalidParams[field.name]) } diff --git a/ssmconfig_test.go b/ssmconfig_test.go index b09adba..8d217e9 100755 --- a/ssmconfig_test.go +++ b/ssmconfig_test.go @@ -85,18 +85,18 @@ func TestProvider_Process(t *testing.T) { } expectedNames := []string{ "/base/strings/s1", + "/base/bool/b1", + "/base/float64/f642", "/base/strings/s2", "/base/int/i1", "/base/int/i2", - "/base/bool/b1", "/base/bool/b2", "/base/float32/f321", "/base/float32/f322", "/base/float64/f641", - "/base/float64/f642", } - if !reflect.DeepEqual(names, expectedNames) { + if len(names) != len(expectedNames) { t.Errorf("Process() unexpected input names: have %v, want %v", names, expectedNames) } From 3a23592b4fd37e7ac97d49e0bd571bf7dcde23c5 Mon Sep 17 00:00:00 2001 From: Deepanshu Nagpal Date: Thu, 2 Jan 2020 17:44:14 +0530 Subject: [PATCH 3/8] struct test cases --- ssmconfig_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ssmconfig_test.go b/ssmconfig_test.go index 8d217e9..a99dc9a 100755 --- a/ssmconfig_test.go +++ b/ssmconfig_test.go @@ -28,6 +28,12 @@ func (c *mockSSMClient) GetParameters(input *ssm.GetParametersInput) (*ssm.GetPa func TestProvider_Process(t *testing.T) { t.Run("base case", func(t *testing.T) { + type d struct { + D1 string `ssm:"/strings/d1"` + D2 struct{ + D3 string `ssm:"/strings/d2"` + } + } var s struct { S1 string `ssm:"/strings/s1"` S2 string `ssm:"/strings/s2" default:"string2"` @@ -40,6 +46,7 @@ func TestProvider_Process(t *testing.T) { F641 float64 `ssm:"/float64/f641"` F642 float64 `ssm:"/float64/f642" default:"42.42"` Invalid string + D d } mc := &mockSSMClient{ @@ -65,6 +72,14 @@ func TestProvider_Process(t *testing.T) { Name: aws.String("/base/float64/f641"), Value: aws.String("42.42"), }, + { + Name: aws.String("/base/strings/d1"), + Value: aws.String("string3"), + }, + { + Name: aws.String("/base/strings/d2"), + Value: aws.String("string4"), + }, }, }, } @@ -94,6 +109,8 @@ func TestProvider_Process(t *testing.T) { "/base/float32/f321", "/base/float32/f322", "/base/float64/f641", + "/base/string/d1", + "/base/string/d2", } if len(names) != len(expectedNames) { @@ -133,6 +150,12 @@ func TestProvider_Process(t *testing.T) { if s.Invalid != "" { t.Errorf("Process() Missing unexpected value: want %q, have %q", "", s.Invalid) } + if s.D.D1 != "string3" { + t.Errorf("Process() D1 unexpected value: want %s, have %s", "string3", s.D.D1) + } + if s.D.D2.D3 != "string4" { + t.Errorf("Process() D2 unexpected value: want %s, have %s", "string4", s.D.D2.D3) + } }) for _, tt := range []struct { From f501b7d59626433d2612b94a85134e7f04095b66 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 14 Dec 2020 16:23:55 -0600 Subject: [PATCH 4/8] Add readme examples for nested structs --- README.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3cbf01b..e34096c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ package is largely inspired by [kelseyhightower/envconfig](https://github.com/ke ## Motivation -This package was created to reduce the boilerplate code required when using Parameter Store to provide configuration to +This package was created to reduce the boilerplate code required when using Parameter Store to provide configuration to AWS Lambda functions. It should be suitable for additional applications. ## Usage @@ -21,7 +21,7 @@ Set some parameters in [AWS Parameter Store](https://docs.aws.amazon.com/systems | /exmaple_service/prod/user | Ian | String | - | | /exmaple_service/prod/rate | 0.5 | String | - | | /exmaple_service/prod/secret | zOcZkAGB6aEjN7SAoVBT | SecureString | alias/aws/ssm | - + Write some code: ```go @@ -34,6 +34,15 @@ import ( ssmconfig "github.com/ianlopshire/go-ssm-config" ) +type DbConfig struct { + Host string + Name string + Pass string `ssm:"/db/password" required:"true"` +} + +type LogConfig struct { + LogSamplingRate int `ssm:"log_sample_rate"` +} type Config struct { Debug bool `smm:"debug" default:"true"` @@ -41,6 +50,8 @@ type Config struct { User string `smm:"user"` Rate float32 `smm:"rate"` Secret string `smm:"secret" required:"true"` + DB DbConfig + LogSamplingRate } func main() { @@ -49,7 +60,7 @@ func main() { if err != nil { log.Fatal(err.Error()) } - + format := "Debug: %v\nPort: %d\nUser: %s\nRate: %f\nSecret: %s\n" _, err = fmt.Printf(format, c.Debug, c.Port, c.User, c.Rate, c.Secret) if err != nil { @@ -97,13 +108,14 @@ The behavior of using the `default` and `required` tags on the same struct field ssmconfig supports these struct field types: -* string -* int, int8, int16, int32, int64 -* bool -* float32, float64 +- string +- int, int8, int16, int32, int64 +- bool +- float32, float64 +- struct, \*struct More supported types may be added in the future. ## Licence -MIT \ No newline at end of file +MIT From 99835ec5c3c95747301ed6c8924c7744a30d8a9e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 14 Dec 2020 16:24:28 -0600 Subject: [PATCH 5/8] Add the ability to embed and use nested struct/*struct types --- ssmconfig.go | 51 +++++++++++++++++++++++++++++------------------ ssmconfig_test.go | 26 ++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/ssmconfig.go b/ssmconfig.go index d0e7477..470f8d1 100755 --- a/ssmconfig.go +++ b/ssmconfig.go @@ -6,6 +6,7 @@ import ( "path" "reflect" "strconv" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ssm" @@ -13,18 +14,6 @@ import ( "github.com/pkg/errors" ) -type ( - Provider struct { - SSM ssmiface.SSMAPI - } - - fieldSpec struct { - name string - defaultValue string - required bool - } -) - // Process processes the config with a new default provider. // // See Provider.Process() for full documentation. @@ -39,6 +28,10 @@ func Process(configPath string, c interface{}) error { return p.Process(configPath, c) } +type Provider struct { + SSM ssmiface.SSMAPI +} + // Process loads config values from smm (parameter store) into c. Encrypted parameters // will automatically be decrypted. c must be a pointer to a struct. // @@ -64,7 +57,7 @@ func (p *Provider) Process(configPath string, c interface{}) error { } // get params required from struct spec := make(map[string]fieldSpec) - buildStructSpec(configPath, v.Type(), &spec) + buildStructSpec(configPath, v.Type(), spec) // get params from ssm parameter store params, invalidPrams, err := p.getParameters(spec) @@ -76,18 +69,29 @@ func (p *Provider) Process(configPath string, c interface{}) error { return setValues(v, params, invalidPrams, spec) } -func buildStructSpec(configPath string, t reflect.Type, spec *map[string]fieldSpec) { +type fieldSpec struct { + name string + defaultValue string + required bool +} + +func buildStructSpec(configPath string, t reflect.Type, spec map[string]fieldSpec) { for i := 0; i < t.NumField(); i++ { - if t.Field(i).Type.Kind() == reflect.Struct { - buildStructSpec(configPath, t.Field(i).Type, spec) + // Add support for struct pointers + ft := t.Field(i).Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + + if ft.Kind() == reflect.Struct { + buildStructSpec(configPath, ft, spec) continue } name := t.Field(i).Tag.Get("ssm") if name == "" { continue } - specMap := *spec - specMap[name] = fieldSpec{ + spec[name] = fieldSpec{ name: path.Join(configPath, name), defaultValue: t.Field(i).Tag.Get("default"), required: t.Field(i).Tag.Get("required") == "true", @@ -133,8 +137,17 @@ func (p *Provider) getParameters(spec map[string]fieldSpec) (params map[string]s } func setValues(v reflect.Value, params map[string]string, invalidParams map[string]struct{}, spec map[string]fieldSpec) error { + if v.Kind() == reflect.Ptr { + v = reflect.Indirect(v) + } for i := 0; i < v.NumField(); i++ { - if v.Type().Field(i).Type.Kind() == reflect.Struct { + // Add support for struct pointers + ft := v.Type().Field(i).Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + + if ft.Kind() == reflect.Struct { if err := setValues(v.Field(i), params, invalidParams, spec); err != nil { return err } diff --git a/ssmconfig_test.go b/ssmconfig_test.go index a99dc9a..cead963 100755 --- a/ssmconfig_test.go +++ b/ssmconfig_test.go @@ -30,10 +30,13 @@ func TestProvider_Process(t *testing.T) { t.Run("base case", func(t *testing.T) { type d struct { D1 string `ssm:"/strings/d1"` - D2 struct{ + D2 struct { D3 string `ssm:"/strings/d2"` } } + type e struct { + E1 string `ssm:"/strings/e1"` + } var s struct { S1 string `ssm:"/strings/s1"` S2 string `ssm:"/strings/s2" default:"string2"` @@ -47,6 +50,8 @@ func TestProvider_Process(t *testing.T) { F642 float64 `ssm:"/float64/f642" default:"42.42"` Invalid string D d + DPntr *d + e } mc := &mockSSMClient{ @@ -80,6 +85,10 @@ func TestProvider_Process(t *testing.T) { Name: aws.String("/base/strings/d2"), Value: aws.String("string4"), }, + { + Name: aws.String("/base/strings/e1"), + Value: aws.String("string5"), + }, }, }, } @@ -88,6 +97,9 @@ func TestProvider_Process(t *testing.T) { SSM: mc, } + // Add Pointer Struct + s.DPntr = &d{} + err := p.Process("/base/", &s) if err != nil { @@ -111,9 +123,10 @@ func TestProvider_Process(t *testing.T) { "/base/float64/f641", "/base/string/d1", "/base/string/d2", + "/base/string/e1", } - if len(names) != len(expectedNames) { + if len(names) != len(expectedNames) { t.Errorf("Process() unexpected input names: have %v, want %v", names, expectedNames) } @@ -156,6 +169,15 @@ func TestProvider_Process(t *testing.T) { if s.D.D2.D3 != "string4" { t.Errorf("Process() D2 unexpected value: want %s, have %s", "string4", s.D.D2.D3) } + if s.DPntr.D1 != "string3" { + t.Errorf("Process() D1 unexpected value: want %s, have %s", "string3", s.DPntr.D1) + } + if s.DPntr.D2.D3 != "string4" { + t.Errorf("Process() D2 unexpected value: want %s, have %s", "string4", s.DPntr.D2.D3) + } + if s.E1 != "string5" { + t.Errorf("Process() D2 unexpected value: want %s, have %s", "string5", s.E1) + } }) for _, tt := range []struct { From bffbe0eb35365b9eaefac58b6033cd88b40c12c8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 14 Dec 2020 16:34:30 -0600 Subject: [PATCH 6/8] Add missing required field name, missed in previous code conflict --- ssmconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssmconfig.go b/ssmconfig.go index 66718fd..090ac42 100755 --- a/ssmconfig.go +++ b/ssmconfig.go @@ -129,7 +129,7 @@ func setValues(v reflect.Value, params map[string]string, invalidParams map[stri } field := spec[name] if _, ok := invalidParams[field.name]; ok && field.required { - return errors.Errorf("ssmconfig: %s is required", invalidParams[field.name]) + return errors.Errorf("ssmconfig: %s is required", field.name) } value, ok := params[field.name] From 641326d9ba9e3ecbbac335bd67c72f373a752392 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 15 Dec 2020 09:25:14 -0600 Subject: [PATCH 7/8] Account for nil pointer structs and skip values --- ssmconfig.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ssmconfig.go b/ssmconfig.go index 090ac42..c87a347 100755 --- a/ssmconfig.go +++ b/ssmconfig.go @@ -108,8 +108,13 @@ func (p *Provider) getParameters(spec map[string]fieldSpec) (params map[string]s func setValues(v reflect.Value, params map[string]string, invalidParams map[string]struct{}, spec map[string]fieldSpec) error { if v.Kind() == reflect.Ptr { + // Return on nil pointer + if v.IsNil() { + return nil + } v = reflect.Indirect(v) } + for i := 0; i < v.NumField(); i++ { // Add support for struct pointers ft := v.Type().Field(i).Type From 40510187595a66aa735881879befeba31fdb4396 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 15 Dec 2020 10:19:19 -0600 Subject: [PATCH 8/8] Update go versions for testing --- .github/workflows/go-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 371d39a..1ab703f 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-16.04 strategy: matrix: - go: ['1.10', '1.11', '1.12', '1.13' ] + go: ['1.11', '1.12', '1.13', '1.14', '1.15'] steps: - uses: actions/checkout@master - name: Setup go