From ffa05118ef0e9cbc020ecb9e1beb4b901d28a209 Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Fri, 8 Jan 2021 17:12:02 +0800 Subject: [PATCH 1/8] Add json and yaml format --- configs/example.yaml | 13 +++++- internal/routes/apollo/routes.go | 4 ++ internal/routes/apollo/routes_test.go | 66 +++++++++++++++++++++++++-- pkg/watcher/watcher.go | 29 +++++++++++- 4 files changed, 104 insertions(+), 8 deletions(-) diff --git a/configs/example.yaml b/configs/example.yaml index 8934656..5d1c22d 100644 --- a/configs/example.yaml +++ b/configs/example.yaml @@ -18,4 +18,15 @@ myAppID: yml: > plain text xml: > - plain text \ No newline at end of file + plain text + yaml: > + plain text + json: > + [ + { + "hello": "hall" + }, + { + "world": "hall" + } + ] \ No newline at end of file diff --git a/internal/routes/apollo/routes.go b/internal/routes/apollo/routes.go index 3b34305..e7f825f 100644 --- a/internal/routes/apollo/routes.go +++ b/internal/routes/apollo/routes.go @@ -116,7 +116,11 @@ func (a *Apollo) getNamespace(appID string, cluster string, namespace string) (w func (a *Apollo) getNamespaceConfig(extension string, namespace watcher.Namespace) (interface{}, error) { switch extension { case ".yml": + return map[string]string{"content": namespace.Yml}, nil + case ".yaml": return map[string]string{"content": namespace.Yaml}, nil + case ".json": + return map[string]string{"content": namespace.JSON}, nil case ".xml": return map[string]string{"content": namespace.XML}, nil case ".properties": diff --git a/internal/routes/apollo/routes_test.go b/internal/routes/apollo/routes_test.go index 65c7ba8..2019e27 100644 --- a/internal/routes/apollo/routes_test.go +++ b/internal/routes/apollo/routes_test.go @@ -30,16 +30,36 @@ var stubConfigs = []watcher.ConfigMap{ dynamic: p6spy: false primary: master +`, Yml: `spring: + datasource: + dynamic: + p6spy: false + primary: master `, XML: "", + JSON: `[ +{ + "abc":"lbs-test", +}, +{ + "def":"456", +} +] +`, }, "ns2": { ReleaseKey: "abc", Properties: map[string]string{}, - Yaml: `[raw] + Yml: `[raw] +key = value +`, + Yaml: `[raw] key = value `, XML: "plain text", + JSON: `[raw] +key = value +`, }, }, }, @@ -186,8 +206,8 @@ func TestGetNamespaceConfig(t *testing.T) { ) }) - t.Run("get yml", func(t *testing.T) { - cfg, err := a.getNamespaceConfig(".yml", stubConfigs[0]["app"]["cluster"]["ns"]) + t.Run("get yaml", func(t *testing.T) { + cfg, err := a.getNamespaceConfig(".yaml", stubConfigs[0]["app"]["cluster"]["ns"]) require.Nil(t, err) c, ok := cfg.(map[string]string) @@ -197,6 +217,8 @@ func TestGetNamespaceConfig(t *testing.T) { require.True(t, found) y := make(map[interface{}]interface{}) + t.Log(content) + t.Log(stubConfigs[0]["app"]["cluster"]["ns"].Yaml) err = yaml.Unmarshal([]byte(content), y) require.Nil(t, err) b, err := yaml.Marshal(y) @@ -227,8 +249,42 @@ func TestGetNamespaceConfig(t *testing.T) { ) }) - t.Run("get invalid yml", func(t *testing.T) { - cfg, err := a.getNamespaceConfig(".yml", stubConfigs[0]["app"]["cluster"]["ns2"]) + t.Run("get json", func(t *testing.T) { + cfg, err := a.getNamespaceConfig(".json", stubConfigs[0]["app"]["cluster"]["ns"]) + require.Nil(t, err) + + c, ok := cfg.(map[string]string) + require.True(t, ok) + + content, found := c["content"] + require.True(t, found) + + require.Equal( + t, + stubConfigs[0]["app"]["cluster"]["ns"].JSON, + content, + ) + }) + + t.Run("get invalid json", func(t *testing.T) { + cfg, err := a.getNamespaceConfig(".json", stubConfigs[0]["app"]["cluster"]["ns2"]) + require.Nil(t, err) + + c, ok := cfg.(map[string]string) + require.True(t, ok) + + content, found := c["content"] + require.True(t, found) + + require.Equal( + t, + stubConfigs[0]["app"]["cluster"]["ns2"].JSON, + content, + ) + }) + + t.Run("get invalid yaml", func(t *testing.T) { + cfg, err := a.getNamespaceConfig(".yaml", stubConfigs[0]["app"]["cluster"]["ns2"]) require.Nil(t, err) c, ok := cfg.(map[string]string) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index cf349f4..664c85d 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -2,6 +2,7 @@ package watcher import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -19,7 +20,9 @@ import ( type Namespace struct { ReleaseKey string `yaml:"releaseKey" json:"releaseKey"` Properties map[string]string `yaml:"properties" json:"properties"` - Yaml string `yaml:"yml" json:"yml"` + Yml string `yaml:"yml" json:"yml"` + Yaml string `yaml:"yaml" json:"yaml"` + JSON string `yaml:"json" json:"json"` XML string `yaml:"xml" json:"xml"` } @@ -156,7 +159,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { if nsKey == "" { return fmt.Errorf("invalid namespace name '%s' in %s/%s", nsKey, appKey, clusterKey) } - if ns.Properties == nil && ns.Yaml == "" && ns.XML == "" { + if ns.Properties == nil && ns.Yml == "" && ns.Yaml == "" && ns.XML == "" { return fmt.Errorf("invalid namespace '%s' in %s/%s", nsKey, appKey, clusterKey) } for configKey := range ns.Properties { @@ -164,6 +167,17 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { return fmt.Errorf("invalid config key '%s' in %s/%s/%s", configKey, appKey, clusterKey, nsKey) } } + // validate Yml + if ns.Yaml != "" { + cfg := make(map[interface{}]interface{}) + if err := yaml.Unmarshal([]byte(ns.Yml), &cfg); err != nil { + log.Get().Warn(fmt.Sprintf( + "failed to parse yaml config for namespace '%s' in %s/%s: %s", + nsKey, appKey, clusterKey, err.Error(), + )) + } + } + // validate Yaml if ns.Yaml != "" { cfg := make(map[interface{}]interface{}) @@ -174,6 +188,17 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { )) } } + + // validate JSON + if ns.JSON != "" { + var cfg []map[string]interface{} + if err := json.Unmarshal([]byte(ns.JSON), &cfg); err != nil { + log.Get().Warn(fmt.Sprintf( + "failed to parse json config for namespace '%s' in %s/%s: %s", + nsKey, appKey, clusterKey, err.Error(), + )) + } + } } } } From 2f368d665c7ec06ec2502b340bd4609305875b71 Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Fri, 8 Jan 2021 17:14:04 +0800 Subject: [PATCH 2/8] Revise example --- configs/example.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/configs/example.yaml b/configs/example.yaml index 5d1c22d..404a805 100644 --- a/configs/example.yaml +++ b/configs/example.yaml @@ -13,6 +13,15 @@ myAppID: dynamic: p6spy: false primary: master + json: > + [ + { + "hello": "hall" + }, + { + "hello": "hall" + } + ] myNamespace3: releaseKey: "20200309212653-7fec91b6d277b5ad" yml: > @@ -22,11 +31,4 @@ myAppID: yaml: > plain text json: > - [ - { - "hello": "hall" - }, - { - "world": "hall" - } - ] \ No newline at end of file + plain text From bac3dc8a7e4bfd0e8c7a213f7399640cb1c62f41 Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Fri, 8 Jan 2021 17:15:07 +0800 Subject: [PATCH 3/8] Clean up test log --- internal/routes/apollo/routes_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/routes/apollo/routes_test.go b/internal/routes/apollo/routes_test.go index 2019e27..45c8103 100644 --- a/internal/routes/apollo/routes_test.go +++ b/internal/routes/apollo/routes_test.go @@ -217,12 +217,9 @@ func TestGetNamespaceConfig(t *testing.T) { require.True(t, found) y := make(map[interface{}]interface{}) - t.Log(content) - t.Log(stubConfigs[0]["app"]["cluster"]["ns"].Yaml) err = yaml.Unmarshal([]byte(content), y) require.Nil(t, err) b, err := yaml.Marshal(y) - t.Log(string(b)) require.Nil(t, err) require.Equal( From 383535d14aadfb694c9b37639838c97c715f12f3 Mon Sep 17 00:00:00 2001 From: snakeyue Date: Fri, 8 Jan 2021 17:51:17 +0800 Subject: [PATCH 4/8] Update pkg/watcher/watcher.go Co-authored-by: Nishaad Ajani --- pkg/watcher/watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 664c85d..fd82404 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -172,7 +172,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { cfg := make(map[interface{}]interface{}) if err := yaml.Unmarshal([]byte(ns.Yml), &cfg); err != nil { log.Get().Warn(fmt.Sprintf( - "failed to parse yaml config for namespace '%s' in %s/%s: %s", + "failed to parse yml config for namespace '%s' in %s/%s: %s", nsKey, appKey, clusterKey, err.Error(), )) } From 91ee24072e2f544b308bf52514e0f43a1554d2d4 Mon Sep 17 00:00:00 2001 From: snakeyue Date: Fri, 8 Jan 2021 17:52:05 +0800 Subject: [PATCH 5/8] Update pkg/watcher/watcher.go Co-authored-by: Nishaad Ajani --- pkg/watcher/watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index fd82404..c4204e9 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -168,7 +168,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { } } // validate Yml - if ns.Yaml != "" { + if ns.Yml != "" { cfg := make(map[interface{}]interface{}) if err := yaml.Unmarshal([]byte(ns.Yml), &cfg); err != nil { log.Get().Warn(fmt.Sprintf( From 94aa9d6793ffe82ff51da62e27d32a239e5f451c Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Fri, 8 Jan 2021 18:05:28 +0800 Subject: [PATCH 6/8] Fix and add test --- pkg/watcher/watcher.go | 2 +- pkg/watcher/watcher_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index c4204e9..d8e3443 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -159,7 +159,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { if nsKey == "" { return fmt.Errorf("invalid namespace name '%s' in %s/%s", nsKey, appKey, clusterKey) } - if ns.Properties == nil && ns.Yml == "" && ns.Yaml == "" && ns.XML == "" { + if ns.Properties == nil && ns.Yml == "" && ns.Yaml == "" && ns.XML == "" && ns.JSON == "" { return fmt.Errorf("invalid namespace '%s' in %s/%s", nsKey, appKey, clusterKey) } for configKey := range ns.Properties { diff --git a/pkg/watcher/watcher_test.go b/pkg/watcher/watcher_test.go index 45c7809..dea08d1 100644 --- a/pkg/watcher/watcher_test.go +++ b/pkg/watcher/watcher_test.go @@ -23,8 +23,10 @@ var stubConfigs = []ConfigMap{ "mysql.uri": "mysql://root@localhost/mysql", "snowflake.uri": "http://192.168.0.1/snowflake", }, + Yml: "", Yaml: "", XML: "", + JSON: "", }, }, }, @@ -38,8 +40,10 @@ var stubConfigs = []ConfigMap{ "mysql.uri": "mysql://root@localhost/mysql2", "snowflake.uri": "http://192.168.0.1/snowflake", }, + Yml: "invalid yml; key = value", Yaml: "invalid yml; key = value", XML: "", + JSON: "", }, }, }, From 2e7c9f0e814431201435f32b160924cd4fc1e62c Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Mon, 11 Jan 2021 14:33:46 +0800 Subject: [PATCH 7/8] Enhance test coverage --- pkg/watcher/watcher.go | 2 +- pkg/watcher/watcher_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index d8e3443..5ddc476 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -191,7 +191,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { // validate JSON if ns.JSON != "" { - var cfg []map[string]interface{} + cfg := make(map[interface{}]interface{}) if err := json.Unmarshal([]byte(ns.JSON), &cfg); err != nil { log.Get().Warn(fmt.Sprintf( "failed to parse json config for namespace '%s' in %s/%s: %s", diff --git a/pkg/watcher/watcher_test.go b/pkg/watcher/watcher_test.go index dea08d1..fc1b231 100644 --- a/pkg/watcher/watcher_test.go +++ b/pkg/watcher/watcher_test.go @@ -43,7 +43,7 @@ var stubConfigs = []ConfigMap{ Yml: "invalid yml; key = value", Yaml: "invalid yml; key = value", XML: "", - JSON: "", + JSON: "invalid json; key = value", }, }, }, From c3151793a0e13fa099464b3f78296dd6c762f1cb Mon Sep 17 00:00:00 2001 From: "kelvin.yue" Date: Mon, 11 Jan 2021 16:18:05 +0800 Subject: [PATCH 8/8] Change to use interface{} for json unmarshal --- pkg/watcher/watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 5ddc476..2255b5e 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -191,7 +191,7 @@ func (w *Watcher) readConfigMap(log nlogger.Provider) error { // validate JSON if ns.JSON != "" { - cfg := make(map[interface{}]interface{}) + var cfg interface{} if err := json.Unmarshal([]byte(ns.JSON), &cfg); err != nil { log.Get().Warn(fmt.Sprintf( "failed to parse json config for namespace '%s' in %s/%s: %s",