AWS: Parameter Store - Adds the ability to use secrets

pkg/manifest/manifest.go

@@ -21,10 +21,12 @@ type Manifest struct {
 	Params      Params      `yaml:"params,omitempty"`
 	Resources   Resources   `yaml:"resources,omitempty"`
 	Services    Services    `yaml:"services,omitempty"`
+	Secrets     Secrets     `yaml:"secrets,omitempty"`
 	Timers      Timers      `yaml:"timers,omitempty"`
 
 	attributes map[string]bool
 	env        map[string]string
+	secrets    map[string]string
 }
 
 func init() {
@@ -66,6 +68,14 @@ func Load(data []byte, env map[string]string) (*Manifest, error) {
 		return nil, err
 	}
 
+	if err := m.CombineSecrets(); err != nil {
+		return nil, err
+	}
+
+	if err := m.ValidateSecrets(); err != nil {
+		return nil, err
+	}
+
 	return &m, nil
 }
 
@@ -137,6 +147,21 @@ func (m *Manifest) CombineEnv() error {
 	return nil
 }
 
+// used only for tests
+func (m *Manifest) SetSecrets(secrets map[string]string) {
+	m.secrets = secrets
+}
+
+func (m *Manifest) CombineSecrets() error {
+	for i, s := range m.Services {
+		me := make([]string, len(m.Secrets))
+		copy(me, m.Secrets)
+		m.Services[i].Secrets = append(me, s.Secrets...)
+	}
+
+	return nil
+}
+
 func (m *Manifest) Service(name string) (*Service, error) {
 	for _, s := range m.Services {
 		if s.Name == name {
@@ -194,6 +219,53 @@ func (m *Manifest) ServiceEnvironment(service string) (map[string]string, error)
 	return env, nil
 }
 
+func (m *Manifest) ServiceSecrets(service string) (map[string]string, error) {
+	s, err := m.Service(service)
+	if err != nil {
+		return nil, err
+	}
+
+	secrets := map[string]string{}
+
+	missing := []string{}
+
+	for _, e := range s.Secrets {
+		parts := strings.SplitN(e, "=", 2)
+
+		switch len(parts) {
+		case 1:
+			if parts[0] == "*" {
+				for k, v := range m.secrets {
+					secrets[k] = v
+				}
+			} else {
+				v, ok := m.secrets[parts[0]]
+				if !ok {
+					missing = append(missing, parts[0])
+				}
+				secrets[parts[0]] = v
+			}
+		case 2:
+			v, ok := m.secrets[parts[0]]
+			if ok {
+				secrets[parts[0]] = v
+			} else {
+				secrets[parts[0]] = parts[1]
+			}
+		default:
+			return nil, fmt.Errorf("invalid secrets declaration: %s", e)
+		}
+	}
+
+	if len(missing) > 0 {
+		sort.Strings(missing)
+
+		return nil, fmt.Errorf("required secrets: %s", strings.Join(missing, ", "))
+	}
+
+	return secrets, nil
+}
+
 // ValidateEnv returns an error if required env vars for a service are not available
 // It also filters m.env to the union of all service env vars defined in the manifest
 func (m *Manifest) ValidateEnv() error {
@@ -219,6 +291,31 @@ func (m *Manifest) ValidateEnv() error {
 	return nil
 }
 
+// ValidateSecrets returns an error if required secrets vars for a service are not available
+// It also filters m.secrets to the union of all service secrets vars defined in the manifest
+func (m *Manifest) ValidateSecrets() error {
+	keys := map[string]bool{}
+
+	for _, s := range m.Services {
+		secrets, err := m.ServiceSecrets(s.Name)
+		if err != nil {
+			return err
+		}
+
+		for k := range secrets {
+			keys[k] = true
+		}
+	}
+
+	for k := range m.secrets {
+		if !keys[k] {
+			delete(m.secrets, k)
+		}
+	}
+
+	return nil
+}
+
 func (m *Manifest) ApplyDefaults() error {
 	for i, s := range m.Services {
 		if s.Build.Path == "" && s.Image == "" {

pkg/manifest/manifest_test.go

@@ -42,6 +42,7 @@ func TestManifestLoad(t *testing.T) {
 					"DEVELOPMENT=false",
 					"SECRET",
 				},
+				Secrets: []string{},
 				Health: manifest.ServiceHealth{
 					Grace:    10,
 					Path:     "/",
@@ -74,6 +75,7 @@ func TestManifestLoad(t *testing.T) {
 				Environment: []string{
 					"SECRET",
 				},
+				Secrets: []string{},
 				Port: manifest.ServicePort{Port: 2000, Scheme: "https"},
 				Scale: manifest.ServiceScale{
 					Count:  manifest.ServiceScaleCount{Min: 1, Max: 1},
@@ -91,6 +93,7 @@ func TestManifestLoad(t *testing.T) {
 				Command: "foo",
 				Domains: []string{"baz.example.org", "qux.example.org"},
 				Drain:   60,
+				Secrets: []string{},
 				Health: manifest.ServiceHealth{
 					Grace:    2,
 					Interval: 5,
@@ -114,6 +117,7 @@ func TestManifestLoad(t *testing.T) {
 				},
 				Command: "",
 				Drain:   30,
+				Secrets: []string{},
 				Health: manifest.ServiceHealth{
 					Grace:    5,
 					Interval: 5,
@@ -135,6 +139,7 @@ func TestManifestLoad(t *testing.T) {
 				},
 				Command: "",
 				Drain:   30,
+				Secrets: []string{},
 				Health: manifest.ServiceHealth{
 					Grace:    5,
 					Interval: 5,
@@ -177,6 +182,7 @@ func TestManifestLoad(t *testing.T) {
 				Environment: []string{
 					"SECRET",
 				},
+				Secrets: []string{},
 				Port: manifest.ServicePort{Port: 2000, Scheme: "https"},
 				Scale: manifest.ServiceScale{
 					Count:  manifest.ServiceScaleCount{Min: 1, Max: 1},
@@ -200,6 +206,7 @@ func TestManifestLoad(t *testing.T) {
 					Path:     ".",
 				},
 				Drain: 30,
+				Secrets: []string{},
 				Health: manifest.ServiceHealth{
 					Grace:    5,
 					Path:     "/",
@@ -335,6 +342,9 @@ func TestManifestLoadSimple(t *testing.T) {
 					"REQUIRED",
 					"DEFAULT=true",
 				},
+				Secrets: manifest.Secrets{
+					"DATABASE=sql://localhost",
+				},
 				Health: manifest.ServiceHealth{
 					Grace:    5,
 					Interval: 5,
@@ -351,7 +361,7 @@ func TestManifestLoadSimple(t *testing.T) {
 		},
 	}
 
-	n.SetAttributes([]string{"services", "services.web", "services.web.build", "services.web.environment"})
+	n.SetAttributes([]string{"services", "services.web", "services.web.build", "services.web.environment", "services.web.secrets"})
 	n.SetEnv(map[string]string{"REQUIRED": "test"})
 
 	// env processing that normally happens as part of load

pkg/manifest/secrets.go

@@ -0,0 +1,3 @@
+package manifest
+
+type Secrets []string

pkg/manifest/service.go

@@ -16,6 +16,7 @@ type Service struct {
 	Domains     ServiceDomains `yaml:"domain,omitempty"`
 	Drain       int            `yaml:"drain,omitempty"`
 	Environment Environment    `yaml:"environment,omitempty"`
+	Secrets     Secrets        `yaml:"secrets,omitempty"`
 	Health      ServiceHealth  `yaml:"health,omitempty"`
 	Image       string         `yaml:"image,omitempty"`
 	Init        bool           `yaml:"init,omitempty"`
@@ -135,6 +136,24 @@ func (s Service) EnvironmentKeys() string {
 	return strings.Join(keys, ",")
 }
 
+func (s Service) SecretstKeys() string {
+	kh := map[string]bool{}
+
+	for _, e := range s.Secrets {
+		kh[strings.Split(e, "=")[0]] = true
+	}
+
+	keys := []string{}
+
+	for k := range kh {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
+	return strings.Join(keys, ",")
+}
+
 func (s Service) GetName() string {
 	return s.Name
 }

pkg/manifest/testdata/env.yml

@@ -8,6 +8,8 @@ services:
   q-train-intent:
     environment:
       - QUEUE_NAME=train-intent
+    secrets:
+      - DATABASE=test://example/db
   q-delete-intent:
     environment:
       - QUEUE_NAME=delete-intent

pkg/manifest/testdata/simple.yml

@@ -4,3 +4,5 @@ services:
     environment:
       - REQUIRED
       - DEFAULT=true
+    secrets:
+      - DATABASE=sql://localhost

provider/aws/formation/service.json.tmpl

@@ -507,6 +507,12 @@
                 { "Ref": "AWS::NoValue" }
               ],
               "Ulimits": [ { "Name": "nofile", "SoftLimit": "1024000", "HardLimit": "1024000" } ]
+              "Secrets": [
+                {{ range $k, $v := .Secrets }}
+                  { "Name": "{{$k}}", "ValueFrom": {{ safe $v }} },
+                {{ end }}
+                { "Ref": "AWS::NoValue" }
+              ]
             }
           ],
           "Cpu": { "Fn::If": [ "Fargate", { "Ref": "Cpu" }, { "Ref": "AWS::NoValue" } ] },