feat: 1p secret provider to return the entire entry as JSON. fixes #1498

 (#1540)

bin/.op-2.19.0.pkg

@@ -0,0 +1 @@
+hermit

bin/op

@@ -0,0 +1 @@
+.op-2.19.0.pkg

common/configuration/1password_provider.go

@@ -1,16 +1,18 @@
 package configuration
 
 import (
-	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"regexp"
 	"strings"
 
+	"github.com/alecthomas/types/optional"
+
 	"github.com/TBD54566975/ftl/internal/exec"
-	"github.com/TBD54566975/ftl/internal/log"
+	"github.com/TBD54566975/ftl/internal/slices"
 )
 
 // OnePasswordProvider is a configuration provider that reads passwords from
@@ -25,6 +27,19 @@ func (OnePasswordProvider) Role() Secrets                               { return
 func (o OnePasswordProvider) Key() string                               { return "op" }
 func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil }
 
+// Load returns either a single field if the op:// reference specifies a field, or all fields if not.
+//
+// A single value/password:
+// op://Personal/With Spaces/username
+// op --format json item get --vault Personal "With Spaces" --fields=username
+// { id, value, ... }
+// "value"
+//
+// All fields:
+// op://Personal/With Spaces
+// op --format json item get --vault Personal "With Spaces"
+// { fields: [ { id, value, ... } ], ... }
+// { id: value, ... }
 func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {
 	_, err := exec.LookPath("op")
 	if err != nil {
@@ -36,16 +51,46 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([
 		return nil, fmt.Errorf("1Password secret reference must be a base64 encoded string: %w", err)
 	}
 
-	output, err := exec.Capture(ctx, ".", "op", "read", "-n", string(decoded))
+	parsedRef, err := decodeSecretRef(string(decoded))
+	if err != nil {
+		return nil, fmt.Errorf("1Password secret reference invalid: %w", err)
+	}
+
+	args := []string{"--format", "json", "item", "get", "--vault", parsedRef.Vault, parsedRef.Item}
+	v, fieldSpecified := parsedRef.Field.Get()
+	if fieldSpecified {
+		args = append(args, "--fields", v)
+	}
+	output, err := exec.Capture(ctx, ".", "op", args...)
 	if err != nil {
-		lines := bytes.Split(output, []byte("\n"))
-		logger := log.FromContext(ctx)
-		for _, line := range lines {
-			logger.Warnf("%s", line)
+		return nil, fmt.Errorf("run `op` with args %v: %w", args, err)
+	}
+
+	if fieldSpecified {
+		v, err := decodeSingle(output)
+		if err != nil {
+			return nil, err
 		}
-		return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err)
+
+		return json.Marshal(v.Value)
+	}
+
+	full, err := decodeFull(output)
+	if err != nil {
+		return nil, err
 	}
-	return json.Marshal(string(output))
+
+	// Filter out anything without a value
+	filtered := slices.Filter(full, func(e entry) bool {
+		return e.Value != ""
+	})
+	// Map to id: value
+	var mapped = make(map[string]string, len(filtered))
+	for _, e := range filtered {
+		mapped[e.ID] = e.Value
+	}
+
+	return json.Marshal(mapped)
 }
 
 func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) {
@@ -61,3 +106,93 @@ func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (
 }
 
 func (o OnePasswordProvider) Writer() bool { return o.OnePassword }
+
+type entry struct {
+	ID    string `json:"id"`
+	Value string `json:"value"`
+}
+
+type fullResponse struct {
+	Fields []entry `json:"fields"`
+}
+
+// Decode a full item response from op
+func decodeFull(output []byte) ([]entry, error) {
+	var full fullResponse
+	if err := json.Unmarshal(output, &full); err != nil {
+		return nil, fmt.Errorf("error decoding op full response: %w", err)
+	}
+	return full.Fields, nil
+}
+
+// Decode a single field from op
+func decodeSingle(output []byte) (*entry, error) {
+	var single entry
+	if err := json.Unmarshal(output, &single); err != nil {
+		return nil, fmt.Errorf("error decoding op single response: %w", err)
+	}
+	return &single, nil
+}
+
+// Custom parser for 1Password secret references because the format is not a standard URL, and we also need to
+// allow users to omit the field name so that we can support secrets with multiple fields.
+//
+// Does not support "section-name".
+//
+//	op://<vault-name>/<item-name>[/<field-name>]
+//
+// Secret references are case-insensitive and support the following characters:
+//
+//	alphanumeric characters (a-z, A-Z, 0-9), -, _, . and the whitespace character
+//
+// If an item or field name includes a / or an unsupported character, use the item
+// or field's unique identifier (ID) instead of its name.
+//
+// See https://developer.1password.com/docs/cli/secrets-reference-syntax/
+type secretRef struct {
+	Vault string
+	Item  string
+	Field optional.Option[string]
+}
+
+var validCharsRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_\. ]+$`)
+
+func decodeSecretRef(ref string) (*secretRef, error) {
+
+	// Take out and check the "op://" prefix
+	const prefix = "op://"
+	if !strings.HasPrefix(ref, prefix) {
+		return nil, fmt.Errorf("must start with \"op://\"")
+	}
+	ref = ref[len(prefix):]
+
+	parts := strings.Split(ref, "/")
+
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("must have at least 2 parts")
+	}
+	if len(parts) > 3 {
+		return nil, fmt.Errorf("must have at most 3 parts")
+	}
+
+	for _, part := range parts {
+		if part == "" {
+			return nil, fmt.Errorf("url parts must not be empty")
+		}
+
+		if !validCharsRegex.MatchString(part) {
+			return nil, fmt.Errorf("url part %q contains unsupported characters. regex: %q", part, validCharsRegex)
+		}
+	}
+
+	secret := secretRef{
+		Vault: parts[0],
+		Item:  parts[1],
+		Field: optional.None[string](),
+	}
+	if len(parts) == 3 {
+		secret.Field = optional.Some(parts[2])
+	}
+
+	return &secret, nil
+}

common/configuration/1password_provider_test.go

@@ -0,0 +1,82 @@
+package configuration
+
+import (
+	"github.com/alecthomas/types/optional"
+	"reflect"
+	"testing"
+)
+
+func TestDecodeSecretRef(t *testing.T) {
+	tests := []struct {
+		name    string
+		ref     string
+		want    *secretRef
+		wantErr bool
+	}{
+		{
+			name: "simple with field",
+			ref:  "op://development/Access Keys/access_key_id",
+			want: &secretRef{
+				Vault: "development",
+				Item:  "Access Keys",
+				Field: optional.Some("access_key_id"),
+			},
+		},
+		{
+			name: "simple without field",
+			ref:  "op://vault/item",
+			want: &secretRef{
+				Vault: "vault",
+				Item:  "item",
+				Field: optional.None[string](),
+			},
+		},
+		{
+			name: "lots of spaces",
+			ref:  "op://My Awesome Vault/My Awesome Item/My Awesome Field",
+			want: &secretRef{
+				Vault: "My Awesome Vault",
+				Item:  "My Awesome Item",
+				Field: optional.Some("My Awesome Field"),
+			},
+		},
+		{
+			name:    "missing op://",
+			ref:     "development/Access Keys/access_key_id",
+			wantErr: true,
+		},
+		{
+			name:    "empty parts",
+			ref:     "op://development//access_key_id",
+			wantErr: true,
+		},
+		{
+			name:    "invalid characters",
+			ref:     "op://development/aws/acce$s",
+			wantErr: true,
+		},
+		{
+			name:    "too many parts",
+			ref:     "op://development/Access Keys/access_key_id/extra",
+			wantErr: true,
+		},
+		{
+			name:    "too few parts",
+			ref:     "op://development",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := decodeSecretRef(tt.ref)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("decodeSecretRef() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("decodeSecretRef() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}