From cfd2f1ffcc62f4ffbb3e4ab317949c454d08be95 Mon Sep 17 00:00:00 2001 From: gak Date: Fri, 17 May 2024 10:31:32 +1000 Subject: [PATCH 1/8] chore: hermit op --- bin/.op-2.19.0.pkg | 1 + bin/op | 1 + 2 files changed, 2 insertions(+) create mode 120000 bin/.op-2.19.0.pkg create mode 120000 bin/op diff --git a/bin/.op-2.19.0.pkg b/bin/.op-2.19.0.pkg new file mode 120000 index 0000000000..383f4511d4 --- /dev/null +++ b/bin/.op-2.19.0.pkg @@ -0,0 +1 @@ +hermit \ No newline at end of file diff --git a/bin/op b/bin/op new file mode 120000 index 0000000000..5de1cd899f --- /dev/null +++ b/bin/op @@ -0,0 +1 @@ +.op-2.19.0.pkg \ No newline at end of file From e459427fba1fc69f3709fe58d3b1904d088777dd Mon Sep 17 00:00:00 2001 From: gak Date: Fri, 17 May 2024 13:58:13 +1000 Subject: [PATCH 2/8] wip --- cmd/ftl/cmd_secret.go | 14 +++ common/configuration/1password_provider.go | 91 +++++++++++++++++-- .../configuration/1password_provider_test.go | 82 +++++++++++++++++ common/configuration/manager.go | 21 +++++ ftl-project.toml | 12 +++ 5 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 common/configuration/1password_provider_test.go diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index 0d8a03c83b..f756d9df89 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/TBD54566975/ftl/internal/log" "io" "os" @@ -84,22 +85,35 @@ Returns a JSON-encoded secret value. } func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { + logger := log.FromContext(ctx) + + logger.Tracef("new secrets manager sr=%v", sr) sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } + + logger.Tracef("1sm = %v", sm) + var value any + + logger.Tracef("1Getting secret %s", s.Ref) err = sm.Get(ctx, s.Ref, &value) if err != nil { return err } + logger.Tracef("1encoding json") enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") + + logger.Tracef("1Encoding secret %s", s.Ref) err = enc.Encode(value) if err != nil { return fmt.Errorf("%s: %w", s.Ref, err) } + + logger.Tracef("1Returning secret %s", s.Ref) return nil } diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index e053fac282..3a8a072fe8 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -1,16 +1,17 @@ package configuration import ( - "bytes" "context" "encoding/base64" "encoding/json" "fmt" "net/url" + "regexp" "strings" + "github.com/alecthomas/types/optional" + "github.com/TBD54566975/ftl/internal/exec" - "github.com/TBD54566975/ftl/internal/log" ) // OnePasswordProvider is a configuration provider that reads passwords from @@ -36,15 +37,23 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return nil, fmt.Errorf("1Password secret reference must be a base64 encoded string: %w", err) } - output, err := exec.Capture(ctx, ".", "op", "read", "-n", string(decoded)) + parsedRef, err := decodeSecretRef(string(decoded)) if err != nil { - lines := bytes.Split(output, []byte("\n")) - logger := log.FromContext(ctx) - for _, line := range lines { - logger.Warnf("%s", line) - } - return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err) + return nil, fmt.Errorf("1Password secret reference invalid: %w", err) } + + //output, err := exec.Capture(ctx, ".", "op", "read", "-n", string(decoded)) + //if err != nil { + // lines := bytes.Split(output, []byte("\n")) + // logger := log.FromContext(ctx) + // for _, line := range lines { + // logger.Warnf("%s", line) + // } + // return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err) + //} + + output, err := exec.Capture(ctx, ".", "op", "get", "item", parsedRef.Vault, parsedRef.Item) + return json.Marshal(string(output)) } @@ -56,8 +65,72 @@ func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) ( if !strings.HasPrefix(opref, "op://") { return nil, fmt.Errorf("1Password secret reference must start with \"op://\"") } + encoded := base64.RawURLEncoding.EncodeToString([]byte(opref)) return &url.URL{Scheme: "op", Host: encoded}, nil } func (o OnePasswordProvider) Writer() bool { return o.OnePassword } + +// Custom parser for 1Password secret references because the format is not a standard URL, and we also need to +// allow users to omit the field name so that we can support secrets with multiple fields. +// +// Does not support "section-name". +// +// op:///[/] +// +// Secret references are case-insensitive and support the following characters: +// +// alphanumeric characters (a-z, A-Z, 0-9), -, _, . and the whitespace character +// +// If an item or field name includes a / or an unsupported character, use the item +// or field's unique identifier (ID) instead of its name. +// +// See https://developer.1password.com/docs/cli/secrets-reference-syntax/ +type secretRef struct { + Vault string + Item string + Field optional.Option[string] +} + +var validCharsRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_\. ]+$`) + +func decodeSecretRef(ref string) (*secretRef, error) { + + // Take out and check the "op://" prefix + const prefix = "op://" + if !strings.HasPrefix(ref, prefix) { + return nil, fmt.Errorf("must start with \"op://\"") + } + ref = ref[len(prefix):] + + parts := strings.Split(ref, "/") + + if len(parts) < 2 { + return nil, fmt.Errorf("must have at least 2 parts") + } + if len(parts) > 3 { + return nil, fmt.Errorf("must have at most 3 parts") + } + + for _, part := range parts { + if part == "" { + return nil, fmt.Errorf("url parts must not be empty") + } + + if !validCharsRegex.MatchString(part) { + return nil, fmt.Errorf("url part %q contains unsupported characters. regex: %q", part, validCharsRegex) + } + } + + secret := secretRef{ + Vault: parts[0], + Item: parts[1], + Field: optional.None[string](), + } + if len(parts) == 3 { + secret.Field = optional.Some(parts[2]) + } + + return &secret, nil +} diff --git a/common/configuration/1password_provider_test.go b/common/configuration/1password_provider_test.go new file mode 100644 index 0000000000..52185ae77f --- /dev/null +++ b/common/configuration/1password_provider_test.go @@ -0,0 +1,82 @@ +package configuration + +import ( + "github.com/alecthomas/types/optional" + "reflect" + "testing" +) + +func TestDecodeSecretRef(t *testing.T) { + tests := []struct { + name string + ref string + want *secretRef + wantErr bool + }{ + { + name: "simple with field", + ref: "op://development/Access Keys/access_key_id", + want: &secretRef{ + Vault: "development", + Item: "Access Keys", + Field: optional.Some("access_key_id"), + }, + }, + { + name: "simple without field", + ref: "op://vault/item", + want: &secretRef{ + Vault: "vault", + Item: "item", + Field: optional.None[string](), + }, + }, + { + name: "lots of spaces", + ref: "op://My Awesome Vault/My Awesome Item/My Awesome Field", + want: &secretRef{ + Vault: "My Awesome Vault", + Item: "My Awesome Item", + Field: optional.Some("My Awesome Field"), + }, + }, + { + name: "missing op://", + ref: "development/Access Keys/access_key_id", + wantErr: true, + }, + { + name: "empty parts", + ref: "op://development//access_key_id", + wantErr: true, + }, + { + name: "invalid characters", + ref: "op://development/aws/acce$s", + wantErr: true, + }, + { + name: "too many parts", + ref: "op://development/Access Keys/access_key_id/extra", + wantErr: true, + }, + { + name: "too few parts", + ref: "op://development", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeSecretRef(tt.ref) + if (err != nil) != tt.wantErr { + t.Errorf("decodeSecretRef() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("decodeSecretRef() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 569295f2f2..35e896d6dc 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/TBD54566975/ftl/internal/log" "os" "strings" @@ -89,25 +90,45 @@ func (m *Manager[R]) Mutable() error { // getData returns a data value for a configuration from the active providers. // The data can be unmarshalled from JSON. func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { + logger := log.FromContext(ctx) + + logger.Tracef("resolving %s", ref) key, err := m.resolver.Get(ctx, ref) + + logger.Tracef("key: %v error: %v", key, err) + + logger.Tracef("checking for %s in module scope", ref) // Try again at the global scope if the value is not found in module scope. if ref.Module.Ok() && errors.Is(err, ErrNotFound) { ref.Module = optional.None[string]() + + logger.Tracef("resolving %s", ref) key, err = m.resolver.Get(ctx, ref) + + logger.Tracef("error: %v", err) if err != nil { return nil, err } } else if err != nil { + logger.Tracef("error 2: %v", err) return nil, err } + + logger.Tracef("provider for %s: %s", ref, key.Scheme) provider, ok := m.providers[key.Scheme] + + logger.Tracef("provider ok %v", ok) if !ok { return nil, fmt.Errorf("no provider for scheme %q", key.Scheme) } + + logger.Tracef("providers loading %s from %s", ref, key) data, err := provider.Load(ctx, ref, key) if err != nil { return nil, fmt.Errorf("%s: %w", ref, err) } + + logger.Tracef("loaded %s from %s", ref, key) return data, nil } diff --git a/ftl-project.toml b/ftl-project.toml index da3d2836ff..3be055cfce 100644 --- a/ftl-project.toml +++ b/ftl-project.toml @@ -1,12 +1,24 @@ module-dirs = ["examples/go"] external-dirs = [] +ftl-min-version = "" [global] + [global.configuration] + moo = "envar://moo" + [global.secrets] + ftl = "keychain://ftl" + p = "op://Personal/TestAbc/username" + s = "op://b3A6Ly9QZXJzb25hbC9UZXN0QWJjL3Bhc3N3b3Jk" + test = "inline://ImFyc3Qi" + z = "op://b3A6Ly9QZXJzb25hbC9UZXN0QWJj" [modules] [modules.echo] [modules.echo.configuration] default = "inline://ImFub255bW91cyI" +[executables] + ftl = "" + [commands] startup = ["echo 'FTL startup command ⚡️'"] From 72c7456758b57254acce212ed71595c008d6a2a8 Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 09:37:59 +1000 Subject: [PATCH 3/8] wip: return map with field: value --- common/configuration/1password_provider.go | 78 +++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 3a8a072fe8..19aaa383fc 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/TBD54566975/ftl/internal/slices" "net/url" "regexp" "strings" @@ -12,6 +13,7 @@ import ( "github.com/alecthomas/types/optional" "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/log" ) // OnePasswordProvider is a configuration provider that reads passwords from @@ -27,6 +29,8 @@ func (o OnePasswordProvider) Key() string { return func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + logger := log.FromContext(ctx) + _, err := exec.LookPath("op") if err != nil { return nil, fmt.Errorf("1Password CLI tool \"op\" not found: %w", err) @@ -52,9 +56,52 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ // return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err) //} - output, err := exec.Capture(ctx, ".", "op", "get", "item", parsedRef.Vault, parsedRef.Item) + // A single password: op --format json item get --vault Personal "With Spaces" --fields=username + // { id, value } + // All fields: op --format json item get --vault Personal "With Spaces" + // { fields: [ { id, value } ] } + + args := []string{"--format", "json", "item", "get", "--vault", parsedRef.Vault, parsedRef.Item} + + v, fieldSpecified := parsedRef.Field.Get() + if fieldSpecified { + args = append(args, "--fields", v) + } + + output, err := exec.Capture(ctx, ".", "op", args...) + if err != nil { + return nil, fmt.Errorf("run `op` with args %v: %w", args, err) + } - return json.Marshal(string(output)) + logger.Debugf("output: %s", output) + logger.Debugf("fieldSpecified: %v", fieldSpecified) + + if fieldSpecified { + v, err := decodeSingleResponse(output) + if err != nil { + return nil, err + } + logger.Debugf("decoed v: %v", v) + + return json.Marshal(v.Value) + } else { + v, err := decodeFullResponse(output) + if err != nil { + return nil, err + } + + // Filter out anything without a value + filtered := slices.Filter(v, func(e entry) bool { + return e.Value != "" + }) + // Map to id: value + var mapped = make(map[string]string) + for _, e := range filtered { + mapped[e.Id] = e.Value + } + + return json.Marshal(mapped) + } } func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { @@ -72,6 +119,33 @@ func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) ( func (o OnePasswordProvider) Writer() bool { return o.OnePassword } +type entry struct { + Id string `json:"id"` + Value string `json:"value"` +} + +type full struct { + Fields []entry `json:"fields"` +} + +// Decode a full response from op +func decodeFullResponse(output []byte) ([]entry, error) { + var full full + if err := json.Unmarshal(output, &full); err != nil { + return nil, fmt.Errorf("error decoding op full response: %w", err) + } + return full.Fields, nil +} + +// Decode a single response from op +func decodeSingleResponse(output []byte) (*entry, error) { + var single entry + if err := json.Unmarshal(output, &single); err != nil { + return nil, fmt.Errorf("error decoding op single response: %w", err) + } + return &single, nil +} + // Custom parser for 1Password secret references because the format is not a standard URL, and we also need to // allow users to omit the field name so that we can support secrets with multiple fields. // From 23fcac01c3741e3fa643b98b7706e47a9e9e6061 Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 09:55:06 +1000 Subject: [PATCH 4/8] wip: clean --- common/configuration/1password_provider.go | 34 +++++----------------- common/configuration/manager.go | 13 --------- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 19aaa383fc..a99d27d422 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -13,7 +13,6 @@ import ( "github.com/alecthomas/types/optional" "github.com/TBD54566975/ftl/internal/exec" - "github.com/TBD54566975/ftl/internal/log" ) // OnePasswordProvider is a configuration provider that reads passwords from @@ -29,8 +28,6 @@ func (o OnePasswordProvider) Key() string { return func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { - logger := log.FromContext(ctx) - _, err := exec.LookPath("op") if err != nil { return nil, fmt.Errorf("1Password CLI tool \"op\" not found: %w", err) @@ -46,21 +43,10 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return nil, fmt.Errorf("1Password secret reference invalid: %w", err) } - //output, err := exec.Capture(ctx, ".", "op", "read", "-n", string(decoded)) - //if err != nil { - // lines := bytes.Split(output, []byte("\n")) - // logger := log.FromContext(ctx) - // for _, line := range lines { - // logger.Warnf("%s", line) - // } - // return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err) - //} - // A single password: op --format json item get --vault Personal "With Spaces" --fields=username - // { id, value } + // { id, value, ... } // All fields: op --format json item get --vault Personal "With Spaces" - // { fields: [ { id, value } ] } - + // { fields: [ { id, value, ... } ], ... } args := []string{"--format", "json", "item", "get", "--vault", parsedRef.Vault, parsedRef.Item} v, fieldSpecified := parsedRef.Field.Get() @@ -73,19 +59,15 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return nil, fmt.Errorf("run `op` with args %v: %w", args, err) } - logger.Debugf("output: %s", output) - logger.Debugf("fieldSpecified: %v", fieldSpecified) - if fieldSpecified { - v, err := decodeSingleResponse(output) + v, err := decodeSingle(output) if err != nil { return nil, err } - logger.Debugf("decoed v: %v", v) return json.Marshal(v.Value) } else { - v, err := decodeFullResponse(output) + v, err := decodeFull(output) if err != nil { return nil, err } @@ -128,8 +110,8 @@ type full struct { Fields []entry `json:"fields"` } -// Decode a full response from op -func decodeFullResponse(output []byte) ([]entry, error) { +// Decode a full item response from op +func decodeFull(output []byte) ([]entry, error) { var full full if err := json.Unmarshal(output, &full); err != nil { return nil, fmt.Errorf("error decoding op full response: %w", err) @@ -137,8 +119,8 @@ func decodeFullResponse(output []byte) ([]entry, error) { return full.Fields, nil } -// Decode a single response from op -func decodeSingleResponse(output []byte) (*entry, error) { +// Decode a single field from op +func decodeSingle(output []byte) (*entry, error) { var single entry if err := json.Unmarshal(output, &single); err != nil { return nil, fmt.Errorf("error decoding op single response: %w", err) diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 35e896d6dc..272164c40b 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -92,20 +92,12 @@ func (m *Manager[R]) Mutable() error { func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { logger := log.FromContext(ctx) - logger.Tracef("resolving %s", ref) key, err := m.resolver.Get(ctx, ref) - logger.Tracef("key: %v error: %v", key, err) - - logger.Tracef("checking for %s in module scope", ref) // Try again at the global scope if the value is not found in module scope. if ref.Module.Ok() && errors.Is(err, ErrNotFound) { ref.Module = optional.None[string]() - - logger.Tracef("resolving %s", ref) key, err = m.resolver.Get(ctx, ref) - - logger.Tracef("error: %v", err) if err != nil { return nil, err } @@ -114,21 +106,16 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { return nil, err } - logger.Tracef("provider for %s: %s", ref, key.Scheme) provider, ok := m.providers[key.Scheme] - - logger.Tracef("provider ok %v", ok) if !ok { return nil, fmt.Errorf("no provider for scheme %q", key.Scheme) } - logger.Tracef("providers loading %s from %s", ref, key) data, err := provider.Load(ctx, ref, key) if err != nil { return nil, fmt.Errorf("%s: %w", ref, err) } - logger.Tracef("loaded %s from %s", ref, key) return data, nil } From 1fb2e6c5d56a2d42df68131fda668b977ae5f8db Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 10:06:33 +1000 Subject: [PATCH 5/8] wip: cleanup --- cmd/ftl/cmd_secret.go | 14 -------------- common/configuration/1password_provider.go | 22 ++++++++++++++-------- common/configuration/manager.go | 8 -------- 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index f756d9df89..0d8a03c83b 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/TBD54566975/ftl/internal/log" "io" "os" @@ -85,35 +84,22 @@ Returns a JSON-encoded secret value. } func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { - logger := log.FromContext(ctx) - - logger.Tracef("new secrets manager sr=%v", sr) sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } - - logger.Tracef("1sm = %v", sm) - var value any - - logger.Tracef("1Getting secret %s", s.Ref) err = sm.Get(ctx, s.Ref, &value) if err != nil { return err } - logger.Tracef("1encoding json") enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") - - logger.Tracef("1Encoding secret %s", s.Ref) err = enc.Encode(value) if err != nil { return fmt.Errorf("%s: %w", s.Ref, err) } - - logger.Tracef("1Returning secret %s", s.Ref) return nil } diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index a99d27d422..369e95a99f 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "github.com/TBD54566975/ftl/internal/slices" "net/url" "regexp" "strings" @@ -13,6 +12,7 @@ import ( "github.com/alecthomas/types/optional" "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/slices" ) // OnePasswordProvider is a configuration provider that reads passwords from @@ -27,6 +27,19 @@ func (OnePasswordProvider) Role() Secrets { return func (o OnePasswordProvider) Key() string { return "op" } func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } +// Load returns either a single field if the op:// reference specifies a field, or all fields if not. +// +// A single value/password: +// op://Personal/With Spaces +// op --format json item get --vault Personal "With Spaces" --fields=username +// { id, value, ... } +// "value" +// +// All fields: +// op://Personal/With Spaces/username +// op --format json item get --vault Personal "With Spaces" +// { fields: [ { id, value, ... } ], ... } +// { id: value, ... } func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { _, err := exec.LookPath("op") if err != nil { @@ -43,17 +56,11 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return nil, fmt.Errorf("1Password secret reference invalid: %w", err) } - // A single password: op --format json item get --vault Personal "With Spaces" --fields=username - // { id, value, ... } - // All fields: op --format json item get --vault Personal "With Spaces" - // { fields: [ { id, value, ... } ], ... } args := []string{"--format", "json", "item", "get", "--vault", parsedRef.Vault, parsedRef.Item} - v, fieldSpecified := parsedRef.Field.Get() if fieldSpecified { args = append(args, "--fields", v) } - output, err := exec.Capture(ctx, ".", "op", args...) if err != nil { return nil, fmt.Errorf("run `op` with args %v: %w", args, err) @@ -94,7 +101,6 @@ func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) ( if !strings.HasPrefix(opref, "op://") { return nil, fmt.Errorf("1Password secret reference must start with \"op://\"") } - encoded := base64.RawURLEncoding.EncodeToString([]byte(opref)) return &url.URL{Scheme: "op", Host: encoded}, nil } diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 272164c40b..569295f2f2 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/TBD54566975/ftl/internal/log" "os" "strings" @@ -90,10 +89,7 @@ func (m *Manager[R]) Mutable() error { // getData returns a data value for a configuration from the active providers. // The data can be unmarshalled from JSON. func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { - logger := log.FromContext(ctx) - key, err := m.resolver.Get(ctx, ref) - // Try again at the global scope if the value is not found in module scope. if ref.Module.Ok() && errors.Is(err, ErrNotFound) { ref.Module = optional.None[string]() @@ -102,20 +98,16 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { return nil, err } } else if err != nil { - logger.Tracef("error 2: %v", err) return nil, err } - provider, ok := m.providers[key.Scheme] if !ok { return nil, fmt.Errorf("no provider for scheme %q", key.Scheme) } - data, err := provider.Load(ctx, ref, key) if err != nil { return nil, fmt.Errorf("%s: %w", ref, err) } - return data, nil } From 7f2b0499663e27779b185f9d41129ce985c51f02 Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 10:08:03 +1000 Subject: [PATCH 6/8] wip: clean --- ftl-project.toml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ftl-project.toml b/ftl-project.toml index 3be055cfce..da3d2836ff 100644 --- a/ftl-project.toml +++ b/ftl-project.toml @@ -1,24 +1,12 @@ module-dirs = ["examples/go"] external-dirs = [] -ftl-min-version = "" [global] - [global.configuration] - moo = "envar://moo" - [global.secrets] - ftl = "keychain://ftl" - p = "op://Personal/TestAbc/username" - s = "op://b3A6Ly9QZXJzb25hbC9UZXN0QWJjL3Bhc3N3b3Jk" - test = "inline://ImFyc3Qi" - z = "op://b3A6Ly9QZXJzb25hbC9UZXN0QWJj" [modules] [modules.echo] [modules.echo.configuration] default = "inline://ImFub255bW91cyI" -[executables] - ftl = "" - [commands] startup = ["echo 'FTL startup command ⚡️'"] From 2851da837629124cbcd50fb5623876c6df88e35b Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 10:13:17 +1000 Subject: [PATCH 7/8] chore: lint --- common/configuration/1password_provider.go | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 369e95a99f..34f093bea6 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -73,24 +73,24 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ } return json.Marshal(v.Value) - } else { - v, err := decodeFull(output) - if err != nil { - return nil, err - } + } - // Filter out anything without a value - filtered := slices.Filter(v, func(e entry) bool { - return e.Value != "" - }) - // Map to id: value - var mapped = make(map[string]string) - for _, e := range filtered { - mapped[e.Id] = e.Value - } + full, err := decodeFull(output) + if err != nil { + return nil, err + } - return json.Marshal(mapped) + // Filter out anything without a value + filtered := slices.Filter(full, func(e entry) bool { + return e.Value != "" + }) + // Map to id: value + var mapped = make(map[string]string) + for _, e := range filtered { + mapped[e.ID] = e.Value } + + return json.Marshal(mapped) } func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { @@ -108,17 +108,17 @@ func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) ( func (o OnePasswordProvider) Writer() bool { return o.OnePassword } type entry struct { - Id string `json:"id"` + ID string `json:"id"` Value string `json:"value"` } -type full struct { +type fullResponse struct { Fields []entry `json:"fields"` } // Decode a full item response from op func decodeFull(output []byte) ([]entry, error) { - var full full + var full fullResponse if err := json.Unmarshal(output, &full); err != nil { return nil, fmt.Errorf("error decoding op full response: %w", err) } From c10250cb9f765c993bc2e0a0a72a57ebf3f2c641 Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 20 May 2024 11:48:17 +1000 Subject: [PATCH 8/8] fix: alloc on new map, fix example --- common/configuration/1password_provider.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 34f093bea6..45eaacda82 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -30,13 +30,13 @@ func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return // Load returns either a single field if the op:// reference specifies a field, or all fields if not. // // A single value/password: -// op://Personal/With Spaces +// op://Personal/With Spaces/username // op --format json item get --vault Personal "With Spaces" --fields=username // { id, value, ... } // "value" // // All fields: -// op://Personal/With Spaces/username +// op://Personal/With Spaces // op --format json item get --vault Personal "With Spaces" // { fields: [ { id, value, ... } ], ... } // { id: value, ... } @@ -85,7 +85,7 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return e.Value != "" }) // Map to id: value - var mapped = make(map[string]string) + var mapped = make(map[string]string, len(filtered)) for _, e := range filtered { mapped[e.ID] = e.Value }