diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3b128b95..4d57d6ea 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -36,6 +36,13 @@ updates: schedule: interval: daily + - package-ecosystem: gomod + directory: /provider/secretmanager + labels: + - Skip-Changelog + schedule: + interval: daily + - package-ecosystem: github-actions directory: / labels: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6a2f3a9e..82fb2bd3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,7 +11,7 @@ jobs: coverage: strategy: matrix: - module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig' ] + module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig', 'provider/secretmanager' ] name: Coverage runs-on: ubuntu-latest steps: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 512bbb70..e443cb4d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: lint: strategy: matrix: - module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig' ] + module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig', 'provider/secretmanager' ] name: Lint runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 238f2387..1bc7de3a 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -11,7 +11,7 @@ jobs: tag: strategy: matrix: - module: [ 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig' ] + module: [ 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig', 'provider/secretmanager' ] name: Submodules runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1aa95b58..3b9c9d14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: test: strategy: matrix: - module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig' ] + module: [ '', 'provider/file', 'provider/pflag', 'provider/appconfig', 'provider/azappconfig', 'provider/secretmanager' ] go-version: [ 'stable', 'oldstable' ] name: Test runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d856017..66a9ecd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Add env.WithNameSplitter/flag.WithNameSplitter/pflag.WithNameSplitter to split the name of the flag/env (#110). - Add Azure App Configuration Loader (#121). +- Add GCP Secret Manager Loader (#128). ### Changed diff --git a/README.md b/README.md index ca9bfe00..6c9e5225 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ There are providers for the following configuration sources: - [`pflag`](provider/pflag) loads configuration from [spf13/pflag](https://github.com/spf13/pflag). - [`appconfig`](provider/appconfig) loads configuration from [AWS AppConfig](https://aws.amazon.com/systems-manager/features/appconfig/). - [`azappconfig`](provider/azappconfig) loads configuration from [Azure App Configuration](https://azure.microsoft.com/en-us/products/app-configuration). +- [`secretmanager`](provider/secretmanager) loads configuration from [GCP Secret Manager](https://cloud.google.com/security/products/secret-manager). ## Custom Configuration Providers diff --git a/provider/secretmanager/go.mod b/provider/secretmanager/go.mod new file mode 100644 index 00000000..1d5e8a21 --- /dev/null +++ b/provider/secretmanager/go.mod @@ -0,0 +1,45 @@ +module github.com/nil-go/konf/provider/secretmanager + +go 1.21 + +require ( + cloud.google.com/go/compute/metadata v0.2.3 + cloud.google.com/go/secretmanager v1.11.5 + github.com/stretchr/testify v1.8.4 + google.golang.org/api v0.165.0 + google.golang.org/grpc v1.61.1 +) + +require ( + cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go/iam v1.1.6 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel v1.23.0 // indirect + go.opentelemetry.io/otel/metric v1.23.0 // indirect + go.opentelemetry.io/otel/trace v1.23.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/oauth2 v0.17.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/provider/secretmanager/go.sum b/provider/secretmanager/go.sum new file mode 100644 index 00000000..c5a55a82 --- /dev/null +++ b/provider/secretmanager/go.sum @@ -0,0 +1,194 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= +cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/otel v1.23.0 h1:Df0pqjqExIywbMCMTxkAwzjLZtRf+bBKLbUcpxO2C9E= +go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= +go.opentelemetry.io/otel/metric v1.23.0 h1:pazkx7ss4LFVVYSxYew7L5I6qvLXHA0Ap2pwV+9Cnpo= +go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.23.0 h1:37Ik5Ib7xfYVb4V1UtnT97T1jI+AoIYkJyPkuL4iJgI= +go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= +golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.165.0 h1:zd5d4JIIIaYYsfVy1HzoXYZ9rWCSBxxAglbczzo7Bgc= +google.golang.org/api v0.165.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 h1:g/4bk7P6TPMkAUbUhquq98xey1slwvuVJPosdBqYJlU= +google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= +google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= +google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/provider/secretmanager/internal/assert/assert.go b/provider/secretmanager/internal/assert/assert.go new file mode 100644 index 00000000..8b030a3b --- /dev/null +++ b/provider/secretmanager/internal/assert/assert.go @@ -0,0 +1,36 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +package assert + +import ( + "reflect" + "testing" +) + +func Equal[T any](tb testing.TB, expected, actual T) { + tb.Helper() + + if !reflect.DeepEqual(expected, actual) { + tb.Errorf("expected: %v; actual: %v", expected, actual) + } +} + +func NoError(tb testing.TB, err error) { + tb.Helper() + + if err != nil { + tb.Errorf("unexpected error: %v", err) + } +} + +func EqualError(tb testing.TB, err error, message string) { + tb.Helper() + + switch { + case err == nil: + tb.Errorf("expected: %v; actual: ", message) + case err.Error() != message: + tb.Errorf("expected: %v; actual: %v", message, err.Error()) + } +} diff --git a/provider/secretmanager/internal/maps/insert.go b/provider/secretmanager/internal/maps/insert.go new file mode 100644 index 00000000..6115560c --- /dev/null +++ b/provider/secretmanager/internal/maps/insert.go @@ -0,0 +1,30 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +package maps + +// Insert recursively inserts the given value into the dst maps. +// Key conflicts are resolved by preferring the given value. +func Insert(dst map[string]any, keys []string, value any) { + next := dst + for _, key := range keys[:len(keys)-1] { + val, exist := next[key] + if !exist { + // Create a map[string]any if the key does not exist. + m := make(map[string]any) + next[key] = m + next = m + + continue + } + + sub, ok := val.(map[string]any) + if !ok { + // Override if the val is not map[string]any. + sub = make(map[string]any) + next[key] = sub + } + next = sub + } + next[keys[len(keys)-1]] = value +} diff --git a/provider/secretmanager/internal/maps/insert_test.go b/provider/secretmanager/internal/maps/insert_test.go new file mode 100644 index 00000000..6c94d78d --- /dev/null +++ b/provider/secretmanager/internal/maps/insert_test.go @@ -0,0 +1,74 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +package maps_test + +import ( + "testing" + + "github.com/nil-go/konf/provider/secretmanager/internal/assert" + "github.com/nil-go/konf/provider/secretmanager/internal/maps" +) + +func TestInsert(t *testing.T) { + t.Parallel() + + testcases := []struct { + description string + keys []string + val any + dst map[string]any + expected map[string]any + }{ + { + description: "empty", + keys: []string{"p", "k"}, + val: "v", + dst: map[string]any{}, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + }, + }, + }, + { + description: "override nested keys", + keys: []string{"p", "k"}, + val: "v", + dst: map[string]any{ + "p": map[string]any{ + "k": "a", + }, + }, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + }, + }, + }, + { + description: "override non-map", + keys: []string{"p", "k"}, + val: "v", + dst: map[string]any{ + "p": "a", + }, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + }, + }, + }, + } + + for _, testcase := range testcases { + testcase := testcase + + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + + maps.Insert(testcase.dst, testcase.keys, testcase.val) + assert.Equal(t, testcase.expected, testcase.dst) + }) + } +} diff --git a/provider/secretmanager/option.go b/provider/secretmanager/option.go new file mode 100644 index 00000000..5a8d6b3a --- /dev/null +++ b/provider/secretmanager/option.go @@ -0,0 +1,81 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +//nolint:ireturn +package secretmanager + +import ( + "log/slog" + "time" + + "google.golang.org/api/option" + "google.golang.org/api/option/internaloption" +) + +// WithProject provides GCP project ID. +// +// By default, it fetches project ID from metadata server. +func WithProject(project string) Option { + return &optionFunc{ + fn: func(options *options) { + options.client.project = project + }, + } +} + +// WithFilter provides [filter] that will be used to select a set of secrets. +// +// [filter]: // https://cloud.google.com/secret-manager/docs/filtering +func WithFilter(filter string) Option { + return &optionFunc{ + fn: func(options *options) { + options.client.filter = filter + }, + } +} + +// WithNameSplitter provides the function used to split secret names into nested keys. +// If it returns an nil/[]string{}/[]string{""}, the secret will be ignored. +// +// For example, with the default splitter, an secret name like "PARENT-CHILD-KEY" +// would be split into "PARENT", "CHILD", and "KEY". +func WithNameSplitter(splitter func(string) []string) Option { + return &optionFunc{ + fn: func(options *options) { + options.splitter = splitter + }, + } +} + +// WithPollInterval provides the interval for polling the configuration. +// +// The default interval is 1 minute. +func WithPollInterval(interval time.Duration) Option { + return &optionFunc{ + fn: func(options *options) { + options.pollInterval = interval + }, + } +} + +// WithLogHandler provides the slog.Handler for logs from watch. +// +// By default, it uses handler from slog.Default(). +func WithLogHandler(handler slog.Handler) Option { + return &optionFunc{ + fn: func(options *options) { + if handler != nil { + options.logger = slog.New(handler) + } + }, + } +} + +type ( + Option = option.ClientOption + optionFunc struct { + internaloption.EmbeddableAdapter + fn func(options *options) + } + options SecretManager +) diff --git a/provider/secretmanager/secretmanager.go b/provider/secretmanager/secretmanager.go new file mode 100644 index 00000000..aef1f0ab --- /dev/null +++ b/provider/secretmanager/secretmanager.go @@ -0,0 +1,231 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +// Package secretmanager loads configuration from GCP [Secret Manager]. +// +// [Secret Manager]: https://cloud.google.com/security/products/secret-manager +package secretmanager + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" + + "cloud.google.com/go/compute/metadata" + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + imaps "github.com/nil-go/konf/provider/secretmanager/internal/maps" +) + +// SecretManager is a Provider that loads configuration from GCP Secret Manager. +// +// To create a new SecretManager, call [New]. +type SecretManager struct { + pollInterval time.Duration + splitter func(string) []string + logger *slog.Logger + + client *clientProxy +} + +// New creates a SecretManager with the given endpoint and Option(s). +func New(opts ...Option) SecretManager { + option := &options{ + client: &clientProxy{}, + } + for _, opt := range opts { + switch o := opt.(type) { + case *optionFunc: + o.fn(option) + default: + option.client.opts = append(option.client.opts, o) + } + } + + if option.pollInterval <= 0 { + option.pollInterval = time.Minute + } + if option.splitter == nil { + option.splitter = func(s string) []string { return strings.Split(s, "-") } + } + if option.logger == nil { + option.logger = slog.Default() + } + + return SecretManager(*option) +} + +func (a SecretManager) Load() (map[string]any, error) { + values, _, err := a.load(context.Background()) + + return values, err +} + +func (a SecretManager) Watch(ctx context.Context, onChange func(map[string]any)) error { + ticker := time.NewTicker(a.pollInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + values, changed, err := a.load(ctx) + if err != nil { + a.logger.WarnContext( + ctx, "Error when reloading from GCP Secret Manager", + "project", a.client.project, + "filter", a.client.filter, + "error", err, + ) + + continue + } + + if changed { + onChange(values) + } + case <-ctx.Done(): + return nil + } + } +} + +func (a SecretManager) load(ctx context.Context) (map[string]any, bool, error) { + resp, changed, err := a.client.load(ctx) + if !changed || err != nil { + return nil, false, err + } + + values := make(map[string]any) + for key, value := range resp { + keys := a.splitter(key) + if len(keys) == 0 || len(keys) == 1 && keys[0] == "" { + continue + } + + imaps.Insert(values, keys, value) + } + + return values, true, nil +} + +func (a SecretManager) String() string { + return "secretManager:" + a.client.project +} + +type clientProxy struct { + project string + filter string + opts []option.ClientOption + + client *secretmanager.Client + clientOnce sync.Once + + lastETags atomic.Pointer[map[string]string] +} + +func (p *clientProxy) load(ctx context.Context) (map[string]string, bool, error) { //nolint:cyclop,funlen + client, err := p.loadClient(ctx) + if err != nil { + return nil, false, err + } + + eTags := make(map[string]string) + iter := client.ListSecrets(ctx, &secretmanagerpb.ListSecretsRequest{ + Parent: "projects/" + p.project, + Filter: p.filter, + }) + for resp, e := iter.Next(); !errors.Is(e, iterator.Done); resp, e = iter.Next() { + if e != nil { + return nil, false, fmt.Errorf("list secrets on %s: %w", p.project, e) + } + + eTags[resp.GetName()] = resp.GetEtag() + } + + var changed bool + if last := p.lastETags.Load(); last == nil || !maps.Equal(*last, eTags) { + changed = true + p.lastETags.Store(&eTags) + } + if !changed { + return nil, false, nil + } + + secretsCh := make(chan *secretmanagerpb.AccessSecretVersionResponse, len(eTags)) + errChan := make(chan error, len(eTags)) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var waitGroup sync.WaitGroup + waitGroup.Add(len(eTags)) + for name := range eTags { + name := name + + go func() { + defer waitGroup.Done() + + resp, e := p.client.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{ + Name: name + "/versions/latest", + }) + if e != nil { + errChan <- fmt.Errorf("access secret %s: %w", strings.Split(name, "/")[3], e) + cancel() + + return + } + secretsCh <- resp + }() + } + waitGroup.Wait() + close(secretsCh) + close(errChan) + + for e := range errChan { + if !errors.Is(e, ctx.Err()) { + err = errors.Join(e) + } + } + if err != nil { + return nil, false, err + } + + values := make(map[string]string, len(eTags)) + for resp := range secretsCh { + data := resp.GetPayload().GetData() + values[strings.Split(resp.GetName(), "/")[3]] = unsafe.String(unsafe.SliceData(data), len(data)) + } + + return values, true, nil +} + +func (p *clientProxy) loadClient(ctx context.Context) (*secretmanager.Client, error) { + var err error + + p.clientOnce.Do(func() { + if p.project == "" { + if p.project, err = metadata.ProjectID(); err != nil { + err = fmt.Errorf("get GCP project ID: %w", err) + + return + } + } + + if p.client, err = secretmanager.NewClient(ctx, p.opts...); err != nil { + err = fmt.Errorf("create GCP secret manager client: %w", err) + + return + } + }) + + return p.client, err +} diff --git a/provider/secretmanager/secretmanager_test.go b/provider/secretmanager/secretmanager_test.go new file mode 100644 index 00000000..1577d6db --- /dev/null +++ b/provider/secretmanager/secretmanager_test.go @@ -0,0 +1,367 @@ +// Copyright (c) 2024 The konf authors +// Use of this source code is governed by a MIT license found in the LICENSE file. + +//go:build !race + +package secretmanager_test + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + pb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + "github.com/nil-go/konf/provider/secretmanager" + "github.com/nil-go/konf/provider/secretmanager/internal/assert" +) + +func TestSecretManager_Load(t *testing.T) { + t.Parallel() + + testcases := []struct { + description string + opts []option.ClientOption + service pb.SecretManagerServiceServer + expected map[string]any + err string + }{ + { + description: "secrets", + service: &secretManagerService{ + values: map[string]string{ + "projects/test/secrets/p-k": "v", + "projects/test/secrets/p-d": ".", + }, + }, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + "d": ".", + }, + }, + }, + { + description: "with filter", + opts: []option.ClientOption{ + secretmanager.WithFilter(`name ~ "p-*"`), + }, + service: &secretManagerService{ + values: map[string]string{ + "projects/test/secrets/p-k": "v", + }, + assert: func(m proto.Message) { + switch request := m.(type) { + case *pb.ListSecretsRequest: + assert.Equal(t, "projects/test", request.GetParent()) + assert.Equal(t, `name ~ "p-*"`, request.GetFilter()) + case *pb.AccessSecretVersionRequest: + assert.Equal(t, "projects/test/secrets/p-k/versions/latest", request.GetName()) + } + }, + }, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + }, + }, + }, + { + description: "with nil splitter", + opts: []option.ClientOption{ + secretmanager.WithNameSplitter(func(string) []string { return nil }), + }, + service: &secretManagerService{ + values: map[string]string{ + "projects/test/secrets/p-k": "v", + }, + }, + expected: map[string]any{}, + }, + { + description: "with empty splitter", + opts: []option.ClientOption{ + secretmanager.WithNameSplitter(func(string) []string { return []string{""} }), + }, + service: &secretManagerService{ + values: map[string]string{ + "projects/test/secrets/p-k": "v", + }, + }, + expected: map[string]any{}, + }, + { + description: "list secrets error", + service: &faultySecretManagerService{method: "ListSecrets"}, + err: "list secrets on test: rpc error: code = Unknown desc = list secrets error", + }, + { + description: "access secret error", + service: &faultySecretManagerService{method: "AccessSecretVersion"}, + err: "access secret p-k: rpc error: code = Unknown desc = access secret error", + }, + } + + for _, testcase := range testcases { + testcase := testcase + + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + + conn, closer := grpcServer(t, testcase.service) + defer closer() + + loader := secretmanager.New(append( + testcase.opts, + secretmanager.WithProject("test"), + option.WithGRPCConn(conn), + )...) + var values map[string]any + values, err := loader.Load() + if testcase.err != "" { + assert.EqualError(t, err, testcase.err) + } else { + assert.NoError(t, err) + assert.Equal(t, testcase.expected, values) + } + }) + } +} + +func TestSecretManager_Watch(t *testing.T) { + t.Parallel() + + testcases := []struct { + description string + opts []option.ClientOption + service pb.SecretManagerServiceServer + expected map[string]any + log string + }{ + { + description: "success", + service: &secretManagerService{ + values: map[string]string{ + "projects/test/secrets/p-k": "v", + "projects/test/secrets/p-d": ".", + }, + }, + expected: map[string]any{ + "p": map[string]any{ + "k": "v", + "d": ".", + }, + }, + }, + { + description: "list secrets error", + service: &faultySecretManagerService{method: "ListSecrets"}, + log: `level=WARN msg="Error when reloading from GCP Secret Manager" project=test filter=""` + + ` error="list secrets on test: rpc error: code = Unknown desc = list secrets error"` + "\n", + }, + { + description: "access secret error", + service: &faultySecretManagerService{method: "AccessSecretVersion"}, + log: `level=WARN msg="Error when reloading from GCP Secret Manager" project=test filter=""` + + ` error="access secret p-k: rpc error: code = Unknown desc = access secret error"` + "\n", + }, + } + + for _, testcase := range testcases { + testcase := testcase + + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + + conn, closer := grpcServer(t, testcase.service) + defer closer() + + buf := new(buffer) + loader := secretmanager.New(append( + testcase.opts, + secretmanager.WithProject("test"), + option.WithGRPCConn(conn), + secretmanager.WithLogHandler(logHandler(buf)), + secretmanager.WithPollInterval(10*time.Millisecond), + )...) + + var values atomic.Value + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var waitGroup sync.WaitGroup + waitGroup.Add(1) + go func() { + waitGroup.Done() + + err := loader.Watch(ctx, func(changed map[string]any) { + values.Store(changed) + }) + assert.NoError(t, err) + }() + waitGroup.Wait() + + time.Sleep(15 * time.Millisecond) // wait for the first tick, but not the second + if val, ok := values.Load().(map[string]any); ok { + assert.Equal(t, testcase.expected, val) + } else { + assert.Equal(t, testcase.log, buf.String()) + } + }) + } +} + +func grpcServer(t *testing.T, service pb.SecretManagerServiceServer) (*grpc.ClientConn, func()) { + t.Helper() + + temp, err := os.MkdirTemp("", "*") + require.NoError(t, err) + endpoint := temp + "/load.sock" + + server := grpc.NewServer() + pb.RegisterSecretManagerServiceServer(server, service) + + var waitGroup sync.WaitGroup + waitGroup.Add(1) + go func() { + _ = os.RemoveAll(endpoint) + listener, e := net.Listen("unix", endpoint) + assert.NoError(t, e) + waitGroup.Done() + + assert.NoError(t, server.Serve(listener)) + }() + waitGroup.Wait() + + conn, err := grpc.Dial("unix:"+endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(t, err) + + return conn, func() { + _ = conn.Close() + server.GracefulStop() + } +} + +func TestSecretManager_String(t *testing.T) { + t.Parallel() + + loader := secretmanager.New(secretmanager.WithProject("test")) + assert.Equal(t, "secretManager:test", loader.String()) +} + +type secretManagerService struct { + pb.UnimplementedSecretManagerServiceServer + + values map[string]string + assert func(proto.Message) +} + +func (s *secretManagerService) ListSecrets( + _ context.Context, + request *pb.ListSecretsRequest, +) (*pb.ListSecretsResponse, error) { + if s.assert != nil { + s.assert(request) + } + + resp := &pb.ListSecretsResponse{TotalSize: int32(len(s.values))} + for name := range s.values { + resp.Secrets = append(resp.Secrets, &pb.Secret{Name: name}) + } + + return resp, nil +} + +func (s *secretManagerService) AccessSecretVersion( + _ context.Context, + request *pb.AccessSecretVersionRequest, +) (*pb.AccessSecretVersionResponse, error) { + if s.assert != nil { + s.assert(request) + } + + name := request.GetName() + + return &pb.AccessSecretVersionResponse{ + Name: strings.Replace(name, "/versions/latest", "/versions/1", 1), + Payload: &pb.SecretPayload{Data: []byte(s.values[strings.TrimSuffix(name, "/versions/latest")])}, + }, nil +} + +type faultySecretManagerService struct { + pb.UnimplementedSecretManagerServiceServer + + method string +} + +func (f *faultySecretManagerService) ListSecrets( + context.Context, + *pb.ListSecretsRequest, +) (*pb.ListSecretsResponse, error) { + if f.method == "ListSecrets" { + return nil, errors.New("list secrets error") + } + + return &pb.ListSecretsResponse{Secrets: []*pb.Secret{{Name: "projects/test/secrets/p-k"}}}, nil +} + +func (f *faultySecretManagerService) AccessSecretVersion( + context.Context, + *pb.AccessSecretVersionRequest, +) (*pb.AccessSecretVersionResponse, error) { + if f.method == "AccessSecretVersion" { + return nil, errors.New("access secret error") + } + + return &pb.AccessSecretVersionResponse{}, nil +} + +func logHandler(buf *buffer) *slog.TextHandler { + return slog.NewTextHandler(buf, &slog.HandlerOptions{ + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + if len(groups) == 0 && attr.Key == slog.TimeKey { + return slog.Attr{} + } + + return attr + }, + }) +} + +type buffer struct { + b bytes.Buffer + m sync.RWMutex +} + +func (b *buffer) Read(p []byte) (int, error) { + b.m.RLock() + defer b.m.RUnlock() + + return b.b.Read(p) +} + +func (b *buffer) Write(p []byte) (int, error) { + b.m.Lock() + defer b.m.Unlock() + + return b.b.Write(p) +} + +func (b *buffer) String() string { + b.m.RLock() + defer b.m.RUnlock() + + return b.b.String() +}