-
Notifications
You must be signed in to change notification settings - Fork 9
/
Copy pathgcs.go
222 lines (187 loc) · 4.89 KB
/
gcs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
// Copyright (c) 2025 The konf authors
// Use of this source code is governed by a MIT license found in the LICENSE file.
// Package gcs loads configuration from GCP [Cloud Storage].
//
// It requires following roles on the target GCS object:
// - roles/storage.objectViewer
//
// # Change notification
//
// By default, it periodically polls the configuration only.
// It also listens to change events by register it to PubSub notifier with [Pub/Sub notifications for Cloud Storage].
//
// Only OBJECT_FINALIZE events trigger polling the configuration and other type of events are ignored.
//
// [Cloud Storage]: https://cloud.google.com/storage
// [Pub/Sub notifications for Cloud Storage]: https://cloud.google.com/storage/docs/pubsub-notifications
package gcs
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync/atomic"
"time"
"cloud.google.com/go/storage"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
)
// GCS is a Provider that loads configuration from GCP Cloud Storage.
//
// To create a new GCS, call [New].
type GCS struct {
pollInterval time.Duration
unmarshal func([]byte, any) error
onStatus func(bool, error)
changedCh chan struct{}
client clientProxy
}
// New creates a GCS with the given endpoint and Option(s).
func New(uri string, opts ...Option) *GCS {
uri = strings.TrimPrefix(uri, "gs:")
uri = strings.TrimLeft(uri, "/")
bucket, object, _ := strings.Cut(uri, "/")
option := &options{
client: clientProxy{
bucket: bucket,
object: object,
},
changedCh: make(chan struct{}, 1),
}
for _, opt := range opts {
switch o := opt.(type) {
case *optionFunc:
o.fn(option)
default:
option.client.opts = append(option.client.opts, o)
}
}
return (*GCS)(option)
}
var errNil = errors.New("nil GCS")
func (g *GCS) Load() (map[string]any, error) {
if g == nil {
return nil, errNil
}
values, _, err := g.load(context.Background())
return values, err
}
func (g *GCS) Watch(ctx context.Context, onChange func(map[string]any)) error { //nolint:cyclop
if g == nil {
return errNil
}
if g.changedCh == nil {
g.changedCh = make(chan struct{}, 1)
}
defer func() {
if g.client.client != nil {
_ = g.client.client.Close()
}
}()
pollInterval := time.Minute
if g.pollInterval > 0 {
pollInterval = g.pollInterval
}
ticker := time.NewTicker(pollInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
g.changed()
case <-g.changedCh:
values, changed, err := g.load(ctx)
if g.onStatus != nil {
g.onStatus(changed, err)
}
if changed {
onChange(values)
}
case <-ctx.Done():
return nil
}
}
}
func (g *GCS) changed() {
select {
case g.changedCh <- struct{}{}:
default:
// Ignore if the channel is full since it's already triggered.
}
}
func (g *GCS) load(ctx context.Context) (map[string]any, bool, error) {
resp, changed, err := g.client.load(ctx)
if !changed || err != nil {
return nil, false, err
}
unmarshal := g.unmarshal
if unmarshal == nil {
unmarshal = json.Unmarshal
}
var values map[string]any
if e := unmarshal(resp, &values); e != nil {
return nil, false, fmt.Errorf("unmarshal: %w", e)
}
return values, true, nil
}
func (g *GCS) OnEvent(attributes map[string]string) error {
if g == nil {
return errNil
}
if attributes["bucketId"] == g.client.bucket &&
attributes["objectId"] == g.client.object {
if attributes["eventType"] == "OBJECT_FINALIZE" {
g.changed()
}
return nil
}
return fmt.Errorf("unsupported gcs event: %w", errors.ErrUnsupported)
}
func (g *GCS) Status(onStatus func(bool, error)) {
g.onStatus = onStatus
}
func (g *GCS) String() string {
return "gs://" + g.client.bucket + "/" + g.client.object
}
type clientProxy struct {
bucket string
object string
client *storage.Client
opts []option.ClientOption
lastGeneration atomic.Int64
}
func (p *clientProxy) load(ctx context.Context) ([]byte, bool, error) {
if p.client == nil {
var err error
if p.client, err = storage.NewClient(ctx, append(p.opts, storage.WithJSONReads())...); err != nil {
return nil, false, fmt.Errorf("create GCS client: %w", err)
}
}
object := p.client.Bucket(p.bucket).Object(p.object)
if generation := p.lastGeneration.Load(); generation > 0 {
object = object.If(storage.Conditions{GenerationNotMatch: generation})
}
reader, err := object.NewReader(ctx)
if err != nil {
var ge *googleapi.Error
if errors.As(err, &ge) && ge.Code == http.StatusNotModified {
return nil, false, nil
}
return nil, false, fmt.Errorf("create object reader: %w", err)
}
defer func() {
// Ignore error: it could do nothing on this error.
_ = reader.Close()
}()
if reader.Attrs.Generation == p.lastGeneration.Load() {
return nil, false, nil
}
p.lastGeneration.Store(reader.Attrs.Generation)
bytes, err := io.ReadAll(reader)
if err != nil {
return nil, false, fmt.Errorf("read object: %w", err)
}
return bytes, true, nil
}