From 60067c4bafc13108345544fd4645be9988a84bf5 Mon Sep 17 00:00:00 2001 From: thisisaaronland Date: Sun, 27 Jun 2021 00:03:50 -0700 Subject: [PATCH] update to use whosonfirst/go-cache v0.5.0 and friends --- go.mod | 5 +- go.sum | 4 + repo/indexer.go | 9 +- repo/readercache.go | 165 ------------------ .../whosonfirst/go-cache-blob/blob.go | 14 +- .../whosonfirst/go-cache-blob/doc.go | 2 + .../whosonfirst/go-cache-blob/go.mod | 3 +- .../whosonfirst/go-cache-blob/go.sum | 4 + .../whosonfirst/go-cache-blob/testing.go | 110 ++++++++++++ .../github.com/whosonfirst/go-cache/README.md | 12 +- .../github.com/whosonfirst/go-cache/cache.go | 4 +- vendor/github.com/whosonfirst/go-cache/doc.go | 3 + vendor/github.com/whosonfirst/go-cache/fs.go | 8 +- vendor/github.com/whosonfirst/go-cache/go.mod | 1 + vendor/github.com/whosonfirst/go-cache/go.sum | 2 + .../whosonfirst/go-cache/gocache.go | 15 +- .../github.com/whosonfirst/go-cache/multi.go | 62 ++----- .../github.com/whosonfirst/go-cache/null.go | 4 +- .../github.com/whosonfirst/go-cache/string.go | 56 ++++++ .../whosonfirst/go-cache/testing.go | 106 +++++++++++ .../github.com/whosonfirst/go-cache/utils.go | 59 ------- vendor/modules.txt | 5 +- 22 files changed, 348 insertions(+), 305 deletions(-) delete mode 100644 repo/readercache.go create mode 100644 vendor/github.com/whosonfirst/go-cache-blob/doc.go create mode 100644 vendor/github.com/whosonfirst/go-cache-blob/testing.go create mode 100644 vendor/github.com/whosonfirst/go-cache/doc.go create mode 100644 vendor/github.com/whosonfirst/go-cache/string.go create mode 100644 vendor/github.com/whosonfirst/go-cache/testing.go delete mode 100644 vendor/github.com/whosonfirst/go-cache/utils.go diff --git a/go.mod b/go.mod index 3d727c1f..7e384858 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,9 @@ require ( github.com/rs/cors v1.7.0 github.com/sfomuseum/go-flags v0.8.2 github.com/tidwall/gjson v1.8.0 - github.com/whosonfirst/go-cache v0.4.0 - github.com/whosonfirst/go-cache-blob v0.1.2 + github.com/whosonfirst/go-cache v0.5.0 + github.com/whosonfirst/go-cache-blob v0.2.0 + github.com/whosonfirst/go-ioutil v1.0.0 github.com/whosonfirst/go-reader v0.5.0 github.com/whosonfirst/go-whosonfirst-iterate v1.1.2 github.com/whosonfirst/go-whosonfirst-uri v1.0.1 diff --git a/go.sum b/go.sum index d99aad60..d4ee85b8 100644 --- a/go.sum +++ b/go.sum @@ -324,8 +324,12 @@ github.com/whosonfirst/go-cache v0.3.0 h1:QwboIxWu3Ab2gPikTMAJT2kukRr5c5yr/+yoMF github.com/whosonfirst/go-cache v0.3.0/go.mod h1:uYn6IpxBxuSdULs15zkHHkOHTW8i4AS4A3jTv8wcbGg= github.com/whosonfirst/go-cache v0.4.0 h1:KTUA6YHbsjh3bFIJpJ1+8uNJDwnaL5+3Et1dCj8xYBQ= github.com/whosonfirst/go-cache v0.4.0/go.mod h1:uYn6IpxBxuSdULs15zkHHkOHTW8i4AS4A3jTv8wcbGg= +github.com/whosonfirst/go-cache v0.5.0 h1:s8o2FafYxI1TGJGqwi3G0XVD5yY1dvzwSe9dE5C+KfA= +github.com/whosonfirst/go-cache v0.5.0/go.mod h1:YawO5K9kQa3XnrtdfwZ6OS4gwuO87S06FqIPblMqZf4= github.com/whosonfirst/go-cache-blob v0.1.2 h1:i+b9t02UdJXvw+mPlUz1WbLYKsJ+TsnHVWFh3eCkMcc= github.com/whosonfirst/go-cache-blob v0.1.2/go.mod h1:HIGjOdg1tGaIv5ADlsKb0gel992vrklmykoYJd+axC0= +github.com/whosonfirst/go-cache-blob v0.2.0 h1:IagILVwbUKEKrGjKmjs/wEb/Q28uwLp0fkYh+IsYnyc= +github.com/whosonfirst/go-cache-blob v0.2.0/go.mod h1:CXgKyLBwkinhTLnuPs7/HVxPr0DSWGL7UoZF0ehmszo= github.com/whosonfirst/go-ioutil v0.0.1 h1:cCrEYen6NDvHfjzV2q4u/VB21u2kTOwDnUGRlMI8Z9o= github.com/whosonfirst/go-ioutil v0.0.1/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-ioutil v1.0.0 h1:sYpgJx7Wsp76e7PFGa8KKQBvWQW3+HMCWSJbKdD5m14= diff --git a/repo/indexer.go b/repo/indexer.go index d012afc3..59493be8 100644 --- a/repo/indexer.go +++ b/repo/indexer.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "github.com/whosonfirst/go-cache" + "github.com/whosonfirst/go-ioutil" "github.com/whosonfirst/go-whosonfirst-findingaid" "github.com/whosonfirst/go-whosonfirst-iterate/iterator" "io" @@ -118,7 +119,11 @@ func (fa *Indexer) IndexReader(ctx context.Context, fh io.Reader) error { } br := bytes.NewReader(enc) - br_cl := io.NopCloser(br) + rsc, err := ioutil.NewReadSeekCloser(br) + + if err != nil { + return err + } key, err := cacheKeyFromRelPath(rsp.URI) @@ -126,7 +131,7 @@ func (fa *Indexer) IndexReader(ctx context.Context, fh io.Reader) error { return err } - _, err = fa.cache.Set(ctx, key, br_cl) + _, err = fa.cache.Set(ctx, key, rsc) if err != nil { return err diff --git a/repo/readercache.go b/repo/readercache.go deleted file mode 100644 index c70ba30d..00000000 --- a/repo/readercache.go +++ /dev/null @@ -1,165 +0,0 @@ -package repo - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "github.com/tidwall/gjson" - "github.com/whosonfirst/go-cache" - "github.com/whosonfirst/go-reader" - "github.com/whosonfirst/go-whosonfirst-uri" - "io" - _ "log" - "net/url" - "strconv" -) - -func init() { - ctx := context.Background() - cache.RegisterCache(ctx, "readercache", NewReaderCache) -} - -// Note to self: This needs to define it's own internal cache for cache MISSes that -// can be determined to always fail. - -// ReaderCache defines an implementation of the whosonfirst/go-cache.Cache interface that contains an internal -// whosonfirst/go-reader.Reader instance for retrieving (and storing) queries that return a cache MISS. -type ReaderCache struct { - cache.Cache - reader reader.Reader - cache cache.Cache -} - -// NewReaderCache returns a ReaderCache instance that implements the whosonfirst/go-cache.Cache interface. -func NewReaderCache(ctx context.Context, uri string) (cache.Cache, error) { - - u, err := url.Parse(uri) - - if err != nil { - return nil, err - } - - q := u.Query() - - r_uri := q.Get("reader") - - if r_uri == "" { - return nil, errors.New("Missing reader parameter") - } - - r, err := reader.NewReader(ctx, r_uri) - - if err != nil { - return nil, err - } - - c_uri := q.Get("cache") - - if c_uri == "" { - return nil, errors.New("Missing cache parameter") - } - - c, err := cache.NewCache(ctx, c_uri) - - if err != nil { - return nil, err - } - - rc := &ReaderCache{ - reader: r, - cache: c, - } - - return rc, nil -} - -// Name returns the name of the whosonfirst/go-cache.Cache implementation. -func (c *ReaderCache) Name() string { - return "readercache" -} - -// Returns a io.ReadCloser instance containing a JSON-encoded repo.FindingAidResponse document. -func (c *ReaderCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { - - c_fh, err := c.cache.Get(ctx, key) - - if err == nil { - return c_fh, nil - } - - id, err := strconv.ParseInt(key, 10, 64) - - if err != nil { - return nil, err - } - - rel_path, err := uri.Id2RelPath(id) - - if err != nil { - return nil, err - } - - r_fh, err := c.reader.Read(ctx, rel_path) - - if err != nil { - return nil, err - } - - defer r_fh.Close() - - body, err := io.ReadAll(r_fh) - - if err != nil { - return nil, err - } - - repo_rsp := gjson.GetBytes(body, "properties.wof:repo") - - if !repo_rsp.Exists() { - return nil, errors.New("Invalid WOF record") - } - - wof_repo := repo_rsp.String() - - fa_rsp := FindingAidResponse{ - ID: id, - URI: rel_path, - Repo: wof_repo, - } - - enc, err := json.Marshal(fa_rsp) - - if err != nil { - return nil, err - } - - br := bytes.NewReader(enc) - rsp := io.NopCloser(br) - - return c.Set(ctx, key, rsp) -} - -func (c *ReaderCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { - return c.cache.Set(ctx, key, fh) -} - -func (c *ReaderCache) Unset(ctx context.Context, key string) error { - return c.cache.Unset(ctx, key) -} - -func (c *ReaderCache) Hits() int64 { - return c.cache.Hits() -} - -func (c *ReaderCache) Misses() int64 { - return c.cache.Misses() -} - -func (c *ReaderCache) Evictions() int64 { - return c.cache.Evictions() -} - -func (c *ReaderCache) Size() int64 { - return c.cache.Size() -} diff --git a/vendor/github.com/whosonfirst/go-cache-blob/blob.go b/vendor/github.com/whosonfirst/go-cache-blob/blob.go index 590df99a..58e6bbf5 100644 --- a/vendor/github.com/whosonfirst/go-cache-blob/blob.go +++ b/vendor/github.com/whosonfirst/go-cache-blob/blob.go @@ -1,13 +1,13 @@ -package cache +package blob import ( "bufio" "bytes" "context" wof_cache "github.com/whosonfirst/go-cache" + "github.com/whosonfirst/go-ioutil" "gocloud.dev/blob" "io" - "io/ioutil" "sync/atomic" ) @@ -57,7 +57,7 @@ func (c *BlobCache) Name() string { return "blob" } -func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { fh, err := c.bucket.NewReader(ctx, key, nil) @@ -67,10 +67,10 @@ func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadCloser, error) } atomic.AddInt64(&c.hits, 1) - return fh, nil + return ioutil.NewReadSeekCloser(fh) } -func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { var wr_opts *blob.WriterOptions @@ -79,7 +79,7 @@ func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.R if v != nil { wr_opts = v.(*blob.WriterOptions) } - + bucket_wr, err := c.bucket.NewWriter(ctx, key, wr_opts) if err != nil { @@ -114,7 +114,7 @@ func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.R atomic.AddInt64(&c.sets, 1) r.Reset(b.Bytes()) - return ioutil.NopCloser(r), nil + return ioutil.NewReadSeekCloser(r) } func (c *BlobCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache-blob/doc.go b/vendor/github.com/whosonfirst/go-cache-blob/doc.go new file mode 100644 index 00000000..ce51d378 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache-blob/doc.go @@ -0,0 +1,2 @@ +// package blob implements the Who's On First `go-cache` interface using Google's GoCloud `Blob` objects. +package blob diff --git a/vendor/github.com/whosonfirst/go-cache-blob/go.mod b/vendor/github.com/whosonfirst/go-cache-blob/go.mod index 4feb2649..42756f3c 100644 --- a/vendor/github.com/whosonfirst/go-cache-blob/go.mod +++ b/vendor/github.com/whosonfirst/go-cache-blob/go.mod @@ -3,6 +3,7 @@ module github.com/whosonfirst/go-cache-blob go 1.16 require ( - github.com/whosonfirst/go-cache v0.3.0 + github.com/whosonfirst/go-cache v0.5.0 + github.com/whosonfirst/go-ioutil v1.0.0 gocloud.dev v0.23.0 ) diff --git a/vendor/github.com/whosonfirst/go-cache-blob/go.sum b/vendor/github.com/whosonfirst/go-cache-blob/go.sum index 71235926..89fc1335 100644 --- a/vendor/github.com/whosonfirst/go-cache-blob/go.sum +++ b/vendor/github.com/whosonfirst/go-cache-blob/go.sum @@ -308,6 +308,10 @@ github.com/whosonfirst/go-cache v0.1.0 h1:w6o4pWT/q2q/uclV72EU3D9CCKOgU+w+D+GKpa github.com/whosonfirst/go-cache v0.1.0/go.mod h1:M5Iche+26mVKFph49/pxS5W4ULZ9Dr01mebPqswG8QU= github.com/whosonfirst/go-cache v0.3.0 h1:QwboIxWu3Ab2gPikTMAJT2kukRr5c5yr/+yoMFRo8vQ= github.com/whosonfirst/go-cache v0.3.0/go.mod h1:uYn6IpxBxuSdULs15zkHHkOHTW8i4AS4A3jTv8wcbGg= +github.com/whosonfirst/go-cache v0.5.0 h1:s8o2FafYxI1TGJGqwi3G0XVD5yY1dvzwSe9dE5C+KfA= +github.com/whosonfirst/go-cache v0.5.0/go.mod h1:YawO5K9kQa3XnrtdfwZ6OS4gwuO87S06FqIPblMqZf4= +github.com/whosonfirst/go-ioutil v1.0.0 h1:sYpgJx7Wsp76e7PFGa8KKQBvWQW3+HMCWSJbKdD5m14= +github.com/whosonfirst/go-ioutil v1.0.0/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-whosonfirst-cache v0.1.0 h1:ryWTsHj7gAEjwHC/WjsjROpFflsz3SCa7W9Qnk4EU7k= github.com/whosonfirst/go-whosonfirst-cache v0.1.0/go.mod h1:P5Tx34j2M50qX/kTfXY516apwGzJGkFSBYQI7TGArKw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/vendor/github.com/whosonfirst/go-cache-blob/testing.go b/vendor/github.com/whosonfirst/go-cache-blob/testing.go new file mode 100644 index 00000000..2a1d5847 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache-blob/testing.go @@ -0,0 +1,110 @@ +package blob + +// Eventually this will be deprecated and all of these methods will be publicly +// available in go-cache/testing.go + +import ( + "bytes" + "context" + "fmt" + "github.com/whosonfirst/go-cache" + "io" + _ "log" +) + +type testCacheOptions struct { + AllowCacheMiss bool +} + +func testCache(ctx context.Context, c cache.Cache, opts *testCacheOptions) error { + + k := "test" + v := "test" + + fh1, err := cache.ReadSeekCloserFromString(v) + + if err != nil { + return fmt.Errorf("Failed to create ReadSeekCloser, %v", err) + } + + fh2, err := c.Set(ctx, k, fh1) + + if err != nil { + return fmt.Errorf("Failed to set %s (%s), %v", k, v, err) + } + + equals, err := compareReadSeekClosers(fh1, fh2) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 2 failed equality test") + } + + fh3, err := c.Get(ctx, k) + + if err != nil && !cache.IsCacheMiss(err) { + return fmt.Errorf("Failed to get cache value for '%s', %v", k, err) + } + + if err != nil { + + if !opts.AllowCacheMiss { + return fmt.Errorf("Unexpected cache miss for '%s'", k) + } + + } else { + + equals, err = compareReadSeekClosers(fh1, fh3) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles 1 and 3, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 3 failed equality test") + } + } + + return nil +} + +func compareReadSeekClosers(fh1 io.ReadSeekCloser, fh2 io.ReadSeekCloser) (bool, error) { + + fh1.Seek(0, 0) + fh2.Seek(0, 0) + + defer func() { + fh1.Seek(0, 0) + fh2.Seek(0, 0) + }() + + b1, err := io.ReadAll(fh1) + + if err != nil { + return false, fmt.Errorf("Failed to read fh1, %v", err) + } + + fh1.Seek(0, 0) + + b2, err := io.ReadAll(fh2) + + if err != nil { + return false, fmt.Errorf("Failed to read fh2, %v", err) + } + + fh2.Seek(0, 0) + + // log.Printf("1 '%s'\n", string(b1)) + // log.Printf("2 '%s'\n", string(b2)) + + equals := bytes.Compare(b1, b2) + + if equals != 0 { + return false, nil + } + + return true, nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/README.md b/vendor/github.com/whosonfirst/go-cache/README.md index a7bc5e8b..82b96df0 100644 --- a/vendor/github.com/whosonfirst/go-cache/README.md +++ b/vendor/github.com/whosonfirst/go-cache/README.md @@ -1,6 +1,6 @@ # go-cache -There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadCloser` instances. +There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadSeekCloser` instances. _This package supersedes [go-whosonfirst-cache](https://github.com/whosonfirst/go-whosonfirst-cache) which will be retired soon._ @@ -29,9 +29,9 @@ func main() { log.Fatal(err) } - str, _ := cache.SetString(c, "some-key", "some-value") + str, _ := cache.SetString(ctx, c, "some-key", "some-value") - str2, _ := cache.GetString(c, "some-key") + str2, _ := cache.GetString(ctx, c, "some-key") log.Println(str2) } @@ -41,7 +41,7 @@ Two things to note: * The use of the `fs://` scheme rather than the more conventional `file://`. This is deliberate so as not to overlap with the [Go Cloud](https://gocloud.dev/howto/blob/) `Blob` package's file handler. -* The use of the `cache.GetString` and `cache.SetString` methods. The `cache.Cache` interface expects `io.ReadCloser` instances so these methods are shortcuts to hide the boilerplate code necessary to work with `io.ReadCloser` interfaces. +* The use of the `cache.GetString` and `cache.SetString` methods. The `cache.Cache` interface expects `io.ReadSeekCloser` instances so these methods are shortcuts to hide the boilerplate code necessary to work with `io.ReadSeekCloser` interfaces. There is also a handy `null://` cache which doesn't do anything at all (expect implement the `cache.Cache` interface). For example: @@ -58,8 +58,8 @@ There is also a handy `null://` cache which doesn't do anything at all (expect i type Cache interface { Name() string Close(context.Context) error - Get(context.Context, string) (io.ReadCloser, error) - Set(context.Context, string, io.ReadCloser) (io.ReadCloser, error) + Get(context.Context, string) (io.ReadSeekCloser, error) + Set(context.Context, string, io.ReadSeekCloser) (io.ReadSeekCloser, error) Unset(context.Context, string) error Hits() int64 Misses() int64 diff --git a/vendor/github.com/whosonfirst/go-cache/cache.go b/vendor/github.com/whosonfirst/go-cache/cache.go index b9c90976..52893b75 100644 --- a/vendor/github.com/whosonfirst/go-cache/cache.go +++ b/vendor/github.com/whosonfirst/go-cache/cache.go @@ -11,8 +11,8 @@ import ( type Cache interface { Close(context.Context) error Name() string - Get(context.Context, string) (io.ReadCloser, error) - Set(context.Context, string, io.ReadCloser) (io.ReadCloser, error) + Get(context.Context, string) (io.ReadSeekCloser, error) + Set(context.Context, string, io.ReadSeekCloser) (io.ReadSeekCloser, error) Unset(context.Context, string) error Hits() int64 Misses() int64 diff --git a/vendor/github.com/whosonfirst/go-cache/doc.go b/vendor/github.com/whosonfirst/go-cache/doc.go new file mode 100644 index 00000000..edcb1ceb --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/doc.go @@ -0,0 +1,3 @@ +// package cache provides a storage-independent interface, and common implementations, for caching things. +// There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadSeekCloser` instances. +package cache diff --git a/vendor/github.com/whosonfirst/go-cache/fs.go b/vendor/github.com/whosonfirst/go-cache/fs.go index caec34cc..38d411b1 100644 --- a/vendor/github.com/whosonfirst/go-cache/fs.go +++ b/vendor/github.com/whosonfirst/go-cache/fs.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "errors" + "github.com/whosonfirst/go-ioutil" "io" _ "log" "net/url" @@ -81,7 +82,7 @@ func (c *FSCache) Name() string { return "fs" } -func (c *FSCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *FSCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -126,7 +127,7 @@ func (c *FSCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { return fh, nil } -func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { c.mu.Lock() defer c.mu.Unlock() @@ -173,7 +174,8 @@ func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea return nil, err } - return NewReadCloser(b.Bytes()), nil + br := bytes.NewReader(b.Bytes()) + return ioutil.NewReadSeekCloser(br) } func (c *FSCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/go.mod b/vendor/github.com/whosonfirst/go-cache/go.mod index 9476799b..15fe5abf 100644 --- a/vendor/github.com/whosonfirst/go-cache/go.mod +++ b/vendor/github.com/whosonfirst/go-cache/go.mod @@ -5,4 +5,5 @@ go 1.16 require ( github.com/aaronland/go-roster v0.0.2 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/whosonfirst/go-ioutil v1.0.0 ) diff --git a/vendor/github.com/whosonfirst/go-cache/go.sum b/vendor/github.com/whosonfirst/go-cache/go.sum index 1cf3ac56..04e6ea8d 100644 --- a/vendor/github.com/whosonfirst/go-cache/go.sum +++ b/vendor/github.com/whosonfirst/go-cache/go.sum @@ -5,5 +5,7 @@ github.com/aaronland/go-roster v0.0.2/go.mod h1:AcovpxlG1XxJxX2Fjqlm63fEIBhCjEIB github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/whosonfirst/go-ioutil v1.0.0 h1:sYpgJx7Wsp76e7PFGa8KKQBvWQW3+HMCWSJbKdD5m14= +github.com/whosonfirst/go-ioutil v1.0.0/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-whosonfirst-cache v0.1.0 h1:ryWTsHj7gAEjwHC/WjsjROpFflsz3SCa7W9Qnk4EU7k= github.com/whosonfirst/go-whosonfirst-cache v0.1.0/go.mod h1:P5Tx34j2M50qX/kTfXY516apwGzJGkFSBYQI7TGArKw= diff --git a/vendor/github.com/whosonfirst/go-cache/gocache.go b/vendor/github.com/whosonfirst/go-cache/gocache.go index 21b5b5a9..faea49f9 100644 --- a/vendor/github.com/whosonfirst/go-cache/gocache.go +++ b/vendor/github.com/whosonfirst/go-cache/gocache.go @@ -3,10 +3,11 @@ package cache // https://godoc.org/github.com/patrickmn/go-cache import ( + "bytes" "context" gocache "github.com/patrickmn/go-cache" + "github.com/whosonfirst/go-ioutil" "io" - "io/ioutil" "net/url" "strconv" "sync/atomic" @@ -104,7 +105,7 @@ func (c *GoCache) Name() string { return "gocache" } -func (c *GoCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *GoCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { // to do: timings that don't slow everything down the way // go-whosonfirst-timer does now (20170915/thisisaaronland) @@ -119,11 +120,12 @@ func (c *GoCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { atomic.AddInt64(&c.hits, 1) body := data.([]byte) + br := bytes.NewReader(body) - return NewReadCloser(body), nil + return ioutil.NewReadSeekCloser(br) } -func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { /* @@ -137,7 +139,7 @@ func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea defer fh.Close() - body, err := ioutil.ReadAll(fh) + body, err := io.ReadAll(fh) if err != nil { return nil, err @@ -146,7 +148,8 @@ func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea c.cache.Set(key, body, gocache.DefaultExpiration) atomic.AddInt64(&c.keys, 1) - return NewReadCloser(body), nil + fh.Seek(0, 0) + return fh, nil } func (c *GoCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/multi.go b/vendor/github.com/whosonfirst/go-cache/multi.go index 6b810774..a72f2820 100644 --- a/vendor/github.com/whosonfirst/go-cache/multi.go +++ b/vendor/github.com/whosonfirst/go-cache/multi.go @@ -1,7 +1,6 @@ package cache import ( - "bytes" "context" "fmt" "io" @@ -82,7 +81,7 @@ func (mc *MultiCache) Name() string { return "multi" } -func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { for _, c := range mc.caches { @@ -94,73 +93,40 @@ func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadCloser, error atomic.AddInt64(&mc.hits, 1) - // START OF... + for _, c := range mc.caches { - body, err := io.ReadAll(fh) + // Only set caches that come *before* this cache - if err != nil { - return nil, err - } - - go func(body []byte) { - - br := bytes.NewReader(body) - - for _, c := range mc.caches { - - // Only set caches that come *before* this cache - - if c.Name() == mc.Name() { - break - } - - cl := io.NopCloser(br) - - c.Set(ctx, key, cl) - br.Seek(0, 0) + if c.Name() == mc.Name() { + break } - }(body) - // END OF... - - br := bytes.NewReader(body) - cl := io.NopCloser(br) + c.Set(ctx, key, fh) + fh.Seek(0, 0) + } - return cl, nil + return fh, nil } atomic.AddInt64(&mc.misses, 1) return nil, new(CacheMiss) } -func (mc *MultiCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { - - body, err := io.ReadAll(fh) - - if err != nil { - return nil, err - } - - br := bytes.NewReader(body) +func (mc *MultiCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { for _, c := range mc.caches { - br.Seek(0, 0) - cl := io.NopCloser(br) - - _, err := c.Set(ctx, key, cl) + _, err := c.Set(ctx, key, fh) if err != nil { return nil, err } + + fh.Seek(0, 0) } atomic.AddInt64(&mc.size, 1) - - br.Seek(0, 0) - cl := io.NopCloser(br) - - return cl, nil + return fh, nil } func (mc *MultiCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/null.go b/vendor/github.com/whosonfirst/go-cache/null.go index 52127603..9310655a 100644 --- a/vendor/github.com/whosonfirst/go-cache/null.go +++ b/vendor/github.com/whosonfirst/go-cache/null.go @@ -31,12 +31,12 @@ func (c *NullCache) Name() string { return "null" } -func (c *NullCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *NullCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { atomic.AddInt64(&c.misses, 1) return nil, new(CacheMiss) } -func (c *NullCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *NullCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { return fh, nil } diff --git a/vendor/github.com/whosonfirst/go-cache/string.go b/vendor/github.com/whosonfirst/go-cache/string.go new file mode 100644 index 00000000..7611d2fc --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/string.go @@ -0,0 +1,56 @@ +package cache + +import ( + "context" + "github.com/whosonfirst/go-ioutil" + "io" + "strings" +) + +func ReadSeekCloserFromString(v string) (io.ReadSeekCloser, error) { + sr := strings.NewReader(v) + return ioutil.NewReadSeekCloser(sr) +} + +func SetString(ctx context.Context, c Cache, k string, v string) (string, error) { + + fh, err := ReadSeekCloserFromString(v) + + if err != nil { + return "", err + } + + fh, err = c.Set(ctx, k, fh) + + if err != nil { + return "", err + } + + defer fh.Close() + + return toString(fh) +} + +func GetString(ctx context.Context, c Cache, k string) (string, error) { + + fh, err := c.Get(ctx, k) + + if err != nil { + return "", err + } + + defer fh.Close() + + return toString(fh) +} + +func toString(fh io.Reader) (string, error) { + + b, err := io.ReadAll(fh) + + if err != nil { + return "", err + } + + return string(b), nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/testing.go b/vendor/github.com/whosonfirst/go-cache/testing.go new file mode 100644 index 00000000..06aa4093 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/testing.go @@ -0,0 +1,106 @@ +package cache + +import ( + "bytes" + "context" + "fmt" + "io" + _ "log" +) + +type testCacheOptions struct { + AllowCacheMiss bool +} + +func testCache(ctx context.Context, c Cache, opts *testCacheOptions) error { + + k := "test" + v := "test" + + fh1, err := ReadSeekCloserFromString(v) + + if err != nil { + return fmt.Errorf("Failed to create ReadSeekCloser, %v", err) + } + + fh2, err := c.Set(ctx, k, fh1) + + if err != nil { + return fmt.Errorf("Failed to set %s (%s), %v", k, v, err) + } + + equals, err := compareReadSeekClosers(fh1, fh2) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 2 failed equality test") + } + + fh3, err := c.Get(ctx, k) + + if err != nil && !IsCacheMiss(err) { + return fmt.Errorf("Failed to get cache value for '%s', %v", k, err) + } + + if err != nil { + + if !opts.AllowCacheMiss { + return fmt.Errorf("Unexpected cache miss for '%s'", k) + } + + } else { + + equals, err = compareReadSeekClosers(fh1, fh3) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles 1 and 3, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 3 failed equality test") + } + } + + return nil +} + +func compareReadSeekClosers(fh1 io.ReadSeekCloser, fh2 io.ReadSeekCloser) (bool, error) { + + fh1.Seek(0, 0) + fh2.Seek(0, 0) + + defer func() { + fh1.Seek(0, 0) + fh2.Seek(0, 0) + }() + + b1, err := io.ReadAll(fh1) + + if err != nil { + return false, fmt.Errorf("Failed to read fh1, %v", err) + } + + fh1.Seek(0, 0) + + b2, err := io.ReadAll(fh2) + + if err != nil { + return false, fmt.Errorf("Failed to read fh2, %v", err) + } + + fh2.Seek(0, 0) + + // log.Printf("1 '%s'\n", string(b1)) + // log.Printf("2 '%s'\n", string(b2)) + + equals := bytes.Compare(b1, b2) + + if equals != 0 { + return false, nil + } + + return true, nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/utils.go b/vendor/github.com/whosonfirst/go-cache/utils.go deleted file mode 100644 index 532869db..00000000 --- a/vendor/github.com/whosonfirst/go-cache/utils.go +++ /dev/null @@ -1,59 +0,0 @@ -package cache - -import ( - "bytes" - "context" - "io" - "io/ioutil" -) - -func NewReadCloser(b []byte) io.ReadCloser { - r := bytes.NewReader(b) - return ioutil.NopCloser(r) -} - -func NewReadCloserFromString(s string) io.ReadCloser { - return NewReadCloser([]byte(s)) -} - -func SetString(c Cache, k string, v string) (string, error) { - - ctx := context.Background() - - r := NewReadCloserFromString(v) - fh, err := c.Set(ctx, k, r) - - if err != nil { - return "", err - } - - defer fh.Close() - - return toString(fh) -} - -func GetString(c Cache, k string) (string, error) { - - ctx := context.Background() - - fh, err := c.Get(ctx, k) - - if err != nil { - return "", err - } - - defer fh.Close() - - return toString(fh) -} - -func toString(fh io.Reader) (string, error) { - - b, err := ioutil.ReadAll(fh) - - if err != nil { - return "", err - } - - return string(b), nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7a711da7..e440a2c8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -39,13 +39,14 @@ github.com/tidwall/match github.com/tidwall/pretty # github.com/whosonfirst/algnhsa v0.1.0 github.com/whosonfirst/algnhsa -# github.com/whosonfirst/go-cache v0.4.0 +# github.com/whosonfirst/go-cache v0.5.0 ## explicit github.com/whosonfirst/go-cache -# github.com/whosonfirst/go-cache-blob v0.1.2 +# github.com/whosonfirst/go-cache-blob v0.2.0 ## explicit github.com/whosonfirst/go-cache-blob # github.com/whosonfirst/go-ioutil v1.0.0 +## explicit github.com/whosonfirst/go-ioutil # github.com/whosonfirst/go-reader v0.5.0 ## explicit