From 2808dc4482ec2686bd0abc4944f2a952477abb76 Mon Sep 17 00:00:00 2001 From: Gabe <7622243+decentralgabe@users.noreply.github.com> Date: Tue, 5 Dec 2023 18:26:53 +0100 Subject: [PATCH] More caching (#62) * fix caching * fix cache * remove comment * fix test * clean up go mod * update cache value to be response * faster --- impl/cmd/cli/identity.go | 3 +- impl/config/config.go | 4 +- impl/config/config.toml | 2 +- impl/go.mod | 8 +--- impl/go.sum | 21 +--------- impl/internal/cli/store.go | 3 +- impl/internal/did/client_test.go | 4 -- impl/internal/did/did_test.go | 3 +- impl/internal/did/testdata_test.go | 3 +- impl/pkg/server/server_pkarr_test.go | 4 -- impl/pkg/server/server_test.go | 3 +- impl/pkg/service/pkarr.go | 58 +++++++++++++++++++++------- impl/pkg/service/pkarr_test.go | 4 -- impl/pkg/storage/pkarr.go | 2 +- impl/pkg/storage/storage_test.go | 3 +- 15 files changed, 62 insertions(+), 63 deletions(-) diff --git a/impl/cmd/cli/identity.go b/impl/cmd/cli/identity.go index d6ad6811..79735413 100644 --- a/impl/cmd/cli/identity.go +++ b/impl/cmd/cli/identity.go @@ -2,9 +2,10 @@ package main import ( "context" - "encoding/json" "fmt" + "github.com/goccy/go-json" + "github.com/miekg/dns" "github.com/mr-tron/base58" "github.com/sirupsen/logrus" diff --git a/impl/config/config.go b/impl/config/config.go index 55fa1265..97c37eab 100644 --- a/impl/config/config.go +++ b/impl/config/config.go @@ -57,7 +57,7 @@ type DHTServiceConfig struct { type PKARRServiceConfig struct { RepublishCRON string `toml:"republish_cron"` - CacheTTLMinutes int64 `toml:"cache_ttl_minutes"` + CacheTTLSeconds int `toml:"cache_ttl_seconds"` } func GetDefaultConfig() Config { @@ -76,7 +76,7 @@ func GetDefaultConfig() Config { }, PkarrConfig: PKARRServiceConfig{ RepublishCRON: "0 */2 * * *", - CacheTTLMinutes: 10, + CacheTTLSeconds: 600, }, } } diff --git a/impl/config/config.toml b/impl/config/config.toml index f0e57594..849f0520 100644 --- a/impl/config/config.toml +++ b/impl/config/config.toml @@ -12,4 +12,4 @@ bootstrap_peers = ["router.magnets.im:6881", "router.bittorrent.com:6881", "dht. [pkarr] republish_cron = "0 */2 * * *" # every 2 hours -cache_ttl_minutes = 10 \ No newline at end of file +cache_ttl_seconds = 600 # 10 minutes diff --git a/impl/go.mod b/impl/go.mod index aacf7237..6af99037 100644 --- a/impl/go.mod +++ b/impl/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/BurntSushi/toml v0.3.1 github.com/TBD54566975/ssi-sdk v0.0.4-alpha.0.20231121023732-c496504a93c4 + github.com/allegro/bigcache/v3 v3.1.0 github.com/anacrolix/dht/v2 v2.20.0 github.com/anacrolix/log v0.14.0 github.com/anacrolix/torrent v1.52.5 @@ -45,7 +46,6 @@ require ( github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect - github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.6 // indirect @@ -62,7 +62,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/uuid v1.4.0 // indirect - github.com/gowebpki/jcs v1.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/hyperledger/aries-framework-go v0.3.2 // indirect @@ -71,7 +70,6 @@ require ( github.com/hyperledger/aries-framework-go/component/models v0.0.0-20230501135648-a9a7ad029347 // indirect github.com/hyperledger/aries-framework-go/spi v0.0.0-20230427134832-0c9969493bd3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jellydator/ttlcache/v3 v3.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kilic/bls12-381 v0.1.1-0.20210503002446-7b7597926c69 // indirect @@ -86,7 +84,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -94,7 +91,6 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect - github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/piprate/json-gold v0.5.0 // indirect @@ -106,7 +102,6 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -131,5 +126,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/impl/go.sum b/impl/go.sum index 4bd370e9..4c585aec 100644 --- a/impl/go.sum +++ b/impl/go.sum @@ -62,6 +62,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk= +github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/anacrolix/chansync v0.3.0 h1:lRu9tbeuw3wl+PhMu/r+JJCRu5ArFXIluOgdF0ao6/U= github.com/anacrolix/chansync v0.3.0/go.mod h1:DZsatdsdXxD0WiwcGl0nJVwyjCKMDv+knl1q2iBjA2k= github.com/anacrolix/dht/v2 v2.20.0 h1:eDx9lfE9iCSf5sPK0290GToHURNhEFuUGN8iyvhvJDk= @@ -114,8 +116,6 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -141,7 +141,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -292,10 +291,6 @@ github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORR github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= -github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= -github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= -github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -324,8 +319,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g= -github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -388,8 +381,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -413,8 +404,6 @@ github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivnc github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= -github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= -github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -487,8 +476,6 @@ github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:X github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -916,8 +903,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= -gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -940,8 +925,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/impl/internal/cli/store.go b/impl/internal/cli/store.go index 1f2d4c24..80a70047 100644 --- a/impl/internal/cli/store.go +++ b/impl/internal/cli/store.go @@ -1,9 +1,10 @@ package cli import ( - "encoding/json" "os" + "github.com/goccy/go-json" + "github.com/sirupsen/logrus" "github.com/TBD54566975/did-dht-method/internal" diff --git a/impl/internal/did/client_test.go b/impl/internal/did/client_test.go index d4ed4b1f..2a0b96f0 100644 --- a/impl/internal/did/client_test.go +++ b/impl/internal/did/client_test.go @@ -2,7 +2,6 @@ package did import ( "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,9 +28,6 @@ func TestClient(t *testing.T) { err = client.PutDocument(doc.ID, *bep44Put) assert.NoError(t, err) - // wait for the record to be published - time.Sleep(10 * time.Second) - gotDID, _, err := client.GetDIDDocument(doc.ID) assert.NoError(t, err) assert.EqualValues(t, doc, gotDID) diff --git a/impl/internal/did/did_test.go b/impl/internal/did/did_test.go index 8e7da96d..ffeeb945 100644 --- a/impl/internal/did/did_test.go +++ b/impl/internal/did/did_test.go @@ -2,9 +2,10 @@ package did import ( "crypto/ed25519" - "encoding/json" "testing" + "github.com/goccy/go-json" + "github.com/TBD54566975/ssi-sdk/crypto" "github.com/TBD54566975/ssi-sdk/crypto/jwx" "github.com/TBD54566975/ssi-sdk/did" diff --git a/impl/internal/did/testdata_test.go b/impl/internal/did/testdata_test.go index c870a4b6..92d4db52 100644 --- a/impl/internal/did/testdata_test.go +++ b/impl/internal/did/testdata_test.go @@ -2,9 +2,10 @@ package did import ( "embed" - "encoding/json" "testing" + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" ) diff --git a/impl/pkg/server/server_pkarr_test.go b/impl/pkg/server/server_pkarr_test.go index bf2ff97c..9d11268c 100644 --- a/impl/pkg/server/server_pkarr_test.go +++ b/impl/pkg/server/server_pkarr_test.go @@ -8,7 +8,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,9 +54,6 @@ func TestPKARRRouter(t *testing.T) { req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", testServerURL, suffix), nil) c = newRequestContextWithParams(w, req, map[string]string{IDParam: suffix}) - // wait for the record to be published - time.Sleep(10 * time.Second) - pkarrRouter.GetRecord(c) assert.True(t, is2xxResponse(w.Code)) diff --git a/impl/pkg/server/server_test.go b/impl/pkg/server/server_test.go index 0b47f136..a4c6d562 100644 --- a/impl/pkg/server/server_test.go +++ b/impl/pkg/server/server_test.go @@ -2,13 +2,14 @@ package server import ( "bytes" - "encoding/json" "io" "net/http" "net/http/httptest" "os" "testing" + "github.com/goccy/go-json" + "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/impl/pkg/service/pkarr.go b/impl/pkg/service/pkarr.go index fdc2d0b0..bac3003c 100644 --- a/impl/pkg/service/pkarr.go +++ b/impl/pkg/service/pkarr.go @@ -6,10 +6,12 @@ import ( "errors" "time" + "github.com/goccy/go-json" + "github.com/TBD54566975/ssi-sdk/util" + "github.com/allegro/bigcache/v3" "github.com/anacrolix/dht/v2/bep44" "github.com/anacrolix/torrent/bencode" - "github.com/jellydator/ttlcache/v3" "github.com/sirupsen/logrus" "github.com/TBD54566975/did-dht-method/config" @@ -23,7 +25,7 @@ type PkarrService struct { cfg *config.Config db *storage.Storage dht *dht.DHT - cache *ttlcache.Cache[string, storage.PkarrRecord] + cache *bigcache.BigCache scheduler *dhtint.Scheduler } @@ -41,10 +43,12 @@ func NewPkarrService(cfg *config.Config, db *storage.Storage) (*PkarrService, er } // create and start cache and scheduler - ttl := time.Duration(cfg.PkarrConfig.CacheTTLMinutes) * time.Minute - cache := ttlcache.New[string, storage.PkarrRecord]( - ttlcache.WithTTL[string, storage.PkarrRecord](ttl), - ) + ttl := time.Duration(cfg.PkarrConfig.CacheTTLSeconds) * time.Second + // TODO(gabe): consider setting size limits on the cache + cache, err := bigcache.New(context.Background(), bigcache.DefaultConfig(ttl)) + if err != nil { + return nil, util.LoggingErrorMsg(err, "failed to instantiate cache") + } scheduler := dhtint.NewScheduler() service := PkarrService{ cfg: cfg, @@ -53,7 +57,6 @@ func NewPkarrService(cfg *config.Config, db *storage.Storage) (*PkarrService, er cache: cache, scheduler: &scheduler, } - go cache.Start() if err = scheduler.Schedule(cfg.PkarrConfig.RepublishCRON, service.republish); err != nil { return nil, util.LoggingErrorMsg(err, "failed to start republisher") } @@ -106,7 +109,17 @@ func (s *PkarrService) PublishPkarr(ctx context.Context, id string, request Publ if err := s.db.WriteRecord(record); err != nil { return err } - s.cache.Set(id, record, ttlcache.DefaultTTL) + recordBytes, err := json.Marshal(GetPkarrResponse{ + V: request.V, + Seq: request.Seq, + Sig: request.Sig, + }) + if err != nil { + return err + } + if err = s.cache.Set(id, recordBytes); err != nil { + return err + } // return here and put it in the DHT asynchronously go s.dht.Put(ctx, bep44.Put{ @@ -146,11 +159,13 @@ func fromPkarrRecord(record storage.PkarrRecord) (*GetPkarrResponse, error) { // GetPkarr returns the full Pkarr record (including sig data) for the given z-base-32 encoded ID func (s *PkarrService) GetPkarr(ctx context.Context, id string) (*GetPkarrResponse, error) { // first do a cache lookup - if s.cache.Has(id) { - cacheItem := s.cache.Get(id) - until := time.Until(cacheItem.ExpiresAt()) - logrus.Debugf("resolved pkarr record<%s> from cache, with remaining TTL: %s", id, until.String()) - return fromPkarrRecord(cacheItem.Value()) + if got, err := s.cache.Get(id); err == nil { + var resp GetPkarrResponse + if err = json.Unmarshal(got, &resp); err != nil { + return nil, err + } + logrus.Debugf("resolved pkarr record[%s] from cache", id) + return &resp, nil } // next do a dht lookup @@ -167,6 +182,8 @@ func (s *PkarrService) GetPkarr(ctx context.Context, id string) (*GetPkarrRespon logrus.Debugf("resolved pkarr<%s> from storage", id) return fromPkarrRecord(*record) } + + // prepare the record for return bBytes, err := got.V.MarshalBencode() if err != nil { return nil, err @@ -175,11 +192,22 @@ func (s *PkarrService) GetPkarr(ctx context.Context, id string) (*GetPkarrRespon if err = bencode.Unmarshal(bBytes, &payload); err != nil { return nil, err } - return &GetPkarrResponse{ + resp := GetPkarrResponse{ V: []byte(payload), Seq: got.Seq, Sig: got.Sig, - }, nil + } + + // add the record to cache, do it here to avoid duplicate calculations + recordBytes, err := json.Marshal(resp) + if err != nil { + return nil, util.LoggingErrorMsgf(err, "failed to marshal pkarr record<%s> for cache", id) + } + if err = s.cache.Set(id, recordBytes); err != nil { + return nil, util.LoggingErrorMsgf(err, "failed to set pkarr record<%s> in cache", id) + } + + return &resp, nil } // TODO(gabe) make this more efficient. create a publish schedule based on each individual record, not all records diff --git a/impl/pkg/service/pkarr_test.go b/impl/pkg/service/pkarr_test.go index 7430679a..21f31b91 100644 --- a/impl/pkg/service/pkarr_test.go +++ b/impl/pkg/service/pkarr_test.go @@ -3,7 +3,6 @@ package service import ( "context" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -92,9 +91,6 @@ func TestPKARRService(t *testing.T) { }) assert.NoError(t, err) - // wait for the record to be published - time.Sleep(10 * time.Second) - got, err := svc.GetPkarr(context.Background(), suffix) assert.NoError(t, err) assert.NotEmpty(t, got) diff --git a/impl/pkg/storage/pkarr.go b/impl/pkg/storage/pkarr.go index d2339f52..16d688d7 100644 --- a/impl/pkg/storage/pkarr.go +++ b/impl/pkg/storage/pkarr.go @@ -1,7 +1,7 @@ package storage import ( - "encoding/json" + "github.com/goccy/go-json" ) const ( diff --git a/impl/pkg/storage/storage_test.go b/impl/pkg/storage/storage_test.go index 2ad558d7..58a48844 100644 --- a/impl/pkg/storage/storage_test.go +++ b/impl/pkg/storage/storage_test.go @@ -1,10 +1,11 @@ package storage import ( - "encoding/json" "os" "testing" + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" )