Skip to content

Commit

Permalink
republishing and storage (#21)
Browse files Browse the repository at this point in the history
* republishing and storage

* fix republishing

* Testing

* server tests

* readme for the impl

* fix tests
  • Loading branch information
decentralgabe authored Oct 23, 2023
1 parent f7466b4 commit f93a1e2
Show file tree
Hide file tree
Showing 23 changed files with 641 additions and 285 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ in Go.

## Build & Run

From the root directory run:
From the `impl` directory run:

```
docker build . -t did-dht -f build/Dockerfile
Expand Down
41 changes: 41 additions & 0 deletions impl/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Server Implementation

- Heavily a work-in-progress
- Designed to be run as a single instance

## Config

# TOML Config File

Config is managed using a [TOML](https://toml.io/en/) [file](../../config/dev.toml). There are sets of configuration values for the server
(e.g. which port to listen on), the services (e.g. which database to use), and each service.

Each service may define specific configuration, such as which DID methods are enabled for the DID service.

A full config example is [provided here](../../config/kitchensink.toml).

## Usage

How it works:

1. On startup the service loads default values into the `ServiceConfig`
2. Checks for a TOML config file:
- If exists, load toml file
- If does not exist, it uses a default config defined in the code inline
3. Loads the `config/.env` file and adds the env variables defined in this file to the final `ServiceConfig`

## Build & Run

Run:

```
docker build . -t did-dht -f build/Dockerfile
```

and then

```
docker run -p8305:8305 did-dht
```


6 changes: 3 additions & 3 deletions impl/cmd/cli/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ var identityAddCmd = &cobra.Command{
Answer: rrds,
}
// generate put request
putReq, err := dht.CreatePKARRPublishRequest(pubKey, privKey, msg)
putReq, err := dht.CreatePKARRPublishRequest(privKey, msg)
if err != nil {
logrus.WithError(err).Error("failed to create put request")
return err
Expand Down Expand Up @@ -133,8 +133,8 @@ var identityAddCmd = &cobra.Command{

var identityGetCmd = &cobra.Command{
Use: "get",
Short: "Get an identity",
Long: `Get an identity by its id.`,
Short: "GetRecord an identity",
Long: `GetRecord an identity by its id.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
id := args[0]
Expand Down
19 changes: 12 additions & 7 deletions impl/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@ func (e EnvironmentVariable) String() string {
}

type Config struct {
ServerConfig ServiceConfig `toml:"server"`
DHTConfig DHTServiceConfig `toml:"dht"`
ServerConfig ServerConfig `toml:"server"`
DHTConfig DHTServiceConfig `toml:"dht"`
PKARRConfig PKARRServiceConfig `toml:"pkarr"`
}

type ServiceConfig struct {
type ServerConfig struct {
Environment Environment `toml:"env"`
APIHost string `toml:"api_host"`
APIPort int `toml:"api_port"`
BaseURL string `toml:"base_url"`
LogLocation string `toml:"log_location"`
LogLevel string `toml:"log_level"`
DBFile string `toml:"db_file"`
Expand All @@ -53,24 +55,27 @@ type DHTServiceConfig struct {
BootstrapPeers []string `toml:"bootstrap_peers"`
}

type GossipServiceConfig struct {
// if set, the API will only accept signed messages
EnforceSignedMessages bool `toml:"enforce_signed_messages"`
type PKARRServiceConfig struct {
RepublishCRON string `toml:"republish_cron"`
}

func GetDefaultConfig() Config {
return Config{
ServerConfig: ServiceConfig{
ServerConfig: ServerConfig{
Environment: EnvironmentDev,
APIHost: "0.0.0.0",
APIPort: 8305,
BaseURL: "http://localhost:8305",
LogLocation: "log",
LogLevel: "debug",
DBFile: "diddht.db",
},
DHTConfig: DHTServiceConfig{
BootstrapPeers: GetDefaultBootstrapPeers(),
},
PKARRConfig: PKARRServiceConfig{
RepublishCRON: "0 */2 * * *",
},
}
}

Expand Down
5 changes: 4 additions & 1 deletion impl/config/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,7 @@ db_file = "diddht.db"

[dht]
bootstrap_peers = ["router.magnets.im:6881", "router.bittorrent.com:6881", "dht.transmissionbt.com:6881",
"router.utorrent.com:6881", "router.nuh.dev:6881"]
"router.utorrent.com:6881", "router.nuh.dev:6881"]

[pkarr]
republish_cron = "0 */2 * * *" # every 2 hours
2 changes: 1 addition & 1 deletion impl/docs/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions impl/docs/swagger.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ paths:
get:
consumes:
- application/octet-stream
description: Get a PKARR from the DHT
description: GetRecord a PKARR record from the DHT
parameters:
- description: ID to get
in: path
Expand Down Expand Up @@ -50,15 +50,15 @@ paths:
description: Internal server error
schema:
type: string
summary: Get a PKARR from the DHT
summary: GetRecord a PKARR record from the DHT
tags:
- Relay
put:
consumes:
- application/octet-stream
description: Put a PKARR record into the DHT
description: PutRecord a PKARR record into the DHT
parameters:
- description: ID to put
- description: ID of the record to put
in: path
name: id
required: true
Expand All @@ -82,7 +82,7 @@ paths:
description: Internal server error
schema:
type: string
summary: Put a PKARR record into the DHT
summary: PutRecord a PKARR record into the DHT
tags:
- Relay
/health:
Expand Down
3 changes: 3 additions & 0 deletions impl/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-co-op/gocron v1.35.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/spec v0.20.9 // indirect
Expand Down Expand Up @@ -98,6 +99,7 @@ require (
github.com/piprate/json-gold v0.5.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/pquerna/cachecontrol v0.1.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
Expand All @@ -112,6 +114,7 @@ require (
github.com/swaggo/swag v1.8.12 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
Expand Down
7 changes: 7 additions & 0 deletions impl/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,8 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod
github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/go-co-op/gocron v1.35.2 h1:lG3rdA9TqBBC/PtT2ukQqgLm6jEepnAzz3+OQetvPTE=
github.com/go-co-op/gocron v1.35.2/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
Expand Down Expand Up @@ -454,9 +456,12 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
Expand Down Expand Up @@ -548,6 +553,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
Expand Down
45 changes: 45 additions & 0 deletions impl/internal/dht/scheduler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package dht

import (
"time"

"github.com/go-co-op/gocron"
"github.com/pkg/errors"
)

// Scheduler runs crob jobs asynchronously, designed to just schedule one job
type Scheduler struct {
scheduler *gocron.Scheduler
job *gocron.Job
}

// NewScheduler creates a new scheduler
func NewScheduler() Scheduler {
s := gocron.NewScheduler(time.UTC)
s.SingletonModeAll()
return Scheduler{scheduler: s}
}

// Schedule schedules a job to run and starts it asynchronously
func (s *Scheduler) Schedule(_ string, job func()) error {
if s.job != nil {
return errors.New("job already scheduled")
}
j, err := s.scheduler.Cron("* * * * *").Do(job)
if err != nil {
return err
}
s.job = j
s.Start()
return nil
}

// Start starts the scheduler
func (s *Scheduler) Start() {
s.scheduler.StartAsync()
}

// Stop stops the scheduler
func (s *Scheduler) Stop() {
s.scheduler.Stop()
}
12 changes: 4 additions & 8 deletions impl/pkg/dht/dht.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/anacrolix/dht/v2/bep44"
"github.com/anacrolix/dht/v2/exts/getput"
"github.com/anacrolix/torrent/types/infohash"
"github.com/sirupsen/logrus"

dhtint "github.com/TBD54566975/did-dht-method/internal/dht"
"github.com/TBD54566975/did-dht-method/internal/util"
Expand All @@ -25,8 +24,7 @@ func NewDHT(bootstrapPeers []string) (*DHT, error) {
c.StartingNodes = func() ([]dht.Addr, error) { return dht.ResolveHostPorts(bootstrapPeers) }
s, err := dht.NewServer(c)
if err != nil {
logrus.WithError(err).Error("failed to create dht server")
return nil, err
return nil, errutil.LoggingErrorMsg(err, "failed to create dht server")
}
return &DHT{Server: s}, nil
}
Expand All @@ -47,8 +45,7 @@ func (d *DHT) Put(ctx context.Context, request bep44.Put) (string, error) {
func (d *DHT) Get(ctx context.Context, key string) (*getput.GetResult, error) {
z32Decoded, err := util.Z32Decode(key)
if err != nil {
logrus.WithError(err).Error("failed to decode key")
return nil, err
return nil, errutil.LoggingErrorMsg(err, "failed to decode key")
}
res, t, err := getput.Get(ctx, infohash.HashBytes(z32Decoded), d.Server, nil, nil)
if err != nil {
Expand All @@ -58,13 +55,12 @@ func (d *DHT) Get(ctx context.Context, key string) (*getput.GetResult, error) {
}

// GetFull returns the full BEP-44 result for the given key from the DHT, using our modified
// implementation of getput.Get. IT should only be used when it's needed to get the signature
// implementation of getput.Get. It should ONLY be used when it's needed to get the signature
// data for a record.
func (d *DHT) GetFull(ctx context.Context, key string) (*dhtint.FullGetResult, error) {
z32Decoded, err := util.Z32Decode(key)
if err != nil {
logrus.WithError(err).Error("failed to decode key")
return nil, err
return nil, errutil.LoggingErrorMsg(err, "failed to decode key")
}
res, t, err := dhtint.Get(ctx, infohash.HashBytes(z32Decoded), d.Server, nil, nil)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion impl/pkg/dht/pkarr.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@ import (
// },
// }
// }
func CreatePKARRPublishRequest(publicKey ed25519.PublicKey, privateKey ed25519.PrivateKey, msg dns.Msg) (*bep44.Put, error) {
func CreatePKARRPublishRequest(privateKey ed25519.PrivateKey, msg dns.Msg) (*bep44.Put, error) {
packed, err := msg.Pack()
if err != nil {
return nil, util.LoggingErrorMsg(err, "failed to pack records")
}
publicKey := privateKey.Public().(ed25519.PublicKey)
put := &bep44.Put{
V: packed,
K: (*[32]byte)(publicKey),
Expand Down
8 changes: 3 additions & 5 deletions impl/pkg/dht/pkarr_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package dht

import (
"context"
"crypto/ed25519"
"testing"

"github.com/TBD54566975/ssi-sdk/crypto"
Expand All @@ -22,7 +21,7 @@ func TestGetPutPKARRDHT(t *testing.T) {
d, err := NewDHT(config.GetDefaultBootstrapPeers())
require.NoError(t, err)

pubKey, privKey, err := util.GenerateKeypair()
_, privKey, err := util.GenerateKeypair()
require.NoError(t, err)

txtRecord := dns.TXT{
Expand All @@ -44,7 +43,7 @@ func TestGetPutPKARRDHT(t *testing.T) {
},
Answer: []dns.RR{&txtRecord},
}
put, err := CreatePKARRPublishRequest(pubKey, privKey, msg)
put, err := CreatePKARRPublishRequest(privKey, msg)
require.NoError(t, err)

id, err := d.Put(context.Background(), *put)
Expand Down Expand Up @@ -105,8 +104,7 @@ func TestGetPutDIDDHT(t *testing.T) {
didDocPacket, err := didID.ToDNSPacket(*doc)
require.NoError(t, err)

key := privKey.Public().(ed25519.PublicKey)
putReq, err := CreatePKARRPublishRequest(key, privKey, *didDocPacket)
putReq, err := CreatePKARRPublishRequest(privKey, *didDocPacket)
require.NoError(t, err)

gotID, err := dht.Put(context.Background(), *putReq)
Expand Down
Loading

0 comments on commit f93a1e2

Please sign in to comment.