Skip to content

Commit

Permalink
all: restructure
Browse files Browse the repository at this point in the history
  • Loading branch information
@n8maninger
n8maninger committed Mar 3, 2024
1 parent 4bff688 commit 10a63c0
Show file tree
Hide file tree
Showing 18 changed files with 710 additions and 1,088 deletions.
44 changes: 28 additions & 16 deletions cmd/fsd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ import (
"go.sia.tech/fsd/config"
shttp "go.sia.tech/fsd/http"
"go.sia.tech/fsd/ipfs"
"go.sia.tech/fsd/persist/badger"
"go.sia.tech/fsd/sia"
"go.sia.tech/fsd/renterd"
"go.sia.tech/fsd/renterd/downloader"
"go.sia.tech/jape"
"go.sia.tech/renterd/bus"
"go.sia.tech/renterd/worker"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -119,13 +121,8 @@ func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
defer cancel()

db, err := badger.OpenDatabase(filepath.Join(dir, "fsd.badgerdb"), log.Named("badger"))
if err != nil {
log.Fatal("failed to open badger database", zap.Error(err))
}
defer db.Close()

var privateKey crypto.PrivKey
var err error
if cfg.IPFS.PrivateKey != "" {
buf, err := hex.DecodeString(strings.TrimPrefix(cfg.IPFS.PrivateKey, "ed25519:"))
if err != nil {
Expand All @@ -150,15 +147,30 @@ func main() {
}
defer ds.Close()

bs := sia.NewBlockStore(db, cfg.Renterd, log.Named("blockstore"))
workerClient := worker.NewClient(cfg.Renterd.WorkerAddress, cfg.Renterd.WorkerPassword)
busClient := bus.NewClient(cfg.Renterd.BusAddress, cfg.Renterd.BusPassword)

inode, err := ipfs.NewNode(ctx, privateKey, cfg.IPFS, ds, bs)
bd, err := downloader.NewBlockDownloader(cfg.Renterd.Bucket, 4096, workerClient, log.Named("downloader"))
if err != nil {
log.Fatal("failed to start ipfs node", zap.Error(err))
log.Fatal("failed to create block downloader", zap.Error(err))
}
bd.StartWorkers(ctx, 1000)

bs, err := renterd.NewBlockStore(
renterd.WithBucket(cfg.Renterd.Bucket),
renterd.WithWorker(workerClient),
renterd.WithBus(busClient),
renterd.WithDownloader(bd),
renterd.WithLog(log.Named("blockstore")))
if err != nil {
log.Fatal("failed to create blockstore", zap.Error(err))
}
defer inode.Close()

snode := sia.New(db, inode, cfg.Renterd, log.Named("sia"))
ipfs, err := ipfs.NewNode(ctx, privateKey, cfg.IPFS, ds, bs)
if err != nil {
log.Fatal("failed to start ipfs node", zap.Error(err))
}
defer ipfs.Close()

apiListener, err := net.Listen("tcp", cfg.API.Address)
if err != nil {
Expand All @@ -173,12 +185,12 @@ func main() {
defer gatewayListener.Close()

apiServer := &http.Server{
Handler: jape.BasicAuth(cfg.API.Password)(shttp.NewAPIHandler(inode, snode, cfg, log.Named("api"))),
Handler: jape.BasicAuth(cfg.API.Password)(shttp.NewAPIHandler(ipfs, cfg, log.Named("api"))),
}
defer apiServer.Close()

gatewayServer := &http.Server{
Handler: shttp.NewIPFSGatewayHandler(inode, snode, cfg, log.Named("gateway")),
Handler: shttp.NewIPFSGatewayHandler(ipfs, cfg, log.Named("gateway")),
}
defer gatewayServer.Close()

Expand All @@ -201,7 +213,7 @@ func main() {
prettyKey := "ed25519:" + hex.EncodeToString(buf)

log.Info("fsd started",
zap.Stringer("peerID", inode.PeerID()),
zap.Stringer("peerID", ipfs.PeerID()),
zap.String("privateKey", prettyKey),
zap.String("apiAddress", apiListener.Addr().String()),
zap.String("gatewayAddress", gatewayListener.Addr().String()),
Expand Down
13 changes: 3 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ module go.sia.tech/fsd
go 1.21

require (
github.com/dgraph-io/badger/v4 v4.2.0
github.com/gogo/protobuf v1.3.2
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/ipfs/boxo v0.15.0
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.1
Expand All @@ -17,7 +16,7 @@ require (
github.com/multiformats/go-multicodec v0.9.0
github.com/multiformats/go-multihash v0.2.3
go.sia.tech/jape v0.11.1
go.sia.tech/renterd v1.0.5
go.sia.tech/renterd v1.0.6-0.20240228125621-9a935c3b8f30
go.uber.org/zap v1.26.0
gopkg.in/yaml.v3 v3.0.1
lukechampine.com/frand v1.4.2
Expand All @@ -37,10 +36,7 @@ require (
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
Expand All @@ -49,11 +45,9 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
github.com/google/uuid v1.5.0 // indirect
Expand All @@ -62,7 +56,6 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/huin/goupnp v1.3.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
Expand Down
18 changes: 2 additions & 16 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
Expand Down Expand Up @@ -85,21 +84,13 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
Expand Down Expand Up @@ -148,8 +139,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand All @@ -176,8 +165,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -687,8 +674,8 @@ go.sia.tech/jape v0.11.1 h1:M7IP+byXL7xOqzxcHUQuXW+q3sYMkYzmMlMw+q8ZZw0=
go.sia.tech/jape v0.11.1/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4=
go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU=
go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo=
go.sia.tech/renterd v1.0.5 h1:qb7tLBthF5ocHaSqgy+yGBkdFKOT4NAEEx8BIX3zzCs=
go.sia.tech/renterd v1.0.5/go.mod h1:0y3agucuCIp1L7W2tRjksvcJIsTZfoC9TZc3h8ie1kU=
go.sia.tech/renterd v1.0.6-0.20240228125621-9a935c3b8f30 h1:ptKXoEbZkCyHUVJs/fGDVb2BixKiEwLlvHHXnA36+Xc=
go.sia.tech/renterd v1.0.6-0.20240228125621-9a935c3b8f30/go.mod h1:0y3agucuCIp1L7W2tRjksvcJIsTZfoC9TZc3h8ie1kU=
go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca h1:aZMg2AKevn7jKx+wlusWQfwSM5pNU9aGtRZme29q3O4=
go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca/go.mod h1:h/1afFwpxzff6/gG5i1XdAgPK7dEY6FaibhK7N5F86Y=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down Expand Up @@ -837,7 +824,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
77 changes: 5 additions & 72 deletions http/api.go
View file
Original file line number Diff line number Diff line change
@@ -1,86 +1,22 @@
package http

import (
"bufio"
"net/http"

"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p/core/peer"
"go.sia.tech/fsd/config"
"go.sia.tech/fsd/ipfs"
"go.sia.tech/fsd/sia"
"go.sia.tech/jape"
"go.uber.org/zap"
)

type (
apiServer struct {
ipfs *ipfs.Node
sia *sia.Node
log *zap.Logger
}
)

func (as *apiServer) handlePin(jc jape.Context) {
ctx := jc.Request.Context()
var cidStr string
if err := jc.DecodeParam("cid", &cidStr); err != nil {
return
}
c, err := cid.Parse(cidStr)
if err != nil {
jc.Error(err, http.StatusBadRequest)
return
}

// TODO: break this out for better support, the current implementation will
// not properly handle anything but standard unixfs files with the default
// block size
rr, err := as.ipfs.DownloadCID(ctx, c, nil)
if err != nil {
jc.Error(err, http.StatusInternalServerError)
return
}
defer rr.Close()

var opts sia.UnixFSOptions
switch c.Version() {
case 1:
prefix := c.Prefix()
opts.CIDBuilder = cid.V1Builder{Codec: prefix.Codec, MhType: prefix.MhType, MhLength: prefix.MhLength}
case 0:
opts.CIDBuilder = cid.V0Builder{}
}

br := bufio.NewReaderSize(rr, 256<<20) // 256 MiB
c, err = as.sia.UploadFile(jc.Request.Context(), br, opts)
if err != nil {
jc.Error(err, http.StatusInternalServerError)
return
}

// return the calculated cid
jc.Encode(c.String())
}

func (as *apiServer) handleCIDVerify(jc jape.Context) {
ctx := jc.Request.Context()
var cidStr string
if err := jc.DecodeParam("cid", &cidStr); err != nil {
return
}
c, err := cid.Parse(cidStr)
if err != nil {
jc.Error(err, http.StatusBadRequest)
return
}

if err := as.sia.VerifyCID(ctx, c); err != nil {
jc.Error(err, http.StatusInternalServerError)
return
}
}

func (as *apiServer) handleListPeers(jc jape.Context) {
jc.Encode(as.ipfs.Peers())
}
Expand All @@ -94,18 +30,15 @@ func (as *apiServer) handleAddPeer(jc jape.Context) {
}

// NewAPIHandler returns a new http.Handler that handles requests to the api
func NewAPIHandler(ipfs *ipfs.Node, sia *sia.Node, cfg config.Config, log *zap.Logger) http.Handler {
func NewAPIHandler(ipfs *ipfs.Node, cfg config.Config, log *zap.Logger) http.Handler {
s := &apiServer{
ipfs: ipfs,
sia: sia,
log: log,
}
return jape.Mux(map[string]jape.Handler{
"POST /api/cid/:cid/verify": s.handleCIDVerify,
"POST /api/unixfs/calculate": s.handleUnixFSCalculate,
"POST /api/unixfs/upload": s.handleUnixFSUpload,
"POST /api/pin/:cid": s.handlePin,
"GET /api/peers": s.handleListPeers,
"PUT /api/peers": s.handleAddPeer,
"POST /api/unixfs/upload": s.handleUnixFSUpload,
// "POST /api/pin/:cid": s.handlePin,
"GET /api/peers": s.handleListPeers,
"PUT /api/peers": s.handleAddPeer,
})
}
Loading

0 comments on commit 10a63c0

Please sign in to comment.