GraphD: an in-memory Graph Service

[ericvolp12]

WIP but GraphD (working title) is an in-memory graph daemon to handle lookups for followers, following, blockers, blocking, muters, muting, etc.

Internally represents relationships between UID-mapped DIDs.

Should be very fast and fairly light on memory, at 42M relationships it's using around 2GB of RAM and can grab 142k followers and resolve them to DIDs in ~20ms.

[whyrusleeping]

Made startup go a good deal faster with this:

     follows := make(map[uint64]map[uint64]struct{})
	following := make(map[uint64]map[uint64]struct{})

	for fileScanner.Scan() {
		if totalFollows%1_000_000 == 0 {
			slog.Info("loaded follows", "total", totalFollows, "duration", time.Since(start))
		}

		// actorDid,rkey,targetDid,createdAt,insertedAt
		followTxt := fileScanner.Text()
		parts := strings.Split(followTxt, ",")
		actorDID := parts[0]
		targetDID := parts[2]

		actorUID, ok := graph.GetUid(actorDID)
		if !ok {
			actorUID = nextUID
			nextUID++
			graph.SetUid(actorDID, actorUID)
			graph.SetDid(actorUID, actorDID)
		}
		targetUID, ok := graph.GetUid(targetDID)
		if !ok {
			targetUID = nextUID
			nextUID++
			graph.SetUid(targetDID, targetUID)
			graph.SetDid(targetUID, targetDID)
		}

		totalFollows++

		f, ok := follows[actorUID]
		if !ok {
			f = make(map[uint64]struct{})
			follows[actorUID] = f
		}
		f[targetUID] = struct{}{}

		f, ok = following[targetUID]
		if !ok {
			f = make(map[uint64]struct{})
			following[targetUID] = f
		}
		f[actorUID] = struct{}{}
	}

	graph.BulkLoad(follows, following)

and

func (g *Graph) BulkLoad(follows, following map[uint64]map[uint64]struct{}) {
	for uid, m := range follows {
		g.follows.Store(uid, &FollowMap{data: m})
	}

	for uid, m := range following {
		g.following.Store(uid, &FollowMap{data: m})
	}
}