Make BGS spidering optional

cmd/bigsky/main.go

@@ -100,6 +100,11 @@ func run(args []string) {
 			Name:  "aggregation",
 			Value: false,
 		},
+		&cli.BoolFlag{
+			Name:    "spidering",
+			Value:   true,
+			EnvVars: []string{"BGS_SPIDERING"},
+		},
 		&cli.StringFlag{
 			Name:  "api-listen",
 			Value: ":2470",
@@ -278,7 +283,7 @@ func Bigsky(cctx *cli.Context) error {
 
 	notifman := &notifs.NullNotifs{}
 
-	ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("aggregation"))
+	ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("spidering"), cctx.Bool("aggregation"))
 	if err != nil {
 		return err
 	}

indexer/indexer.go

@@ -51,13 +51,14 @@ type Indexer struct {
 	LimitMux sync.RWMutex
 
 	doAggregations bool
+	doSpider       bool
 
 	SendRemoteFollow       func(context.Context, string, uint) error
 	CreateExternalUser     func(context.Context, string) (*models.ActorInfo, error)
 	ApplyPDSClientSettings func(*xrpc.Client)
 }
 
-func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate bool) (*Indexer, error) {
+func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate, spider bool) (*Indexer, error) {
 	db.AutoMigrate(&models.FeedPost{})
 	db.AutoMigrate(&models.ActorInfo{})
 	db.AutoMigrate(&models.FollowRecord{})
@@ -72,6 +73,7 @@ func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events
 		didr:           didr,
 		Limiters:       make(map[uint]*rate.Limiter),
 		doAggregations: aggregate,
+		doSpider:       spider,
 		SendRemoteFollow: func(context.Context, string, uint) error {
 			return nil
 		},
@@ -181,10 +183,11 @@ func (ix *Indexer) handleRepoOp(ctx context.Context, evt *repomgr.RepoEvent, op
 				return fmt.Errorf("handle recordCreate: %w", err)
 			}
 		}
-		if err := ix.crawlRecordReferences(ctx, op); err != nil {
-			return err
+		if ix.doSpider {
+			if err := ix.crawlRecordReferences(ctx, op); err != nil {
+				return err
+			}
 		}
-
 	case repomgr.EvtKindDeleteRecord:
 		if ix.doAggregations {
 			if err := ix.handleRecordDelete(ctx, evt, op, true); err != nil {

indexer/posts_test.go

@@ -61,7 +61,7 @@ func testIndexer(t *testing.T) *testIx {
 
 	didr := testPLC(t)
 
-	ix, err := NewIndexer(maindb, notifman, evtman, didr, repoman, false, true)
+	ix, err := NewIndexer(maindb, notifman, evtman, didr, repoman, false, true, true)
 	if err != nil {
 		t.Fatal(err)
 	}

pds/server.go

@@ -80,7 +80,7 @@ func NewServer(db *gorm.DB, cs *carstore.CarStore, serkey *did.PrivKey, handleSu
 	repoman := repomgr.NewRepoManager(cs, kmgr)
 	notifman := notifs.NewNotificationManager(db, repoman.GetRecord)
 
-	ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true)
+	ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true, true)
 	if err != nil {
 		return nil, err
 	}

testing/utils.go

@@ -436,7 +436,7 @@ func SetupBGS(ctx context.Context, didr plc.PLCClient) (*TestBGS, error) {
 
 	evtman := events.NewEventManager(diskpersist)
 
-	ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true)
+	ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true, true)
 	if err != nil {
 		return nil, err
 	}