diff --git a/api/atproto/admindefs.go b/api/atproto/admindefs.go index f674117d6..c6e424236 100644 --- a/api/atproto/admindefs.go +++ b/api/atproto/admindefs.go @@ -10,17 +10,18 @@ import ( // AdminDefs_AccountView is a "accountView" in the com.atproto.admin.defs schema. type AdminDefs_AccountView struct { - DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` - Did string `json:"did" cborgen:"did"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` - InvitedBy *ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` - Invites []*ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` - InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` - RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords,omitempty" cborgen:"relatedRecords,omitempty"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + Invites []*ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords,omitempty" cborgen:"relatedRecords,omitempty"` + ThreatSignatures []*AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` } // AdminDefs_RepoBlobRef is a "repoBlobRef" in the com.atproto.admin.defs schema. @@ -46,3 +47,9 @@ type AdminDefs_StatusAttr struct { Applied bool `json:"applied" cborgen:"applied"` Ref *string `json:"ref,omitempty" cborgen:"ref,omitempty"` } + +// AdminDefs_ThreatSignature is a "threatSignature" in the com.atproto.admin.defs schema. +type AdminDefs_ThreatSignature struct { + Property string `json:"property" cborgen:"property"` + Value string `json:"value" cborgen:"value"` +} diff --git a/api/atproto/cbor_gen.go b/api/atproto/cbor_gen.go index c1f325ebc..d0b57ccbb 100644 --- a/api/atproto/cbor_gen.go +++ b/api/atproto/cbor_gen.go @@ -129,21 +129,24 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RepoStrongRef: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": @@ -180,7 +183,9 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -492,21 +497,24 @@ func (t *SyncSubscribeRepos_Commit) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Commit: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Ops ([]*atproto.SyncSubscribeRepos_RepoOp) (slice) case "ops": @@ -767,7 +775,9 @@ func (t *SyncSubscribeRepos_Commit) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -901,21 +911,24 @@ func (t *SyncSubscribeRepos_Handle) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Handle: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -978,7 +991,9 @@ func (t *SyncSubscribeRepos_Handle) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1126,21 +1141,24 @@ func (t *SyncSubscribeRepos_Identity) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Identity: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -1213,7 +1231,9 @@ func (t *SyncSubscribeRepos_Identity) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1377,21 +1397,24 @@ func (t *SyncSubscribeRepos_Account) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Account: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -1482,7 +1505,9 @@ func (t *SyncSubscribeRepos_Account) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1585,21 +1610,24 @@ func (t *SyncSubscribeRepos_Info) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Info: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Name (string) (string) case "name": @@ -1635,7 +1663,9 @@ func (t *SyncSubscribeRepos_Info) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1775,21 +1805,24 @@ func (t *SyncSubscribeRepos_Migrate) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Migrate: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -1862,7 +1895,9 @@ func (t *SyncSubscribeRepos_Migrate) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1967,21 +2002,24 @@ func (t *SyncSubscribeRepos_RepoOp) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_RepoOp: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (util.LexLink) (struct) case "cid": @@ -2027,7 +2065,9 @@ func (t *SyncSubscribeRepos_RepoOp) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2138,21 +2178,24 @@ func (t *SyncSubscribeRepos_Tombstone) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SyncSubscribeRepos_Tombstone: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 4) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -2204,7 +2247,9 @@ func (t *SyncSubscribeRepos_Tombstone) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2301,21 +2346,24 @@ func (t *LabelDefs_SelfLabels) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelDefs_SelfLabels: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -2390,7 +2438,9 @@ func (t *LabelDefs_SelfLabels) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2456,21 +2506,24 @@ func (t *LabelDefs_SelfLabel) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelDefs_SelfLabel: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 3) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Val (string) (string) case "val": @@ -2485,7 +2538,9 @@ func (t *LabelDefs_SelfLabel) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2790,21 +2845,24 @@ func (t *LabelDefs_Label) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelDefs_Label: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 3) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": @@ -2986,7 +3044,9 @@ func (t *LabelDefs_Label) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3077,21 +3137,24 @@ func (t *LabelSubscribeLabels_Labels) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelSubscribeLabels_Labels: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Seq (int64) (int64) case "seq": { @@ -3170,7 +3233,9 @@ func (t *LabelSubscribeLabels_Labels) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3273,21 +3338,24 @@ func (t *LabelSubscribeLabels_Info) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelSubscribeLabels_Info: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Name (string) (string) case "name": @@ -3323,7 +3391,9 @@ func (t *LabelSubscribeLabels_Info) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3527,21 +3597,24 @@ func (t *LabelDefs_LabelValueDefinition) UnmarshalCBOR(r io.Reader) (err error) return fmt.Errorf("LabelDefs_LabelValueDefinition: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 14) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Blurs (string) (string) case "blurs": @@ -3681,7 +3754,9 @@ func (t *LabelDefs_LabelValueDefinition) UnmarshalCBOR(r io.Reader) (err error) default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3793,21 +3868,24 @@ func (t *LabelDefs_LabelValueDefinitionStrings) UnmarshalCBOR(r io.Reader) (err return fmt.Errorf("LabelDefs_LabelValueDefinitionStrings: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Lang (string) (string) case "lang": @@ -3844,7 +3922,9 @@ func (t *LabelDefs_LabelValueDefinitionStrings) UnmarshalCBOR(r io.Reader) (err default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/atproto/repoapplyWrites.go b/api/atproto/repoapplyWrites.go index 939530c14..436a9f13d 100644 --- a/api/atproto/repoapplyWrites.go +++ b/api/atproto/repoapplyWrites.go @@ -25,6 +25,16 @@ type RepoApplyWrites_Create struct { Value *util.LexiconTypeDecoder `json:"value" cborgen:"value"` } +// RepoApplyWrites_CreateResult is a "createResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_CreateResult +type RepoApplyWrites_CreateResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#createResult" cborgen:"$type,const=com.atproto.repo.applyWrites#createResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + // RepoApplyWrites_Delete is a "delete" in the com.atproto.repo.applyWrites schema. // // Operation which deletes an existing record. @@ -36,13 +46,20 @@ type RepoApplyWrites_Delete struct { Rkey string `json:"rkey" cborgen:"rkey"` } +// RepoApplyWrites_DeleteResult is a "deleteResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_DeleteResult +type RepoApplyWrites_DeleteResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#deleteResult" cborgen:"$type,const=com.atproto.repo.applyWrites#deleteResult"` +} + // RepoApplyWrites_Input is the input argument to a com.atproto.repo.applyWrites call. type RepoApplyWrites_Input struct { // repo: The handle or DID of the repo (aka, current account). Repo string `json:"repo" cborgen:"repo"` // swapCommit: If provided, the entire operation will fail if the current repo commit CID does not match this value. Used to prevent conflicting repo mutations. SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` - // validate: Can be set to 'false' to skip Lexicon schema validation of record data, for all operations. + // validate: Can be set to 'false' to skip Lexicon schema validation of record data across all operations, 'true' to require it, or leave unset to validate only for known Lexicons. Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` Writes []*RepoApplyWrites_Input_Writes_Elem `json:"writes" cborgen:"writes"` } @@ -90,6 +107,55 @@ func (t *RepoApplyWrites_Input_Writes_Elem) UnmarshalJSON(b []byte) error { } } +// RepoApplyWrites_Output is the output of a com.atproto.repo.applyWrites call. +type RepoApplyWrites_Output struct { + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Results []*RepoApplyWrites_Output_Results_Elem `json:"results,omitempty" cborgen:"results,omitempty"` +} + +type RepoApplyWrites_Output_Results_Elem struct { + RepoApplyWrites_CreateResult *RepoApplyWrites_CreateResult + RepoApplyWrites_UpdateResult *RepoApplyWrites_UpdateResult + RepoApplyWrites_DeleteResult *RepoApplyWrites_DeleteResult +} + +func (t *RepoApplyWrites_Output_Results_Elem) MarshalJSON() ([]byte, error) { + if t.RepoApplyWrites_CreateResult != nil { + t.RepoApplyWrites_CreateResult.LexiconTypeID = "com.atproto.repo.applyWrites#createResult" + return json.Marshal(t.RepoApplyWrites_CreateResult) + } + if t.RepoApplyWrites_UpdateResult != nil { + t.RepoApplyWrites_UpdateResult.LexiconTypeID = "com.atproto.repo.applyWrites#updateResult" + return json.Marshal(t.RepoApplyWrites_UpdateResult) + } + if t.RepoApplyWrites_DeleteResult != nil { + t.RepoApplyWrites_DeleteResult.LexiconTypeID = "com.atproto.repo.applyWrites#deleteResult" + return json.Marshal(t.RepoApplyWrites_DeleteResult) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *RepoApplyWrites_Output_Results_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.repo.applyWrites#createResult": + t.RepoApplyWrites_CreateResult = new(RepoApplyWrites_CreateResult) + return json.Unmarshal(b, t.RepoApplyWrites_CreateResult) + case "com.atproto.repo.applyWrites#updateResult": + t.RepoApplyWrites_UpdateResult = new(RepoApplyWrites_UpdateResult) + return json.Unmarshal(b, t.RepoApplyWrites_UpdateResult) + case "com.atproto.repo.applyWrites#deleteResult": + t.RepoApplyWrites_DeleteResult = new(RepoApplyWrites_DeleteResult) + return json.Unmarshal(b, t.RepoApplyWrites_DeleteResult) + + default: + return fmt.Errorf("closed enums must have a matching value") + } +} + // RepoApplyWrites_Update is a "update" in the com.atproto.repo.applyWrites schema. // // Operation which updates an existing record. @@ -102,11 +168,22 @@ type RepoApplyWrites_Update struct { Value *util.LexiconTypeDecoder `json:"value" cborgen:"value"` } +// RepoApplyWrites_UpdateResult is a "updateResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_UpdateResult +type RepoApplyWrites_UpdateResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#updateResult" cborgen:"$type,const=com.atproto.repo.applyWrites#updateResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + // RepoApplyWrites calls the XRPC method "com.atproto.repo.applyWrites". -func RepoApplyWrites(ctx context.Context, c *xrpc.Client, input *RepoApplyWrites_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.applyWrites", nil, input, nil); err != nil { - return err +func RepoApplyWrites(ctx context.Context, c *xrpc.Client, input *RepoApplyWrites_Input) (*RepoApplyWrites_Output, error) { + var out RepoApplyWrites_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.applyWrites", nil, input, &out); err != nil { + return nil, err } - return nil + return &out, nil } diff --git a/api/atproto/repocreateRecord.go b/api/atproto/repocreateRecord.go index fbc654800..4c3935ea3 100644 --- a/api/atproto/repocreateRecord.go +++ b/api/atproto/repocreateRecord.go @@ -23,14 +23,16 @@ type RepoCreateRecord_Input struct { Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` // swapCommit: Compare and swap with the previous commit by CID. SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` - // validate: Can be set to 'false' to skip Lexicon schema validation of record data. + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` } // RepoCreateRecord_Output is the output of a com.atproto.repo.createRecord call. type RepoCreateRecord_Output struct { - Cid string `json:"cid" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` } // RepoCreateRecord calls the XRPC method "com.atproto.repo.createRecord". diff --git a/api/atproto/repodefs.go b/api/atproto/repodefs.go new file mode 100644 index 000000000..66ea2aa53 --- /dev/null +++ b/api/atproto/repodefs.go @@ -0,0 +1,11 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package atproto + +// schema: com.atproto.repo.defs + +// RepoDefs_CommitMeta is a "commitMeta" in the com.atproto.repo.defs schema. +type RepoDefs_CommitMeta struct { + Cid string `json:"cid" cborgen:"cid"` + Rev string `json:"rev" cborgen:"rev"` +} diff --git a/api/atproto/repodeleteRecord.go b/api/atproto/repodeleteRecord.go index f474f0254..5ce66c9c7 100644 --- a/api/atproto/repodeleteRecord.go +++ b/api/atproto/repodeleteRecord.go @@ -24,11 +24,17 @@ type RepoDeleteRecord_Input struct { SwapRecord *string `json:"swapRecord,omitempty" cborgen:"swapRecord,omitempty"` } +// RepoDeleteRecord_Output is the output of a com.atproto.repo.deleteRecord call. +type RepoDeleteRecord_Output struct { + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` +} + // RepoDeleteRecord calls the XRPC method "com.atproto.repo.deleteRecord". -func RepoDeleteRecord(ctx context.Context, c *xrpc.Client, input *RepoDeleteRecord_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.deleteRecord", nil, input, nil); err != nil { - return err +func RepoDeleteRecord(ctx context.Context, c *xrpc.Client, input *RepoDeleteRecord_Input) (*RepoDeleteRecord_Output, error) { + var out RepoDeleteRecord_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.deleteRecord", nil, input, &out); err != nil { + return nil, err } - return nil + return &out, nil } diff --git a/api/atproto/repoputRecord.go b/api/atproto/repoputRecord.go index 52bf1d4cf..5b8a77343 100644 --- a/api/atproto/repoputRecord.go +++ b/api/atproto/repoputRecord.go @@ -25,14 +25,16 @@ type RepoPutRecord_Input struct { SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` // swapRecord: Compare and swap with the previous record by CID. WARNING: nullable and optional field; may cause problems with golang implementation SwapRecord *string `json:"swapRecord" cborgen:"swapRecord"` - // validate: Can be set to 'false' to skip Lexicon schema validation of record data. + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` } // RepoPutRecord_Output is the output of a com.atproto.repo.putRecord call. type RepoPutRecord_Output struct { - Cid string `json:"cid" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` } // RepoPutRecord calls the XRPC method "com.atproto.repo.putRecord". diff --git a/api/bsky/actordefs.go b/api/bsky/actordefs.go index feda2f128..ee9b9367e 100644 --- a/api/bsky/actordefs.go +++ b/api/bsky/actordefs.go @@ -35,6 +35,8 @@ type ActorDefs_BskyAppProgressGuide struct { type ActorDefs_BskyAppStatePref struct { LexiconTypeID string `json:"$type,const=app.bsky.actor.defs#bskyAppStatePref" cborgen:"$type,const=app.bsky.actor.defs#bskyAppStatePref"` ActiveProgressGuide *ActorDefs_BskyAppProgressGuide `json:"activeProgressGuide,omitempty" cborgen:"activeProgressGuide,omitempty"` + // nuxs: Storage for NUXs the user has encountered. + Nuxs []*ActorDefs_Nux `json:"nuxs,omitempty" cborgen:"nuxs,omitempty"` // queuedNudges: An array of tokens which identify nudges (modals, popups, tours, highlight dots) that should be shown to the user. QueuedNudges []string `json:"queuedNudges,omitempty" cborgen:"queuedNudges,omitempty"` } @@ -132,6 +134,18 @@ type ActorDefs_MutedWordsPref struct { Items []*ActorDefs_MutedWord `json:"items" cborgen:"items"` } +// ActorDefs_Nux is a "nux" in the app.bsky.actor.defs schema. +// +// A new user experiences (NUX) storage object +type ActorDefs_Nux struct { + Completed bool `json:"completed" cborgen:"completed"` + // data: Arbitrary data for the NUX. The structure is defined by the NUX itself. Limited to 300 characters. + Data *string `json:"data,omitempty" cborgen:"data,omitempty"` + // expiresAt: The date and time at which the NUX will expire and should be considered completed. + ExpiresAt *string `json:"expiresAt,omitempty" cborgen:"expiresAt,omitempty"` + Id string `json:"id" cborgen:"id"` +} + // ActorDefs_PersonalDetailsPref is a "personalDetailsPref" in the app.bsky.actor.defs schema. // // RECORDTYPE: ActorDefs_PersonalDetailsPref @@ -311,6 +325,7 @@ type ActorDefs_ProfileViewDetailed struct { IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt,omitempty"` JoinedViaStarterPack *GraphDefs_StarterPackViewBasic `json:"joinedViaStarterPack,omitempty" cborgen:"joinedViaStarterPack,omitempty"` Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + PinnedPost *comatprototypes.RepoStrongRef `json:"pinnedPost,omitempty" cborgen:"pinnedPost,omitempty"` PostsCount *int64 `json:"postsCount,omitempty" cborgen:"postsCount,omitempty"` Viewer *ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` } diff --git a/api/bsky/actorprofile.go b/api/bsky/actorprofile.go index b60ea27b1..e94a0e2d3 100644 --- a/api/bsky/actorprofile.go +++ b/api/bsky/actorprofile.go @@ -31,7 +31,8 @@ type ActorProfile struct { DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` JoinedViaStarterPack *comatprototypes.RepoStrongRef `json:"joinedViaStarterPack,omitempty" cborgen:"joinedViaStarterPack,omitempty"` // labels: Self-label values, specific to the Bluesky application, on the overall account. - Labels *ActorProfile_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + Labels *ActorProfile_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + PinnedPost *comatprototypes.RepoStrongRef `json:"pinnedPost,omitempty" cborgen:"pinnedPost,omitempty"` } // Self-label values, specific to the Bluesky application, on the overall account. diff --git a/api/bsky/cbor_gen.go b/api/bsky/cbor_gen.go index 4287bcd51..383265fda 100644 --- a/api/bsky/cbor_gen.go +++ b/api/bsky/cbor_gen.go @@ -338,21 +338,24 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPost: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Tags ([]string) (slice) case "tags": @@ -627,7 +630,9 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -728,21 +733,24 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedRepost: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -788,7 +796,9 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -893,21 +903,24 @@ func (t *FeedPost_Entity) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPost_Entity: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Type (string) (string) case "type": @@ -953,7 +966,9 @@ func (t *FeedPost_Entity) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1028,21 +1043,24 @@ func (t *FeedPost_ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPost_ReplyRef: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Root (atproto.RepoStrongRef) (struct) case "root": @@ -1086,7 +1104,9 @@ func (t *FeedPost_ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1174,21 +1194,24 @@ func (t *FeedPost_TextSlice) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPost_TextSlice: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.End (int64) (int64) case "end": { @@ -1244,7 +1267,9 @@ func (t *FeedPost_TextSlice) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1332,21 +1357,24 @@ func (t *EmbedImages) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedImages: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -1410,7 +1438,9 @@ func (t *EmbedImages) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1488,21 +1518,24 @@ func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedExternal: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -1537,7 +1570,9 @@ func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1673,21 +1708,24 @@ func (t *EmbedExternal_External) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedExternal_External: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Uri (string) (string) case "uri": @@ -1744,7 +1782,9 @@ func (t *EmbedExternal_External) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1850,21 +1890,24 @@ func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedImages_Image: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Alt (string) (string) case "alt": @@ -1919,7 +1962,9 @@ func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2027,21 +2072,24 @@ func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphFollow: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -2078,7 +2126,9 @@ func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2091,7 +2141,7 @@ func (t *ActorProfile) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 8 + fieldCount := 9 if t.Avatar == nil { fieldCount-- @@ -2121,6 +2171,10 @@ func (t *ActorProfile) MarshalCBOR(w io.Writer) error { fieldCount-- } + if t.PinnedPost == nil { + fieldCount-- + } + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -2233,6 +2287,25 @@ func (t *ActorProfile) MarshalCBOR(w io.Writer) error { } } + // t.PinnedPost (atproto.RepoStrongRef) (struct) + if t.PinnedPost != nil { + + if len("pinnedPost") > 1000000 { + return xerrors.Errorf("Value in field \"pinnedPost\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pinnedPost"))); err != nil { + return err + } + if _, err := cw.WriteString(string("pinnedPost")); err != nil { + return err + } + + if err := t.PinnedPost.MarshalCBOR(cw); err != nil { + return err + } + } + // t.Description (string) (string) if t.Description != nil { @@ -2341,21 +2414,24 @@ func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("ActorProfile: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 20) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -2448,6 +2524,26 @@ func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { t.CreatedAt = (*string)(&sval) } } + // t.PinnedPost (atproto.RepoStrongRef) (struct) + case "pinnedPost": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.PinnedPost = new(atproto.RepoStrongRef) + if err := t.PinnedPost.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.PinnedPost pointer: %w", err) + } + } + + } // t.Description (string) (string) case "description": @@ -2513,7 +2609,9 @@ func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2591,21 +2689,24 @@ func (t *EmbedRecord) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedRecord: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -2640,7 +2741,9 @@ func (t *EmbedRecord) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2741,21 +2844,24 @@ func (t *FeedLike) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedLike: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -2801,7 +2907,9 @@ func (t *FeedLike) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -2886,21 +2994,24 @@ func (t *RichtextFacet) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RichtextFacet: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Index (bsky.RichtextFacet_ByteSlice) (struct) case "index": @@ -2973,7 +3084,9 @@ func (t *RichtextFacet) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3061,21 +3174,24 @@ func (t *RichtextFacet_ByteSlice) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RichtextFacet_ByteSlice: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.ByteEnd (int64) (int64) case "byteEnd": { @@ -3131,7 +3247,9 @@ func (t *RichtextFacet_ByteSlice) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3216,21 +3334,24 @@ func (t *RichtextFacet_Link) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RichtextFacet_Link: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Uri (string) (string) case "uri": @@ -3256,7 +3377,9 @@ func (t *RichtextFacet_Link) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3341,21 +3464,24 @@ func (t *RichtextFacet_Mention) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RichtextFacet_Mention: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -3381,7 +3507,9 @@ func (t *RichtextFacet_Mention) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3466,21 +3594,24 @@ func (t *RichtextFacet_Tag) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RichtextFacet_Tag: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Tag (string) (string) case "tag": @@ -3506,7 +3637,9 @@ func (t *RichtextFacet_Tag) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3600,21 +3733,24 @@ func (t *EmbedRecordWithMedia) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedRecordWithMedia: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -3669,7 +3805,9 @@ func (t *EmbedRecordWithMedia) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3770,21 +3908,24 @@ func (t *FeedDefs_NotFoundPost) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedDefs_NotFoundPost: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Uri (string) (string) case "uri": @@ -3828,7 +3969,9 @@ func (t *FeedDefs_NotFoundPost) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -3936,21 +4079,24 @@ func (t *GraphBlock) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphBlock: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -3987,7 +4133,9 @@ func (t *GraphBlock) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -4240,21 +4388,24 @@ func (t *GraphList) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphList: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 17) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Name (string) (string) case "name": @@ -4422,7 +4573,9 @@ func (t *GraphList) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -4553,21 +4706,24 @@ func (t *GraphListitem) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphListitem: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.List (string) (string) case "list": @@ -4615,7 +4771,9 @@ func (t *GraphListitem) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -4891,21 +5049,24 @@ func (t *FeedGenerator) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedGenerator: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 19) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -5096,7 +5257,9 @@ func (t *FeedGenerator) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -5204,21 +5367,24 @@ func (t *GraphListblock) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphListblock: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -5255,7 +5421,9 @@ func (t *GraphListblock) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -5343,21 +5511,24 @@ func (t *EmbedDefs_AspectRatio) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedDefs_AspectRatio: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Width (int64) (int64) case "width": { @@ -5413,7 +5584,9 @@ func (t *EmbedDefs_AspectRatio) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -5595,21 +5768,24 @@ func (t *FeedThreadgate) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedThreadgate: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 13) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Post (string) (string) case "post": @@ -5735,7 +5911,9 @@ func (t *FeedThreadgate) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -5820,21 +5998,24 @@ func (t *FeedThreadgate_ListRule) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedThreadgate_ListRule: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.List (string) (string) case "list": @@ -5860,7 +6041,9 @@ func (t *FeedThreadgate_ListRule) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -5922,21 +6105,24 @@ func (t *FeedThreadgate_MentionRule) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedThreadgate_MentionRule: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -5951,7 +6137,9 @@ func (t *FeedThreadgate_MentionRule) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -6013,21 +6201,24 @@ func (t *FeedThreadgate_FollowingRule) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedThreadgate_FollowingRule: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -6042,7 +6233,9 @@ func (t *FeedThreadgate_FollowingRule) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -6108,21 +6301,24 @@ func (t *GraphStarterpack_FeedItem) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphStarterpack_FeedItem: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 3) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Uri (string) (string) case "uri": @@ -6137,7 +6333,9 @@ func (t *GraphStarterpack_FeedItem) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -6371,21 +6569,24 @@ func (t *GraphStarterpack) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GraphStarterpack: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 17) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.List (string) (string) case "list": @@ -6552,7 +6753,9 @@ func (t *GraphStarterpack) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -6677,21 +6880,24 @@ func (t *LabelerService) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelerService: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -6757,7 +6963,9 @@ func (t *LabelerService) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -6873,21 +7081,24 @@ func (t *LabelerDefs_LabelerPolicies) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LabelerDefs_LabelerPolicies: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 21) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LabelValues ([]*string) (slice) case "labelValues": @@ -6990,7 +7201,9 @@ func (t *LabelerDefs_LabelerPolicies) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -7161,21 +7374,24 @@ func (t *EmbedVideo) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedVideo: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Alt (string) (string) case "alt": @@ -7300,7 +7516,9 @@ func (t *EmbedVideo) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -7382,21 +7600,24 @@ func (t *EmbedVideo_Caption) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EmbedVideo_Caption: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 4) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.File (util.LexBlob) (struct) case "file": @@ -7431,7 +7652,9 @@ func (t *EmbedVideo_Caption) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -7613,21 +7836,24 @@ func (t *FeedPostgate) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPostgate: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 21) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Post (string) (string) case "post": @@ -7753,7 +7979,9 @@ func (t *FeedPostgate) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -7815,21 +8043,24 @@ func (t *FeedPostgate_DisableRule) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPostgate_DisableRule: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -7844,7 +8075,9 @@ func (t *FeedPostgate_DisableRule) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/bsky/embedrecord.go b/api/bsky/embedrecord.go index dbeff69c1..baddc252f 100644 --- a/api/bsky/embedrecord.go +++ b/api/bsky/embedrecord.go @@ -68,6 +68,7 @@ type EmbedRecord_ViewRecord struct { IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + QuoteCount *int64 `json:"quoteCount,omitempty" cborgen:"quoteCount,omitempty"` ReplyCount *int64 `json:"replyCount,omitempty" cborgen:"replyCount,omitempty"` RepostCount *int64 `json:"repostCount,omitempty" cborgen:"repostCount,omitempty"` Uri string `json:"uri" cborgen:"uri"` diff --git a/api/bsky/feeddefs.go b/api/bsky/feeddefs.go index c2f0c1e91..d3c0d3784 100644 --- a/api/bsky/feeddefs.go +++ b/api/bsky/feeddefs.go @@ -39,6 +39,7 @@ type FeedDefs_FeedViewPost struct { type FeedDefs_FeedViewPost_Reason struct { FeedDefs_ReasonRepost *FeedDefs_ReasonRepost + FeedDefs_ReasonPin *FeedDefs_ReasonPin } func (t *FeedDefs_FeedViewPost_Reason) MarshalJSON() ([]byte, error) { @@ -46,6 +47,10 @@ func (t *FeedDefs_FeedViewPost_Reason) MarshalJSON() ([]byte, error) { t.FeedDefs_ReasonRepost.LexiconTypeID = "app.bsky.feed.defs#reasonRepost" return json.Marshal(t.FeedDefs_ReasonRepost) } + if t.FeedDefs_ReasonPin != nil { + t.FeedDefs_ReasonPin.LexiconTypeID = "app.bsky.feed.defs#reasonPin" + return json.Marshal(t.FeedDefs_ReasonPin) + } return nil, fmt.Errorf("cannot marshal empty enum") } func (t *FeedDefs_FeedViewPost_Reason) UnmarshalJSON(b []byte) error { @@ -58,6 +63,9 @@ func (t *FeedDefs_FeedViewPost_Reason) UnmarshalJSON(b []byte) error { case "app.bsky.feed.defs#reasonRepost": t.FeedDefs_ReasonRepost = new(FeedDefs_ReasonRepost) return json.Unmarshal(b, t.FeedDefs_ReasonRepost) + case "app.bsky.feed.defs#reasonPin": + t.FeedDefs_ReasonPin = new(FeedDefs_ReasonPin) + return json.Unmarshal(b, t.FeedDefs_ReasonPin) default: return nil @@ -117,6 +125,7 @@ type FeedDefs_PostView struct { IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + QuoteCount *int64 `json:"quoteCount,omitempty" cborgen:"quoteCount,omitempty"` Record *util.LexiconTypeDecoder `json:"record" cborgen:"record"` ReplyCount *int64 `json:"replyCount,omitempty" cborgen:"replyCount,omitempty"` RepostCount *int64 `json:"repostCount,omitempty" cborgen:"repostCount,omitempty"` @@ -184,6 +193,13 @@ func (t *FeedDefs_PostView_Embed) UnmarshalJSON(b []byte) error { } } +// FeedDefs_ReasonPin is a "reasonPin" in the app.bsky.feed.defs schema. +// +// RECORDTYPE: FeedDefs_ReasonPin +type FeedDefs_ReasonPin struct { + LexiconTypeID string `json:"$type,const=app.bsky.feed.defs#reasonPin" cborgen:"$type,const=app.bsky.feed.defs#reasonPin"` +} + // FeedDefs_ReasonRepost is a "reasonRepost" in the app.bsky.feed.defs schema. // // RECORDTYPE: FeedDefs_ReasonRepost @@ -297,6 +313,7 @@ type FeedDefs_SkeletonFeedPost struct { type FeedDefs_SkeletonFeedPost_Reason struct { FeedDefs_SkeletonReasonRepost *FeedDefs_SkeletonReasonRepost + FeedDefs_SkeletonReasonPin *FeedDefs_SkeletonReasonPin } func (t *FeedDefs_SkeletonFeedPost_Reason) MarshalJSON() ([]byte, error) { @@ -304,6 +321,10 @@ func (t *FeedDefs_SkeletonFeedPost_Reason) MarshalJSON() ([]byte, error) { t.FeedDefs_SkeletonReasonRepost.LexiconTypeID = "app.bsky.feed.defs#skeletonReasonRepost" return json.Marshal(t.FeedDefs_SkeletonReasonRepost) } + if t.FeedDefs_SkeletonReasonPin != nil { + t.FeedDefs_SkeletonReasonPin.LexiconTypeID = "app.bsky.feed.defs#skeletonReasonPin" + return json.Marshal(t.FeedDefs_SkeletonReasonPin) + } return nil, fmt.Errorf("cannot marshal empty enum") } func (t *FeedDefs_SkeletonFeedPost_Reason) UnmarshalJSON(b []byte) error { @@ -316,12 +337,22 @@ func (t *FeedDefs_SkeletonFeedPost_Reason) UnmarshalJSON(b []byte) error { case "app.bsky.feed.defs#skeletonReasonRepost": t.FeedDefs_SkeletonReasonRepost = new(FeedDefs_SkeletonReasonRepost) return json.Unmarshal(b, t.FeedDefs_SkeletonReasonRepost) + case "app.bsky.feed.defs#skeletonReasonPin": + t.FeedDefs_SkeletonReasonPin = new(FeedDefs_SkeletonReasonPin) + return json.Unmarshal(b, t.FeedDefs_SkeletonReasonPin) default: return nil } } +// FeedDefs_SkeletonReasonPin is a "skeletonReasonPin" in the app.bsky.feed.defs schema. +// +// RECORDTYPE: FeedDefs_SkeletonReasonPin +type FeedDefs_SkeletonReasonPin struct { + LexiconTypeID string `json:"$type,const=app.bsky.feed.defs#skeletonReasonPin" cborgen:"$type,const=app.bsky.feed.defs#skeletonReasonPin"` +} + // FeedDefs_SkeletonReasonRepost is a "skeletonReasonRepost" in the app.bsky.feed.defs schema. // // RECORDTYPE: FeedDefs_SkeletonReasonRepost @@ -440,6 +471,7 @@ type FeedDefs_ThreadgateView struct { type FeedDefs_ViewerState struct { EmbeddingDisabled *bool `json:"embeddingDisabled,omitempty" cborgen:"embeddingDisabled,omitempty"` Like *string `json:"like,omitempty" cborgen:"like,omitempty"` + Pinned *bool `json:"pinned,omitempty" cborgen:"pinned,omitempty"` ReplyDisabled *bool `json:"replyDisabled,omitempty" cborgen:"replyDisabled,omitempty"` Repost *string `json:"repost,omitempty" cborgen:"repost,omitempty"` ThreadMuted *bool `json:"threadMuted,omitempty" cborgen:"threadMuted,omitempty"` diff --git a/api/bsky/feedgetAuthorFeed.go b/api/bsky/feedgetAuthorFeed.go index c582ea77a..32522d5b2 100644 --- a/api/bsky/feedgetAuthorFeed.go +++ b/api/bsky/feedgetAuthorFeed.go @@ -19,14 +19,15 @@ type FeedGetAuthorFeed_Output struct { // FeedGetAuthorFeed calls the XRPC method "app.bsky.feed.getAuthorFeed". // // filter: Combinations of post/repost types to include in response. -func FeedGetAuthorFeed(ctx context.Context, c *xrpc.Client, actor string, cursor string, filter string, limit int64) (*FeedGetAuthorFeed_Output, error) { +func FeedGetAuthorFeed(ctx context.Context, c *xrpc.Client, actor string, cursor string, filter string, includePins bool, limit int64) (*FeedGetAuthorFeed_Output, error) { var out FeedGetAuthorFeed_Output params := map[string]interface{}{ - "actor": actor, - "cursor": cursor, - "filter": filter, - "limit": limit, + "actor": actor, + "cursor": cursor, + "filter": filter, + "includePins": includePins, + "limit": limit, } if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getAuthorFeed", params, nil, &out); err != nil { return nil, err diff --git a/api/bsky/feedgetPostThread.go b/api/bsky/feedgetPostThread.go index aea994e1a..0cb7bc46d 100644 --- a/api/bsky/feedgetPostThread.go +++ b/api/bsky/feedgetPostThread.go @@ -15,7 +15,8 @@ import ( // FeedGetPostThread_Output is the output of a app.bsky.feed.getPostThread call. type FeedGetPostThread_Output struct { - Thread *FeedGetPostThread_Output_Thread `json:"thread" cborgen:"thread"` + Thread *FeedGetPostThread_Output_Thread `json:"thread" cborgen:"thread"` + Threadgate *FeedDefs_ThreadgateView `json:"threadgate,omitempty" cborgen:"threadgate,omitempty"` } type FeedGetPostThread_Output_Thread struct { diff --git a/api/bsky/feedgetQuotes.go b/api/bsky/feedgetQuotes.go new file mode 100644 index 000000000..d060d666d --- /dev/null +++ b/api/bsky/feedgetQuotes.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.feed.getQuotes + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// FeedGetQuotes_Output is the output of a app.bsky.feed.getQuotes call. +type FeedGetQuotes_Output struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Posts []*FeedDefs_PostView `json:"posts" cborgen:"posts"` + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedGetQuotes calls the XRPC method "app.bsky.feed.getQuotes". +// +// cid: If supplied, filters to quotes of specific version (by CID) of the post record. +// uri: Reference (AT-URI) of post record +func FeedGetQuotes(ctx context.Context, c *xrpc.Client, cid string, cursor string, limit int64, uri string) (*FeedGetQuotes_Output, error) { + var out FeedGetQuotes_Output + + params := map[string]interface{}{ + "cid": cid, + "cursor": cursor, + "limit": limit, + "uri": uri, + } + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getQuotes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetSuggestedFollowsByActor.go b/api/bsky/graphgetSuggestedFollowsByActor.go index 58a334113..0dc57b929 100644 --- a/api/bsky/graphgetSuggestedFollowsByActor.go +++ b/api/bsky/graphgetSuggestedFollowsByActor.go @@ -12,6 +12,8 @@ import ( // GraphGetSuggestedFollowsByActor_Output is the output of a app.bsky.graph.getSuggestedFollowsByActor call. type GraphGetSuggestedFollowsByActor_Output struct { + // isFallback: If true, response has fallen-back to generic results, and is not scoped using relativeToDid + IsFallback *bool `json:"isFallback,omitempty" cborgen:"isFallback,omitempty"` Suggestions []*ActorDefs_ProfileView `json:"suggestions" cborgen:"suggestions"` } diff --git a/api/bsky/unspeccedgetConfig.go b/api/bsky/unspeccedgetConfig.go new file mode 100644 index 000000000..7bc728341 --- /dev/null +++ b/api/bsky/unspeccedgetConfig.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.unspecced.getConfig + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// UnspeccedGetConfig_Output is the output of a app.bsky.unspecced.getConfig call. +type UnspeccedGetConfig_Output struct { + CheckEmailConfirmed *bool `json:"checkEmailConfirmed,omitempty" cborgen:"checkEmailConfirmed,omitempty"` +} + +// UnspeccedGetConfig calls the XRPC method "app.bsky.unspecced.getConfig". +func UnspeccedGetConfig(ctx context.Context, c *xrpc.Client) (*UnspeccedGetConfig_Output, error) { + var out UnspeccedGetConfig_Output + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.unspecced.getConfig", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestionsSkeleton.go b/api/bsky/unspeccedgetSuggestionsSkeleton.go index 6a49b2cbe..f01ab8e96 100644 --- a/api/bsky/unspeccedgetSuggestionsSkeleton.go +++ b/api/bsky/unspeccedgetSuggestionsSkeleton.go @@ -14,6 +14,8 @@ import ( type UnspeccedGetSuggestionsSkeleton_Output struct { Actors []*UnspeccedDefs_SkeletonSearchActor `json:"actors" cborgen:"actors"` Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // relativeToDid: DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer. + RelativeToDid *string `json:"relativeToDid,omitempty" cborgen:"relativeToDid,omitempty"` } // UnspeccedGetSuggestionsSkeleton calls the XRPC method "app.bsky.unspecced.getSuggestionsSkeleton". diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 66b989efa..766bc3180 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -230,21 +230,24 @@ func (t *CreateOp) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("CreateOp: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Sig (string) (string) case "sig": @@ -335,7 +338,9 @@ func (t *CreateOp) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/chat/cbor_gen.go b/api/chat/cbor_gen.go index 0cb37eaac..44d077e75 100644 --- a/api/chat/cbor_gen.go +++ b/api/chat/cbor_gen.go @@ -97,21 +97,24 @@ func (t *ActorDeclaration) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("ActorDeclaration: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 13) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": @@ -137,7 +140,9 @@ func (t *ActorDeclaration) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/ozone/communicationcreateTemplate.go b/api/ozone/communicationcreateTemplate.go index aef239863..a43c2bba0 100644 --- a/api/ozone/communicationcreateTemplate.go +++ b/api/ozone/communicationcreateTemplate.go @@ -16,6 +16,8 @@ type CommunicationCreateTemplate_Input struct { ContentMarkdown string `json:"contentMarkdown" cborgen:"contentMarkdown"` // createdBy: DID of the user who is creating the template. CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` // name: Name of the template. Name string `json:"name" cborgen:"name"` // subject: Subject of the message, used in emails. diff --git a/api/ozone/communicationdefs.go b/api/ozone/communicationdefs.go index 8184894c2..8c59f4a06 100644 --- a/api/ozone/communicationdefs.go +++ b/api/ozone/communicationdefs.go @@ -11,6 +11,8 @@ type CommunicationDefs_TemplateView struct { CreatedAt string `json:"createdAt" cborgen:"createdAt"` Disabled bool `json:"disabled" cborgen:"disabled"` Id string `json:"id" cborgen:"id"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` // lastUpdatedBy: DID of the user who last updated the template. LastUpdatedBy string `json:"lastUpdatedBy" cborgen:"lastUpdatedBy"` // name: Name of the template. diff --git a/api/ozone/communicationupdateTemplate.go b/api/ozone/communicationupdateTemplate.go index 78b30a8ef..88b860847 100644 --- a/api/ozone/communicationupdateTemplate.go +++ b/api/ozone/communicationupdateTemplate.go @@ -17,6 +17,8 @@ type CommunicationUpdateTemplate_Input struct { Disabled *bool `json:"disabled,omitempty" cborgen:"disabled,omitempty"` // id: ID of the template to be updated. Id string `json:"id" cborgen:"id"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` // name: Name of the template. Name *string `json:"name,omitempty" cborgen:"name,omitempty"` // subject: Subject of the message, used in emails. diff --git a/api/ozone/moderationdefs.go b/api/ozone/moderationdefs.go index 96c07f3c6..7c8e939c2 100644 --- a/api/ozone/moderationdefs.go +++ b/api/ozone/moderationdefs.go @@ -240,8 +240,10 @@ type ModerationDefs_ModEventTag struct { // // RECORDTYPE: ModerationDefs_ModEventTakedown type ModerationDefs_ModEventTakedown struct { - LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#modEventTakedown" cborgen:"$type,const=tools.ozone.moderation.defs#modEventTakedown"` - Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#modEventTakedown" cborgen:"$type,const=tools.ozone.moderation.defs#modEventTakedown"` + // acknowledgeAccountSubjects: If true, all other reports on content authored by this account will be resolved (acknowledged). + AcknowledgeAccountSubjects *bool `json:"acknowledgeAccountSubjects,omitempty" cborgen:"acknowledgeAccountSubjects,omitempty"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` // durationInHours: Indicates how long the takedown should be in effect before automatically expiring. DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"` } @@ -748,15 +750,18 @@ type ModerationDefs_RecordView struct { } // ModerationDefs_RecordViewDetail is a "recordViewDetail" in the tools.ozone.moderation.defs schema. +// +// RECORDTYPE: ModerationDefs_RecordViewDetail type ModerationDefs_RecordViewDetail struct { - Blobs []*ModerationDefs_BlobView `json:"blobs" cborgen:"blobs"` - Cid string `json:"cid" cborgen:"cid"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` - Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` - Repo *ModerationDefs_RepoView `json:"repo" cborgen:"repo"` - Uri string `json:"uri" cborgen:"uri"` - Value *util.LexiconTypeDecoder `json:"value" cborgen:"value"` + LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#recordViewDetail" cborgen:"$type,const=tools.ozone.moderation.defs#recordViewDetail"` + Blobs []*ModerationDefs_BlobView `json:"blobs" cborgen:"blobs"` + Cid string `json:"cid" cborgen:"cid"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` + Repo *ModerationDefs_RepoView `json:"repo" cborgen:"repo"` + Uri string `json:"uri" cborgen:"uri"` + Value *util.LexiconTypeDecoder `json:"value" cborgen:"value"` } // ModerationDefs_RecordViewNotFound is a "recordViewNotFound" in the tools.ozone.moderation.defs schema. @@ -771,34 +776,39 @@ type ModerationDefs_RecordViewNotFound struct { // // RECORDTYPE: ModerationDefs_RepoView type ModerationDefs_RepoView struct { - LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#repoView" cborgen:"$type,const=tools.ozone.moderation.defs#repoView"` - DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` - Did string `json:"did" cborgen:"did"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` - InvitedBy *comatprototypes.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` - InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` - Moderation *ModerationDefs_Moderation `json:"moderation" cborgen:"moderation"` - RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#repoView" cborgen:"$type,const=tools.ozone.moderation.defs#repoView"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *comatprototypes.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + Moderation *ModerationDefs_Moderation `json:"moderation" cborgen:"moderation"` + RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + ThreatSignatures []*comatprototypes.AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` } // ModerationDefs_RepoViewDetail is a "repoViewDetail" in the tools.ozone.moderation.defs schema. +// +// RECORDTYPE: ModerationDefs_RepoViewDetail type ModerationDefs_RepoViewDetail struct { - DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` - Did string `json:"did" cborgen:"did"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` - InvitedBy *comatprototypes.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` - Invites []*comatprototypes.ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` - InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` - Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` - Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` - RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#repoViewDetail" cborgen:"$type,const=tools.ozone.moderation.defs#repoViewDetail"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *comatprototypes.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + Invites []*comatprototypes.ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + Labels []*comatprototypes.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` + RelatedRecords []*util.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + ThreatSignatures []*comatprototypes.AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` } // ModerationDefs_RepoViewNotFound is a "repoViewNotFound" in the tools.ozone.moderation.defs schema. diff --git a/api/ozone/moderationemitEvent.go b/api/ozone/moderationemitEvent.go index f5d3f0541..09f8f3497 100644 --- a/api/ozone/moderationemitEvent.go +++ b/api/ozone/moderationemitEvent.go @@ -34,6 +34,7 @@ type ModerationEmitEvent_Input_Event struct { ModerationDefs_ModEventMuteReporter *ModerationDefs_ModEventMuteReporter ModerationDefs_ModEventUnmuteReporter *ModerationDefs_ModEventUnmuteReporter ModerationDefs_ModEventReverseTakedown *ModerationDefs_ModEventReverseTakedown + ModerationDefs_ModEventResolveAppeal *ModerationDefs_ModEventResolveAppeal ModerationDefs_ModEventEmail *ModerationDefs_ModEventEmail ModerationDefs_ModEventTag *ModerationDefs_ModEventTag ModerationDefs_AccountEvent *ModerationDefs_AccountEvent @@ -86,6 +87,10 @@ func (t *ModerationEmitEvent_Input_Event) MarshalJSON() ([]byte, error) { t.ModerationDefs_ModEventReverseTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventReverseTakedown" return json.Marshal(t.ModerationDefs_ModEventReverseTakedown) } + if t.ModerationDefs_ModEventResolveAppeal != nil { + t.ModerationDefs_ModEventResolveAppeal.LexiconTypeID = "tools.ozone.moderation.defs#modEventResolveAppeal" + return json.Marshal(t.ModerationDefs_ModEventResolveAppeal) + } if t.ModerationDefs_ModEventEmail != nil { t.ModerationDefs_ModEventEmail.LexiconTypeID = "tools.ozone.moderation.defs#modEventEmail" return json.Marshal(t.ModerationDefs_ModEventEmail) @@ -148,6 +153,9 @@ func (t *ModerationEmitEvent_Input_Event) UnmarshalJSON(b []byte) error { case "tools.ozone.moderation.defs#modEventReverseTakedown": t.ModerationDefs_ModEventReverseTakedown = new(ModerationDefs_ModEventReverseTakedown) return json.Unmarshal(b, t.ModerationDefs_ModEventReverseTakedown) + case "tools.ozone.moderation.defs#modEventResolveAppeal": + t.ModerationDefs_ModEventResolveAppeal = new(ModerationDefs_ModEventResolveAppeal) + return json.Unmarshal(b, t.ModerationDefs_ModEventResolveAppeal) case "tools.ozone.moderation.defs#modEventEmail": t.ModerationDefs_ModEventEmail = new(ModerationDefs_ModEventEmail) return json.Unmarshal(b, t.ModerationDefs_ModEventEmail) diff --git a/api/ozone/moderationgetRecords.go b/api/ozone/moderationgetRecords.go new file mode 100644 index 000000000..86b14e2a6 --- /dev/null +++ b/api/ozone/moderationgetRecords.go @@ -0,0 +1,68 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.moderation.getRecords + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/xrpc" +) + +// ModerationGetRecords_Output is the output of a tools.ozone.moderation.getRecords call. +type ModerationGetRecords_Output struct { + Records []*ModerationGetRecords_Output_Records_Elem `json:"records" cborgen:"records"` +} + +type ModerationGetRecords_Output_Records_Elem struct { + ModerationDefs_RecordViewDetail *ModerationDefs_RecordViewDetail + ModerationDefs_RecordViewNotFound *ModerationDefs_RecordViewNotFound +} + +func (t *ModerationGetRecords_Output_Records_Elem) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RecordViewDetail != nil { + t.ModerationDefs_RecordViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#recordViewDetail" + return json.Marshal(t.ModerationDefs_RecordViewDetail) + } + if t.ModerationDefs_RecordViewNotFound != nil { + t.ModerationDefs_RecordViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#recordViewNotFound" + return json.Marshal(t.ModerationDefs_RecordViewNotFound) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *ModerationGetRecords_Output_Records_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#recordViewDetail": + t.ModerationDefs_RecordViewDetail = new(ModerationDefs_RecordViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RecordViewDetail) + case "tools.ozone.moderation.defs#recordViewNotFound": + t.ModerationDefs_RecordViewNotFound = new(ModerationDefs_RecordViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RecordViewNotFound) + + default: + return nil + } +} + +// ModerationGetRecords calls the XRPC method "tools.ozone.moderation.getRecords". +func ModerationGetRecords(ctx context.Context, c *xrpc.Client, uris []string) (*ModerationGetRecords_Output, error) { + var out ModerationGetRecords_Output + + params := map[string]interface{}{ + "uris": uris, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.moderation.getRecords", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetRepos.go b/api/ozone/moderationgetRepos.go new file mode 100644 index 000000000..af51a97f3 --- /dev/null +++ b/api/ozone/moderationgetRepos.go @@ -0,0 +1,68 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.moderation.getRepos + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/xrpc" +) + +// ModerationGetRepos_Output is the output of a tools.ozone.moderation.getRepos call. +type ModerationGetRepos_Output struct { + Repos []*ModerationGetRepos_Output_Repos_Elem `json:"repos" cborgen:"repos"` +} + +type ModerationGetRepos_Output_Repos_Elem struct { + ModerationDefs_RepoViewDetail *ModerationDefs_RepoViewDetail + ModerationDefs_RepoViewNotFound *ModerationDefs_RepoViewNotFound +} + +func (t *ModerationGetRepos_Output_Repos_Elem) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RepoViewDetail != nil { + t.ModerationDefs_RepoViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#repoViewDetail" + return json.Marshal(t.ModerationDefs_RepoViewDetail) + } + if t.ModerationDefs_RepoViewNotFound != nil { + t.ModerationDefs_RepoViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#repoViewNotFound" + return json.Marshal(t.ModerationDefs_RepoViewNotFound) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *ModerationGetRepos_Output_Repos_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#repoViewDetail": + t.ModerationDefs_RepoViewDetail = new(ModerationDefs_RepoViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RepoViewDetail) + case "tools.ozone.moderation.defs#repoViewNotFound": + t.ModerationDefs_RepoViewNotFound = new(ModerationDefs_RepoViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RepoViewNotFound) + + default: + return nil + } +} + +// ModerationGetRepos calls the XRPC method "tools.ozone.moderation.getRepos". +func ModerationGetRepos(ctx context.Context, c *xrpc.Client, dids []string) (*ModerationGetRepos_Output, error) { + var out ModerationGetRepos_Output + + params := map[string]interface{}{ + "dids": dids, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.moderation.getRepos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationqueryEvents.go b/api/ozone/moderationqueryEvents.go index fee7adc39..1be68412f 100644 --- a/api/ozone/moderationqueryEvents.go +++ b/api/ozone/moderationqueryEvents.go @@ -20,21 +20,24 @@ type ModerationQueryEvents_Output struct { // // addedLabels: If specified, only events where all of these labels were added are returned // addedTags: If specified, only events where all of these tags were added are returned +// collections: If specified, only events where the subject belongs to the given collections will be returned. When subjectType is set to 'account', this will be ignored. // comment: If specified, only events with comments containing the keyword are returned // createdAfter: Retrieve events created after a given timestamp // createdBefore: Retrieve events created before a given timestamp // hasComment: If true, only events with comments are returned -// includeAllUserRecords: If true, events on all record types (posts, lists, profile etc.) owned by the did are returned +// includeAllUserRecords: If true, events on all record types (posts, lists, profile etc.) or records from given 'collections' param, owned by the did are returned. // removedLabels: If specified, only events where all of these labels were removed are returned // removedTags: If specified, only events where all of these tags were removed are returned // sortDirection: Sort direction for the events. Defaults to descending order of created at timestamp. +// subjectType: If specified, only events where the subject is of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. // types: The types of events (fully qualified string in the format of tools.ozone.moderation.defs#modEvent) to filter by. If not specified, all events are returned. -func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []string, addedTags []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, types []string) (*ModerationQueryEvents_Output, error) { +func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []string, addedTags []string, collections []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, subjectType string, types []string) (*ModerationQueryEvents_Output, error) { var out ModerationQueryEvents_Output params := map[string]interface{}{ "addedLabels": addedLabels, "addedTags": addedTags, + "collections": collections, "comment": comment, "createdAfter": createdAfter, "createdBefore": createdBefore, @@ -48,6 +51,7 @@ func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []st "reportTypes": reportTypes, "sortDirection": sortDirection, "subject": subject, + "subjectType": subjectType, "types": types, } if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.moderation.queryEvents", params, nil, &out); err != nil { diff --git a/api/ozone/moderationqueryStatuses.go b/api/ozone/moderationqueryStatuses.go index fe00aad86..969d77e9d 100644 --- a/api/ozone/moderationqueryStatuses.go +++ b/api/ozone/moderationqueryStatuses.go @@ -19,7 +19,9 @@ type ModerationQueryStatuses_Output struct { // ModerationQueryStatuses calls the XRPC method "tools.ozone.moderation.queryStatuses". // // appealed: Get subjects in unresolved appealed status +// collections: If specified, subjects belonging to the given collections will be returned. When subjectType is set to 'account', this will be ignored. // comment: Search subjects by keyword from comments +// includeAllUserRecords: All subjects, or subjects from given 'collections' param, belonging to the account specified in the 'subject' param will be returned. // includeMuted: By default, we don't include muted subjects in the results. Set this to true to include them. // lastReviewedBy: Get all subject statuses that were reviewed by a specific moderator // onlyMuted: When set to true, only muted subjects and reporters will be returned. @@ -28,30 +30,35 @@ type ModerationQueryStatuses_Output struct { // reviewState: Specify when fetching subjects in a certain state // reviewedAfter: Search subjects reviewed after a given timestamp // reviewedBefore: Search subjects reviewed before a given timestamp +// subject: The subject to get the status for. +// subjectType: If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. // takendown: Get subjects that were taken down -func ModerationQueryStatuses(ctx context.Context, c *xrpc.Client, appealed bool, comment string, cursor string, excludeTags []string, ignoreSubjects []string, includeMuted bool, lastReviewedBy string, limit int64, onlyMuted bool, reportedAfter string, reportedBefore string, reviewState string, reviewedAfter string, reviewedBefore string, sortDirection string, sortField string, subject string, tags []string, takendown bool) (*ModerationQueryStatuses_Output, error) { +func ModerationQueryStatuses(ctx context.Context, c *xrpc.Client, appealed bool, collections []string, comment string, cursor string, excludeTags []string, ignoreSubjects []string, includeAllUserRecords bool, includeMuted bool, lastReviewedBy string, limit int64, onlyMuted bool, reportedAfter string, reportedBefore string, reviewState string, reviewedAfter string, reviewedBefore string, sortDirection string, sortField string, subject string, subjectType string, tags []string, takendown bool) (*ModerationQueryStatuses_Output, error) { var out ModerationQueryStatuses_Output params := map[string]interface{}{ - "appealed": appealed, - "comment": comment, - "cursor": cursor, - "excludeTags": excludeTags, - "ignoreSubjects": ignoreSubjects, - "includeMuted": includeMuted, - "lastReviewedBy": lastReviewedBy, - "limit": limit, - "onlyMuted": onlyMuted, - "reportedAfter": reportedAfter, - "reportedBefore": reportedBefore, - "reviewState": reviewState, - "reviewedAfter": reviewedAfter, - "reviewedBefore": reviewedBefore, - "sortDirection": sortDirection, - "sortField": sortField, - "subject": subject, - "tags": tags, - "takendown": takendown, + "appealed": appealed, + "collections": collections, + "comment": comment, + "cursor": cursor, + "excludeTags": excludeTags, + "ignoreSubjects": ignoreSubjects, + "includeAllUserRecords": includeAllUserRecords, + "includeMuted": includeMuted, + "lastReviewedBy": lastReviewedBy, + "limit": limit, + "onlyMuted": onlyMuted, + "reportedAfter": reportedAfter, + "reportedBefore": reportedBefore, + "reviewState": reviewState, + "reviewedAfter": reviewedAfter, + "reviewedBefore": reviewedBefore, + "sortDirection": sortDirection, + "sortField": sortField, + "subject": subject, + "subjectType": subjectType, + "tags": tags, + "takendown": takendown, } if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.moderation.queryStatuses", params, nil, &out); err != nil { return nil, err diff --git a/api/ozone/setaddValues.go b/api/ozone/setaddValues.go new file mode 100644 index 000000000..1835d5e92 --- /dev/null +++ b/api/ozone/setaddValues.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.addValues + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetAddValues_Input is the input argument to a tools.ozone.set.addValues call. +type SetAddValues_Input struct { + // name: Name of the set to add values to + Name string `json:"name" cborgen:"name"` + // values: Array of string values to add to the set + Values []string `json:"values" cborgen:"values"` +} + +// SetAddValues calls the XRPC method "tools.ozone.set.addValues". +func SetAddValues(ctx context.Context, c *xrpc.Client, input *SetAddValues_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.set.addValues", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/setdefs.go b/api/ozone/setdefs.go new file mode 100644 index 000000000..2181b12fd --- /dev/null +++ b/api/ozone/setdefs.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.defs + +// SetDefs_Set is a "set" in the tools.ozone.set.defs schema. +type SetDefs_Set struct { + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Name string `json:"name" cborgen:"name"` +} + +// SetDefs_SetView is a "setView" in the tools.ozone.set.defs schema. +type SetDefs_SetView struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Name string `json:"name" cborgen:"name"` + SetSize int64 `json:"setSize" cborgen:"setSize"` + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` +} diff --git a/api/ozone/setdeleteSet.go b/api/ozone/setdeleteSet.go new file mode 100644 index 000000000..b5d192364 --- /dev/null +++ b/api/ozone/setdeleteSet.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.deleteSet + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetDeleteSet_Input is the input argument to a tools.ozone.set.deleteSet call. +type SetDeleteSet_Input struct { + // name: Name of the set to delete + Name string `json:"name" cborgen:"name"` +} + +// SetDeleteSet_Output is the output of a tools.ozone.set.deleteSet call. +type SetDeleteSet_Output struct { +} + +// SetDeleteSet calls the XRPC method "tools.ozone.set.deleteSet". +func SetDeleteSet(ctx context.Context, c *xrpc.Client, input *SetDeleteSet_Input) (*SetDeleteSet_Output, error) { + var out SetDeleteSet_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.set.deleteSet", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setdeleteValues.go b/api/ozone/setdeleteValues.go new file mode 100644 index 000000000..34b62898a --- /dev/null +++ b/api/ozone/setdeleteValues.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.deleteValues + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetDeleteValues_Input is the input argument to a tools.ozone.set.deleteValues call. +type SetDeleteValues_Input struct { + // name: Name of the set to delete values from + Name string `json:"name" cborgen:"name"` + // values: Array of string values to delete from the set + Values []string `json:"values" cborgen:"values"` +} + +// SetDeleteValues calls the XRPC method "tools.ozone.set.deleteValues". +func SetDeleteValues(ctx context.Context, c *xrpc.Client, input *SetDeleteValues_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.set.deleteValues", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/setgetValues.go b/api/ozone/setgetValues.go new file mode 100644 index 000000000..50e77fdda --- /dev/null +++ b/api/ozone/setgetValues.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.getValues + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetGetValues_Output is the output of a tools.ozone.set.getValues call. +type SetGetValues_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Set *SetDefs_SetView `json:"set" cborgen:"set"` + Values []string `json:"values" cborgen:"values"` +} + +// SetGetValues calls the XRPC method "tools.ozone.set.getValues". +func SetGetValues(ctx context.Context, c *xrpc.Client, cursor string, limit int64, name string) (*SetGetValues_Output, error) { + var out SetGetValues_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + "name": name, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.set.getValues", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setquerySets.go b/api/ozone/setquerySets.go new file mode 100644 index 000000000..f2f31effb --- /dev/null +++ b/api/ozone/setquerySets.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.querySets + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetQuerySets_Output is the output of a tools.ozone.set.querySets call. +type SetQuerySets_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Sets []*SetDefs_SetView `json:"sets" cborgen:"sets"` +} + +// SetQuerySets calls the XRPC method "tools.ozone.set.querySets". +// +// sortDirection: Defaults to ascending order of name field. +func SetQuerySets(ctx context.Context, c *xrpc.Client, cursor string, limit int64, namePrefix string, sortBy string, sortDirection string) (*SetQuerySets_Output, error) { + var out SetQuerySets_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + "namePrefix": namePrefix, + "sortBy": sortBy, + "sortDirection": sortDirection, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.set.querySets", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setupsertSet.go b/api/ozone/setupsertSet.go new file mode 100644 index 000000000..9f6cc5376 --- /dev/null +++ b/api/ozone/setupsertSet.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.set.upsertSet + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SetUpsertSet calls the XRPC method "tools.ozone.set.upsertSet". +func SetUpsertSet(ctx context.Context, c *xrpc.Client, input *SetDefs_Set) (*SetDefs_SetView, error) { + var out SetDefs_SetView + if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.set.upsertSet", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturedefs.go b/api/ozone/signaturedefs.go new file mode 100644 index 000000000..7336836a6 --- /dev/null +++ b/api/ozone/signaturedefs.go @@ -0,0 +1,11 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.signature.defs + +// SignatureDefs_SigDetail is a "sigDetail" in the tools.ozone.signature.defs schema. +type SignatureDefs_SigDetail struct { + Property string `json:"property" cborgen:"property"` + Value string `json:"value" cborgen:"value"` +} diff --git a/api/ozone/signaturefindCorrelation.go b/api/ozone/signaturefindCorrelation.go new file mode 100644 index 000000000..e07e67fe4 --- /dev/null +++ b/api/ozone/signaturefindCorrelation.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.signature.findCorrelation + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// SignatureFindCorrelation_Output is the output of a tools.ozone.signature.findCorrelation call. +type SignatureFindCorrelation_Output struct { + Details []*SignatureDefs_SigDetail `json:"details" cborgen:"details"` +} + +// SignatureFindCorrelation calls the XRPC method "tools.ozone.signature.findCorrelation". +func SignatureFindCorrelation(ctx context.Context, c *xrpc.Client, dids []string) (*SignatureFindCorrelation_Output, error) { + var out SignatureFindCorrelation_Output + + params := map[string]interface{}{ + "dids": dids, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.signature.findCorrelation", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturefindRelatedAccounts.go b/api/ozone/signaturefindRelatedAccounts.go new file mode 100644 index 000000000..b32bba8e3 --- /dev/null +++ b/api/ozone/signaturefindRelatedAccounts.go @@ -0,0 +1,40 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.signature.findRelatedAccounts + +import ( + "context" + + comatprototypes "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/xrpc" +) + +// SignatureFindRelatedAccounts_Output is the output of a tools.ozone.signature.findRelatedAccounts call. +type SignatureFindRelatedAccounts_Output struct { + Accounts []*SignatureFindRelatedAccounts_RelatedAccount `json:"accounts" cborgen:"accounts"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// SignatureFindRelatedAccounts_RelatedAccount is a "relatedAccount" in the tools.ozone.signature.findRelatedAccounts schema. +type SignatureFindRelatedAccounts_RelatedAccount struct { + Account *comatprototypes.AdminDefs_AccountView `json:"account" cborgen:"account"` + Similarities []*SignatureDefs_SigDetail `json:"similarities,omitempty" cborgen:"similarities,omitempty"` +} + +// SignatureFindRelatedAccounts calls the XRPC method "tools.ozone.signature.findRelatedAccounts". +func SignatureFindRelatedAccounts(ctx context.Context, c *xrpc.Client, cursor string, did string, limit int64) (*SignatureFindRelatedAccounts_Output, error) { + var out SignatureFindRelatedAccounts_Output + + params := map[string]interface{}{ + "cursor": cursor, + "did": did, + "limit": limit, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.signature.findRelatedAccounts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturesearchAccounts.go b/api/ozone/signaturesearchAccounts.go new file mode 100644 index 000000000..08da3eec9 --- /dev/null +++ b/api/ozone/signaturesearchAccounts.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package ozone + +// schema: tools.ozone.signature.searchAccounts + +import ( + "context" + + comatprototypes "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/xrpc" +) + +// SignatureSearchAccounts_Output is the output of a tools.ozone.signature.searchAccounts call. +type SignatureSearchAccounts_Output struct { + Accounts []*comatprototypes.AdminDefs_AccountView `json:"accounts" cborgen:"accounts"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// SignatureSearchAccounts calls the XRPC method "tools.ozone.signature.searchAccounts". +func SignatureSearchAccounts(ctx context.Context, c *xrpc.Client, cursor string, limit int64, values []string) (*SignatureSearchAccounts_Output, error) { + var out SignatureSearchAccounts_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + "values": values, + } + if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.signature.searchAccounts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/atproto/data/cbor_gen.go b/atproto/data/cbor_gen.go index 89c5e0a5c..18280b707 100644 --- a/atproto/data/cbor_gen.go +++ b/atproto/data/cbor_gen.go @@ -78,21 +78,24 @@ func (t *GenericRecord) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("GenericRecord: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Type (string) (string) case "$type": @@ -107,7 +110,9 @@ func (t *GenericRecord) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -196,21 +201,24 @@ func (t *LegacyBlobSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LegacyBlobSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": @@ -236,7 +244,9 @@ func (t *LegacyBlobSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -359,21 +369,24 @@ func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("BlobSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Ref (data.CIDLink) (struct) case "ref": @@ -435,7 +448,9 @@ func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/atproto/data/parse.go b/atproto/data/parse.go index 5924ff2a3..2f1f4305e 100644 --- a/atproto/data/parse.go +++ b/atproto/data/parse.go @@ -111,6 +111,18 @@ func parseMap(obj map[string]any) (any, error) { return nil, fmt.Errorf("$type field must contain a non-empty string") } } + // legacy blob type + if len(obj) == 2 { + if _, ok := obj["mimeType"]; ok { + if _, ok := obj["cid"]; ok { + b, err := parseLegacyBlob(obj) + if err != nil { + return nil, err + } + return *b, nil + } + } + } out := make(map[string]any, len(obj)) for k, val := range obj { if len(k) > MAX_OBJECT_KEY_LEN { @@ -213,6 +225,30 @@ func parseBlob(obj map[string]any) (*Blob, error) { }, nil } +func parseLegacyBlob(obj map[string]any) (*Blob, error) { + if len(obj) != 2 { + return nil, fmt.Errorf("legacy blobs expected to have 2 fields") + } + var err error + mimeType, ok := obj["mimeType"].(string) + if !ok { + return nil, fmt.Errorf("blob 'mimeType' missing or not a string") + } + cidStr, ok := obj["cid"] + if !ok { + return nil, fmt.Errorf("blob 'cid' missing") + } + c, err := cid.Parse(cidStr) + if err != nil { + return nil, fmt.Errorf("invalid CID: %w", err) + } + return &Blob{ + Size: -1, + MimeType: mimeType, + Ref: CIDLink(c), + }, nil +} + func parseObject(obj map[string]any) (map[string]any, error) { out, err := parseMap(obj) if err != nil { diff --git a/atproto/identity/cache_directory.go b/atproto/identity/cache_directory.go index 55951bbf0..2df1ac5c5 100644 --- a/atproto/identity/cache_directory.go +++ b/atproto/identity/cache_directory.go @@ -7,8 +7,6 @@ import ( "time" "github.com/bluesky-social/indigo/atproto/syntax" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/hashicorp/golang-lru/v2/expirable" ) @@ -35,40 +33,10 @@ type IdentityEntry struct { Err error } -var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_handle_cache_hits", - Help: "Number of cache hits for ATProto handle lookups", -}) - -var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_handle_cache_misses", - Help: "Number of cache misses for ATProto handle lookups", -}) - -var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_identity_cache_hits", - Help: "Number of cache hits for ATProto identity lookups", -}) - -var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_identity_cache_misses", - Help: "Number of cache misses for ATProto identity lookups", -}) - -var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_identity_requests_coalesced", - Help: "Number of identity requests coalesced", -}) - -var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ - Name: "atproto_directory_handle_requests_coalesced", - Help: "Number of handle requests coalesced", -}) - var _ Directory = (*CacheDirectory)(nil) // Capacity of zero means unlimited size. Similarly, ttl of zero means unlimited duration. -func NewCacheDirectory(inner Directory, capacity int, hitTTL, errTTL time.Duration, invalidHandleTTL time.Duration) CacheDirectory { +func NewCacheDirectory(inner Directory, capacity int, hitTTL, errTTL, invalidHandleTTL time.Duration) CacheDirectory { return CacheDirectory{ ErrTTL: errTTL, InvalidHandleTTL: invalidHandleTTL, @@ -124,6 +92,9 @@ func (d *CacheDirectory) updateHandle(ctx context.Context, h syntax.Handle) Hand } func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + if h.IsInvalidHandle() { + return "", fmt.Errorf("invalid handle") + } entry, ok := d.handleCache.Get(h) if ok && !d.IsHandleStale(&entry) { handleCacheHits.Inc() diff --git a/atproto/identity/doc.go b/atproto/identity/doc.go index 911b2811c..32bef3313 100644 --- a/atproto/identity/doc.go +++ b/atproto/identity/doc.go @@ -1,8 +1,6 @@ /* Package identity provides types and routines for resolving handles and DIDs from the network -The two main abstractions are a Catalog interface for identity service implementations, and an Identity structure which represents core identity information relevant to atproto. The Catalog interface can be nested, somewhat like HTTP middleware, to provide caching, observability, or other bespoke needs in more complex systems. - -Much of the implementation of this SDK is based on existing code in indigo:api/extra.go +The two main abstractions are a Directory interface for identity service implementations, and an Identity struct which represents core identity information relevant to atproto. The Directory interface can be nested, somewhat like HTTP middleware, to provide caching, observability, or other bespoke needs in more complex systems. */ package identity diff --git a/atproto/identity/handle.go b/atproto/identity/handle.go index 024ac6e32..77a157f85 100644 --- a/atproto/identity/handle.go +++ b/atproto/identity/handle.go @@ -168,6 +168,10 @@ func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) var dnsErr error var did syntax.DID + if handle.IsInvalidHandle() { + return "", fmt.Errorf("invalid handle") + } + if !handle.AllowedTLD() { return "", ErrHandleReservedTLD } diff --git a/atproto/identity/identity.go b/atproto/identity/identity.go index 02e66f22c..c0453b2af 100644 --- a/atproto/identity/identity.go +++ b/atproto/identity/identity.go @@ -65,11 +65,16 @@ func DefaultDirectory() Directory { base := BaseDirectory{ PLCURL: DefaultPLCURL, HTTPClient: http.Client{ - Timeout: time.Second * 15, + Timeout: time.Second * 10, + Transport: &http.Transport{ + // would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 1000, + MaxIdleConns: 100, + }, }, Resolver: net.Resolver{ Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{Timeout: time.Second * 5} + d := net.Dialer{Timeout: time.Second * 3} return d.DialContext(ctx, network, address) }, }, diff --git a/atproto/identity/metrics.go b/atproto/identity/metrics.go new file mode 100644 index 000000000..d6061c9ce --- /dev/null +++ b/atproto/identity/metrics.go @@ -0,0 +1,36 @@ +package identity + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_cache_hits", + Help: "Number of cache hits for ATProto handle lookups", +}) + +var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_cache_misses", + Help: "Number of cache misses for ATProto handle lookups", +}) + +var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_cache_hits", + Help: "Number of cache hits for ATProto identity lookups", +}) + +var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_cache_misses", + Help: "Number of cache misses for ATProto identity lookups", +}) + +var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_requests_coalesced", + Help: "Number of identity requests coalesced", +}) + +var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_requests_coalesced", + Help: "Number of handle requests coalesced", +}) diff --git a/atproto/identity/redisdir/live_test.go b/atproto/identity/redisdir/live_test.go new file mode 100644 index 000000000..433c45b8c --- /dev/null +++ b/atproto/identity/redisdir/live_test.go @@ -0,0 +1,136 @@ +package redisdir + +import ( + "context" + "log/slog" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "golang.org/x/time/rate" + + "github.com/stretchr/testify/assert" +) + +var redisLocalTestURL string = "redis://localhost:6379/0" + +// NOTE: this hits the open internet! marked as skip below by default +func testDirectoryLive(t *testing.T, d identity.Directory) { + assert := assert.New(t) + ctx := context.Background() + + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + pdsSuffix := "host.bsky.network" + + resp, err := d.LookupHandle(ctx, handle) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + dh, err := resp.DeclaredHandle() + assert.NoError(err) + assert.Equal(handle, dh) + pk, err := resp.PublicKey() + assert.NoError(err) + assert.NotNil(pk) + + resp, err = d.LookupDID(ctx, did) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + + _, err = d.LookupHandle(ctx, syntax.Handle("fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrHandleNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:web:fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrDIDNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrDIDNotFound) + + _, err = d.LookupHandle(ctx, syntax.HandleInvalid) + assert.Error(err) +} + +func TestRedisDirectory(t *testing.T) { + t.Skip("TODO: skipping live network test") + assert := assert.New(t) + ctx := context.Background() + inner := identity.BaseDirectory{} + d, err := NewRedisDirectory(&inner, redisLocalTestURL, time.Hour*1, time.Hour*1, time.Hour*1, 1000) + if err != nil { + t.Fatal(err) + } + + err = d.Purge(ctx, syntax.Handle("atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.Handle("fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.DID("did:web:fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + + for i := 0; i < 3; i = i + 1 { + testDirectoryLive(t, d) + } +} + +func TestRedisCoalesce(t *testing.T) { + t.Skip("TODO: skipping live network test") + + assert := assert.New(t) + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + + base := identity.BaseDirectory{ + PLCURL: "https://plc.directory", + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + // Limit the number of requests we can make to the PLC to 1 per second + PLCLimiter: rate.NewLimiter(1, 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social"}, + } + dir, err := NewRedisDirectory(&base, redisLocalTestURL, time.Hour*1, time.Hour*1, time.Hour*1, 1000) + if err != nil { + t.Fatal(err) + } + // All 60 routines launch at the same time, so they should all miss the cache initially + routines := 60 + wg := sync.WaitGroup{} + + // Cancel the context after 2 seconds, if we're coalescing correctly, we should only make 1 request + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + for i := 0; i < routines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ident, err := dir.LookupDID(ctx, did) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(handle, ident.Handle) + + ident, err = dir.LookupHandle(ctx, handle) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(did, ident.DID) + }() + } + wg.Wait() +} diff --git a/atproto/identity/redisdir/redis_directory.go b/atproto/identity/redisdir/redis_directory.go index 3ea9f354a..f1d67f256 100644 --- a/atproto/identity/redisdir/redis_directory.go +++ b/atproto/identity/redisdir/redis_directory.go @@ -2,6 +2,7 @@ package redisdir import ( "context" + "errors" "fmt" "sync" "time" @@ -20,9 +21,10 @@ var redisDirPrefix string = "dir/" // // Includes an in-process LRU cache as well (provided by the redis client library), for hot key (identities). type RedisDirectory struct { - Inner identity.Directory - ErrTTL time.Duration - HitTTL time.Duration + Inner identity.Directory + ErrTTL time.Duration + HitTTL time.Duration + InvalidHandleTTL time.Duration handleCache *cache.Cache identityCache *cache.Cache @@ -32,8 +34,9 @@ type RedisDirectory struct { type handleEntry struct { Updated time.Time - DID syntax.DID - Err error + // needs to be pointer type, because unmarshalling empty string would be an error + DID *syntax.DID + Err error } type identityEntry struct { @@ -49,7 +52,9 @@ var _ identity.Directory = (*RedisDirectory)(nil) // `redisURL` contains all the redis connection config options. // `hitTTL` and `errTTL` define how long successful and errored identity metadata should be cached (respectively). errTTL is expected to be shorted than hitTTL. // `lruSize` is the size of the in-process cache, for each of the handle and identity caches. 10000 is a reasonable default. -func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL time.Duration, lruSize int) (*RedisDirectory, error) { +// +// NOTE: Errors returned may be inconsistent with the base directory, or between calls. This is because cached errors are serialized/deserialized and that may break equality checks. +func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL, invalidHandleTTL time.Duration, lruSize int) (*RedisDirectory, error) { opt, err := redis.ParseURL(redisURL) if err != nil { return nil, err @@ -69,11 +74,12 @@ func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL LocalCache: cache.NewTinyLFU(lruSize, hitTTL), }) return &RedisDirectory{ - Inner: inner, - ErrTTL: errTTL, - HitTTL: hitTTL, - handleCache: handleCache, - identityCache: identityCache, + Inner: inner, + ErrTTL: errTTL, + HitTTL: hitTTL, + InvalidHandleTTL: invalidHandleTTL, + handleCache: handleCache, + identityCache: identityCache, }, nil } @@ -88,16 +94,19 @@ func (d *RedisDirectory) isIdentityStale(e *identityEntry) bool { if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { return true } + if e.Identity != nil && e.Identity.Handle.IsInvalidHandle() && time.Since(e.Updated) > d.InvalidHandleTTL { + return true + } return false } -func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*handleEntry, error) { +func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) handleEntry { h = h.Normalize() ident, err := d.Inner.LookupHandle(ctx, h) if err != nil { he := handleEntry{ Updated: time.Now(), - DID: "", + DID: nil, Err: err, } err = d.handleCache.Set(&cache.Item{ @@ -107,9 +116,11 @@ func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*ha TTL: d.ErrTTL, }) if err != nil { - return nil, err + he.DID = nil + he.Err = fmt.Errorf("identity cache write: %w", err) + return he } - return &he, nil + return he } entry := identityEntry{ @@ -119,7 +130,7 @@ func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*ha } he := handleEntry{ Updated: time.Now(), - DID: ident.DID, + DID: &ident.DID, Err: nil, } @@ -130,7 +141,9 @@ func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*ha TTL: d.HitTTL, }) if err != nil { - return nil, err + he.DID = nil + he.Err = fmt.Errorf("identity cache write: %w", err) + return he } err = d.handleCache.Set(&cache.Item{ Ctx: ctx, @@ -139,20 +152,31 @@ func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*ha TTL: d.HitTTL, }) if err != nil { - return nil, err + he.DID = nil + he.Err = fmt.Errorf("identity cache write: %w", err) + return he } - return &he, nil + return he } func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + if h.IsInvalidHandle() { + return "", errors.New("invalid handle") + } var entry handleEntry err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), &entry) if err != nil && err != cache.ErrCacheMiss { - return "", err + return "", fmt.Errorf("identity cache read: %w", err) } - if err != cache.ErrCacheMiss && !d.isHandleStale(&entry) { + if err == nil && !d.isHandleStale(&entry) { // if no error... handleCacheHits.Inc() - return entry.DID, entry.Err + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } } handleCacheMisses.Inc() @@ -167,32 +191,41 @@ func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (sy // The result should now be in the cache err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), entry) if err != nil && err != cache.ErrCacheMiss { - return "", err + return "", fmt.Errorf("identity cache read: %w", err) } - if err != cache.ErrCacheMiss && !d.isHandleStale(&entry) { - return entry.DID, entry.Err + if err == nil && !d.isHandleStale(&entry) { // if no error... + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } } - return "", fmt.Errorf("identity not found in cache after coalesce returned") + return "", errors.New("identity not found in cache after coalesce returned") case <-ctx.Done(): return "", ctx.Err() } } - var did syntax.DID // Update the Handle Entry from PLC and cache the result - newEntry, err := d.updateHandle(ctx, h) - if err == nil && newEntry != nil { - did = newEntry.DID - } + newEntry := d.updateHandle(ctx, h) + // Cleanup the coalesce map and close the results channel d.handleLookupChans.Delete(h.String()) // Callers waiting will now get the result from the cache close(res) - return did, err + if newEntry.Err != nil { + return "", newEntry.Err + } + if newEntry.DID != nil { + return *newEntry.DID, nil + } + return "", errors.New("unexpected control-flow error") } -func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*identityEntry, error) { +func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) identityEntry { ident, err := d.Inner.LookupDID(ctx, did) // persist the identity lookup error, instead of processing it immediately entry := identityEntry{ @@ -202,10 +235,10 @@ func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*identi } var he *handleEntry // if *not* an error, then also update the handle cache - if nil == err && !ident.Handle.IsInvalidHandle() { + if err == nil && !ident.Handle.IsInvalidHandle() { he = &handleEntry{ Updated: time.Now(), - DID: did, + DID: &did, Err: nil, } } @@ -217,7 +250,9 @@ func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*identi TTL: d.HitTTL, }) if err != nil { - return nil, err + entry.Identity = nil + entry.Err = fmt.Errorf("identity cache write: %v", err) + return entry } if he != nil { err = d.handleCache.Set(&cache.Item{ @@ -227,21 +262,28 @@ func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*identi TTL: d.HitTTL, }) if err != nil { - return nil, err + entry.Identity = nil + entry.Err = fmt.Errorf("identity cache write: %v", err) + return entry } } - return &entry, nil + return entry } func (d *RedisDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identity.Identity, error) { + id, _, err := d.LookupDIDWithCacheState(ctx, did) + return id, err +} + +func (d *RedisDirectory) LookupDIDWithCacheState(ctx context.Context, did syntax.DID) (*identity.Identity, bool, error) { var entry identityEntry err := d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) if err != nil && err != cache.ErrCacheMiss { - return nil, err + return nil, false, fmt.Errorf("identity cache read: %v", err) } - if err != cache.ErrCacheMiss && !d.isIdentityStale(&entry) { + if err == nil && !d.isIdentityStale(&entry) { // if no error... identityCacheHits.Inc() - return entry.Identity, entry.Err + return entry.Identity, true, entry.Err } identityCacheMisses.Inc() @@ -256,72 +298,89 @@ func (d *RedisDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identi // The result should now be in the cache err = d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) if err != nil && err != cache.ErrCacheMiss { - return nil, err + return nil, false, fmt.Errorf("identity cache read: %v", err) } - if err != cache.ErrCacheMiss && !d.isIdentityStale(&entry) { - return entry.Identity, entry.Err + if err == nil && !d.isIdentityStale(&entry) { // if no error... + return entry.Identity, false, entry.Err } - return nil, fmt.Errorf("identity not found in cache after coalesce returned") + return nil, false, errors.New("identity not found in cache after coalesce returned") case <-ctx.Done(): - return nil, ctx.Err() + return nil, false, ctx.Err() } } - var doc *identity.Identity // Update the Identity Entry from PLC and cache the result - newEntry, err := d.updateDID(ctx, did) - if err == nil && newEntry != nil { - doc = newEntry.Identity - } + newEntry := d.updateDID(ctx, did) + // Cleanup the coalesce map and close the results channel d.didLookupChans.Delete(did.String()) // Callers waiting will now get the result from the cache close(res) - return doc, err + if newEntry.Err != nil { + return nil, false, newEntry.Err + } + if newEntry.Identity != nil { + return newEntry.Identity, false, nil + } + return nil, false, errors.New("unexpected control-flow error") } func (d *RedisDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*identity.Identity, error) { + ident, _, err := d.LookupHandleWithCacheState(ctx, h) + return ident, err +} + +func (d *RedisDirectory) LookupHandleWithCacheState(ctx context.Context, h syntax.Handle) (*identity.Identity, bool, error) { + h = h.Normalize() did, err := d.ResolveHandle(ctx, h) if err != nil { - return nil, err + return nil, false, err } - ident, err := d.LookupDID(ctx, did) + ident, hit, err := d.LookupDIDWithCacheState(ctx, did) if err != nil { - return nil, err + return nil, hit, err } declared, err := ident.DeclaredHandle() if err != nil { - return nil, err + return nil, hit, err } if declared != h { - return nil, fmt.Errorf("handle does not match that declared in DID document") + return nil, hit, identity.ErrHandleMismatch } - return ident, nil + return ident, hit, nil } func (d *RedisDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*identity.Identity, error) { handle, err := a.AsHandle() - if nil == err { // if not an error, is a handle + if err == nil { // if not an error, is a handle return d.LookupHandle(ctx, handle) } did, err := a.AsDID() - if nil == err { // if not an error, is a DID + if err == nil { // if not an error, is a DID return d.LookupDID(ctx, did) } - return nil, fmt.Errorf("at-identifier neither a Handle nor a DID") + return nil, errors.New("at-identifier neither a Handle nor a DID") } func (d *RedisDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error { handle, err := a.AsHandle() - if nil == err { // if not an error, is a handle + if err == nil { // if not an error, is a handle handle = handle.Normalize() - return d.handleCache.Delete(ctx, handle.String()) + err = d.handleCache.Delete(ctx, redisDirPrefix+handle.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err } did, err := a.AsDID() - if nil == err { // if not an error, is a DID - return d.identityCache.Delete(ctx, did.String()) + if err == nil { // if not an error, is a DID + err = d.identityCache.Delete(ctx, redisDirPrefix+did.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err } - return fmt.Errorf("at-identifier neither a Handle nor a DID") + return errors.New("at-identifier neither a Handle nor a DID") } diff --git a/atproto/lexicon/catalog.go b/atproto/lexicon/catalog.go new file mode 100644 index 000000000..7797fad93 --- /dev/null +++ b/atproto/lexicon/catalog.go @@ -0,0 +1,117 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// Interface type for a resolver or container of lexicon schemas, and methods for validating generic data against those schemas. +type Catalog interface { + // Looks up a schema refrence (NSID string with optional fragment) to a Schema object. + Resolve(ref string) (*Schema, error) +} + +// Trivial in-memory Lexicon Catalog implementation. +type BaseCatalog struct { + schemas map[string]Schema +} + +// Creates a new empty BaseCatalog +func NewBaseCatalog() BaseCatalog { + return BaseCatalog{ + schemas: make(map[string]Schema), + } +} + +func (c *BaseCatalog) Resolve(ref string) (*Schema, error) { + if ref == "" { + return nil, fmt.Errorf("tried to resolve empty string name") + } + // default to #main if name doesn't have a fragment + if !strings.Contains(ref, "#") { + ref = ref + "#main" + } + s, ok := c.schemas[ref] + if !ok { + return nil, fmt.Errorf("schema not found in catalog: %s", ref) + } + return &s, nil +} + +// Inserts a schema loaded from a JSON file in to the catalog. +func (c *BaseCatalog) AddSchemaFile(sf SchemaFile) error { + base := sf.ID + for frag, def := range sf.Defs { + if len(frag) == 0 || strings.Contains(frag, "#") || strings.Contains(frag, ".") { + // TODO: more validation here? + return fmt.Errorf("schema name invalid: %s", frag) + } + name := base + "#" + frag + if _, ok := c.schemas[name]; ok { + return fmt.Errorf("catalog already contained a schema with name: %s", name) + } + // "A file can have at most one definition with one of the "primary" types. Primary types should always have the name main. It is possible for main to describe a non-primary type." + switch s := def.Inner.(type) { + case SchemaRecord, SchemaQuery, SchemaProcedure, SchemaSubscription: + if frag != "main" { + return fmt.Errorf("record, query, procedure, and subscription types must be 'main', not: %s", frag) + } + case SchemaToken: + // add fully-qualified name to token + s.fullName = name + def.Inner = s + } + def.SetBase(base) + if err := def.CheckSchema(); err != nil { + return err + } + s := Schema{ + ID: name, + Revision: sf.Revision, + Def: def.Inner, + } + c.schemas[name] = s + } + return nil +} + +// Recursively loads all '.json' files from a directory in to the catalog. +func (c *BaseCatalog) LoadDirectory(dirPath string) error { + return filepath.WalkDir(dirPath, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.HasSuffix(p, ".json") { + return nil + } + slog.Debug("loading Lexicon schema file", "path", p) + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + b, err := io.ReadAll(f) + if err != nil { + return err + } + + var sf SchemaFile + if err = json.Unmarshal(b, &sf); err != nil { + return err + } + if err = c.AddSchemaFile(sf); err != nil { + return err + } + return nil + }) +} diff --git a/atproto/lexicon/cmd/lextool/main.go b/atproto/lexicon/cmd/lextool/main.go new file mode 100644 index 000000000..85f379360 --- /dev/null +++ b/atproto/lexicon/cmd/lextool/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/atproto/lexicon" + + "github.com/urfave/cli/v2" +) + +func main() { + app := cli.App{ + Name: "lex-tool", + Usage: "informal debugging CLI tool for atproto lexicons", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "parse-schema", + Usage: "parse an individual lexicon schema file (JSON)", + Action: runParseSchema, + }, + &cli.Command{ + Name: "load-directory", + Usage: "try recursively loading all the schemas from a directory", + Action: runLoadDirectory, + }, + &cli.Command{ + Name: "validate-record", + Usage: "fetch from network, validate against catalog", + Action: runValidateRecord, + }, + &cli.Command{ + Name: "validate-firehose", + Usage: "subscribe to a firehose, validate every known record against catalog", + Action: runValidateFirehose, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + app.RunAndExitOnError() +} + +func runParseSchema(cctx *cli.Context) error { + p := cctx.Args().First() + if p == "" { + return fmt.Errorf("need to provide path to a schema file as an argument") + } + + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + b, err := io.ReadAll(f) + if err != nil { + return err + } + + var sf lexicon.SchemaFile + if err := json.Unmarshal(b, &sf); err != nil { + return err + } + out, err := json.MarshalIndent(sf, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func runLoadDirectory(cctx *cli.Context) error { + p := cctx.Args().First() + if p == "" { + return fmt.Errorf("need to provide directory path as an argument") + } + + c := lexicon.NewBaseCatalog() + err := c.LoadDirectory(p) + if err != nil { + return err + } + + fmt.Println("success!") + return nil +} diff --git a/atproto/lexicon/cmd/lextool/net.go b/atproto/lexicon/cmd/lextool/net.go new file mode 100644 index 000000000..1600aaee8 --- /dev/null +++ b/atproto/lexicon/cmd/lextool/net.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + + "github.com/bluesky-social/indigo/atproto/data" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v2" +) + +func runValidateRecord(cctx *cli.Context) error { + ctx := context.Background() + args := cctx.Args().Slice() + if len(args) != 2 { + return fmt.Errorf("expected two args (catalog path and AT-URI)") + } + p := args[0] + if p == "" { + return fmt.Errorf("need to provide directory path as an argument") + } + + cat := lexicon.NewBaseCatalog() + err := cat.LoadDirectory(p) + if err != nil { + return err + } + + aturi, err := syntax.ParseATURI(args[1]) + if err != nil { + return err + } + if aturi.RecordKey() == "" { + return fmt.Errorf("need a full, not partial, AT-URI: %s", aturi) + } + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, aturi.Authority()) + if err != nil { + return fmt.Errorf("resolving AT-URI authority: %v", err) + } + pdsURL := ident.PDSEndpoint() + if pdsURL == "" { + return fmt.Errorf("could not resolve PDS endpoint for AT-URI account: %s", ident.DID.String()) + } + + slog.Info("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", + pdsURL, ident.DID, aturi.Collection(), aturi.RecordKey()) + resp, err := http.Get(url) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetch failed") + } + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + body, err := data.UnmarshalJSON(respBytes) + record, ok := body["value"].(map[string]any) + if !ok { + return fmt.Errorf("fetched record was not an object") + } + + slog.Info("validating", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + err = lexicon.ValidateRecord(&cat, record, aturi.Collection().String(), lexicon.LenientMode) + if err != nil { + return err + } + fmt.Println("success!") + return nil +} + +func runValidateFirehose(cctx *cli.Context) error { + p := cctx.Args().First() + if p == "" { + return fmt.Errorf("need to provide directory path as an argument") + } + + cat := lexicon.NewBaseCatalog() + err := cat.LoadDirectory(p) + if err != nil { + return err + } + + return fmt.Errorf("UNIMPLEMENTED") +} diff --git a/atproto/lexicon/docs.go b/atproto/lexicon/docs.go new file mode 100644 index 000000000..754179933 --- /dev/null +++ b/atproto/lexicon/docs.go @@ -0,0 +1,4 @@ +/* +Package atproto/lexicon provides generic Lexicon schema parsing and run-time validation. +*/ +package lexicon diff --git a/atproto/lexicon/examples_test.go b/atproto/lexicon/examples_test.go new file mode 100644 index 000000000..fe2cd0568 --- /dev/null +++ b/atproto/lexicon/examples_test.go @@ -0,0 +1,41 @@ +package lexicon + +import ( + "fmt" + + atdata "github.com/bluesky-social/indigo/atproto/data" +) + +func ExampleValidateRecord() { + + // First load Lexicon schema JSON files from local disk. + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + panic("failed to load lexicons") + } + + // Parse record JSON data using atproto/data helper + recordJSON := `{ + "$type": "example.lexicon.record", + "integer": 123, + "formats": { + "did": "did:web:example.com", + "aturi": "at://handle.example.com/com.example.nsid/asdf123", + "datetime": "2023-10-30T22:25:23Z", + "language": "en", + "tid": "3kznmn7xqxl22" + } + }` + + recordData, err := atdata.UnmarshalJSON([]byte(recordJSON)) + if err != nil { + panic("failed to parse record JSON") + } + + if err := ValidateRecord(&cat, recordData, "example.lexicon.record", 0); err != nil { + fmt.Printf("Schema validation failed: %v\n", err) + } else { + fmt.Println("Success!") + } + // Output: Success! +} diff --git a/atproto/lexicon/extract.go b/atproto/lexicon/extract.go new file mode 100644 index 000000000..51d09ba93 --- /dev/null +++ b/atproto/lexicon/extract.go @@ -0,0 +1,20 @@ +package lexicon + +import ( + "encoding/json" +) + +// Helper type for extracting record type from JSON +type genericSchemaDef struct { + Type string `json:"type"` +} + +// Parses the top-level $type field from generic atproto JSON data +func ExtractTypeJSON(b []byte) (string, error) { + var gsd genericSchemaDef + if err := json.Unmarshal(b, &gsd); err != nil { + return "", err + } + + return gsd.Type, nil +} diff --git a/atproto/lexicon/interop_language_test.go b/atproto/lexicon/interop_language_test.go new file mode 100644 index 000000000..fa9f347ee --- /dev/null +++ b/atproto/lexicon/interop_language_test.go @@ -0,0 +1,95 @@ +package lexicon + +import ( + "encoding/json" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +type LexiconFixture struct { + Name string `json:"name"` + Lexicon json.RawMessage `json:"lexicon"` +} + +func TestInteropLexiconValid(t *testing.T) { + + f, err := os.Open("testdata/lexicon-valid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []LexiconFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, f := range fixtures { + testLexiconFixtureValid(t, f) + } +} + +func testLexiconFixtureValid(t *testing.T, fixture LexiconFixture) { + assert := assert.New(t) + + var schema SchemaFile + if err := json.Unmarshal(fixture.Lexicon, &schema); err != nil { + t.Fatal(err) + } + + outBytes, err := json.Marshal(schema) + if err != nil { + t.Fatal(err) + } + + var beforeMap map[string]any + if err := json.Unmarshal(fixture.Lexicon, &beforeMap); err != nil { + t.Fatal(err) + } + + var afterMap map[string]any + if err := json.Unmarshal(outBytes, &afterMap); err != nil { + t.Fatal(err) + } + + assert.Equal(beforeMap, afterMap) +} + +func TestInteropLexiconInvalid(t *testing.T) { + + f, err := os.Open("testdata/lexicon-invalid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []LexiconFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, f := range fixtures { + testLexiconFixtureInvalid(t, f) + } +} + +func testLexiconFixtureInvalid(t *testing.T, fixture LexiconFixture) { + assert := assert.New(t) + + var schema SchemaFile + err := json.Unmarshal(fixture.Lexicon, &schema) + assert.Error(err) +} diff --git a/atproto/lexicon/interop_record_test.go b/atproto/lexicon/interop_record_test.go new file mode 100644 index 000000000..cb6546998 --- /dev/null +++ b/atproto/lexicon/interop_record_test.go @@ -0,0 +1,92 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "io" + "os" + "testing" + + "github.com/bluesky-social/indigo/atproto/data" + + "github.com/stretchr/testify/assert" +) + +type RecordFixture struct { + Name string `json:"name"` + RecordKey string `json:"rkey"` + Data json.RawMessage `json:"data"` +} + +func TestInteropRecordValid(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + f, err := os.Open("testdata/record-data-valid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []RecordFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, fixture := range fixtures { + fmt.Println(fixture.Name) + d, err := data.UnmarshalJSON(fixture.Data) + if err != nil { + t.Fatal(err) + } + + assert.NoError(ValidateRecord(&cat, d, "example.lexicon.record", 0)) + } +} + +func TestInteropRecordInvalid(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + f, err := os.Open("testdata/record-data-invalid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []RecordFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, fixture := range fixtures { + fmt.Println(fixture.Name) + d, err := data.UnmarshalJSON(fixture.Data) + if err != nil { + t.Fatal(err) + } + err = ValidateRecord(&cat, d, "example.lexicon.record", 0) + if err == nil { + fmt.Println(" FAIL") + } + assert.Error(err) + } +} diff --git a/atproto/lexicon/language.go b/atproto/lexicon/language.go new file mode 100644 index 000000000..734d0031e --- /dev/null +++ b/atproto/lexicon/language.go @@ -0,0 +1,923 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/bluesky-social/indigo/atproto/data" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/rivo/uniseg" +) + +// Serialization helper type for top-level Lexicon schema JSON objects (files) +type SchemaFile struct { + Lexicon int `json:"lexicon,const=1"` + ID string `json:"id"` + Revision *int `json:"revision,omitempty"` + Description *string `json:"description,omitempty"` + Defs map[string]SchemaDef `json:"defs"` +} + +// enum type to represent any of the schema fields +type SchemaDef struct { + Inner any +} + +// Checks that the schema definition itself is valid (recursively). +func (s *SchemaDef) CheckSchema() error { + switch v := s.Inner.(type) { + case SchemaRecord: + return v.CheckSchema() + case SchemaQuery: + return v.CheckSchema() + case SchemaProcedure: + return v.CheckSchema() + case SchemaSubscription: + return v.CheckSchema() + case SchemaNull: + return v.CheckSchema() + case SchemaBoolean: + return v.CheckSchema() + case SchemaInteger: + return v.CheckSchema() + case SchemaString: + return v.CheckSchema() + case SchemaBytes: + return v.CheckSchema() + case SchemaCIDLink: + return v.CheckSchema() + case SchemaArray: + return v.CheckSchema() + case SchemaObject: + return v.CheckSchema() + case SchemaBlob: + return v.CheckSchema() + case SchemaParams: + return v.CheckSchema() + case SchemaToken: + return v.CheckSchema() + case SchemaRef: + return v.CheckSchema() + case SchemaUnion: + return v.CheckSchema() + case SchemaUnknown: + return v.CheckSchema() + default: + return fmt.Errorf("unhandled schema type: %v", reflect.TypeOf(v)) + } +} + +// Helper to recurse down the definition tree and set full references on any sub-schemas which need to embed that metadata +func (s *SchemaDef) SetBase(base string) { + switch v := s.Inner.(type) { + case SchemaRecord: + for i, val := range v.Record.Properties { + val.SetBase(base) + v.Record.Properties[i] = val + } + s.Inner = v + case SchemaQuery: + for i, val := range v.Parameters.Properties { + val.SetBase(base) + v.Parameters.Properties[i] = val + } + if v.Output != nil && v.Output.Schema != nil { + v.Output.Schema.SetBase(base) + } + s.Inner = v + case SchemaProcedure: + for i, val := range v.Parameters.Properties { + val.SetBase(base) + v.Parameters.Properties[i] = val + } + if v.Input != nil && v.Input.Schema != nil { + v.Input.Schema.SetBase(base) + } + if v.Output != nil && v.Output.Schema != nil { + v.Output.Schema.SetBase(base) + } + s.Inner = v + case SchemaSubscription: + for i, val := range v.Parameters.Properties { + val.SetBase(base) + v.Parameters.Properties[i] = val + } + if v.Message != nil { + v.Message.Schema.SetBase(base) + } + s.Inner = v + case SchemaArray: + v.Items.SetBase(base) + s.Inner = v + case SchemaObject: + for i, val := range v.Properties { + val.SetBase(base) + v.Properties[i] = val + } + s.Inner = v + case SchemaParams: + for i, val := range v.Properties { + val.SetBase(base) + v.Properties[i] = val + } + s.Inner = v + case SchemaRef: + // add fully-qualified name + if strings.HasPrefix(v.Ref, "#") { + v.fullRef = base + v.Ref + } else { + v.fullRef = v.Ref + } + s.Inner = v + case SchemaUnion: + // add fully-qualified name + for _, ref := range v.Refs { + if strings.HasPrefix(ref, "#") { + ref = base + ref + } + v.fullRefs = append(v.fullRefs, ref) + } + s.Inner = v + } + return +} + +func (s SchemaDef) MarshalJSON() ([]byte, error) { + return json.Marshal(s.Inner) +} + +func (s *SchemaDef) UnmarshalJSON(b []byte) error { + t, err := ExtractTypeJSON(b) + if err != nil { + return err + } + // TODO: should we call CheckSchema here, instead of in lexicon loading? + switch t { + case "record": + v := new(SchemaRecord) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "query": + v := new(SchemaQuery) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "procedure": + v := new(SchemaProcedure) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "subscription": + v := new(SchemaSubscription) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "null": + v := new(SchemaNull) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "boolean": + v := new(SchemaBoolean) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "integer": + v := new(SchemaInteger) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "string": + v := new(SchemaString) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "bytes": + v := new(SchemaBytes) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "cid-link": + v := new(SchemaCIDLink) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "array": + v := new(SchemaArray) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "object": + v := new(SchemaObject) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "blob": + v := new(SchemaBlob) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "params": + v := new(SchemaParams) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "token": + v := new(SchemaToken) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "ref": + v := new(SchemaRef) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "union": + v := new(SchemaUnion) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "unknown": + v := new(SchemaUnknown) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + default: + return fmt.Errorf("unexpected schema type: %s", t) + } +} + +type SchemaRecord struct { + Type string `json:"type,const=record"` + Description *string `json:"description,omitempty"` + Key string `json:"key"` + Record SchemaObject `json:"record"` +} + +func (s *SchemaRecord) CheckSchema() error { + switch s.Key { + case "tid", "any": + // pass + default: + if !strings.HasPrefix(s.Key, "literal:") { + return fmt.Errorf("invalid record key specifier: %s", s.Key) + } + } + return s.Record.CheckSchema() +} + +type SchemaQuery struct { + Type string `json:"type,const=query"` + Description *string `json:"description,omitempty"` + Parameters SchemaParams `json:"parameters"` + Output *SchemaBody `json:"output"` + Errors []SchemaError `json:"errors,omitempty"` // optional +} + +func (s *SchemaQuery) CheckSchema() error { + if s.Output != nil { + if err := s.Output.CheckSchema(); err != nil { + return err + } + } + for _, e := range s.Errors { + if err := e.CheckSchema(); err != nil { + return err + } + } + return s.Parameters.CheckSchema() +} + +type SchemaProcedure struct { + Type string `json:"type,const=procedure"` + Description *string `json:"description,omitempty"` + Parameters SchemaParams `json:"parameters"` + Output *SchemaBody `json:"output"` // optional + Errors []SchemaError `json:"errors,omitempty"` // optional + Input *SchemaBody `json:"input"` // optional +} + +func (s *SchemaProcedure) CheckSchema() error { + if s.Input != nil { + if err := s.Input.CheckSchema(); err != nil { + return err + } + } + if s.Output != nil { + if err := s.Output.CheckSchema(); err != nil { + return err + } + } + for _, e := range s.Errors { + if err := e.CheckSchema(); err != nil { + return err + } + } + return s.Parameters.CheckSchema() +} + +type SchemaSubscription struct { + Type string `json:"type,const=subscription"` + Description *string `json:"description,omitempty"` + Parameters SchemaParams `json:"parameters"` + Message *SchemaMessage `json:"message,omitempty"` // TODO(specs): is this really optional? +} + +func (s *SchemaSubscription) CheckSchema() error { + if s.Message != nil { + if err := s.Message.CheckSchema(); err != nil { + return err + } + } + return s.Parameters.CheckSchema() +} + +type SchemaBody struct { + Description *string `json:"description,omitempty"` + Encoding string `json:"encoding"` // required, mimetype + Schema *SchemaDef `json:"schema"` // optional; type:object, type:ref, or type:union +} + +func (s *SchemaBody) CheckSchema() error { + // TODO: any validation of encoding? + if s.Schema != nil { + switch s.Schema.Inner.(type) { + case SchemaObject, SchemaRef, SchemaUnion: + // pass + default: + return fmt.Errorf("body type can only have object, ref, or union schema") + } + if err := s.Schema.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaMessage struct { + Description *string `json:"description,omitempty"` + Schema SchemaDef `json:"schema"` // required; type:union only +} + +func (s *SchemaMessage) CheckSchema() error { + if _, ok := s.Schema.Inner.(SchemaUnion); !ok { + return fmt.Errorf("message must have schema type union") + } + return s.Schema.CheckSchema() +} + +type SchemaError struct { + Name string `json:"name"` + Description *string `json:"description"` +} + +func (s *SchemaError) CheckSchema() error { + return nil +} +func (s *SchemaError) Validate(d any) error { + e, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("expected an object in error position") + } + n, ok := e["error"] + if !ok { + return fmt.Errorf("expected error type") + } + if n != s.Name { + return fmt.Errorf("error type mis-match: %s", n) + } + return nil +} + +type SchemaNull struct { + Type string `json:"type,const=null"` + Description *string `json:"description,omitempty"` +} + +func (s *SchemaNull) CheckSchema() error { + return nil +} + +func (s *SchemaNull) Validate(d any) error { + if d != nil { + return fmt.Errorf("expected null data, got: %s", reflect.TypeOf(d)) + } + return nil +} + +type SchemaBoolean struct { + Type string `json:"type,const=bool"` + Description *string `json:"description,omitempty"` + Default *bool `json:"default,omitempty"` + Const *bool `json:"const,omitempty"` +} + +func (s *SchemaBoolean) CheckSchema() error { + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + return nil +} + +func (s *SchemaBoolean) Validate(d any) error { + v, ok := d.(bool) + if !ok { + return fmt.Errorf("expected a boolean") + } + if s.Const != nil && v != *s.Const { + return fmt.Errorf("boolean val didn't match constant (%v): %v", *s.Const, v) + } + return nil +} + +type SchemaInteger struct { + Type string `json:"type,const=integer"` + Description *string `json:"description,omitempty"` + Minimum *int `json:"minimum,omitempty"` + Maximum *int `json:"maximum,omitempty"` + Enum []int `json:"enum,omitempty"` + Default *int `json:"default,omitempty"` + Const *int `json:"const,omitempty"` +} + +func (s *SchemaInteger) CheckSchema() error { + // TODO: enforce min/max against enum, default, const + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + if s.Minimum != nil && s.Maximum != nil && *s.Maximum < *s.Minimum { + return fmt.Errorf("schema max < min") + } + return nil +} + +func (s *SchemaInteger) Validate(d any) error { + v64, ok := d.(int64) + if !ok { + return fmt.Errorf("expected an integer") + } + v := int(v64) + if s.Const != nil && v != *s.Const { + return fmt.Errorf("integer val didn't match constant (%d): %d", *s.Const, v) + } + if (s.Minimum != nil && v < *s.Minimum) || (s.Maximum != nil && v > *s.Maximum) { + return fmt.Errorf("integer val outside specified range: %d", v) + } + if len(s.Enum) != 0 { + inEnum := false + for _, e := range s.Enum { + if e == v { + inEnum = true + break + } + } + if !inEnum { + return fmt.Errorf("integer val not in required enum: %d", v) + } + } + return nil +} + +type SchemaString struct { + Type string `json:"type,const=string"` + Description *string `json:"description,omitempty"` + Format *string `json:"format,omitempty"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` + MinGraphemes *int `json:"minGraphemes,omitempty"` + MaxGraphemes *int `json:"maxGraphemes,omitempty"` + KnownValues []string `json:"knownValues,omitempty"` + Enum []string `json:"enum,omitempty"` + Default *string `json:"default,omitempty"` + Const *string `json:"const,omitempty"` +} + +func (s *SchemaString) CheckSchema() error { + // TODO: enforce min/max against enum, default, const + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if s.MinGraphemes != nil && s.MaxGraphemes != nil && *s.MaxGraphemes < *s.MinGraphemes { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) || + (s.MinGraphemes != nil && *s.MinGraphemes < 0) || + (s.MaxGraphemes != nil && *s.MaxGraphemes < 0) { + return fmt.Errorf("string schema min or max below zero") + } + if s.Format != nil { + switch *s.Format { + case "at-identifier", "at-uri", "cid", "datetime", "did", "handle", "nsid", "uri", "language", "tid", "record-key": + // pass + default: + return fmt.Errorf("unknown string format: %s", *s.Format) + } + } + return nil +} + +func (s *SchemaString) Validate(d any, flags ValidateFlags) error { + v, ok := d.(string) + if !ok { + return fmt.Errorf("expected a string: %v", reflect.TypeOf(d)) + } + if s.Const != nil && v != *s.Const { + return fmt.Errorf("string val didn't match constant (%s): %s", *s.Const, v) + } + // TODO: is this actually counting UTF-8 length? + if (s.MinLength != nil && len(v) < *s.MinLength) || (s.MaxLength != nil && len(v) > *s.MaxLength) { + return fmt.Errorf("string length outside specified range: %d", len(v)) + } + if len(s.Enum) != 0 { + inEnum := false + for _, e := range s.Enum { + if e == v { + inEnum = true + break + } + } + if !inEnum { + return fmt.Errorf("string val not in required enum: %s", v) + } + } + if s.MinGraphemes != nil || s.MaxGraphemes != nil { + lenG := uniseg.GraphemeClusterCount(v) + if (s.MinGraphemes != nil && lenG < *s.MinGraphemes) || (s.MaxGraphemes != nil && lenG > *s.MaxGraphemes) { + return fmt.Errorf("string length (graphemes) outside specified range: %d", lenG) + } + } + if s.Format != nil { + switch *s.Format { + case "at-identifier": + if _, err := syntax.ParseAtIdentifier(v); err != nil { + return err + } + case "at-uri": + if _, err := syntax.ParseATURI(v); err != nil { + return err + } + case "cid": + if _, err := syntax.ParseCID(v); err != nil { + return err + } + case "datetime": + if flags&AllowLenientDatetime != 0 { + if _, err := syntax.ParseDatetimeLenient(v); err != nil { + return err + } + } else { + if _, err := syntax.ParseDatetime(v); err != nil { + return err + } + } + case "did": + if _, err := syntax.ParseDID(v); err != nil { + return err + } + case "handle": + if _, err := syntax.ParseHandle(v); err != nil { + return err + } + case "nsid": + if _, err := syntax.ParseNSID(v); err != nil { + return err + } + case "uri": + if _, err := syntax.ParseURI(v); err != nil { + return err + } + case "language": + if _, err := syntax.ParseLanguage(v); err != nil { + return err + } + case "tid": + if _, err := syntax.ParseTID(v); err != nil { + return err + } + case "record-key": + if _, err := syntax.ParseRecordKey(v); err != nil { + return err + } + } + } + return nil +} + +type SchemaBytes struct { + Type string `json:"type,const=bytes"` + Description *string `json:"description,omitempty"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` +} + +func (s *SchemaBytes) CheckSchema() error { + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) { + return fmt.Errorf("bytes schema min or max below zero") + } + return nil +} + +func (s *SchemaBytes) Validate(d any) error { + v, ok := d.(data.Bytes) + if !ok { + return fmt.Errorf("expecting bytes") + } + if (s.MinLength != nil && len(v) < *s.MinLength) || (s.MaxLength != nil && len(v) > *s.MaxLength) { + return fmt.Errorf("bytes size out of bounds: %d", len(v)) + } + return nil +} + +type SchemaCIDLink struct { + Type string `json:"type,const=cid-link"` + Description *string `json:"description,omitempty"` +} + +func (s *SchemaCIDLink) CheckSchema() error { + return nil +} + +func (s *SchemaCIDLink) Validate(d any) error { + _, ok := d.(data.CIDLink) + if !ok { + return fmt.Errorf("expecting a cid-link") + } + return nil +} + +type SchemaArray struct { + Type string `json:"type,const=array"` + Description *string `json:"description,omitempty"` + Items SchemaDef `json:"items"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` +} + +func (s *SchemaArray) CheckSchema() error { + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) { + return fmt.Errorf("array schema min or max below zero") + } + return s.Items.CheckSchema() +} + +type SchemaObject struct { + Type string `json:"type,const=object"` + Description *string `json:"description,omitempty"` + Properties map[string]SchemaDef `json:"properties"` + Required []string `json:"required,omitempty"` + Nullable []string `json:"nullable,omitempty"` +} + +func (s *SchemaObject) CheckSchema() error { + // TODO: check for set intersection between required and nullable + // TODO: check for set uniqueness of required and nullable + for _, k := range s.Required { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'required' field not in properties: %s", k) + } + } + for _, k := range s.Nullable { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'nullable' field not in properties: %s", k) + } + } + for k, def := range s.Properties { + // TODO: more checks on field name? + if len(k) == 0 { + return fmt.Errorf("empty object schema field name not allowed") + } + if err := def.CheckSchema(); err != nil { + return err + } + } + return nil +} + +// Checks if a field name 'k' is one of the Nullable fields for this object +func (s *SchemaObject) IsNullable(k string) bool { + for _, el := range s.Nullable { + if el == k { + return true + } + } + return false +} + +type SchemaBlob struct { + Type string `json:"type,const=blob"` + Description *string `json:"description,omitempty"` + Accept []string `json:"accept,omitempty"` + MaxSize *int `json:"maxSize,omitempty"` +} + +func (s *SchemaBlob) CheckSchema() error { + // TODO: validate Accept (mimetypes)? + if s.MaxSize != nil && *s.MaxSize <= 0 { + return fmt.Errorf("blob max size less or equal to zero") + } + return nil +} + +func (s *SchemaBlob) Validate(d any, flags ValidateFlags) error { + v, ok := d.(data.Blob) + if !ok { + return fmt.Errorf("expected a blob") + } + if !(flags&AllowLegacyBlob != 0) && v.Size < 0 { + return fmt.Errorf("legacy blobs not allowed") + } + if len(s.Accept) > 0 { + typeOk := false + for _, pat := range s.Accept { + if acceptableMimeType(pat, v.MimeType) { + typeOk = true + break + } + } + if !typeOk { + return fmt.Errorf("blob mimetype doesn't match accepted: %s", v.MimeType) + } + } + if s.MaxSize != nil && int(v.Size) > *s.MaxSize { + return fmt.Errorf("blob size too large: %d", v.Size) + } + return nil +} + +type SchemaParams struct { + Type string `json:"type,const=params"` + Description *string `json:"description,omitempty"` + Properties map[string]SchemaDef `json:"properties"` // boolean, integer, string, or unknown; or an array of these types + Required []string `json:"required,omitempty"` +} + +func (s *SchemaParams) CheckSchema() error { + // TODO: check for set uniqueness of required + for _, k := range s.Required { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'required' field not in properties: %s", k) + } + } + for k, def := range s.Properties { + // TODO: more checks on field name? + if len(k) == 0 { + return fmt.Errorf("empty object schema field name not allowed") + } + switch v := def.Inner.(type) { + case SchemaBoolean, SchemaInteger, SchemaString, SchemaUnknown: + // pass + case SchemaArray: + switch v.Items.Inner.(type) { + case SchemaBoolean, SchemaInteger, SchemaString, SchemaUnknown: + // pass + default: + return fmt.Errorf("params array item type must be boolean, integer, string, or unknown") + } + default: + return fmt.Errorf("params field type must be boolean, integer, string, or unknown") + } + if err := def.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaToken struct { + Type string `json:"type,const=token"` + Description *string `json:"description,omitempty"` + // the fully-qualified identifier of this token + fullName string +} + +func (s *SchemaToken) CheckSchema() error { + if s.fullName == "" { + return fmt.Errorf("expected fully-qualified token name") + } + return nil +} + +func (s *SchemaToken) Validate(d any) error { + str, ok := d.(string) + if !ok { + return fmt.Errorf("expected a string for token, got: %s", reflect.TypeOf(d)) + } + if s.fullName == "" { + return fmt.Errorf("token name was not populated at parse time") + } + if str != s.fullName { + return fmt.Errorf("token name did not match expected: %s", str) + } + return nil +} + +type SchemaRef struct { + Type string `json:"type,const=ref"` + Description *string `json:"description,omitempty"` + Ref string `json:"ref"` + // full path of reference + fullRef string +} + +func (s *SchemaRef) CheckSchema() error { + // TODO: more validation of ref string? + if len(s.Ref) == 0 { + return fmt.Errorf("empty schema ref") + } + if len(s.fullRef) == 0 { + return fmt.Errorf("empty full schema ref") + } + return nil +} + +type SchemaUnion struct { + Type string `json:"type,const=union"` + Description *string `json:"description,omitempty"` + Refs []string `json:"refs"` + Closed *bool `json:"closed,omitempty"` + // fully qualified + fullRefs []string +} + +func (s *SchemaUnion) CheckSchema() error { + // TODO: uniqueness check on refs + for _, ref := range s.Refs { + // TODO: more validation of ref string? + if len(ref) == 0 { + return fmt.Errorf("empty schema ref") + } + } + if len(s.fullRefs) != len(s.Refs) { + return fmt.Errorf("union refs were not expanded") + } + return nil +} + +type SchemaUnknown struct { + Type string `json:"type,const=unknown"` + Description *string `json:"description,omitempty"` +} + +func (s *SchemaUnknown) CheckSchema() error { + return nil +} + +func (s *SchemaUnknown) Validate(d any) error { + _, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("'unknown' data must an object") + } + return nil +} diff --git a/atproto/lexicon/language_test.go b/atproto/lexicon/language_test.go new file mode 100644 index 000000000..cd33396f2 --- /dev/null +++ b/atproto/lexicon/language_test.go @@ -0,0 +1,47 @@ +package lexicon + +import ( + "encoding/json" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicLabelLexicon(t *testing.T) { + assert := assert.New(t) + + f, err := os.Open("testdata/catalog/com_atproto_label_defs.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var schema SchemaFile + if err := json.Unmarshal(jsonBytes, &schema); err != nil { + t.Fatal(err) + } + + outBytes, err := json.Marshal(schema) + if err != nil { + t.Fatal(err) + } + + var beforeMap map[string]any + if err := json.Unmarshal(jsonBytes, &beforeMap); err != nil { + t.Fatal(err) + } + + var afterMap map[string]any + if err := json.Unmarshal(outBytes, &afterMap); err != nil { + t.Fatal(err) + } + + assert.Equal(beforeMap, afterMap) +} diff --git a/atproto/lexicon/lexicon.go b/atproto/lexicon/lexicon.go new file mode 100644 index 000000000..840d2ae7a --- /dev/null +++ b/atproto/lexicon/lexicon.go @@ -0,0 +1,179 @@ +package lexicon + +import ( + "fmt" + "reflect" +) + +// Boolean flags tweaking how Lexicon validation rules are interpreted. +type ValidateFlags int + +const ( + // Flag which allows legacy "blob" data to pass validation. + AllowLegacyBlob = 1 << iota + // Flag which loosens "datetime" string syntax validation. String must still be an ISO datetime, but might be missing timezone (for example) + AllowLenientDatetime + // Flag which requires validation of nested data in open unions. By default nested union types are only validated optimistically (if the type is known in catatalog) for unlisted types. This flag will result in a validation error if the Lexicon can't be resolved from the catalog. + StrictRecursiveValidation +) + +// Combination of agument flags for less formal validation. Recommended for, eg, working with old/legacy data from 2023. +var LenientMode ValidateFlags = AllowLegacyBlob | AllowLenientDatetime + +// Represents a Lexicon schema definition +type Schema struct { + ID string + Revision *int + Def any +} + +// Checks Lexicon schema (fetched from the catalog) for the given record, with optional flags tweaking default validation rules. +// +// 'recordData' is typed as 'any', but is expected to be 'map[string]any' +// 'ref' is a reference to the schema type, as an NSID with optional fragment. For records, the '$type' must match 'ref' +// 'flags' are parameters tweaking Lexicon validation rules. Zero value is default. +func ValidateRecord(cat Catalog, recordData any, ref string, flags ValidateFlags) error { + return validateRecordConfig(cat, recordData, ref, flags) +} + +func validateRecordConfig(cat Catalog, recordData any, ref string, flags ValidateFlags) error { + def, err := cat.Resolve(ref) + if err != nil { + return err + } + s, ok := def.Def.(SchemaRecord) + if !ok { + return fmt.Errorf("schema is not of record type: %s", ref) + } + d, ok := recordData.(map[string]any) + if !ok { + return fmt.Errorf("record data is not object type") + } + t, ok := d["$type"] + if !ok || t != ref { + return fmt.Errorf("record data missing $type, or didn't match expected NSID") + } + return validateObject(cat, s.Record, d, flags) +} + +func validateData(cat Catalog, def any, d any, flags ValidateFlags) error { + switch v := def.(type) { + case SchemaNull: + return v.Validate(d) + case SchemaBoolean: + return v.Validate(d) + case SchemaInteger: + return v.Validate(d) + case SchemaString: + return v.Validate(d, flags) + case SchemaBytes: + return v.Validate(d) + case SchemaCIDLink: + return v.Validate(d) + case SchemaArray: + arr, ok := d.([]any) + if !ok { + return fmt.Errorf("expected an array, got: %s", reflect.TypeOf(d)) + } + return validateArray(cat, v, arr, flags) + case SchemaObject: + obj, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("expected an object, got: %s", reflect.TypeOf(d)) + } + return validateObject(cat, v, obj, flags) + case SchemaBlob: + return v.Validate(d, flags) + case SchemaRef: + // recurse + next, err := cat.Resolve(v.fullRef) + if err != nil { + return err + } + return validateData(cat, next.Def, d, flags) + case SchemaUnion: + return validateUnion(cat, v, d, flags) + case SchemaUnknown: + return v.Validate(d) + case SchemaToken: + return v.Validate(d) + default: + return fmt.Errorf("unhandled schema type: %s", reflect.TypeOf(v)) + } +} + +func validateObject(cat Catalog, s SchemaObject, d map[string]any, flags ValidateFlags) error { + for _, k := range s.Required { + if _, ok := d[k]; !ok { + return fmt.Errorf("required field missing: %s", k) + } + } + for k, def := range s.Properties { + if v, ok := d[k]; ok { + if v == nil && s.IsNullable(k) { + continue + } + err := validateData(cat, def.Inner, v, flags) + if err != nil { + return err + } + } + } + return nil +} + +func validateArray(cat Catalog, s SchemaArray, arr []any, flags ValidateFlags) error { + if (s.MinLength != nil && len(arr) < *s.MinLength) || (s.MaxLength != nil && len(arr) > *s.MaxLength) { + return fmt.Errorf("array length out of bounds: %d", len(arr)) + } + for _, v := range arr { + err := validateData(cat, s.Items.Inner, v, flags) + if err != nil { + return err + } + } + return nil +} + +func validateUnion(cat Catalog, s SchemaUnion, d any, flags ValidateFlags) error { + closed := s.Closed != nil && *s.Closed == true + + obj, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("union data is not object type") + } + typeVal, ok := obj["$type"] + if !ok { + return fmt.Errorf("union data must have $type") + } + t, ok := typeVal.(string) + if !ok { + return fmt.Errorf("union data must have string $type") + } + + for _, ref := range s.fullRefs { + if ref != t { + continue + } + def, err := cat.Resolve(ref) + if err != nil { + return fmt.Errorf("could not resolve known union variant $type: %s", ref) + } + return validateData(cat, def.Def, d, flags) + } + if closed { + return fmt.Errorf("data did not match any variant of closed union: %s", t) + } + + // eagerly attempt validation of the open union type + // TODO: validate reference as NSID with optional fragment + def, err := cat.Resolve(t) + if err != nil { + if flags&StrictRecursiveValidation != 0 { + return fmt.Errorf("could not strictly validate open union variant $type: %s", t) + } + // by default, ignore validation of unknown open union data + return nil + } + return validateData(cat, def.Def, d, flags) +} diff --git a/atproto/lexicon/lexicon_test.go b/atproto/lexicon/lexicon_test.go new file mode 100644 index 000000000..ce5c490ec --- /dev/null +++ b/atproto/lexicon/lexicon_test.go @@ -0,0 +1,47 @@ +package lexicon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicCatalog(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + def, err := cat.Resolve("com.atproto.label.defs#label") + if err != nil { + t.Fatal(err) + } + assert.NoError(validateData( + &cat, + def.Def, + map[string]any{ + "cid": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + "cts": "2000-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "at://did:plc:asdf123/com.atproto.feed.post/asdf123", + "val": "test-label", + }, + 0, + )) + + assert.Error(validateData( + &cat, + def.Def, + map[string]any{ + "cid": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + "cts": "2000-01-01T00:00:00.000Z", + "neg": false, + "uri": "at://did:plc:asdf123/com.atproto.feed.post/asdf123", + "val": "test-label", + }, + 0, + )) +} diff --git a/atproto/lexicon/mimetype.go b/atproto/lexicon/mimetype.go new file mode 100644 index 000000000..0e42d5edb --- /dev/null +++ b/atproto/lexicon/mimetype.go @@ -0,0 +1,18 @@ +package lexicon + +import ( + "strings" +) + +// checks if val matches pattern, with optional trailing glob on pattern. case-sensitive. +func acceptableMimeType(pattern, val string) bool { + if val == "" || pattern == "" { + return false + } + if strings.HasSuffix(pattern, "*") { + prefix := pattern[:len(pattern)-1] + return strings.HasPrefix(val, prefix) + } else { + return pattern == val + } +} diff --git a/atproto/lexicon/mimetype_test.go b/atproto/lexicon/mimetype_test.go new file mode 100644 index 000000000..db2be81d5 --- /dev/null +++ b/atproto/lexicon/mimetype_test.go @@ -0,0 +1,21 @@ +package lexicon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAcceptableMimeType(t *testing.T) { + assert := assert.New(t) + + assert.True(acceptableMimeType("image/*", "image/png")) + assert.True(acceptableMimeType("text/plain", "text/plain")) + + assert.False(acceptableMimeType("image/*", "text/plain")) + assert.False(acceptableMimeType("text/plain", "image/png")) + assert.False(acceptableMimeType("text/plain", "")) + assert.False(acceptableMimeType("", "text/plain")) + + // TODO: application/json, application/json+thing +} diff --git a/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json b/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json new file mode 100644 index 000000000..f8677dc37 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json @@ -0,0 +1,78 @@ +{ + "lexicon": 1, + "id": "com.atproto.label.defs", + "defs": { + "label": { + "description": "Metadata tag on an atproto resource (eg, repo or record)", + "properties": { + "cid": { + "description": "optionally, CID specifying the specific version of 'uri' resource this label applies to", + "format": "cid", + "type": "string" + }, + "cts": { + "description": "timestamp when this label was created", + "format": "datetime", + "type": "string" + }, + "neg": { + "description": "if true, this is a negation label, overwriting a previous label", + "type": "boolean" + }, + "src": { + "description": "DID of the actor who created this label", + "format": "did", + "type": "string" + }, + "uri": { + "description": "AT URI of the record, repository (account), or other resource which this label applies to", + "format": "uri", + "type": "string" + }, + "val": { + "description": "the short string name of the value or type of this label", + "maxLength": 128, + "type": "string" + } + }, + "required": [ + "src", + "uri", + "val", + "cts" + ], + "type": "object" + }, + "selfLabel": { + "description": "Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.", + "properties": { + "val": { + "description": "the short string name of the value or type of this label", + "maxLength": 128, + "type": "string" + } + }, + "required": [ + "val" + ], + "type": "object" + }, + "selfLabels": { + "description": "Metadata tags on an atproto record, published by the author within the record.", + "properties": { + "values": { + "items": { + "ref": "#selfLabel", + "type": "ref" + }, + "maxLength": 10, + "type": "array" + } + }, + "required": [ + "values" + ], + "type": "object" + } + } +} diff --git a/atproto/lexicon/testdata/catalog/query.json b/atproto/lexicon/testdata/catalog/query.json new file mode 100644 index 000000000..b337fc007 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/query.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "example.lexicon.query", + "revision": 1, + "description": "exersizes many lexicon features for the query type", + "defs": { + "main": { + "type": "query", + "description": "a query type", + "parameters": { + "type": "params", + "description": "a params type", + "required": ["string"], + "properties": { + "boolean": { + "type": "boolean", + "description": "field of type boolean" + }, + "integer": { + "type": "integer", + "description": "field of type integer" + }, + "string": { + "type": "string", + "description": "field of type string" + }, + "handle": { + "type": "string", + "format": "handle", + "description": "field of type string, format handle" + }, + "unknown": { + "type": "unknown", + "description": "field of type unknown" + }, + "array": { + "type": "array", + "description": "field of type array", + "items": { "type": "integer" } + } + } + }, + "output": { + "description": "output body type", + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + } + }, + "errors": [ + { + "name": "DemoError", + "description": "demo error value" + }, + { + "name": "AnotherDemoError", + "description": "another demo error value" + } + ] + } + } +} diff --git a/atproto/lexicon/testdata/catalog/record.json b/atproto/lexicon/testdata/catalog/record.json new file mode 100644 index 000000000..b7ef7297d --- /dev/null +++ b/atproto/lexicon/testdata/catalog/record.json @@ -0,0 +1,234 @@ +{ + "lexicon": 1, + "id": "example.lexicon.record", + "revision": 1, + "description": "exersizes many lexicon features for the record type", + "defs": { + "main": { + "type": "record", + "key": "literal:demo", + "description": "a record type with many field", + "record": { + "required": [ "integer" ], + "nullable": [ "nullableString" ], + "properties": { + "null": { + "type": "null", + "description": "field of type null" + }, + "boolean": { + "type": "boolean", + "description": "field of type boolean" + }, + "integer": { + "type": "integer", + "description": "field of type integer" + }, + "string": { + "type": "string", + "description": "field of type string" + }, + "nullableString": { + "type": "string", + "description": "field of type string; value is nullable" + }, + "bytes": { + "type": "bytes", + "description": "field of type bytes" + }, + "cid-link": { + "type": "cid-link", + "description": "field of type cid-link" + }, + "blob": { + "type": "blob", + "description": "field of type blob" + }, + "unknown": { + "type": "unknown", + "description": "field of type unknown" + }, + "array": { + "type": "array", + "description": "field of type array", + "items": { "type": "integer" } + }, + "object": { + "type": "object", + "description": "field of type null", + "properties": { + "a": { "type": "integer" }, + "b": { "type": "integer" } + } + }, + "ref": { + "type": "ref", + "description": "field of type ref", + "ref": "example.lexicon.record#demoToken" + }, + "union": { + "type": "union", + "refs": [ + "example.lexicon.record#demoObject", + "example.lexicon.record#demoObjectTwo" + ] + }, + "formats": { + "type": "ref", + "ref": "example.lexicon.record#stringFormats" + }, + "constInteger": { + "type": "integer", + "const": 42 + }, + "defaultInteger": { + "type": "integer", + "default": 42 + }, + "enumInteger": { + "type": "integer", + "enum": [4, 9, 16, 25] + }, + "rangeInteger": { + "type": "integer", + "minimum": 10, + "maximum": 20 + }, + "lenString": { + "type": "string", + "minLength": 10, + "maxLength": 20 + }, + "graphemeString": { + "type": "string", + "minGraphemes": 10, + "maxGraphemes": 20 + }, + "enumString": { + "type": "string", + "enum": ["fish", "tree", "rock"] + }, + "knownString": { + "type": "string", + "knownValues": ["blue", "green", "red"] + }, + "sizeBytes": { + "type": "bytes", + "minLength": 10, + "maxLength": 20 + }, + "lenArray": { + "type": "array", + "items": { "type": "integer" }, + "minLength": 2, + "maxLength": 5 + }, + "sizeBlob": { + "type": "blob", + "maxSize": 20 + }, + "acceptBlob": { + "type": "blob", + "accept": [ "image/*" ] + }, + "closedUnion": { + "type": "union", + "refs": [ + "example.lexicon.record#demoObject" + ], + "closed": true + } + } + } + }, + "stringFormats": { + "type": "object", + "description": "all the various string format types", + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "a did string" + }, + "handle": { + "type": "string", + "format": "handle", + "description": "a did string" + }, + "atidentifier": { + "type": "string", + "format": "at-identifier", + "description": "an at-identifier string" + }, + "nsid": { + "type": "string", + "format": "nsid", + "description": "an nsid string" + }, + "aturi": { + "type": "string", + "format": "at-uri", + "description": "an at-uri string" + }, + "cid": { + "type": "string", + "format": "cid", + "description": "a cid string (not a cid-link)" + }, + "datetime": { + "type": "string", + "format": "datetime", + "description": "a datetime string" + }, + "language": { + "type": "string", + "format": "language", + "description": "a language string" + }, + "uri": { + "type": "string", + "format": "uri", + "description": "a generic URI field" + }, + "tid": { + "type": "string", + "format": "tid", + "description": "a generic TID field" + }, + "recordkey": { + "type": "string", + "format": "record-key", + "description": "a generic record-key field" + } + } + }, + "demoToken": { + "type": "token", + "description": "an example of what a token looks like" + }, + "demoObject": { + "type": "object", + "description": "smaller object schema for unions", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + }, + "demoObjectTwo": { + "type": "object", + "description": "smaller object schema for unions", + "properties": { + "c": { + "type": "integer" + }, + "d": { + "type": "integer" + } + } + } + } +} diff --git a/atproto/lexicon/testdata/lexicon-invalid.json b/atproto/lexicon/testdata/lexicon-invalid.json new file mode 100644 index 000000000..2c80fedbc --- /dev/null +++ b/atproto/lexicon/testdata/lexicon-invalid.json @@ -0,0 +1,18 @@ +[ +{ + "name": "invalid lexicon field", + "lexicon": { + "lexicon": "one", + "id": "example.lexicon", + "defs": { "demo": { "type": "integer" } } + } +}, +{ + "name": "invalid id field", + "lexicon": { + "lexicon": 1, + "id": 2, + "defs": { "demo": { "type": "integer" } } + } +} +] diff --git a/atproto/lexicon/testdata/lexicon-valid.json b/atproto/lexicon/testdata/lexicon-valid.json new file mode 100644 index 000000000..bfecaeb73 --- /dev/null +++ b/atproto/lexicon/testdata/lexicon-valid.json @@ -0,0 +1,10 @@ +[ +{ + "name": "minimal", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon", + "defs": { "demo": { "type": "integer" } } + } +} +] diff --git a/atproto/lexicon/testdata/record-data-invalid.json b/atproto/lexicon/testdata/record-data-invalid.json new file mode 100644 index 000000000..58c227cf1 --- /dev/null +++ b/atproto/lexicon/testdata/record-data-invalid.json @@ -0,0 +1,223 @@ +[ + { "name": "missing required field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record" } + }, + { "name": "invalid null field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "null": true } }, + { "name": "invalid boolean field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "boolean": "green"} }, + { "name": "invalid integer field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "integer": "green"} }, + { "name": "invalid non-nullable string field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "string": null } }, + { "name": "invalid string field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "string": 2 } }, + { "name": "invalid bytes field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "bytes": "green" } }, + { "name": "invalid bytes: empty object", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "bytes": {}}}, + { "name": "invalid bytes: wrong type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "bytes": { + "bytes": "asdfasdfasdfasdf" + }}}, + { "name": "invalid cid-link field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "cid-link": "green" } }, + { "name": "invalid blob field", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "blob": "green" } }, + { "name": "invalid blob: wrong type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "blob": { + "type": "blob", + "size": 123, + "mimeType": false, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }}}, + { "name": "invalid array", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "array": 123 } + }, + { "name": "invalid array element", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "array": [true, false] } + }, + { "name": "object wrong data type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "object": 123 } + }, + { "name": "object nested wrong data type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "object": {"a": "not-a-number" } } + }, + { "name": "invalid token ref type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "ref": 123 } + }, + { "name": "invalid ref value", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "ref": "example.lexicon.record#wrongToken" } + }, + { "name": "invalid string format handle", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "handle": "123" } } + }, + { "name": "invalid string format did", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "did": "123" } } + }, + { "name": "invalid string format atidentifier", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "atidentifier": "123" } } + }, + { "name": "invalid string format nsid", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "nsid": "123" } } + }, + { "name": "invalid string format aturi", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "aturi": "123" } } + }, + { "name": "invalid string format cid", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "cid": "123" } } + }, + { "name": "invalid string format datetime", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "datetime": "123" } } + }, + { "name": "invalid string format language", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "language": "123" } } + }, + { "name": "invalid string format uri", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "uri": "123" } } + }, + { "name": "invalid string format tid", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "tid": "000" } } + }, + { "name": "invalid string format recordkey", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "formats": { "recordkey": "." } } + }, + { "name": "wrong const value", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "constInteger": 41 } + }, + { "name": "integer not in enum", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "enumInteger": 7 } + }, + { "name": "out of integer range", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "rangeInteger": 9000 } + }, + { "name": "string too short", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "lenString": "." } + }, + { "name": "string too long", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "lenString": "abcdefg-abcdefg-abcdefg" } + }, + { "name": "string too short (graphemes)", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "graphemeString": "👩‍👩‍👦‍👦👩‍👩‍👦‍👦" } + }, + { "name": "string too long (graphemes)", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "graphemeString": "abcdefg-abcdefg-abcdefg" } + }, + { "name": "out of enum string", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "enumString": "unexpected" } + }, + { "name": "bytes too short", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "sizeBytes": { "$bytes": "b25l" }} + }, + { "name": "bytes too long", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "sizeBytes": { "$bytes": "b25lb25lb25lb25lb25lb25lb25lb25lb25lb25lb25l" }} + }, + { "name": "array too short", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "lenArray": [0]} + }, + { "name": "array too long", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "lenArray": [0,0,0,0,0,0,0,0,0,0]} + }, + { "name": "blob too large", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "sizeBlob": { + "$type": "blob", + "size": 12345, + "mimeType": "text/plain", + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }}}, + { "name": "blob wrong type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "acceptBlob": { + "$type": "blob", + "size": 12345, + "mimeType": "text/plain", + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }}}, + { "name": "open union wrong data type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "union": 123 } + }, + { "name": "open union missing $type", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "union": {"a": 1, "b": 2 } } + }, + { "name": "out of closed union", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "closedUnion": { "$type": "example.unknown-lexicon.blah", "a": 1 } } + }, + { "name": "union inner invalid", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "closedUnion": { "$type": "example.lexicon.record#demoObjectTwo", "a": 1 } } + }, + { "name": "union inner invalid", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "integer": 1, "union": { "$type": "example.lexicon.record#demoObject", "a": "not-a-number" } } + }, + { "name": "unknown wrong type (bool)", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "unknown": false } + }, + { "name": "unknown wrong type (bytes)", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "unknown": { "$bytes": "123" } } + }, + { "name": "unknown wrong type (blob)", + "rkey": "demo", + "data": { "$type": "example.lexicon.record", "unknown": { + "$type": "blob", + "mimeType": "text/plain", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }} + } +] diff --git a/atproto/lexicon/testdata/record-data-valid.json b/atproto/lexicon/testdata/record-data-valid.json new file mode 100644 index 000000000..a56489e7d --- /dev/null +++ b/atproto/lexicon/testdata/record-data-valid.json @@ -0,0 +1,108 @@ +[ + { + "name": "minimal", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1 + } + }, + { + "name": "full", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "null": null, + "boolean": true, + "integer": 3, + "string": "blah", + "nullableString": null, + "bytes": { + "$bytes": "123" + }, + "cidlink": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + }, + "blob": { + "$type": "blob", + "mimeType": "text/plain", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "unknown": { + "a": "alphabet", + "b": 3 + }, + "array": [1,2,3], + "object": { + "a": 1, + "b": 2 + }, + "ref": "example.lexicon.record#demoToken", + "union": { + "$type": "example.lexicon.record#demoObject", + "a": 1, + "b": 2 + }, + "formats": { + "did": "did:web:example.com", + "handle": "handle.example.com", + "atidentifier": "handle.example.com", + "aturi": "at://handle.example.com/com.example.nsid/asdf123", + "nsid": "com.example.nsid", + "cid": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq", + "datetime": "2023-10-30T22:25:23Z", + "language": "en", + "tid": "3kznmn7xqxl22", + "recordkey": "simple" + }, + "constInteger": 42, + "defaultInteger": 123, + "enumInteger": 16, + "rangeInteger": 16, + "lenString": "1234567890ABC", + "graphemeString": "🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈", + "enumString": "fish", + "knownString": "blue", + "sizeBytes": { + "$bytes": "asdfasdfasdfasdf" + }, + "lenArray": [1,2,3], + "sizeBlob": { + "$type": "blob", + "mimeType": "text/plain", + "size": 8, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "acceptBlob": { + "$type": "blob", + "mimeType": "image/png", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "closedUnion": { + "$type": "example.lexicon.record#demoObject", + "a": 1 + } + } + }, + { + "name": "unknown as a type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "unknown": { + "$type": "example.lexicon.record#demoObject", + "a": 1, + "b": 2 + } + } + } +] diff --git a/atproto/syntax/atidentifier.go b/atproto/syntax/atidentifier.go index 21a0f8879..ee7e34e53 100644 --- a/atproto/syntax/atidentifier.go +++ b/atproto/syntax/atidentifier.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "strings" ) @@ -10,6 +10,9 @@ type AtIdentifier struct { } func ParseAtIdentifier(raw string) (*AtIdentifier, error) { + if raw == "" { + return nil, errors.New("expected AT account identifier, got empty string") + } if strings.HasPrefix(raw, "did:") { did, err := ParseDID(raw) if err != nil { @@ -34,7 +37,7 @@ func (n AtIdentifier) AsHandle() (Handle, error) { if ok { return handle, nil } - return "", fmt.Errorf("AT Identifier is not a Handle") + return "", errors.New("AT Identifier is not a Handle") } func (n AtIdentifier) IsDID() bool { @@ -47,7 +50,7 @@ func (n AtIdentifier) AsDID() (DID, error) { if ok { return did, nil } - return "", fmt.Errorf("AT Identifier is not a DID") + return "", errors.New("AT Identifier is not a DID") } func (n AtIdentifier) Normalize() AtIdentifier { diff --git a/atproto/syntax/aturi.go b/atproto/syntax/aturi.go index 0084a67c9..0ca801f33 100644 --- a/atproto/syntax/aturi.go +++ b/atproto/syntax/aturi.go @@ -1,6 +1,7 @@ package syntax import ( + "errors" "fmt" "regexp" "strings" @@ -17,11 +18,11 @@ type ATURI string func ParseATURI(raw string) (ATURI, error) { if len(raw) > 8192 { - return "", fmt.Errorf("ATURI is too long (8192 chars max)") + return "", errors.New("ATURI is too long (8192 chars max)") } parts := aturiRegex.FindStringSubmatch(raw) if parts == nil || len(parts) < 2 || parts[0] == "" { - return "", fmt.Errorf("AT-URI syntax didn't validate via regex") + return "", errors.New("AT-URI syntax didn't validate via regex") } // verify authority as either a DID or NSID _, err := ParseAtIdentifier(parts[1]) diff --git a/atproto/syntax/cid.go b/atproto/syntax/cid.go index bd2c7a9b3..c6ad616c6 100644 --- a/atproto/syntax/cid.go +++ b/atproto/syntax/cid.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" "strings" ) @@ -16,18 +16,21 @@ type CID string var cidRegex = regexp.MustCompile(`^[a-zA-Z0-9+=]{8,256}$`) func ParseCID(raw string) (CID, error) { + if raw == "" { + return "", errors.New("expected CID, got empty string") + } if len(raw) > 256 { - return "", fmt.Errorf("CID is too long (256 chars max)") + return "", errors.New("CID is too long (256 chars max)") } if len(raw) < 8 { - return "", fmt.Errorf("CID is too short (8 chars min)") + return "", errors.New("CID is too short (8 chars min)") } if !cidRegex.MatchString(raw) { - return "", fmt.Errorf("CID syntax didn't validate via regex") + return "", errors.New("CID syntax didn't validate via regex") } if strings.HasPrefix(raw, "Qmb") { - return "", fmt.Errorf("CIDv0 not allowed in this version of atproto") + return "", errors.New("CIDv0 not allowed in this version of atproto") } return CID(raw), nil } diff --git a/atproto/syntax/datetime.go b/atproto/syntax/datetime.go index 631918e6a..21b33fcde 100644 --- a/atproto/syntax/datetime.go +++ b/atproto/syntax/datetime.go @@ -1,6 +1,7 @@ package syntax import ( + "errors" "fmt" "regexp" "strings" @@ -24,15 +25,18 @@ type Datetime string var datetimeRegex = regexp.MustCompile(`^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$`) func ParseDatetime(raw string) (Datetime, error) { + if raw == "" { + return "", errors.New("expected datetime, got empty string") + } if len(raw) > 64 { - return "", fmt.Errorf("Datetime too long (max 64 chars)") + return "", errors.New("Datetime too long (max 64 chars)") } if !datetimeRegex.MatchString(raw) { - return "", fmt.Errorf("Datetime syntax didn't validate via regex") + return "", errors.New("Datetime syntax didn't validate via regex") } if strings.HasSuffix(raw, "-00:00") { - return "", fmt.Errorf("Datetime can't use '-00:00' for UTC timezone, must use '+00:00', per ISO-8601") + return "", errors.New("Datetime can't use '-00:00' for UTC timezone, must use '+00:00', per ISO-8601") } // ensure that the datetime actually parses using golang time lib _, err := time.Parse(time.RFC3339Nano, raw) diff --git a/atproto/syntax/did.go b/atproto/syntax/did.go index 567c58ede..ef9c0e04f 100644 --- a/atproto/syntax/did.go +++ b/atproto/syntax/did.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" "strings" ) @@ -16,11 +16,14 @@ type DID string var didRegex = regexp.MustCompile(`^did:[a-z]+:[a-zA-Z0-9._:%-]*[a-zA-Z0-9._-]$`) func ParseDID(raw string) (DID, error) { + if raw == "" { + return "", errors.New("expected DID, got empty string") + } if len(raw) > 2*1024 { - return "", fmt.Errorf("DID is too long (2048 chars max)") + return "", errors.New("DID is too long (2048 chars max)") } if !didRegex.MatchString(raw) { - return "", fmt.Errorf("DID syntax didn't validate via regex") + return "", errors.New("DID syntax didn't validate via regex") } return DID(raw), nil } diff --git a/atproto/syntax/handle.go b/atproto/syntax/handle.go index 63322b7fc..b903698b2 100644 --- a/atproto/syntax/handle.go +++ b/atproto/syntax/handle.go @@ -1,6 +1,7 @@ package syntax import ( + "errors" "fmt" "regexp" "strings" @@ -21,8 +22,11 @@ var ( type Handle string func ParseHandle(raw string) (Handle, error) { + if raw == "" { + return "", errors.New("expected handle, got empty string") + } if len(raw) > 253 { - return "", fmt.Errorf("handle is too long (253 chars max)") + return "", errors.New("handle is too long (253 chars max)") } if !handleRegex.MatchString(raw) { return "", fmt.Errorf("handle syntax didn't validate via regex: %s", raw) diff --git a/atproto/syntax/language.go b/atproto/syntax/language.go index 0d263eec6..59a9d3322 100644 --- a/atproto/syntax/language.go +++ b/atproto/syntax/language.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" ) @@ -15,11 +15,14 @@ type Language string var langRegex = regexp.MustCompile(`^(i|[a-z]{2,3})(-[a-zA-Z0-9]+)*$`) func ParseLanguage(raw string) (Language, error) { + if raw == "" { + return "", errors.New("expected language code, got empty string") + } if len(raw) > 128 { - return "", fmt.Errorf("Language is too long (128 chars max)") + return "", errors.New("Language is too long (128 chars max)") } if !langRegex.MatchString(raw) { - return "", fmt.Errorf("Language syntax didn't validate via regex") + return "", errors.New("Language syntax didn't validate via regex") } return Language(raw), nil } diff --git a/atproto/syntax/nsid.go b/atproto/syntax/nsid.go index a87778eba..ba5c51620 100644 --- a/atproto/syntax/nsid.go +++ b/atproto/syntax/nsid.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" "strings" ) @@ -16,11 +16,14 @@ var nsidRegex = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\. type NSID string func ParseNSID(raw string) (NSID, error) { + if raw == "" { + return "", errors.New("expected NSID, got empty string") + } if len(raw) > 317 { - return "", fmt.Errorf("NSID is too long (317 chars max)") + return "", errors.New("NSID is too long (317 chars max)") } if !nsidRegex.MatchString(raw) { - return "", fmt.Errorf("NSID syntax didn't validate via regex") + return "", errors.New("NSID syntax didn't validate via regex") } return NSID(raw), nil } diff --git a/atproto/syntax/recordkey.go b/atproto/syntax/recordkey.go index 24606803b..c8af66893 100644 --- a/atproto/syntax/recordkey.go +++ b/atproto/syntax/recordkey.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" ) @@ -15,14 +15,17 @@ var recordKeyRegex = regexp.MustCompile(`^[a-zA-Z0-9_~.:-]{1,512}$`) type RecordKey string func ParseRecordKey(raw string) (RecordKey, error) { + if raw == "" { + return "", errors.New("expected record key, got empty string") + } if len(raw) > 512 { - return "", fmt.Errorf("recordkey is too long (512 chars max)") + return "", errors.New("recordkey is too long (512 chars max)") } if raw == "" || raw == "." || raw == ".." { - return "", fmt.Errorf("recordkey can not be empty, '.', or '..'") + return "", errors.New("recordkey can not be empty, '.', or '..'") } if !recordKeyRegex.MatchString(raw) { - return "", fmt.Errorf("recordkey syntax didn't validate via regex") + return "", errors.New("recordkey syntax didn't validate via regex") } return RecordKey(raw), nil } diff --git a/atproto/syntax/tid.go b/atproto/syntax/tid.go index c67b4b216..ebcb853ac 100644 --- a/atproto/syntax/tid.go +++ b/atproto/syntax/tid.go @@ -2,7 +2,7 @@ package syntax import ( "encoding/base32" - "fmt" + "errors" "regexp" "strings" "sync" @@ -27,11 +27,14 @@ type TID string var tidRegex = regexp.MustCompile(`^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$`) func ParseTID(raw string) (TID, error) { + if raw == "" { + return "", errors.New("expected TID, got empty string") + } if len(raw) != 13 { - return "", fmt.Errorf("TID is wrong length (expected 13 chars)") + return "", errors.New("TID is wrong length (expected 13 chars)") } if !tidRegex.MatchString(raw) { - return "", fmt.Errorf("TID syntax didn't validate via regex") + return "", errors.New("TID syntax didn't validate via regex") } return TID(raw), nil } diff --git a/atproto/syntax/uri.go b/atproto/syntax/uri.go index d2cc6ce7e..fbf8807c4 100644 --- a/atproto/syntax/uri.go +++ b/atproto/syntax/uri.go @@ -1,7 +1,7 @@ package syntax import ( - "fmt" + "errors" "regexp" ) @@ -13,12 +13,15 @@ import ( type URI string func ParseURI(raw string) (URI, error) { + if raw == "" { + return "", errors.New("expected URI, got empty string") + } if len(raw) > 8192 { - return "", fmt.Errorf("URI is too long (8192 chars max)") + return "", errors.New("URI is too long (8192 chars max)") } var uriRegex = regexp.MustCompile(`^[a-z][a-z.-]{0,80}:[[:graph:]]+$`) if !uriRegex.MatchString(raw) { - return "", fmt.Errorf("URI syntax didn't validate via regex") + return "", errors.New("URI syntax didn't validate via regex") } return URI(raw), nil } diff --git a/automod/consumer/doc.go b/automod/consumer/doc.go new file mode 100644 index 000000000..fa8ccdcb2 --- /dev/null +++ b/automod/consumer/doc.go @@ -0,0 +1,2 @@ +// Code for consuming from atproto firehose and ozone event stream, pushing events in to automod engine. +package consumer diff --git a/cmd/hepa/consumer.go b/automod/consumer/firehose.go similarity index 50% rename from cmd/hepa/consumer.go rename to automod/consumer/firehose.go index d91cade38..f210b3055 100644 --- a/cmd/hepa/consumer.go +++ b/automod/consumer/firehose.go @@ -1,13 +1,14 @@ -package main +package consumer import ( "bytes" "context" "fmt" + "log/slog" "net/http" "net/url" - "strings" "sync/atomic" + "time" comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/syntax" @@ -21,25 +22,50 @@ import ( "github.com/bluesky-social/indigo/repomgr" "github.com/carlmjohnson/versioninfo" "github.com/gorilla/websocket" + "github.com/redis/go-redis/v9" ) -func (s *Server) RunConsumer(ctx context.Context) error { +// TODO: should probably make this not hepa-specific; or even configurable +var firehoseCursorKey = "hepa/seq" - cur, err := s.ReadLastCursor(ctx) +type FirehoseConsumer struct { + Parallelism int + Logger *slog.Logger + RedisClient *redis.Client + Engine *automod.Engine + Host string + + // TODO: prefilter record collections; or predicate function? + // TODO: enable/disable event types; or predicate function? + + // lastSeq is the most recent event sequence number we've received and begun to handle. + // This number is periodically persisted to redis, if redis is present. + // The value is best-effort (the stream handling itself is concurrent, so event numbers may not be monotonic), + // but nonetheless, you must use atomics when updating or reading this (to avoid data races). + lastSeq int64 +} + +func (fc *FirehoseConsumer) Run(ctx context.Context) error { + + if fc.Engine == nil { + return fmt.Errorf("nil engine") + } + + cur, err := fc.ReadLastCursor(ctx) if err != nil { return err } dialer := websocket.DefaultDialer - u, err := url.Parse(s.relayHost) + u, err := url.Parse(fc.Host) if err != nil { - return fmt.Errorf("invalid relayHost URI: %w", err) + return fmt.Errorf("invalid Host URI: %w", err) } u.Path = "xrpc/com.atproto.sync.subscribeRepos" if cur != 0 { u.RawQuery = fmt.Sprintf("cursor=%d", cur) } - s.logger.Info("subscribing to repo event stream", "upstream", s.relayHost, "cursor", cur) + fc.Logger.Info("subscribing to repo event stream", "upstream", fc.Host, "cursor", cur) con, _, err := dialer.Dial(u.String(), http.Header{ "User-Agent": []string{fmt.Sprintf("hepa/%s", versioninfo.Short())}, }) @@ -49,20 +75,20 @@ func (s *Server) RunConsumer(ctx context.Context) error { rsc := &events.RepoStreamCallbacks{ RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { - atomic.StoreInt64(&s.lastSeq, evt.Seq) - return s.HandleRepoCommit(ctx, evt) + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + return fc.HandleRepoCommit(ctx, evt) }, RepoIdentity: func(evt *comatproto.SyncSubscribeRepos_Identity) error { - atomic.StoreInt64(&s.lastSeq, evt.Seq) - if err := s.engine.ProcessIdentityEvent(ctx, *evt); err != nil { - s.logger.Error("processing repo identity failed", "did", evt.Did, "seq", evt.Seq, "err", err) + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + if err := fc.Engine.ProcessIdentityEvent(ctx, *evt); err != nil { + fc.Logger.Error("processing repo identity failed", "did", evt.Did, "seq", evt.Seq, "err", err) } return nil }, RepoAccount: func(evt *comatproto.SyncSubscribeRepos_Account) error { - atomic.StoreInt64(&s.lastSeq, evt.Seq) - if err := s.engine.ProcessAccountEvent(ctx, *evt); err != nil { - s.logger.Error("processing repo account failed", "did", evt.Did, "seq", evt.Seq, "err", err) + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + if err := fc.Engine.ProcessAccountEvent(ctx, *evt); err != nil { + fc.Logger.Error("processing repo account failed", "did", evt.Did, "seq", evt.Seq, "err", err) } return nil }, @@ -71,49 +97,32 @@ func (s *Server) RunConsumer(ctx context.Context) error { } var scheduler events.Scheduler - if s.firehoseParallelism > 0 { + if fc.Parallelism > 0 { // use a fixed-parallelism scheduler if configured scheduler = parallel.NewScheduler( - s.firehoseParallelism, + fc.Parallelism, 1000, - s.relayHost, + fc.Host, rsc.EventHandler, ) - s.logger.Info("hepa scheduler configured", "scheduler", "parallel", "initial", s.firehoseParallelism) + fc.Logger.Info("hepa scheduler configured", "scheduler", "parallel", "initial", fc.Parallelism) } else { // otherwise use auto-scaling scheduler scaleSettings := autoscaling.DefaultAutoscaleSettings() // start at higher parallelism (somewhat arbitrary) scaleSettings.Concurrency = 4 scaleSettings.MaxConcurrency = 200 - scheduler = autoscaling.NewScheduler(scaleSettings, s.relayHost, rsc.EventHandler) - s.logger.Info("hepa scheduler configured", "scheduler", "autoscaling", "initial", scaleSettings.Concurrency, "max", scaleSettings.MaxConcurrency) + scheduler = autoscaling.NewScheduler(scaleSettings, fc.Host, rsc.EventHandler) + fc.Logger.Info("hepa scheduler configured", "scheduler", "autoscaling", "initial", scaleSettings.Concurrency, "max", scaleSettings.MaxConcurrency) } return events.HandleRepoStream(ctx, con, scheduler) } -// TODO: move this to a "ParsePath" helper in syntax package? -func splitRepoPath(path string) (syntax.NSID, syntax.RecordKey, error) { - parts := strings.SplitN(path, "/", 3) - if len(parts) != 2 { - return "", "", fmt.Errorf("invalid record path: %s", path) - } - collection, err := syntax.ParseNSID(parts[0]) - if err != nil { - return "", "", err - } - rkey, err := syntax.ParseRecordKey(parts[1]) - if err != nil { - return "", "", err - } - return collection, rkey, nil -} - // NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better. -func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { +func (fc *FirehoseConsumer) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { - logger := s.logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq) + logger := fc.Logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq) logger.Debug("received commit event") if evt.TooBig { @@ -173,7 +182,7 @@ func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubsc CID: &recCID, RecordCBOR: *recCBOR, } - err = s.engine.ProcessRecordOp(ctx, op) + err = fc.Engine.ProcessRecordOp(ctx, op) if err != nil { logger.Error("engine failed to process record", "err", err) continue @@ -187,7 +196,7 @@ func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubsc CID: nil, RecordCBOR: nil, } - err = s.engine.ProcessRecordOp(ctx, op) + err = fc.Engine.ProcessRecordOp(ctx, op) if err != nil { logger.Error("engine failed to process record", "err", err) continue @@ -199,3 +208,66 @@ func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubsc return nil } + +func (fc *FirehoseConsumer) ReadLastCursor(ctx context.Context) (int64, error) { + // if redis isn't configured, just skip + if fc.RedisClient == nil { + fc.Logger.Info("redis not configured, skipping cursor read") + return 0, nil + } + + val, err := fc.RedisClient.Get(ctx, firehoseCursorKey).Int64() + if err == redis.Nil { + fc.Logger.Info("no pre-existing cursor in redis") + return 0, nil + } else if err != nil { + return 0, err + } + fc.Logger.Info("successfully found prior subscription cursor seq in redis", "seq", val) + return val, nil +} + +func (fc *FirehoseConsumer) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if fc.RedisClient == nil { + return nil + } + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq <= 0 { + return nil + } + err := fc.RedisClient.Set(ctx, firehoseCursorKey, lastSeq, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (fc *FirehoseConsumer) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if fc.RedisClient == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq >= 1 { + fc.Logger.Info("persisting final cursor seq value", "seq", lastSeq) + err := fc.PersistCursor(ctx) + if err != nil { + fc.Logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + return nil + case <-ticker.C: + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq >= 1 { + err := fc.PersistCursor(ctx) + if err != nil { + fc.Logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + } + } +} diff --git a/automod/consumer/ozone.go b/automod/consumer/ozone.go new file mode 100644 index 000000000..2211cf21a --- /dev/null +++ b/automod/consumer/ozone.go @@ -0,0 +1,187 @@ +package consumer + +import ( + "context" + "fmt" + "log/slog" + "sync/atomic" + "time" + + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/redis/go-redis/v9" +) + +// TODO: should probably make this not hepa-specific; or even configurable +var ozoneCursorKey = "hepa/ozoneTimestamp" + +type OzoneConsumer struct { + Logger *slog.Logger + RedisClient *redis.Client + OzoneClient *xrpc.Client + Engine *automod.Engine + + // same as lastSeq, but for Ozone timestamp cursor. the value is a string. + lastCursor atomic.Value +} + +func (oc *OzoneConsumer) Run(ctx context.Context) error { + + if oc.Engine == nil { + return fmt.Errorf("nil engine") + } + if oc.OzoneClient == nil { + return fmt.Errorf("nil ozoneclient") + } + + cur, err := oc.ReadLastCursor(ctx) + if err != nil { + return err + } + + if cur == "" { + cur = syntax.DatetimeNow().String() + } + since, err := syntax.ParseDatetime(cur) + if err != nil { + return err + } + + oc.Logger.Info("subscribing to ozone event log", "upstream", oc.OzoneClient.Host, "cursor", cur, "since", since) + var limit int64 = 50 + period := time.Second * 5 + + for { + me, err := toolsozone.ModerationQueryEvents( + ctx, + oc.OzoneClient, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + since.String(), // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + true, // includeAllUserRecords bool + limit, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "asc", // sortDirection string + "", // subject string + "", // subjectType string + nil, // types []string + ) + if err != nil { + oc.Logger.Warn("ozone query events failed; sleeping then will retrying", "err", err, "period", period.String()) + time.Sleep(period) + continue + } + + // track if the response contained anything new + anyNewEvents := false + for _, evt := range me.Events { + createdAt, err := syntax.ParseDatetime(evt.CreatedAt) + if err != nil { + return fmt.Errorf("invalid time format for ozone 'createdAt': %w", err) + } + // skip if the timestamp is the exact same + if createdAt == since { + continue + } + anyNewEvents = true + // TODO: is there a race condition here? + if !createdAt.Time().After(since.Time()) { + oc.Logger.Error("out of order ozone event", "createdAt", createdAt, "since", since) + return fmt.Errorf("out of order ozone event") + } + if err = oc.HandleOzoneEvent(ctx, evt); err != nil { + oc.Logger.Error("failed to process ozone event", "event", evt) + } + since = createdAt + oc.lastCursor.Store(since.String()) + } + if !anyNewEvents { + oc.Logger.Debug("... ozone poller sleeping", "period", period.String()) + time.Sleep(period) + } + } +} + +func (oc *OzoneConsumer) HandleOzoneEvent(ctx context.Context, eventView *toolsozone.ModerationDefs_ModEventView) error { + + oc.Logger.Debug("received ozone event", "eventID", eventView.Id, "createdAt", eventView.CreatedAt) + + if err := oc.Engine.ProcessOzoneEvent(ctx, eventView); err != nil { + oc.Logger.Error("engine failed to process ozone event", "err", err) + } + return nil +} + +func (oc *OzoneConsumer) ReadLastCursor(ctx context.Context) (string, error) { + // if redis isn't configured, just skip + if oc.RedisClient == nil { + oc.Logger.Info("redis not configured, skipping ozone cursor read") + return "", nil + } + + val, err := oc.RedisClient.Get(ctx, ozoneCursorKey).Result() + if err == redis.Nil || val == "" { + oc.Logger.Info("no pre-existing ozone cursor in redis") + return "", nil + } else if err != nil { + return "", err + } + oc.Logger.Info("successfully found prior ozone offset timestamp in redis", "cursor", val) + return val, nil +} + +func (oc *OzoneConsumer) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if oc.RedisClient == nil { + return nil + } + lastCursor := oc.lastCursor.Load() + if lastCursor == nil || lastCursor == "" { + return nil + } + err := oc.RedisClient.Set(ctx, ozoneCursorKey, lastCursor, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (oc *OzoneConsumer) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if oc.RedisClient == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + lastCursor := oc.lastCursor.Load() + if lastCursor != nil && lastCursor != "" { + oc.Logger.Info("persisting final ozone cursor timestamp", "cursor", lastCursor) + err := oc.PersistCursor(ctx) + if err != nil { + oc.Logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) + } + } + return nil + case <-ticker.C: + lastCursor := oc.lastCursor.Load() + if lastCursor != nil && lastCursor != "" { + err := oc.PersistCursor(ctx) + if err != nil { + oc.Logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) + } + } + } + } +} diff --git a/automod/consumer/util.go b/automod/consumer/util.go new file mode 100644 index 000000000..b1c34ebaf --- /dev/null +++ b/automod/consumer/util.go @@ -0,0 +1,25 @@ +package consumer + +import ( + "fmt" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// TODO: move this to a "ParsePath" helper in syntax package? +func splitRepoPath(path string) (syntax.NSID, syntax.RecordKey, error) { + parts := strings.SplitN(path, "/", 3) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid record path: %s", path) + } + collection, err := syntax.ParseNSID(parts[0]) + if err != nil { + return "", "", err + } + rkey, err := syntax.ParseRecordKey(parts[1]) + if err != nil { + return "", "", err + } + return collection, rkey, nil +} diff --git a/automod/engine/account_meta.go b/automod/engine/account_meta.go index 5a5a178a2..9e3ca15c1 100644 --- a/automod/engine/account_meta.go +++ b/automod/engine/account_meta.go @@ -6,6 +6,13 @@ import ( "github.com/bluesky-social/indigo/atproto/identity" ) +var ( + ReviewStateEscalated = "escalated" + ReviewStateOpen = "open" + ReviewStateClosed = "closed" + ReviewStateNone = "none" +) + // information about a repo/account/identity, always pre-populated and relevant to many rules type AccountMeta struct { Identity *identity.Identity @@ -25,13 +32,25 @@ type AccountMeta struct { type ProfileSummary struct { HasAvatar bool + AvatarCid *string + BannerCid *string Description *string DisplayName *string } +// opaque fingerprints for correlating abusive accounts +type AbuseSignature struct { + Property string + Value string +} + type AccountPrivate struct { Email string EmailConfirmed bool IndexedAt *time.Time AccountTags []string + // ReviewState will be one of ReviewStateEscalated, ReviewStateOpen, ReviewStateClosed, ReviewStateNone, or "" (unknown) + ReviewState string + Appealed bool + AbuseSignatures []AbuseSignature } diff --git a/automod/engine/cid_from_cdn_test.go b/automod/engine/cid_from_cdn_test.go new file mode 100644 index 000000000..0780403ca --- /dev/null +++ b/automod/engine/cid_from_cdn_test.go @@ -0,0 +1,42 @@ +package engine + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCidFromCdnUrl(t *testing.T) { + assert := assert.New(t) + + fixCid := "abcdefghijk" + + fixtures := []struct { + url string + cid *string + }{ + { + url: "https://cdn.bsky.app/img/avatar/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: &fixCid, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: &fixCid, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize", + cid: nil, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk", + cid: &fixCid, + }, + { + url: "https://cdn.asky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: nil, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.cid, cidFromCdnUrl(&fix.url)) + } +} diff --git a/automod/engine/context.go b/automod/engine/context.go index 16da748df..56e62e536 100644 --- a/automod/engine/context.go +++ b/automod/engine/context.go @@ -169,6 +169,13 @@ func (c *BaseContext) InSet(name, val string) bool { return out } +// Returns a pointer to the underlying automod engine. This usually should NOT be used in rules. +// +// This is an escape hatch for hacking on the system before features get fully integerated in to the content API surface. The Engine API is not stable. +func (c *BaseContext) InternalEngine() *Engine { + return c.engine +} + func NewAccountContext(ctx context.Context, eng *Engine, meta AccountMeta) AccountContext { return AccountContext{ BaseContext: BaseContext{ @@ -263,6 +270,10 @@ func (c *AccountContext) AddAccountLabel(val string) { c.effects.AddAccountLabel(val) } +func (c *AccountContext) AddAccountTag(val string) { + c.effects.AddAccountTag(val) +} + func (c *AccountContext) ReportAccount(reason, comment string) { c.effects.ReportAccount(reason, comment) } @@ -271,6 +282,14 @@ func (c *AccountContext) TakedownAccount() { c.effects.TakedownAccount() } +func (c *AccountContext) EscalateAccount() { + c.effects.EscalateAccount() +} + +func (c *AccountContext) AcknowledgeAccount() { + c.effects.AcknowledgeAccount() +} + func (c *RecordContext) AddRecordFlag(val string) { c.effects.AddRecordFlag(val) } @@ -279,6 +298,10 @@ func (c *RecordContext) AddRecordLabel(val string) { c.effects.AddRecordLabel(val) } +func (c *RecordContext) AddRecordTag(val string) { + c.effects.AddRecordTag(val) +} + func (c *RecordContext) ReportRecord(reason, comment string) { c.effects.ReportRecord(reason, comment) } diff --git a/automod/engine/effects.go b/automod/engine/effects.go index 806909229..279e28f9d 100644 --- a/automod/engine/effects.go +++ b/automod/engine/effects.go @@ -12,6 +12,8 @@ var ( QuotaModReportDay = 2000 // number of takedowns automod can action per day, for all subjects combined (circuit breaker) QuotaModTakedownDay = 200 + // number of misc actions automod can do per day, for all subjects combined (circuit breaker) + QuotaModActionDay = 1000 ) type CounterRef struct { @@ -38,14 +40,22 @@ type Effects struct { CounterDistinctIncrements []CounterDistinctRef // TODO: better variable names // Label values which should be applied to the overall account, as a result of rule execution. AccountLabels []string - // Moderation flags (similar to labels, but private) which should be applied to the overall account, as a result of rule execution. + // Moderation tags (similar to labels, but private) which should be applied to the overall account, as a result of rule execution. + AccountTags []string + // automod flags (metadata) which should be applied to the account as a result of rule execution. AccountFlags []string // Reports which should be filed against this account, as a result of rule execution. AccountReports []ModReport - // If "true", indicates that a rule indicates that the entire account should have a takedown. + // If "true", a rule decided that the entire account should have a takedown. AccountTakedown bool + // If "true", a rule decided that the reported account should be escalated. + AccountEscalate bool + // If "true", a rule decided that the reports on account should be resolved as acknowledged. + AccountAcknowledge bool // Same as "AccountLabels", but at record-level RecordLabels []string + // Same as "AccountTags", but at record-level + RecordTags []string // Same as "AccountFlags", but at record-level RecordFlags []string // Same as "AccountReports", but at record-level @@ -98,6 +108,18 @@ func (e *Effects) AddAccountLabel(val string) { e.AccountLabels = append(e.AccountLabels, val) } +// Enqueues the provided label (string value) to be added to the account at the end of rule processing. +func (e *Effects) AddAccountTag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.AccountTags { + if v == val { + return + } + } + e.AccountTags = append(e.AccountTags, val) +} + // Enqueues the provided flag (string value) to be recorded (in the Engine's flagstore) at the end of rule processing. func (e *Effects) AddAccountFlag(val string) { e.mu.Lock() @@ -130,6 +152,16 @@ func (e *Effects) TakedownAccount() { e.AccountTakedown = true } +// Enqueues the account to be "escalated" for mod review at the end of rule processing. +func (e *Effects) EscalateAccount() { + e.AccountEscalate = true +} + +// Enqueues reports on account to be "acknowledged" (closed) at the end of rule processing. +func (e *Effects) AcknowledgeAccount() { + e.AccountAcknowledge = true +} + // Enqueues the provided label (string value) to be added to the record at the end of rule processing. func (e *Effects) AddRecordLabel(val string) { e.mu.Lock() @@ -142,6 +174,18 @@ func (e *Effects) AddRecordLabel(val string) { e.RecordLabels = append(e.RecordLabels, val) } +// Enqueues the provided tag (string value) to be added to the record at the end of rule processing. +func (e *Effects) AddRecordTag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RecordTags { + if v == val { + return + } + } + e.RecordTags = append(e.RecordTags, val) +} + // Enqueues the provided flag (string value) to be recorded (in the Engine's flagstore) at the end of rule processing. func (e *Effects) AddRecordFlag(val string) { e.mu.Lock() diff --git a/automod/engine/engine.go b/automod/engine/engine.go index fdbf7a949..76eb88c99 100644 --- a/automod/engine/engine.go +++ b/automod/engine/engine.go @@ -23,17 +23,11 @@ const ( notificationEventTimeout = 5 * time.Second ) -type EngineConfig struct { - // if true, sent firehose identity and account events to ozone backend as events - PersistSubjectHistoryOzone bool -} - // runtime for executing rules, managing state, and recording moderation actions. // // NOTE: careful when initializing: several fields must not be nil or zero, even though they are pointer type. type Engine struct { Logger *slog.Logger - Config EngineConfig Directory identity.Directory Rules RuleSet Counters countstore.CountStore @@ -50,6 +44,16 @@ type Engine struct { AdminClient *xrpc.Client // used to fetch blobs from upstream PDS instances BlobClient *http.Client + + // internal configuration + Config EngineConfig +} + +type EngineConfig struct { + // if enabled, account metadata is not hydrated for every event by default + SkipAccountMeta bool + // if true, sent firehose identity and account events to ozone backend as events + PersistSubjectHistoryOzone bool } // Entrypoint for external code pushing #identity events in to the engine. @@ -93,10 +97,18 @@ func (eng *Engine) ProcessIdentityEvent(ctx context.Context, evt comatproto.Sync return fmt.Errorf("identity not found for DID: %s", did.String()) } - am, err := eng.GetAccountMeta(ctx, ident) - if err != nil { - eventErrorCount.WithLabelValues("identity").Inc() - return fmt.Errorf("failed to fetch account metadata: %w", err) + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } } ac := NewAccountContext(ctx, eng, *am) if err := eng.Rules.CallIdentityRules(&ac); err != nil { @@ -161,10 +173,18 @@ func (eng *Engine) ProcessAccountEvent(ctx context.Context, evt comatproto.SyncS return fmt.Errorf("identity not found for DID: %s", did.String()) } - am, err := eng.GetAccountMeta(ctx, ident) - if err != nil { - eventErrorCount.WithLabelValues("account").Inc() - return fmt.Errorf("failed to fetch account metadata: %w", err) + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } } ac := NewAccountContext(ctx, eng, *am) if err := eng.Rules.CallAccountRules(&ac); err != nil { @@ -222,10 +242,18 @@ func (eng *Engine) ProcessRecordOp(ctx context.Context, op RecordOp) error { return fmt.Errorf("identity not found for DID: %s", op.DID) } - am, err := eng.GetAccountMeta(ctx, ident) - if err != nil { - eventErrorCount.WithLabelValues("record").Inc() - return fmt.Errorf("failed to fetch account metadata: %w", err) + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } } rc := NewRecordContext(ctx, eng, *am, op) rc.Logger.Debug("processing record") @@ -340,6 +368,7 @@ func (e *Engine) CanonicalLogLineAccount(c *AccountContext) { c.Logger.Info("canonical-event-line", "accountLabels", c.effects.AccountLabels, "accountFlags", c.effects.AccountFlags, + "accountTags", c.effects.AccountTags, "accountTakedown", c.effects.AccountTakedown, "accountReports", len(c.effects.AccountReports), ) @@ -349,10 +378,12 @@ func (e *Engine) CanonicalLogLineRecord(c *RecordContext) { c.Logger.Info("canonical-event-line", "accountLabels", c.effects.AccountLabels, "accountFlags", c.effects.AccountFlags, + "accountTags", c.effects.AccountTags, "accountTakedown", c.effects.AccountTakedown, "accountReports", len(c.effects.AccountReports), "recordLabels", c.effects.RecordLabels, "recordFlags", c.effects.RecordFlags, + "recordTags", c.effects.RecordTags, "recordTakedown", c.effects.RecordTakedown, "recordReports", len(c.effects.RecordReports), ) @@ -362,6 +393,7 @@ func (e *Engine) CanonicalLogLineNotification(c *NotificationContext) { c.Logger.Info("canonical-event-line", "accountLabels", c.effects.AccountLabels, "accountFlags", c.effects.AccountFlags, + "accountTags", c.effects.AccountTags, "accountTakedown", c.effects.AccountTakedown, "accountReports", len(c.effects.AccountReports), "reject", c.effects.RejectEvent, diff --git a/automod/engine/fetch_account_meta.go b/automod/engine/fetch_account_meta.go index 16dae3f2d..5c55b417f 100644 --- a/automod/engine/fetch_account_meta.go +++ b/automod/engine/fetch_account_meta.go @@ -24,7 +24,7 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) ( // fallback in case client wasn't configured (eg, testing) if e.BskyClient == nil { - logger.Warn("skipping account meta hydration") + logger.Debug("skipping account meta hydration") am := AccountMeta{ Identity: ident, Profile: ProfileSummary{}, @@ -64,7 +64,7 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) ( // most common cause of this is a race between automod and ozone/appview for new accounts. just sleep a couple seconds and retry! var xrpcError *xrpc.Error if err != nil && errors.As(err, &xrpcError) && (xrpcError.StatusCode == 400 || xrpcError.StatusCode == 404) { - logger.Info("account profile lookup initially failed (from bsky appview), will retry", "err", err, "sleepDuration", newAccountRetryDuration) + logger.Debug("account profile lookup initially failed (from bsky appview), will retry", "err", err, "sleepDuration", newAccountRetryDuration) time.Sleep(newAccountRetryDuration) pv, err = appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String()) } @@ -75,6 +75,8 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) ( am.Profile = ProfileSummary{ HasAvatar: pv.Avatar != nil, + AvatarCid: cidFromCdnUrl(pv.Avatar), + BannerCid: cidFromCdnUrl(pv.Banner), Description: pv.Description, DisplayName: pv.DisplayName, } @@ -131,7 +133,29 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) ( if rd.Moderation.SubjectStatus.Takendown != nil && *rd.Moderation.SubjectStatus.Takendown == true { am.Takendown = true } + if rd.Moderation.SubjectStatus.Appealed != nil && *rd.Moderation.SubjectStatus.Appealed == true { + ap.Appealed = true + } ap.AccountTags = dedupeStrings(rd.Moderation.SubjectStatus.Tags) + if rd.Moderation.SubjectStatus.ReviewState != nil { + switch *rd.Moderation.SubjectStatus.ReviewState { + case "#reviewOpen": + ap.ReviewState = ReviewStateOpen + case "#reviewEscalated": + ap.ReviewState = ReviewStateEscalated + case "#reviewClosed": + ap.ReviewState = ReviewStateClosed + case "#reviewNonde": + ap.ReviewState = ReviewStateNone + } + } + } + if rd.ThreatSignatures != nil || len(rd.ThreatSignatures) > 0 { + asigs := make([]AbuseSignature, len(rd.ThreatSignatures)) + for i, sig := range rd.ThreatSignatures { + asigs[i] = AbuseSignature{Property: sig.Property, Value: sig.Value} + } + ap.AbuseSignatures = asigs } am.Private = &ap } diff --git a/automod/engine/metrics.go b/automod/engine/metrics.go index bf71197d4..bc32b8e54 100644 --- a/automod/engine/metrics.go +++ b/automod/engine/metrics.go @@ -25,6 +25,11 @@ var actionNewLabelCount = promauto.NewCounterVec(prometheus.CounterOpts{ Help: "Number of new labels persisted", }, []string{"type", "val"}) +var actionNewTagCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_tags", + Help: "Number of new tags persisted", +}, []string{"type", "val"}) + var actionNewFlagCount = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "automod_new_action_flags", Help: "Number of new flags persisted", @@ -37,7 +42,17 @@ var actionNewReportCount = promauto.NewCounterVec(prometheus.CounterOpts{ var actionNewTakedownCount = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "automod_new_action_takedowns", - Help: "Number of new flags persisted", + Help: "Number of new takedowns", +}, []string{"type"}) + +var actionNewEscalationCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_escalations", + Help: "Number of new subject escalations", +}, []string{"type"}) + +var actionNewAcknowledgeCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_acknowledges", + Help: "Number of new subjects acknowledged", }, []string{"type"}) var accountMetaFetches = promauto.NewCounter(prometheus.CounterOpts{ diff --git a/automod/engine/persist.go b/automod/engine/persist.go index 7d6675bbf..3c4b36fd0 100644 --- a/automod/engine/persist.go +++ b/automod/engine/persist.go @@ -32,7 +32,7 @@ func (eng *Engine) persistCounters(ctx context.Context, eff *Effects) error { return nil } -// Persists account-level moderation actions: new labels, new flags, new takedowns, and reports. +// Persists account-level moderation actions: new labels, new tags, new flags, new takedowns, and reports. // // If necessary, will "purge" identity and account caches, so that state updates will be picked up for subsequent events. // @@ -42,6 +42,11 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { // de-dupe actions newLabels := dedupeLabelActions(c.effects.AccountLabels, c.Account.AccountLabels, c.Account.AccountNegatedLabels) + existingTags := []string{} + if c.Account.Private != nil { + existingTags = c.Account.Private.AccountTags + } + newTags := dedupeTagActions(c.effects.AccountTags, existingTags) newFlags := dedupeFlagActions(c.effects.AccountFlags, c.Account.AccountFlags) // don't report the same account multiple times on the same day for the same reason. this is a quick check; we also query the mod service API just before creating the report. @@ -57,8 +62,28 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { if err != nil { return fmt.Errorf("circuit-breaking takedowns: %w", err) } + newEscalation := c.effects.AccountEscalate + if c.Account.Private != nil && c.Account.Private.ReviewState == ReviewStateEscalated { + // de-dupe account escalation + newEscalation = false + } else { + newEscalation, err = eng.circuitBreakModAction(ctx, newEscalation) + if err != nil { + return fmt.Errorf("circuit-breaking escalation: %w", err) + } + } + newAcknowledge := c.effects.AccountAcknowledge + if c.Account.Private != nil && (c.Account.Private.ReviewState == "closed" || c.Account.Private.ReviewState == "none") { + // de-dupe account escalation + newAcknowledge = false + } else { + newAcknowledge, err = eng.circuitBreakModAction(ctx, newAcknowledge) + if err != nil { + return fmt.Errorf("circuit-breaking acknowledge: %w", err) + } + } - anyModActions := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 + anyModActions := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || len(newReports) > 0 if anyModActions && eng.Notifier != nil { for _, srv := range dedupeStrings(c.effects.NotifyServices) { if err := eng.Notifier.SendAccount(ctx, srv, c); err != nil { @@ -87,7 +112,7 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { xrpcc := eng.OzoneClient if len(newLabels) > 0 { - c.Logger.Info("labeling record", "newLabels", newLabels) + c.Logger.Info("labeling account", "newLabels", newLabels) for _, val := range newLabels { // note: WithLabelValues is a prometheus label, not an atproto label actionNewLabelCount.WithLabelValues("account", val).Inc() @@ -113,6 +138,33 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { } } + if len(newTags) > 0 { + c.Logger.Info("tagging account", "newTags", newTags) + for _, val := range newTags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewTagCount.WithLabelValues("account", val).Inc() + } + comment := "[automod]: auto-tagging account" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTag: &toolsozone.ModerationDefs_ModEventTag{ + Add: newTags, + Remove: []string{}, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to create account tags", "err", err) + } + } + // reports are additionally de-duped when persisting the action, so track with a flag createdReports := false for _, mr := range newReports { @@ -145,9 +197,56 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { if err != nil { c.Logger.Error("failed to execute account takedown", "err", err) } + + // we don't want to escalate if there is a takedown + newEscalation = false + } + + if newEscalation { + c.Logger.Warn("account-escalate") + actionNewEscalationCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-escalation" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account escalation", "err", err) + } + } + + if newAcknowledge { + c.Logger.Warn("account-acknowledge") + actionNewAcknowledgeCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-acknowledge" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account acknowledge", "err", err) + } } - needCachePurge := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || createdReports + needCachePurge := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || createdReports if needCachePurge { return eng.PurgeAccountCaches(ctx, c.Account.Identity.DID) } @@ -155,7 +254,7 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { return nil } -// Persists some record-level state: labels, takedowns, reports. +// Persists some record-level state: labels, tags, takedowns, reports. // // NOTE: this method currently does *not* persist record-level flags to any storage, and does not de-dupe most actions, on the assumption that the record is new (from firehose) and has no existing mod state. func (eng *Engine) persistRecordModActions(c *RecordContext) error { @@ -166,7 +265,9 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { atURI := c.RecordOp.ATURI().String() newLabels := dedupeStrings(c.effects.RecordLabels) - if len(newLabels) > 0 && eng.OzoneClient != nil { + newTags := dedupeStrings(c.effects.RecordTags) + if (len(newLabels) > 0 || len(newTags) > 0) && eng.OzoneClient != nil { + // fetch existing record labels, tags, etc rv, err := toolsozone.ModerationGetRecord(ctx, eng.OzoneClient, c.RecordOp.CID.String(), c.RecordOp.ATURI().String()) if err != nil { // NOTE: there is a frequent 4xx error here from Ozone because this record has not been indexed yet @@ -183,10 +284,15 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { } existingLabels = dedupeStrings(existingLabels) negLabels = dedupeStrings(negLabels) - // fetch existing record labels newLabels = dedupeLabelActions(newLabels, existingLabels, negLabels) + existingTags := []string{} + if rv.Moderation != nil && rv.Moderation.SubjectStatus != nil && rv.Moderation.SubjectStatus.Tags != nil { + existingTags = rv.Moderation.SubjectStatus.Tags + } + newTags = dedupeTagActions(newTags, existingTags) } } + newFlags := dedupeStrings(c.effects.RecordFlags) if len(newFlags) > 0 { // fetch existing flags, and de-dupe @@ -211,7 +317,7 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { return fmt.Errorf("failed to circuit break takedowns: %w", err) } - if newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 { + if newTakedown || len(newLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || len(newReports) > 0 { if eng.Notifier != nil { for _, srv := range dedupeStrings(c.effects.NotifyServices) { if err := eng.Notifier.SendRecord(ctx, srv, c); err != nil { @@ -231,7 +337,7 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { } // exit early - if !newTakedown && len(newLabels) == 0 && len(newReports) == 0 { + if !newTakedown && len(newLabels) == 0 && len(newTags) == 0 && len(newReports) == 0 { return nil } @@ -276,6 +382,31 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { } } + if len(newTags) > 0 { + c.Logger.Info("tagging record", "newTags", newTags) + for _, val := range newTags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewTagCount.WithLabelValues("record", val).Inc() + } + comment := "[automod]: auto-tagging record" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTag: &toolsozone.ModerationDefs_ModEventTag{ + Add: newTags, + Remove: []string{}, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to create record tag", "err", err) + } + } + for _, mr := range newReports { _, err := eng.createRecordReportIfFresh(ctx, xrpcc, c.RecordOp.ATURI(), c.RecordOp.CID, mr) if err != nil { @@ -303,5 +434,6 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { c.Logger.Error("failed to execute record takedown", "err", err) } } + return nil } diff --git a/automod/engine/persisthelpers.go b/automod/engine/persisthelpers.go index c224cc4ab..42ba8934e 100644 --- a/automod/engine/persisthelpers.go +++ b/automod/engine/persisthelpers.go @@ -35,6 +35,23 @@ func dedupeLabelActions(labels, existing, existingNegated []string) []string { return newLabels } +func dedupeTagActions(tags, existing []string) []string { + newTags := []string{} + for _, val := range dedupeStrings(tags) { + exists := false + for _, e := range existing { + if val == e { + exists = true + break + } + } + if !exists { + newTags = append(newTags, val) + } + } + return newTags +} + func dedupeFlagActions(flags, existing []string) []string { newFlags := []string{} for _, val := range dedupeStrings(flags) { @@ -111,6 +128,26 @@ func (eng *Engine) circuitBreakTakedown(ctx context.Context, takedown bool) (boo return takedown, nil } +// Combined circuit breaker for miscellaneous mod actions like: escalate, acknowledge +func (eng *Engine) circuitBreakModAction(ctx context.Context, action bool) (bool, error) { + if !action { + return false, nil + } + c, err := eng.Counters.GetCount(ctx, "automod-quota", "mod-action", countstore.PeriodDay) + if err != nil { + return false, fmt.Errorf("checking mod action quota: %w", err) + } + if c >= QuotaModActionDay { + eng.Logger.Warn("CIRCUIT BREAKER: automod action") + return false, nil + } + err = eng.Counters.Increment(ctx, "automod-quota", "mod-action") + if err != nil { + return false, fmt.Errorf("incrementing mod action quota: %w", err) + } + return action, nil +} + // Creates a moderation report, but checks first if there was a similar recent one, and skips if so. // // Returns a bool indicating if a new report was created. @@ -118,26 +155,27 @@ func (eng *Engine) createReportIfFresh(ctx context.Context, xrpcc *xrpc.Client, // before creating a report, query to see if automod has already reported this account in the past week for the same reason // NOTE: this is running in an inner loop (if there are multiple reports), which is a bit inefficient, but seems acceptable - // ModerationQueryEvents(ctx context.Context, c *xrpc.Client, createdBy string, cursor string, inc ludeAllUserRecords bool, limit int64, sortDirection string, subject string, types []string) resp, err := toolsozone.ModerationQueryEvents( ctx, xrpcc, - nil, - nil, - "", - "", - "", - xrpcc.Auth.Did, - "", - false, - false, - 5, - nil, - nil, - nil, - "", - did.String(), - []string{"tools.ozone.moderation.defs#modEventReport"}, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + xrpcc.Auth.Did, // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 5, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + did.String(), // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string ) if err != nil { @@ -194,26 +232,27 @@ func (eng *Engine) createRecordReportIfFresh(ctx context.Context, xrpcc *xrpc.Cl // before creating a report, query to see if automod has already reported this account in the past week for the same reason // NOTE: this is running in an inner loop (if there are multiple reports), which is a bit inefficient, but seems acceptable - // ModerationQueryEvents(ctx context.Context, c *xrpc.Client, createdBy string, cursor string, inc ludeAllUserRecords bool, limit int64, sortDirection string, subject string, types []string) resp, err := toolsozone.ModerationQueryEvents( ctx, xrpcc, - nil, - nil, - "", - "", - "", - xrpcc.Auth.Did, - "", - false, - false, - 5, - nil, - nil, - nil, - "", - uri.String(), - []string{"tools.ozone.moderation.defs#modEventReport"}, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + xrpcc.Auth.Did, // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 5, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + uri.String(), // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string ) if err != nil { return false, err diff --git a/automod/engine/util.go b/automod/engine/util.go index 195454c1b..e96c411d5 100644 --- a/automod/engine/util.go +++ b/automod/engine/util.go @@ -1,5 +1,10 @@ package engine +import ( + "net/url" + "strings" +) + func dedupeStrings(in []string) []string { var out []string seen := make(map[string]bool) @@ -11,3 +16,22 @@ func dedupeStrings(in []string) []string { } return out } + +// get the cid from a bluesky cdn url +func cidFromCdnUrl(str *string) *string { + if str == nil { + return nil + } + + u, err := url.Parse(*str) + if err != nil || u.Host != "cdn.bsky.app" { + return nil + } + + parts := strings.Split(u.Path, "/") + if len(parts) != 6 { + return nil + } + + return &strings.Split(parts[5], "@")[0] +} diff --git a/automod/helpers/account.go b/automod/helpers/account.go new file mode 100644 index 000000000..2a1a275cb --- /dev/null +++ b/automod/helpers/account.go @@ -0,0 +1,49 @@ +package helpers + +import ( + "time" + + "github.com/bluesky-social/indigo/automod" +) + +// no accounts exist before this time +var atprotoAccountEpoch = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + +// returns true if account creation timestamp is plausible: not-nil, not in distant past, not in the future +func plausibleAccountCreation(when *time.Time) bool { + if when == nil { + return false + } + // this is mostly to check for misconfigurations or null values (eg, UNIX epoch zero means "unknown" not actually 1970) + if !when.After(atprotoAccountEpoch) { + return false + } + // a timestamp in the future would also indicate some misconfiguration + if when.After(time.Now().Add(time.Hour)) { + return false + } + return true +} + +// checks if account was created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' +func AccountIsYoungerThan(c *automod.AccountContext, age time.Duration) bool { + // TODO: consider swapping priority order here (and below) + if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { + return time.Since(*c.Account.CreatedAt) < age + } + if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { + return time.Since(*c.Account.Private.IndexedAt) < age + } + return false +} + +// checks if account was *not* created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' +func AccountIsOlderThan(c *automod.AccountContext, age time.Duration) bool { + if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { + return time.Since(*c.Account.CreatedAt) >= age + } + if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { + return time.Since(*c.Account.Private.IndexedAt) >= age + } + return false +} diff --git a/automod/rules/helpers_test.go b/automod/helpers/account_test.go similarity index 58% rename from automod/rules/helpers_test.go rename to automod/helpers/account_test.go index 0d5e11ef2..c949eb77c 100644 --- a/automod/rules/helpers_test.go +++ b/automod/helpers/account_test.go @@ -1,4 +1,4 @@ -package rules +package helpers import ( "testing" @@ -7,65 +7,9 @@ import ( "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" - "github.com/bluesky-social/indigo/automod/keyword" "github.com/stretchr/testify/assert" ) -func TestTokenizeText(t *testing.T) { - assert := assert.New(t) - - fixtures := []struct { - s string - out []string - }{ - { - s: "1 'Two' three!", - out: []string{"1", "two", "three"}, - }, - { - s: " foo1;bar2,baz3...", - out: []string{"foo1", "bar2", "baz3"}, - }, - { - s: "https://example.com/index.html", - out: []string{"https", "example", "com", "index", "html"}, - }, - } - - for _, fix := range fixtures { - assert.Equal(fix.out, keyword.TokenizeText(fix.s)) - } -} - -func TestExtractURL(t *testing.T) { - assert := assert.New(t) - - fixtures := []struct { - s string - out []string - }{ - { - s: "this is a description with example.com mentioned in the middle", - out: []string{"example.com"}, - }, - { - s: "this is another example with https://en.wikipedia.org/index.html: and archive.org, and https://eff.org/... and bsky.app.", - out: []string{"https://en.wikipedia.org/index.html", "archive.org", "https://eff.org/", "bsky.app"}, - }, - } - - for _, fix := range fixtures { - assert.Equal(fix.out, ExtractTextURLs(fix.s)) - } -} - -func TestHashOfString(t *testing.T) { - assert := assert.New(t) - - // hashing function should be consistent over time - assert.Equal("4e6f69c0e3d10992", HashOfString("dummy-value")) -} - func TestAccountIsYoungerThan(t *testing.T) { assert := assert.New(t) diff --git a/automod/rules/helpers.go b/automod/helpers/bsky.go similarity index 67% rename from automod/rules/helpers.go rename to automod/helpers/bsky.go index 993b3913c..c7416f2dd 100644 --- a/automod/rules/helpers.go +++ b/automod/helpers/bsky.go @@ -1,30 +1,14 @@ -package rules +package helpers import ( "fmt" - "regexp" - "time" appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/keyword" - - "github.com/spaolacci/murmur3" ) -func dedupeStrings(in []string) []string { - var out []string - seen := make(map[string]bool) - for _, v := range in { - if !seen[v] { - out = append(out, v) - seen[v] = true - } - } - return out -} - func ExtractHashtagsPost(post *appbsky.FeedPost) []string { var tags []string for _, tag := range post.Tags { @@ -37,7 +21,7 @@ func ExtractHashtagsPost(post *appbsky.FeedPost) []string { } } } - return dedupeStrings(tags) + return DedupeStrings(tags) } func NormalizeHashtag(raw string) string { @@ -103,7 +87,7 @@ func ExtractPostBlobCIDsPost(post *appbsky.FeedPost) []string { } } } - return dedupeStrings(out) + return DedupeStrings(out) } func ExtractBlobCIDsProfile(profile *appbsky.ActorProfile) []string { @@ -114,7 +98,7 @@ func ExtractBlobCIDsProfile(profile *appbsky.ActorProfile) []string { if profile.Banner != nil { out = append(out, profile.Banner.Ref.String()) } - return dedupeStrings(out) + return DedupeStrings(out) } func ExtractTextTokensPost(post *appbsky.FeedPost) []string { @@ -152,13 +136,6 @@ func ExtractTextTokensProfile(profile *appbsky.ActorProfile) []string { return keyword.TokenizeText(s) } -// based on: https://stackoverflow.com/a/48769624, with no trailing period allowed -var urlRegex = regexp.MustCompile(`(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]*[\w/\-&?=%]+`) - -func ExtractTextURLs(raw string) []string { - return urlRegex.FindAllString(raw, -1) -} - func ExtractTextURLsProfile(profile *appbsky.ActorProfile) []string { s := "" if profile.Description != nil { @@ -191,14 +168,6 @@ func IsSelfThread(c *automod.RecordContext, post *appbsky.FeedPost) bool { return false } -// returns a fast, compact hash of a string -// -// current implementation uses murmur3, default seed, and hex encoding -func HashOfString(s string) string { - val := murmur3.Sum64([]byte(s)) - return fmt.Sprintf("%016x", val) -} - func ParentOrRootIsFollower(c *automod.RecordContext, post *appbsky.FeedPost) bool { if post.Reply == nil || IsSelfThread(c, post) { return false @@ -242,44 +211,59 @@ func ParentOrRootIsFollower(c *automod.RecordContext, post *appbsky.FeedPost) bo return false } -// no accounts exist before this time -var atprotoAccountEpoch = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - -// returns true if account creation timestamp is plausible: not-nil, not in distant past, not in the future -func plausibleAccountCreation(when *time.Time) bool { - if when == nil { +func PostParentOrRootIsDid(post *appbsky.FeedPost, did string) bool { + if post.Reply == nil { return false } - // this is mostly to check for misconfigurations or null values (eg, UNIX epoch zero means "unknown" not actually 1970) - if !when.After(atprotoAccountEpoch) { + + rootUri, err := syntax.ParseATURI(post.Reply.Root.Uri) + if err != nil || !rootUri.Authority().IsDID() { return false } - // a timestamp in the future would also indicate some misconfiguration - if when.After(time.Now().Add(time.Hour)) { + + parentUri, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil || !parentUri.Authority().IsDID() { return false } - return true + + return rootUri.Authority().String() == did || parentUri.Authority().String() == did } -// checks if account was created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' -func AccountIsYoungerThan(c *automod.AccountContext, age time.Duration) bool { - // TODO: consider swapping priority order here (and below) - if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { - return time.Since(*c.Account.CreatedAt) < age +func PostParentOrRootIsAnyDid(post *appbsky.FeedPost, dids []string) bool { + if post.Reply == nil { + return false } - if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { - return time.Since(*c.Account.Private.IndexedAt) < age + + for _, did := range dids { + if PostParentOrRootIsDid(post, did) { + return true + } } + return false } -// checks if account was *not* created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' -func AccountIsOlderThan(c *automod.AccountContext, age time.Duration) bool { - if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { - return time.Since(*c.Account.CreatedAt) >= age +func PostMentionsDid(post *appbsky.FeedPost, did string) bool { + facets, err := ExtractFacets(post) + if err != nil { + return false + } + + for _, facet := range facets { + if facet.DID != nil && *facet.DID == did { + return true + } } - if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { - return time.Since(*c.Account.Private.IndexedAt) >= age + + return false +} + +func PostMentionsAnyDid(post *appbsky.FeedPost, dids []string) bool { + for _, did := range dids { + if PostMentionsDid(post, did) { + return true + } } + return false } diff --git a/automod/helpers/bsky_test.go b/automod/helpers/bsky_test.go new file mode 100644 index 000000000..b5d6cb242 --- /dev/null +++ b/automod/helpers/bsky_test.go @@ -0,0 +1,141 @@ +package helpers + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParentOrRootIsDid(t *testing.T) { + assert := assert.New(t) + + post1 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + }, + } + + post2 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + }, + } + + post3 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + }, + } + + post4 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + }, + } + + assert.True(PostParentOrRootIsDid(post1, "did:plc:abc123")) + assert.False(PostParentOrRootIsDid(post1, "did:plc:321abc")) + + assert.True(PostParentOrRootIsDid(post2, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post2, "did:plc:321abc")) + + assert.True(PostParentOrRootIsDid(post3, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post3, "did:plc:321abc")) + + assert.False(PostParentOrRootIsDid(post4, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post4, "did:plc:321abc")) + + didList1 := []string{ + "did:plc:cba321", + "did:web:bsky.app", + "did:plc:abc123", + } + + didList2 := []string{ + "did:plc:321cba", + "did:web:bsky.app", + "did:plc:123abc", + } + + assert.True(PostParentOrRootIsAnyDid(post1, didList1)) + assert.False(PostParentOrRootIsAnyDid(post1, didList2)) +} + +func TestPostMentionsDid(t *testing.T) { + assert := assert.New(t) + + post := &appbsky.FeedPost{ + Text: "@hailey.at what is upppp also hello to @darthbluesky.bsky.social", + Facets: []*appbsky.RichtextFacet{ + { + Features: []*appbsky.RichtextFacet_Features_Elem{ + { + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc123", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 0, + ByteEnd: 9, + }, + }, + { + Features: []*appbsky.RichtextFacet_Features_Elem{ + { + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc456", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 39, + ByteEnd: 63, + }, + }, + }, + } + assert.True(PostMentionsDid(post, "did:plc:abc123")) + assert.False(PostMentionsDid(post, "did:plc:cba321")) + + didList1 := []string{ + "did:plc:cba321", + "did:web:bsky.app", + "did:plc:abc456", + } + + didList2 := []string{ + "did:plc:321cba", + "did:web:bsky.app", + "did:plc:123abc", + } + + assert.True(PostMentionsAnyDid(post, didList1)) + assert.False(PostMentionsAnyDid(post, didList2)) +} diff --git a/automod/helpers/text.go b/automod/helpers/text.go new file mode 100644 index 000000000..412eb9c8c --- /dev/null +++ b/automod/helpers/text.go @@ -0,0 +1,35 @@ +package helpers + +import ( + "fmt" + "regexp" + + "github.com/spaolacci/murmur3" +) + +func DedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} + +// returns a fast, compact hash of a string +// +// current implementation uses murmur3, default seed, and hex encoding +func HashOfString(s string) string { + val := murmur3.Sum64([]byte(s)) + return fmt.Sprintf("%016x", val) +} + +// based on: https://stackoverflow.com/a/48769624, with no trailing period allowed +var urlRegex = regexp.MustCompile(`(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]*[\w/\-&?=%]+`) + +func ExtractTextURLs(raw string) []string { + return urlRegex.FindAllString(raw, -1) +} diff --git a/automod/helpers/text_test.go b/automod/helpers/text_test.go new file mode 100644 index 000000000..ef219155e --- /dev/null +++ b/automod/helpers/text_test.go @@ -0,0 +1,64 @@ +package helpers + +import ( + "testing" + + "github.com/bluesky-social/indigo/automod/keyword" + + "github.com/stretchr/testify/assert" +) + +func TestTokenizeText(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + s string + out []string + }{ + { + s: "1 'Two' three!", + out: []string{"1", "two", "three"}, + }, + { + s: " foo1;bar2,baz3...", + out: []string{"foo1", "bar2", "baz3"}, + }, + { + s: "https://example.com/index.html", + out: []string{"https", "example", "com", "index", "html"}, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, keyword.TokenizeText(fix.s)) + } +} + +func TestExtractURL(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + s string + out []string + }{ + { + s: "this is a description with example.com mentioned in the middle", + out: []string{"example.com"}, + }, + { + s: "this is another example with https://en.wikipedia.org/index.html: and archive.org, and https://eff.org/... and bsky.app.", + out: []string{"https://en.wikipedia.org/index.html", "archive.org", "https://eff.org/", "bsky.app"}, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, ExtractTextURLs(fix.s)) + } +} + +func TestHashOfString(t *testing.T) { + assert := assert.New(t) + + // hashing function should be consistent over time + assert.Equal("4e6f69c0e3d10992", HashOfString("dummy-value")) +} diff --git a/automod/keyword/tokenize.go b/automod/keyword/tokenize.go index 8c477432b..0b5a33ca4 100644 --- a/automod/keyword/tokenize.go +++ b/automod/keyword/tokenize.go @@ -12,18 +12,19 @@ import ( ) var ( - puncChars = regexp.MustCompile(`[[:punct:]]+`) - nonTokenChars = regexp.MustCompile(`[^\pL\pN\s]+`) + puncChars = regexp.MustCompile(`[[:punct:]]+`) + nonTokenChars = regexp.MustCompile(`[^\pL\pN\s]+`) + nonTokenCharsSkipCensorChars = regexp.MustCompile(`[^\pL\pN\s#*_-]`) ) // Splits free-form text in to tokens, including lower-case, unicode normalization, and some unicode folding. // // The intent is for this to work similarly to an NLP tokenizer, as might be used in a fulltext search engine, and enable fast matching to a list of known tokens. It might eventually even do stemming, removing pluralization (trailing "s" for English), etc. -func TokenizeText(text string) []string { +func TokenizeTextWithRegex(text string, nonTokenCharsRegex *regexp.Regexp) []string { // this function needs to be re-defined in every function call to prevent a race condition normFunc := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) - split := strings.ToLower(nonTokenChars.ReplaceAllString(text, " ")) - bare := strings.ToLower(nonTokenChars.ReplaceAllString(split, "")) + split := strings.ToLower(nonTokenCharsRegex.ReplaceAllString(text, " ")) + bare := strings.ToLower(nonTokenCharsRegex.ReplaceAllString(split, "")) norm, _, err := transform.String(normFunc, bare) if err != nil { slog.Warn("unicode normalization error", "err", err) @@ -32,6 +33,14 @@ func TokenizeText(text string) []string { return strings.Fields(norm) } +func TokenizeText(text string) []string { + return TokenizeTextWithRegex(text, nonTokenChars) +} + +func TokenizeTextSkippingCensorChars(text string) []string { + return TokenizeTextWithRegex(text, nonTokenCharsSkipCensorChars) +} + func splitIdentRune(c rune) bool { return !unicode.IsLetter(c) && !unicode.IsNumber(c) } diff --git a/automod/keyword/tokenize_test.go b/automod/keyword/tokenize_test.go index 89d5f79b1..45b477f6d 100644 --- a/automod/keyword/tokenize_test.go +++ b/automod/keyword/tokenize_test.go @@ -1,6 +1,7 @@ package keyword import ( + "regexp" "testing" "github.com/stretchr/testify/assert" @@ -17,6 +18,9 @@ func TestTokenizeText(t *testing.T) { {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, {text: "Gdańsk", out: []string{"gdansk"}}, {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar", out: []string{"foo", "bar"}}, + {text: "foo-bar", out: []string{"foo", "bar"}}, + {text: "foo_bar", out: []string{"foo", "bar"}}, } for _, fix := range fixtures { @@ -24,6 +28,49 @@ func TestTokenizeText(t *testing.T) { } } +func TestTokenizeTextWithCensorChars(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out []string + }{ + {text: "", out: []string{}}, + {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, + {text: "Gdańsk", out: []string{"gdansk"}}, + {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar,foo&bar", out: []string{"foo*bar", "foo", "bar"}}, + {text: "foo-bar,foo&bar", out: []string{"foo-bar", "foo", "bar"}}, + {text: "foo_bar,foo&bar", out: []string{"foo_bar", "foo", "bar"}}, + {text: "foo#bar,foo&bar", out: []string{"foo#bar", "foo", "bar"}}, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeTextSkippingCensorChars(fix.text)) + } +} + +func TestTokenizeTextWithCustomRegex(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out []string + }{ + {text: "", out: []string{}}, + {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, + {text: "Gdańsk", out: []string{"gdansk"}}, + {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar", out: []string{"foo", "bar"}}, + {text: "foo&bar,foo*bar", out: []string{"foo&bar", "foo", "bar"}}, + } + + regex := regexp.MustCompile(`[^\pL\pN\s&]`) + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeTextWithRegex(fix.text, regex)) + } +} + func TestTokenizeIdentifier(t *testing.T) { assert := assert.New(t) diff --git a/automod/rules/gtube.go b/automod/rules/gtube.go index 4684541a4..71922a528 100644 --- a/automod/rules/gtube.go +++ b/automod/rules/gtube.go @@ -16,6 +16,7 @@ func GtubePostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { if strings.Contains(post.Text, gtubeString) { c.AddRecordLabel("spam") c.Notify("slack") + c.AddRecordTag("gtube-record") } return nil } @@ -26,6 +27,7 @@ func GtubeProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) e if profile.Description != nil && strings.Contains(*profile.Description, gtubeString) { c.AddRecordLabel("spam") c.Notify("slack") + c.AddAccountTag("gtuber-account") } return nil } diff --git a/automod/rules/harassment.go b/automod/rules/harassment.go index d5a35008e..2cf7ce194 100644 --- a/automod/rules/harassment.go +++ b/automod/rules/harassment.go @@ -8,18 +8,19 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = HarassmentTargetInteractionPostRule // looks for new accounts, which interact with frequently-harassed accounts, and report them for review func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) { return nil } var interactionDIDs []string - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { return err } @@ -28,7 +29,7 @@ func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky interactionDIDs = append(interactionDIDs, *pf.DID) } } - if post.Reply != nil && !IsSelfThread(c, post) { + if post.Reply != nil && !helpers.IsSelfThread(c, post) { parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) if err != nil { return err @@ -57,7 +58,7 @@ func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky return nil } - interactionDIDs = dedupeStrings(interactionDIDs) + interactionDIDs = helpers.DedupeStrings(interactionDIDs) for _, d := range interactionDIDs { did, err := syntax.ParseDID(d) if err != nil { @@ -76,10 +77,12 @@ func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky if targetAccount == nil { continue } - for _, t := range targetAccount.Private.AccountTags { - if t == "harassment-protection" { - targetIsProtected = true - break + if targetAccount.Private != nil { + for _, t := range targetAccount.Private.AccountTags { + if t == "harassment-protection" { + targetIsProtected = true + break + } } } } @@ -112,7 +115,7 @@ var _ automod.PostRuleFunc = HarassmentTrivialPostRule // looks for new accounts, which frequently post the same type of content func HarassmentTrivialPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { return nil } diff --git a/automod/rules/hashtags.go b/automod/rules/hashtags.go index c6d734807..682ce746a 100644 --- a/automod/rules/hashtags.go +++ b/automod/rules/hashtags.go @@ -5,13 +5,14 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/bluesky-social/indigo/automod/keyword" ) // looks for specific hashtags from known lists func BadHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - for _, tag := range ExtractHashtagsPost(post) { - tag = NormalizeHashtag(tag) + for _, tag := range helpers.ExtractHashtagsPost(post) { + tag = helpers.NormalizeHashtag(tag) // skip some bad-word hashtags which frequently false-positive if tag == "nazi" || tag == "hitler" { continue @@ -35,7 +36,7 @@ var _ automod.PostRuleFunc = BadHashtagsPostRule // if a post is "almost all" hashtags, it might be a form of search spam func TooManyHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - tags := ExtractHashtagsPost(post) + tags := helpers.ExtractHashtagsPost(post) tagChars := 0 for _, tag := range tags { tagChars += len(tag) diff --git a/automod/rules/identity.go b/automod/rules/identity.go index 365d63f95..e74991233 100644 --- a/automod/rules/identity.go +++ b/automod/rules/identity.go @@ -7,11 +7,12 @@ import ( "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) // triggers on first identity event for an account (DID) func NewAccountRule(c *automod.AccountContext) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(c, 4*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 4*time.Hour) { return nil } diff --git a/automod/rules/keyword.go b/automod/rules/keyword.go index abb202600..8d5caa395 100644 --- a/automod/rules/keyword.go +++ b/automod/rules/keyword.go @@ -7,6 +7,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/bluesky-social/indigo/automod/keyword" ) @@ -17,7 +18,7 @@ func BadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { isJapanese = true } } - for _, tok := range ExtractTextTokensPost(post) { + for _, tok := range helpers.ExtractTextTokensPost(post) { word := keyword.SlugIsExplicitSlur(tok) // used very frequently in a reclaimed context if word != "" && word != "faggot" && word != "tranny" && word != "coon" && !(word == "kike" && isJapanese) { @@ -54,7 +55,7 @@ func BadWordProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) //c.Notify("slack") } } - for _, tok := range ExtractTextTokensProfile(profile) { + for _, tok := range helpers.ExtractTextTokensProfile(profile) { // de-pluralize tok = strings.TrimSuffix(tok, "s") if c.InSet("worst-words", tok) { @@ -71,8 +72,8 @@ var _ automod.ProfileRuleFunc = BadWordProfileRule // looks for the specific harassment situation of a replay to another user with only a single word func ReplySingleBadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply != nil && !IsSelfThread(c, post) { - tokens := ExtractTextTokensPost(post) + if post.Reply != nil && !helpers.IsSelfThread(c, post) { + tokens := helpers.ExtractTextTokensPost(post) if len(tokens) != 1 { return nil } diff --git a/automod/rules/mentions.go b/automod/rules/mentions.go index 8155b4a4a..98d419d09 100644 --- a/automod/rules/mentions.go +++ b/automod/rules/mentions.go @@ -8,6 +8,7 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = DistinctMentionsRule @@ -47,7 +48,7 @@ var youngMentionAccountLimit = 12 var _ automod.PostRuleFunc = YoungAccountDistinctMentionsRule func YoungAccountDistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) { return nil } diff --git a/automod/rules/misleading.go b/automod/rules/misleading.go index df4525cfc..31822ccce 100644 --- a/automod/rules/misleading.go +++ b/automod/rules/misleading.go @@ -9,9 +9,10 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) -func isMisleadingURLFacet(facet PostFacet, logger *slog.Logger) bool { +func isMisleadingURLFacet(facet helpers.PostFacet, logger *slog.Logger) bool { linkURL, err := url.Parse(*facet.URL) if err != nil { logger.Warn("invalid link metadata URL", "url", facet.URL) @@ -84,7 +85,7 @@ func MisleadingURLPostRule(c *automod.RecordContext, post *appbsky.FeedPost) err if c.Account.Identity.Handle == "nowbreezing.ntw.app" { return nil } - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { c.Logger.Warn("invalid facets", "err", err) // TODO: or some other "this record is corrupt" indicator? @@ -105,7 +106,7 @@ func MisleadingURLPostRule(c *automod.RecordContext, post *appbsky.FeedPost) err var _ automod.PostRuleFunc = MisleadingMentionPostRule func MisleadingMentionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - facets, err := ExtractFacets(post) + facets, err := helpers.ExtractFacets(post) if err != nil { c.Logger.Warn("invalid facets", "err", err) // TODO: or some other "this record is corrupt" indicator? diff --git a/automod/rules/misleading_test.go b/automod/rules/misleading_test.go index cf8e814af..2e47883a6 100644 --- a/automod/rules/misleading_test.go +++ b/automod/rules/misleading_test.go @@ -11,6 +11,7 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/engine" + "github.com/bluesky-social/indigo/automod/helpers" "github.com/stretchr/testify/assert" ) @@ -118,67 +119,67 @@ func TestIsMisleadingURL(t *testing.T) { logger := slog.Default() fixtures := []struct { - facet PostFacet + facet helpers.PostFacet out bool }{ { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://evil.com"), }, out: true, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://www.atproto.com", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "https://atproto.com", URL: pstr("https://www.atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "[example.com]", URL: pstr("https://www.example.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "example.com...", URL: pstr("https://example.com.evil.com"), }, out: true, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "ATPROTO.com...", URL: pstr("https://atproto.com"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "1234.5678", URL: pstr("https://arxiv.org/abs/1234.5678"), }, out: false, }, { - facet: PostFacet{ + facet: helpers.PostFacet{ Text: "www.techdirt.com…", URL: pstr("https://www.techdirt.com/"), }, diff --git a/automod/rules/nostr.go b/automod/rules/nostr.go index 0291d0668..5f91e7ee6 100644 --- a/automod/rules/nostr.go +++ b/automod/rules/nostr.go @@ -7,13 +7,14 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = NostrSpamPostRule // looks for new accounts, which frequently post the same type of content func NostrSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { return nil } diff --git a/automod/rules/promo.go b/automod/rules/promo.go index 0dad7aaf4..f6fe23a24 100644 --- a/automod/rules/promo.go +++ b/automod/rules/promo.go @@ -9,6 +9,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = AggressivePromotionRule @@ -17,16 +18,16 @@ var _ automod.PostRuleFunc = AggressivePromotionRule // // this rule depends on ReplyCountPostRule() to set counts func AggressivePromotionRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { return nil } - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } - allURLs := ExtractTextURLs(post.Text) + allURLs := helpers.ExtractTextURLs(post.Text) if c.Account.Profile.Description != nil { - profileURLs := ExtractTextURLs(*c.Account.Profile.Description) + profileURLs := helpers.ExtractTextURLs(*c.Account.Profile.Description) allURLs = append(allURLs, profileURLs...) } hasPromo := false diff --git a/automod/rules/quick.go b/automod/rules/quick.go index 77075d94a..ea6a69e36 100644 --- a/automod/rules/quick.go +++ b/automod/rules/quick.go @@ -7,6 +7,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" ) var botLinkStrings = []string{"ainna13762491", "LINK押して", "→ https://tiny", "⇒ http://tiny"} @@ -54,7 +55,7 @@ func SimpleBotPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { var _ automod.IdentityRuleFunc = NewAccountBotEmailRule func NewAccountBotEmailRule(c *automod.AccountContext) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(c, 1*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 1*time.Hour) { return nil } @@ -73,7 +74,7 @@ var _ automod.PostRuleFunc = TrivialSpamPostRule // looks for new accounts, which frequently post the same type of content func TrivialSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 8*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 8*24*time.Hour) { return nil } diff --git a/automod/rules/replies.go b/automod/rules/replies.go index aed986737..e03e9de53 100644 --- a/automod/rules/replies.go +++ b/automod/rules/replies.go @@ -9,13 +9,14 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var _ automod.PostRuleFunc = ReplyCountPostRule // does not count "self-replies" (direct to self, or in own post thread) func ReplyCountPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -47,7 +48,7 @@ var _ automod.PostRuleFunc = IdenticalReplyPostRule // // There can be legitimate situations that trigger this rule, so in most situations should be a "report" not "label" action. func IdenticalReplyPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -55,18 +56,18 @@ func IdenticalReplyPostRule(c *automod.RecordContext, post *appbsky.FeedPost) er if utf8.RuneCountInString(post.Text) <= 10 { return nil } - if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { return nil } // don't count if there is a follow-back relationship - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } // increment before read. use a specific period (IncrementPeriod()) to reduce the number of counters (one per unique post text) period := countstore.PeriodDay - bucket := c.Account.Identity.DID.String() + "/" + HashOfString(post.Text) + bucket := c.Account.Identity.DID.String() + "/" + helpers.HashOfString(post.Text) c.IncrementPeriod("reply-text", bucket, period) count := c.GetCount("reply-text", bucket, period) @@ -91,21 +92,21 @@ var identicalReplySameParentMaxPosts int64 = 50 var _ automod.PostRuleFunc = IdenticalReplyPostSameParentRule func IdenticalReplyPostSameParentRule(c *automod.RecordContext, post *appbsky.FeedPost) error { - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } postCount := c.Account.PostsCount - if AccountIsOlderThan(&c.AccountContext, identicalReplySameParentMaxAge) || postCount >= identicalReplySameParentMaxPosts { + if helpers.AccountIsOlderThan(&c.AccountContext, identicalReplySameParentMaxAge) || postCount >= identicalReplySameParentMaxPosts { return nil } period := countstore.PeriodHour - bucket := c.Account.Identity.DID.String() + "/" + post.Reply.Parent.Uri + "/" + HashOfString(post.Text) + bucket := c.Account.Identity.DID.String() + "/" + post.Reply.Parent.Uri + "/" + helpers.HashOfString(post.Text) c.IncrementPeriod("reply-text-same-post", bucket, period) count := c.GetCount("reply-text-same-post", bucket, period) @@ -126,7 +127,7 @@ var _ automod.PostRuleFunc = YoungAccountDistinctRepliesRule func YoungAccountDistinctRepliesRule(c *automod.RecordContext, post *appbsky.FeedPost) error { // only replies, and skip self-replies (eg, threads) - if post.Reply == nil || IsSelfThread(c, post) { + if post.Reply == nil || helpers.IsSelfThread(c, post) { return nil } @@ -134,12 +135,12 @@ func YoungAccountDistinctRepliesRule(c *automod.RecordContext, post *appbsky.Fee if utf8.RuneCountInString(post.Text) <= 10 { return nil } - if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { return nil } // don't count if there is a follow-back relationship - if ParentOrRootIsFollower(c, post) { + if helpers.ParentOrRootIsFollower(c, post) { return nil } diff --git a/automod/rules/reposts.go b/automod/rules/reposts.go index 75b248461..573146558 100644 --- a/automod/rules/reposts.go +++ b/automod/rules/reposts.go @@ -7,6 +7,7 @@ import ( "github.com/bluesky-social/indigo/automod" "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" ) var dailyRepostThresholdWithoutPost = 30 @@ -18,7 +19,7 @@ var _ automod.RecordRuleFunc = TooManyRepostRule // looks for accounts which do frequent reposts func TooManyRepostRule(c *automod.RecordContext) error { // Don't bother checking reposts from accounts older than 30 days - if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) { return nil } diff --git a/automod/visual/hiveai_rule.go b/automod/visual/hiveai_rule.go index 32bcf6a9a..850ee83b1 100644 --- a/automod/visual/hiveai_rule.go +++ b/automod/visual/hiveai_rule.go @@ -5,7 +5,7 @@ import ( "time" "github.com/bluesky-social/indigo/automod" - "github.com/bluesky-social/indigo/automod/rules" + "github.com/bluesky-social/indigo/automod/helpers" lexutil "github.com/bluesky-social/indigo/lex/util" ) @@ -43,7 +43,7 @@ func (hal *HiveAIClient) HiveLabelBlobRule(c *automod.RecordContext, blob lexuti for _, l := range labels { // NOTE: experimenting with profile reporting for new accounts - if l == "sexual" && c.RecordOp.Collection.String() == "app.bsky.actor.profile" && rules.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + if l == "sexual" && c.RecordOp.Collection.String() == "app.bsky.actor.profile" && helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { c.ReportRecord(automod.ReportReasonSexual, "possible sexual profile (not labeled yet)") c.Logger.Info("skipping record label", "label", l, "reason", "sexual-profile-experiment") } else { diff --git a/backfill/backfill.go b/backfill/backfill.go index f84d97df9..6f199677c 100644 --- a/backfill/backfill.go +++ b/backfill/backfill.go @@ -119,7 +119,7 @@ func DefaultBackfillOptions() *BackfillOptions { ParallelRecordCreates: 100, NSIDFilter: "", SyncRequestsPerSecond: 2, - CheckoutPath: "https://bsky.social/xrpc/com.atproto.sync.getRepo", + CheckoutPath: "https://bsky.network/xrpc/com.atproto.sync.getRepo", } } diff --git a/bgs/bgs.go b/bgs/bgs.go index 192f4ee3d..35dfab9d9 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -107,20 +107,22 @@ type SocketConsumer struct { } type BGSConfig struct { - SSL bool - CompactInterval time.Duration - DefaultRepoLimit int64 - ConcurrencyPerPDS int64 - MaxQueuePerPDS int64 + SSL bool + CompactInterval time.Duration + DefaultRepoLimit int64 + ConcurrencyPerPDS int64 + MaxQueuePerPDS int64 + NumCompactionWorkers int } func DefaultBGSConfig() *BGSConfig { return &BGSConfig{ - SSL: true, - CompactInterval: 4 * time.Hour, - DefaultRepoLimit: 100, - ConcurrencyPerPDS: 100, - MaxQueuePerPDS: 1_000, + SSL: true, + CompactInterval: 4 * time.Hour, + DefaultRepoLimit: 100, + ConcurrencyPerPDS: 100, + MaxQueuePerPDS: 1_000, + NumCompactionWorkers: 2, } } @@ -168,7 +170,9 @@ func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtm return nil, err } - compactor := NewCompactor(nil) + cOpts := DefaultCompactorOptions() + cOpts.NumWorkers = config.NumCompactionWorkers + compactor := NewCompactor(cOpts) compactor.requeueInterval = config.CompactInterval compactor.Start(bgs) bgs.compactor = compactor @@ -349,6 +353,7 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { e.GET("/xrpc/com.atproto.sync.notifyOfUpdate", bgs.HandleComAtprotoSyncNotifyOfUpdate) e.GET("/xrpc/_health", bgs.HandleHealthCheck) e.GET("/_health", bgs.HandleHealthCheck) + e.GET("/", bgs.HandleHomeMessage) admin := e.Group("/admin", bgs.checkAdminAuth) @@ -420,6 +425,23 @@ func (bgs *BGS) HandleHealthCheck(c echo.Context) error { } } +var homeMessage string = ` +d8888b. d888888b d888b .d8888. db dD db db +88 '8D '88' 88' Y8b 88' YP 88 ,8P' '8b d8' +88oooY' 88 88 '8bo. 88,8P '8bd8' +88~~~b. 88 88 ooo 'Y8b. 88'8b 88 +88 8D .88. 88. ~8~ db 8D 88 '88. 88 +Y8888P' Y888888P Y888P '8888Y' YP YD YP + +This is an atproto [https://atproto.com] relay instance, running the 'bigsky' codebase [https://github.com/bluesky-social/indigo] + +The firehose WebSocket path is at: /xrpc/com.atproto.sync.subscribeRepos +` + +func (bgs *BGS) HandleHomeMessage(c echo.Context) error { + return c.String(http.StatusOK, homeMessage) +} + type AuthToken struct { gorm.Model Token string `gorm:"index"` diff --git a/bgs/handlers.go b/bgs/handlers.go index d81ba7c4a..da87c9521 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -195,7 +195,7 @@ func (s *BGS) handleComAtprotoSyncNotifyOfUpdate(ctx context.Context, body *coma func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor int64, limit int) (*comatprototypes.SyncListRepos_Output, error) { // Filter out tombstoned, taken down, and deactivated accounts - q := fmt.Sprintf("id > ? AND NOT tombstoned AND NOT taken_down AND upstream_status != '%s' AND upstream_status != '%s' AND upstream_status != '%s'", + q := fmt.Sprintf("id > ? AND NOT tombstoned AND NOT taken_down AND (upstream_status is NULL OR (upstream_status != '%s' AND upstream_status != '%s' AND upstream_status != '%s'))", events.AccountStatusDeactivated, events.AccountStatusSuspended, events.AccountStatusTakendown) // Load the users diff --git a/carstore/bs.go b/carstore/bs.go index 1f86b6612..e7af35d12 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -4,14 +4,11 @@ import ( "bufio" "bytes" "context" - "encoding/binary" "fmt" "io" "os" "path/filepath" "sort" - "strconv" - "strings" "sync" "sync/atomic" "time" @@ -33,7 +30,6 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -52,15 +48,28 @@ const MaxSliceLength = 2 << 20 const BigShardThreshold = 2 << 20 -type CarStore struct { - meta *gorm.DB +type CarStore interface { + CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) + GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) + GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) + GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) + ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) + NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) + ReadOnlySession(user models.Uid) (*DeltaSession, error) + ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error + Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) + WipeUserData(ctx context.Context, user models.Uid) error +} + +type FileCarStore struct { + meta *CarStoreGormMeta rootDir string lscLk sync.Mutex lastShardCache map[models.Uid]*CarShard } -func NewCarStore(meta *gorm.DB, root string) (*CarStore, error) { +func NewCarStore(meta *gorm.DB, root string) (CarStore, error) { if _, err := os.Stat(root); err != nil { if !os.IsNotExist(err) { return nil, err @@ -77,82 +86,15 @@ func NewCarStore(meta *gorm.DB, root string) (*CarStore, error) { return nil, err } - return &CarStore{ - meta: meta, + return &FileCarStore{ + meta: &CarStoreGormMeta{meta: meta}, rootDir: root, lastShardCache: make(map[models.Uid]*CarShard), }, nil } -type UserInfo struct { - gorm.Model - Head string -} - -type CarShard struct { - ID uint `gorm:"primarykey"` - CreatedAt time.Time - - Root models.DbCID `gorm:"index"` - DataStart int64 - Seq int `gorm:"index:idx_car_shards_seq;index:idx_car_shards_usr_seq,priority:2,sort:desc"` - Path string - Usr models.Uid `gorm:"index:idx_car_shards_usr;index:idx_car_shards_usr_seq,priority:1"` - Rev string -} - -type blockRef struct { - ID uint `gorm:"primarykey"` - Cid models.DbCID `gorm:"index"` - Shard uint `gorm:"index"` - Offset int64 - //User uint `gorm:"index"` -} - -type staleRef struct { - ID uint `gorm:"primarykey"` - Cid *models.DbCID - Cids []byte - Usr models.Uid `gorm:"index"` -} - -func (sr *staleRef) getCids() ([]cid.Cid, error) { - if sr.Cid != nil { - return []cid.Cid{sr.Cid.CID}, nil - } - - return unpackCids(sr.Cids) -} - -func packCids(cids []cid.Cid) []byte { - buf := new(bytes.Buffer) - for _, c := range cids { - buf.Write(c.Bytes()) - } - - return buf.Bytes() -} - -func unpackCids(b []byte) ([]cid.Cid, error) { - br := bytes.NewReader(b) - var out []cid.Cid - for { - _, c, err := cid.CidFromReader(br) - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - - out = append(out, c) - } - - return out, nil -} - type userView struct { - cs *CarStore + cs *FileCarStore user models.Uid cache map[cid.Cid]blockformat.Block @@ -166,17 +108,7 @@ func (uv *userView) HashOnRead(hor bool) { } func (uv *userView) Has(ctx context.Context, k cid.Cid) (bool, error) { - var count int64 - if err := uv.cs.meta. - Model(blockRef{}). - Select("path, block_refs.offset"). - Joins("left join car_shards on block_refs.shard = car_shards.id"). - Where("usr = ? AND cid = ?", uv.user, models.DbCID{CID: k}). - Count(&count).Error; err != nil { - return false, err - } - - return count > 0, nil + return uv.cs.meta.HasUidCid(ctx, uv.user, k) } var CacheHits int64 @@ -197,30 +129,16 @@ func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, erro } atomic.AddInt64(&CacheMiss, 1) - // TODO: for now, im using a join to ensure we only query blocks from the - // correct user. maybe it makes sense to put the user in the blockRef - // directly? tradeoff of time vs space - var info struct { - Path string - Offset int64 - Usr models.Uid - } - if err := uv.cs.meta.Raw(`SELECT - (select path from car_shards where id = block_refs.shard) as path, - block_refs.offset, - (select usr from car_shards where id = block_refs.shard) as usr -FROM block_refs -WHERE - block_refs.cid = ? -LIMIT 1;`, models.DbCID{CID: k}).Scan(&info).Error; err != nil { + path, offset, user, err := uv.cs.meta.LookupBlockRef(ctx, k) + if err != nil { return nil, err } - if info.Path == "" { + if path == "" { return nil, ipld.ErrNotFound{Cid: k} } prefetch := uv.prefetch - if info.Usr != uv.user { + if user != uv.user { blockGetTotalCounterUsrskip.Add(1) prefetch = false } else { @@ -228,9 +146,9 @@ LIMIT 1;`, models.DbCID{CID: k}).Scan(&info).Error; err != nil { } if prefetch { - return uv.prefetchRead(ctx, k, info.Path, info.Offset) + return uv.prefetchRead(ctx, k, path, offset) } else { - return uv.singleRead(ctx, k, info.Path, info.Offset) + return uv.singleRead(ctx, k, path, offset) } } @@ -351,11 +269,11 @@ type DeltaSession struct { baseCid cid.Cid seq int readonly bool - cs *CarStore + cs *FileCarStore lastRev string } -func (cs *CarStore) checkLastShardCache(user models.Uid) *CarShard { +func (cs *FileCarStore) checkLastShardCache(user models.Uid) *CarShard { cs.lscLk.Lock() defer cs.lscLk.Unlock() @@ -367,21 +285,21 @@ func (cs *CarStore) checkLastShardCache(user models.Uid) *CarShard { return nil } -func (cs *CarStore) removeLastShardCache(user models.Uid) { +func (cs *FileCarStore) removeLastShardCache(user models.Uid) { cs.lscLk.Lock() defer cs.lscLk.Unlock() delete(cs.lastShardCache, user) } -func (cs *CarStore) putLastShardCache(ls *CarShard) { +func (cs *FileCarStore) putLastShardCache(ls *CarShard) { cs.lscLk.Lock() defer cs.lscLk.Unlock() cs.lastShardCache[ls.Usr] = ls } -func (cs *CarStore) getLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { +func (cs *FileCarStore) getLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") defer span.End() @@ -390,23 +308,18 @@ func (cs *CarStore) getLastShard(ctx context.Context, user models.Uid) (*CarShar return maybeLs, nil } - var lastShard CarShard - // this is often slow (which is why we're caching it) but could be sped up with an extra index: - // CREATE INDEX idx_car_shards_usr_id ON car_shards (usr, seq DESC); - if err := cs.meta.WithContext(ctx).Model(CarShard{}).Limit(1).Order("seq desc").Find(&lastShard, "usr = ?", user).Error; err != nil { - //if err := cs.meta.Model(CarShard{}).Where("user = ?", user).Last(&lastShard).Error; err != nil { - //if err != gorm.ErrRecordNotFound { + lastShard, err := cs.meta.GetLastShard(ctx, user) + if err != nil { return nil, err - //} } - cs.putLastShardCache(&lastShard) - return &lastShard, nil + cs.putLastShardCache(lastShard) + return lastShard, nil } var ErrRepoBaseMismatch = fmt.Errorf("attempted a delta session on top of the wrong previous head") -func (cs *CarStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { +func (cs *FileCarStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") defer span.End() @@ -438,7 +351,7 @@ func (cs *CarStore) NewDeltaSession(ctx context.Context, user models.Uid, since }, nil } -func (cs *CarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { +func (cs *FileCarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { return &DeltaSession{ base: &userView{ user: user, @@ -452,24 +365,27 @@ func (cs *CarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { }, nil } -func (cs *CarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { +// TODO: incremental is only ever called true, remove the param +func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() var earlySeq int if sinceRev != "" { - var untilShard CarShard - if err := cs.meta.Where("rev >= ? AND usr = ?", sinceRev, user).Order("rev").First(&untilShard).Error; err != nil { - return fmt.Errorf("finding early shard: %w", err) + var err error + earlySeq, err = cs.meta.SeqForRev(ctx, user, sinceRev) + if err != nil { + return err } - earlySeq = untilShard.Seq } - var shards []CarShard - if err := cs.meta.Order("seq desc").Where("usr = ? AND seq >= ?", user, earlySeq).Find(&shards).Error; err != nil { + // TODO: Why does ReadUserCar want shards seq DESC but CompactUserShards wants seq ASC ? + shards, err := cs.meta.GetUserShardsDesc(ctx, user, earlySeq) + if err != nil { return err } + // TODO: incremental is only ever called true, so this is fine and we can remove the error check if !incremental && earlySeq > 0 { // have to do it the ugly way return fmt.Errorf("nyi") @@ -496,7 +412,9 @@ func (cs *CarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev s return nil } -func (cs *CarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Writer) error { +// inner loop part of ReadUserCar +// copy shard blocks from disk to Writer +func (cs *FileCarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "writeShardBlocks") defer span.End() @@ -519,32 +437,8 @@ func (cs *CarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Wri return nil } -func (cs *CarStore) writeBlockFromShard(ctx context.Context, sh *CarShard, w io.Writer, c cid.Cid) error { - fi, err := os.Open(sh.Path) - if err != nil { - return err - } - defer fi.Close() - - rr, err := car.NewCarReader(fi) - if err != nil { - return err - } - - for { - blk, err := rr.Next() - if err != nil { - return err - } - - if blk.Cid() == c { - _, err := LdWrite(w, c.Bytes(), blk.RawData()) - return err - } - } -} - -func (cs *CarStore) iterateShardBlocks(ctx context.Context, sh *CarShard, cb func(blk blockformat.Block) error) error { +// inner loop part of compactBucket +func (cs *FileCarStore) iterateShardBlocks(ctx context.Context, sh *CarShard, cb func(blk blockformat.Block) error) error { fi, err := os.Open(sh.Path) if err != nil { return err @@ -647,7 +541,7 @@ func (ds *DeltaSession) GetSize(ctx context.Context, c cid.Cid) (int, error) { func fnameForShard(user models.Uid, seq int) string { return fmt.Sprintf("sh-%d-%d", user, seq) } -func (cs *CarStore) openNewShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { +func (cs *FileCarStore) openNewShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { // TODO: some overwrite protections fname := filepath.Join(cs.rootDir, fnameForShard(user, seq)) fi, err := os.Create(fname) @@ -658,7 +552,7 @@ func (cs *CarStore) openNewShardFile(ctx context.Context, user models.Uid, seq i return fi, fname, nil } -func (cs *CarStore) writeNewShardFile(ctx context.Context, user models.Uid, seq int, data []byte) (string, error) { +func (cs *FileCarStore) writeNewShardFile(ctx context.Context, user models.Uid, seq int, data []byte) (string, error) { _, span := otel.Tracer("carstore").Start(ctx, "writeNewShardFile") defer span.End() @@ -671,7 +565,7 @@ func (cs *CarStore) writeNewShardFile(ctx context.Context, user models.Uid, seq return fname, nil } -func (cs *CarStore) deleteShardFile(ctx context.Context, sh *CarShard) error { +func (cs *FileCarStore) deleteShardFile(ctx context.Context, sh *CarShard) error { return os.Remove(sh.Path) } @@ -706,7 +600,7 @@ func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) { return hnw, nil } -func (cs *CarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { +func (cs *FileCarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { buf := new(bytes.Buffer) hnw, err := WriteCarHeader(buf, root) @@ -765,127 +659,22 @@ func (cs *CarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, return buf.Bytes(), nil } -func (cs *CarStore) putShard(ctx context.Context, shard *CarShard, brefs []map[string]any, rmcids map[cid.Cid]bool, nocache bool) error { +func (cs *FileCarStore) putShard(ctx context.Context, shard *CarShard, brefs []map[string]any, rmcids map[cid.Cid]bool, nocache bool) error { ctx, span := otel.Tracer("carstore").Start(ctx, "putShard") defer span.End() - // TODO: there should be a way to create the shard and block_refs that - // reference it in the same query, would save a lot of time - tx := cs.meta.WithContext(ctx).Begin() - - if err := tx.WithContext(ctx).Create(shard).Error; err != nil { - return fmt.Errorf("failed to create shard in DB tx: %w", err) - } - - if !nocache { - cs.putLastShardCache(shard) - } - - for _, ref := range brefs { - ref["shard"] = shard.ID - } - - if err := createBlockRefs(ctx, tx, brefs); err != nil { - return fmt.Errorf("failed to create block refs: %w", err) - } - - if len(rmcids) > 0 { - cids := make([]cid.Cid, 0, len(rmcids)) - for c := range rmcids { - cids = append(cids, c) - } - - if err := tx.Create(&staleRef{ - Cids: packCids(cids), - Usr: shard.Usr, - }).Error; err != nil { - return err - } - } - - err := tx.WithContext(ctx).Commit().Error + err := cs.meta.PutShardAndRefs(ctx, shard, brefs, rmcids) if err != nil { - return fmt.Errorf("failed to commit shard DB transaction: %w", err) - } - - return nil -} - -func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) error { - ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs") - defer span.End() - - if err := createInBatches(ctx, tx, brefs, 2000); err != nil { return err } - return nil -} - -func generateInsertQuery(data []map[string]any) (string, []any) { - placeholders := strings.Repeat("(?, ?, ?),", len(data)) - placeholders = placeholders[:len(placeholders)-1] // trim trailing comma - - query := "INSERT INTO block_refs (\"cid\", \"offset\", \"shard\") VALUES " + placeholders - - values := make([]any, 0, 3*len(data)) - for _, entry := range data { - values = append(values, entry["cid"], entry["offset"], entry["shard"]) + if !nocache { + cs.putLastShardCache(shard) } - return query, values -} - -// Function to create in batches -func createInBatches(ctx context.Context, tx *gorm.DB, data []map[string]any, batchSize int) error { - for i := 0; i < len(data); i += batchSize { - batch := data[i:] - if len(batch) > batchSize { - batch = batch[:batchSize] - } - - query, values := generateInsertQuery(batch) - - if err := tx.WithContext(ctx).Exec(query, values...).Error; err != nil { - return err - } - } return nil } -func LdWrite(w io.Writer, d ...[]byte) (int64, error) { - var sum uint64 - for _, s := range d { - sum += uint64(len(s)) - } - - buf := make([]byte, 8) - n := binary.PutUvarint(buf, sum) - nw, err := w.Write(buf[:n]) - if err != nil { - return 0, err - } - - for _, s := range d { - onw, err := w.Write(s) - if err != nil { - return int64(nw), err - } - nw += onw - } - - return int64(nw), nil -} - -func setToSlice(s map[cid.Cid]bool) []cid.Cid { - out := make([]cid.Cid, 0, len(s)) - for c := range s { - out = append(out, c) - } - - return out -} - func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block, skipcids map[cid.Cid]bool) (map[cid.Cid]bool, error) { ctx, span := otel.Tracer("repo").Start(ctx, "BlockDiff") defer span.End() @@ -950,7 +739,7 @@ func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, n return dropset, nil } -func (cs *CarStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { +func (cs *FileCarStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") defer span.End() @@ -998,7 +787,7 @@ func (ds *DeltaSession) CalcDiff(ctx context.Context, skipcids map[cid.Cid]bool) return nil } -func (cs *CarStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { +func (cs *FileCarStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { lastShard, err := cs.getLastShard(ctx, user) if err != nil { return cid.Undef, err @@ -1010,7 +799,7 @@ func (cs *CarStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.C return lastShard.Root.CID, nil } -func (cs *CarStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { +func (cs *FileCarStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { lastShard, err := cs.getLastShard(ctx, user) if err != nil { return "", err @@ -1028,9 +817,9 @@ type UserStat struct { Created time.Time } -func (cs *CarStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { - var shards []CarShard - if err := cs.meta.Order("seq asc").Find(&shards, "usr = ?", usr).Error; err != nil { +func (cs *FileCarStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + shards, err := cs.meta.GetUserShards(ctx, usr) + if err != nil { return nil, err } @@ -1046,9 +835,9 @@ func (cs *CarStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error return out, nil } -func (cs *CarStore) WipeUserData(ctx context.Context, user models.Uid) error { - var shards []*CarShard - if err := cs.meta.Find(&shards, "usr = ?", user).Error; err != nil { +func (cs *FileCarStore) WipeUserData(ctx context.Context, user models.Uid) error { + shards, err := cs.meta.GetUserShards(ctx, user) + if err != nil { return err } @@ -1063,32 +852,23 @@ func (cs *CarStore) WipeUserData(ctx context.Context, user models.Uid) error { return nil } -func (cs *CarStore) deleteShards(ctx context.Context, shs []*CarShard) error { +func (cs *FileCarStore) deleteShards(ctx context.Context, shs []CarShard) error { ctx, span := otel.Tracer("carstore").Start(ctx, "deleteShards") defer span.End() - deleteSlice := func(ctx context.Context, subs []*CarShard) error { - var ids []uint - for _, sh := range subs { - ids = append(ids, sh.ID) - } - - txn := cs.meta.Begin() - - if err := txn.Delete(&CarShard{}, "id in (?)", ids).Error; err != nil { - return err - } - - if err := txn.Delete(&blockRef{}, "shard in (?)", ids).Error; err != nil { - return err + deleteSlice := func(ctx context.Context, subs []CarShard) error { + ids := make([]uint, len(subs)) + for i, sh := range subs { + ids[i] = sh.ID } - if err := txn.Commit().Error; err != nil { + err := cs.meta.DeleteShardsAndRefs(ctx, ids) + if err != nil { return err } for _, sh := range subs { - if err := cs.deleteShardFile(ctx, sh); err != nil { + if err := cs.deleteShardFile(ctx, &sh); err != nil { if !os.IsNotExist(err) { return err } @@ -1198,8 +978,10 @@ func (cb *compBucket) isEmpty() bool { return len(cb.shards) == 0 } -func (cs *CarStore) openNewCompactedShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { +func (cs *FileCarStore) openNewCompactedShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { // TODO: some overwrite protections + // NOTE CreateTemp is used for creating a non-colliding file, but we keep it and don't delete it so don't think of it as "temporary". + // This creates "sh-%d-%d%s" with some random stuff in the last position fi, err := os.CreateTemp(cs.rootDir, fnameForShard(user, seq)) if err != nil { return nil, "", err @@ -1213,35 +995,23 @@ type CompactionTarget struct { NumShards int } -func (cs *CarStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { +func (cs *FileCarStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "GetCompactionTargets") defer span.End() - var targets []CompactionTarget - if err := cs.meta.Raw(`select usr, count(*) as num_shards from car_shards group by usr having count(*) > ? order by num_shards desc`, shardCount).Scan(&targets).Error; err != nil { - return nil, err - } - - return targets, nil + return cs.meta.GetCompactionTargets(ctx, shardCount) } -func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { +// getBlockRefsForShards is a prep function for CompactUserShards +func (cs *FileCarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "getBlockRefsForShards") defer span.End() span.SetAttributes(attribute.Int("shards", len(shardIds))) - chunkSize := 2000 - out := make([]blockRef, 0, len(shardIds)) - for i := 0; i < len(shardIds); i += chunkSize { - sl := shardIds[i:] - if len(sl) > chunkSize { - sl = sl[:chunkSize] - } - - if err := blockRefsForShards(ctx, cs.meta, sl, &out); err != nil { - return nil, fmt.Errorf("getting block refs: %w", err) - } + out, err := cs.meta.GetBlockRefsForShards(ctx, shardIds) + if err != nil { + return nil, err } span.SetAttributes(attribute.Int("refs", len(out))) @@ -1249,31 +1019,6 @@ func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) return out, nil } -func valuesStatementForShards(shards []uint) string { - sb := new(strings.Builder) - for i, v := range shards { - sb.WriteByte('(') - sb.WriteString(strconv.Itoa(int(v))) - sb.WriteByte(')') - if i != len(shards)-1 { - sb.WriteByte(',') - } - } - return sb.String() -} - -func blockRefsForShards(ctx context.Context, db *gorm.DB, shards []uint, obuf *[]blockRef) error { - // Check the database driver - switch db.Dialector.(type) { - case *postgres.Dialector: - sval := valuesStatementForShards(shards) - q := fmt.Sprintf(`SELECT block_refs.* FROM block_refs INNER JOIN (VALUES %s) AS vals(v) ON block_refs.shard = v`, sval) - return db.Raw(q).Scan(obuf).Error - default: - return db.Raw(`SELECT * FROM block_refs WHERE shard IN (?)`, shards).Scan(obuf).Error - } -} - func shardSize(sh *CarShard) (int64, error) { st, err := os.Stat(sh.Path) if err != nil { @@ -1296,21 +1041,17 @@ type CompactionStats struct { DupeCount int `json:"dupeCount"` } -func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { +func (cs *FileCarStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "CompactUserShards") defer span.End() span.SetAttributes(attribute.Int64("user", int64(user))) - var shards []CarShard - if err := cs.meta.WithContext(ctx).Find(&shards, "usr = ?", user).Error; err != nil { + shards, err := cs.meta.GetUserShards(ctx, user) + if err != nil { return nil, err } - sort.Slice(shards, func(i, j int) bool { - return shards[i].Seq < shards[j].Seq - }) - if skipBigShards { // Since we generally expect shards to start bigger and get smaller, // and because we want to avoid compacting non-adjacent shards @@ -1356,8 +1097,8 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip span.SetAttributes(attribute.Int("blockRefs", len(brefs))) - var staleRefs []staleRef - if err := cs.meta.WithContext(ctx).Find(&staleRefs, "usr = ?", user).Error; err != nil { + staleRefs, err := cs.meta.GetUserStaleRefs(ctx, user) + if err != nil { return nil, err } @@ -1486,7 +1227,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip stats.NewShards++ - var todelete []*CarShard + todelete := make([]CarShard, 0, len(b.shards)) for _, s := range b.shards { removedShards[s.ID] = true sh, ok := shardsById[s.ID] @@ -1494,7 +1235,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip return nil, fmt.Errorf("missing shard to delete") } - todelete = append(todelete, &sh) + todelete = append(todelete, sh) } stats.ShardsDeleted += len(todelete) @@ -1514,7 +1255,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip return stats, nil } -func (cs *CarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) error { +func (cs *FileCarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) error { ctx, span := otel.Tracer("carstore").Start(ctx, "deleteStaleRefs") defer span.End() @@ -1546,30 +1287,10 @@ func (cs *CarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs [ } } - txn := cs.meta.Begin() - - if err := txn.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { - return err - } - - // now create a new staleRef with all the refs we couldn't clear out - if len(staleToKeep) > 0 { - if err := txn.Create(&staleRef{ - Usr: uid, - Cids: packCids(staleToKeep), - }).Error; err != nil { - return err - } - } - - if err := txn.Commit().Error; err != nil { - return fmt.Errorf("failed to commit staleRef updates: %w", err) - } - - return nil + return cs.meta.SetStaleRef(ctx, uid, staleToKeep) } -func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compBucket, shardsById map[uint]CarShard, keep map[cid.Cid]bool) error { +func (cs *FileCarStore) compactBucket(ctx context.Context, user models.Uid, b *compBucket, shardsById map[uint]CarShard, keep map[cid.Cid]bool) error { ctx, span := otel.Tracer("carstore").Start(ctx, "compactBucket") defer span.End() diff --git a/carstore/meta_gorm.go b/carstore/meta_gorm.go new file mode 100644 index 000000000..eb9ff7bbc --- /dev/null +++ b/carstore/meta_gorm.go @@ -0,0 +1,346 @@ +package carstore + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/bluesky-social/indigo/models" + "github.com/ipfs/go-cid" + "go.opentelemetry.io/otel" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +type CarStoreGormMeta struct { + meta *gorm.DB +} + +func (cs *CarStoreGormMeta) Init() error { + if err := cs.meta.AutoMigrate(&CarShard{}, &blockRef{}); err != nil { + return err + } + if err := cs.meta.AutoMigrate(&staleRef{}); err != nil { + return err + } + return nil +} + +// Return true if any known record matches (Uid, Cid) +func (cs *CarStoreGormMeta) HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) { + var count int64 + if err := cs.meta. + Model(blockRef{}). + Select("path, block_refs.offset"). + Joins("left join car_shards on block_refs.shard = car_shards.id"). + Where("usr = ? AND cid = ?", user, models.DbCID{CID: k}). + Count(&count).Error; err != nil { + return false, err + } + + return count > 0, nil +} + +// For some Cid, lookup the block ref. +// Return the path of the file written, the offset within the file, and the user associated with the Cid. +func (cs *CarStoreGormMeta) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) { + // TODO: for now, im using a join to ensure we only query blocks from the + // correct user. maybe it makes sense to put the user in the blockRef + // directly? tradeoff of time vs space + var info struct { + Path string + Offset int64 + Usr models.Uid + } + if err := cs.meta.Raw(`SELECT + (select path from car_shards where id = block_refs.shard) as path, + block_refs.offset, + (select usr from car_shards where id = block_refs.shard) as usr +FROM block_refs +WHERE + block_refs.cid = ? +LIMIT 1;`, models.DbCID{CID: k}).Scan(&info).Error; err != nil { + var defaultUser models.Uid + return "", -1, defaultUser, err + } + return info.Path, info.Offset, info.Usr, nil +} + +func (cs *CarStoreGormMeta) GetLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { + var lastShard CarShard + if err := cs.meta.WithContext(ctx).Model(CarShard{}).Limit(1).Order("seq desc").Find(&lastShard, "usr = ?", user).Error; err != nil { + return nil, err + } + return &lastShard, nil +} + +// return all of a users's shards, ascending by Seq +func (cs *CarStoreGormMeta) GetUserShards(ctx context.Context, usr models.Uid) ([]CarShard, error) { + var shards []CarShard + if err := cs.meta.Order("seq asc").Find(&shards, "usr = ?", usr).Error; err != nil { + return nil, err + } + return shards, nil +} + +// return all of a users's shards, descending by Seq +func (cs *CarStoreGormMeta) GetUserShardsDesc(ctx context.Context, usr models.Uid, minSeq int) ([]CarShard, error) { + var shards []CarShard + if err := cs.meta.Order("seq desc").Find(&shards, "usr = ? AND seq >= ?", usr, minSeq).Error; err != nil { + return nil, err + } + return shards, nil +} + +func (cs *CarStoreGormMeta) GetUserStaleRefs(ctx context.Context, user models.Uid) ([]staleRef, error) { + var staleRefs []staleRef + if err := cs.meta.WithContext(ctx).Find(&staleRefs, "usr = ?", user).Error; err != nil { + return nil, err + } + return staleRefs, nil +} + +func (cs *CarStoreGormMeta) SeqForRev(ctx context.Context, user models.Uid, sinceRev string) (int, error) { + var untilShard CarShard + if err := cs.meta.Where("rev >= ? AND usr = ?", sinceRev, user).Order("rev").First(&untilShard).Error; err != nil { + return 0, fmt.Errorf("finding early shard: %w", err) + } + return untilShard.Seq, nil +} + +func (cs *CarStoreGormMeta) GetCompactionTargets(ctx context.Context, minShardCount int) ([]CompactionTarget, error) { + var targets []CompactionTarget + if err := cs.meta.Raw(`select usr, count(*) as num_shards from car_shards group by usr having count(*) > ? order by num_shards desc`, minShardCount).Scan(&targets).Error; err != nil { + return nil, err + } + + return targets, nil +} + +func (cs *CarStoreGormMeta) PutShardAndRefs(ctx context.Context, shard *CarShard, brefs []map[string]any, rmcids map[cid.Cid]bool) error { + // TODO: there should be a way to create the shard and block_refs that + // reference it in the same query, would save a lot of time + tx := cs.meta.WithContext(ctx).Begin() + + if err := tx.WithContext(ctx).Create(shard).Error; err != nil { + return fmt.Errorf("failed to create shard in DB tx: %w", err) + } + + for _, ref := range brefs { + ref["shard"] = shard.ID + } + + if err := createBlockRefs(ctx, tx, brefs); err != nil { + return fmt.Errorf("failed to create block refs: %w", err) + } + + if len(rmcids) > 0 { + cids := make([]cid.Cid, 0, len(rmcids)) + for c := range rmcids { + cids = append(cids, c) + } + + if err := tx.Create(&staleRef{ + Cids: packCids(cids), + Usr: shard.Usr, + }).Error; err != nil { + return err + } + } + + err := tx.WithContext(ctx).Commit().Error + if err != nil { + return fmt.Errorf("failed to commit shard DB transaction: %w", err) + } + return nil +} + +func (cs *CarStoreGormMeta) DeleteShardsAndRefs(ctx context.Context, ids []uint) error { + txn := cs.meta.Begin() + + if err := txn.Delete(&CarShard{}, "id in (?)", ids).Error; err != nil { + txn.Rollback() + return err + } + + if err := txn.Delete(&blockRef{}, "shard in (?)", ids).Error; err != nil { + txn.Rollback() + return err + } + + return txn.Commit().Error +} + +func (cs *CarStoreGormMeta) GetBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { + chunkSize := 2000 + out := make([]blockRef, 0, len(shardIds)) + for i := 0; i < len(shardIds); i += chunkSize { + sl := shardIds[i:] + if len(sl) > chunkSize { + sl = sl[:chunkSize] + } + + if err := blockRefsForShards(ctx, cs.meta, sl, &out); err != nil { + return nil, fmt.Errorf("getting block refs: %w", err) + } + } + return out, nil +} + +// blockRefsForShards is an inner loop helper for GetBlockRefsForShards +func blockRefsForShards(ctx context.Context, db *gorm.DB, shards []uint, obuf *[]blockRef) error { + // Check the database driver + switch db.Dialector.(type) { + case *postgres.Dialector: + sval := valuesStatementForShards(shards) + q := fmt.Sprintf(`SELECT block_refs.* FROM block_refs INNER JOIN (VALUES %s) AS vals(v) ON block_refs.shard = v`, sval) + return db.Raw(q).Scan(obuf).Error + default: + return db.Raw(`SELECT * FROM block_refs WHERE shard IN (?)`, shards).Scan(obuf).Error + } +} + +// valuesStatementForShards builds a postgres compatible statement string from int literals +func valuesStatementForShards(shards []uint) string { + sb := new(strings.Builder) + for i, v := range shards { + sb.WriteByte('(') + sb.WriteString(strconv.Itoa(int(v))) + sb.WriteByte(')') + if i != len(shards)-1 { + sb.WriteByte(',') + } + } + return sb.String() +} + +func (cs *CarStoreGormMeta) SetStaleRef(ctx context.Context, uid models.Uid, staleToKeep []cid.Cid) error { + txn := cs.meta.Begin() + + if err := txn.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { + return err + } + + // now create a new staleRef with all the refs we couldn't clear out + if len(staleToKeep) > 0 { + if err := txn.Create(&staleRef{ + Usr: uid, + Cids: packCids(staleToKeep), + }).Error; err != nil { + return err + } + } + + if err := txn.Commit().Error; err != nil { + return fmt.Errorf("failed to commit staleRef updates: %w", err) + } + return nil +} + +type CarShard struct { + ID uint `gorm:"primarykey"` + CreatedAt time.Time + + Root models.DbCID `gorm:"index"` + DataStart int64 + Seq int `gorm:"index:idx_car_shards_seq;index:idx_car_shards_usr_seq,priority:2,sort:desc"` + Path string + Usr models.Uid `gorm:"index:idx_car_shards_usr;index:idx_car_shards_usr_seq,priority:1"` + Rev string +} + +type blockRef struct { + ID uint `gorm:"primarykey"` + Cid models.DbCID `gorm:"index"` + Shard uint `gorm:"index"` + Offset int64 + //User uint `gorm:"index"` +} + +type staleRef struct { + ID uint `gorm:"primarykey"` + Cid *models.DbCID + Cids []byte + Usr models.Uid `gorm:"index"` +} + +func (sr *staleRef) getCids() ([]cid.Cid, error) { + if sr.Cid != nil { + return []cid.Cid{sr.Cid.CID}, nil + } + + return unpackCids(sr.Cids) +} + +func unpackCids(b []byte) ([]cid.Cid, error) { + br := bytes.NewReader(b) + var out []cid.Cid + for { + _, c, err := cid.CidFromReader(br) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + out = append(out, c) + } + + return out, nil +} + +func packCids(cids []cid.Cid) []byte { + buf := new(bytes.Buffer) + for _, c := range cids { + buf.Write(c.Bytes()) + } + + return buf.Bytes() +} + +func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs") + defer span.End() + + if err := createInBatches(ctx, tx, brefs, 2000); err != nil { + return err + } + + return nil +} + +// Function to create in batches +func createInBatches(ctx context.Context, tx *gorm.DB, brefs []map[string]any, batchSize int) error { + for i := 0; i < len(brefs); i += batchSize { + batch := brefs[i:] + if len(batch) > batchSize { + batch = batch[:batchSize] + } + + query, values := generateInsertQuery(batch) + + if err := tx.WithContext(ctx).Exec(query, values...).Error; err != nil { + return err + } + } + return nil +} + +func generateInsertQuery(brefs []map[string]any) (string, []any) { + placeholders := strings.Repeat("(?, ?, ?),", len(brefs)) + placeholders = placeholders[:len(placeholders)-1] // trim trailing comma + + query := "INSERT INTO block_refs (\"cid\", \"offset\", \"shard\") VALUES " + placeholders + + values := make([]any, 0, 3*len(brefs)) + for _, entry := range brefs { + values = append(values, entry["cid"], entry["offset"], entry["shard"]) + } + + return query, values +} diff --git a/carstore/repo_test.go b/carstore/repo_test.go index 084f16f36..a4d2c8cb8 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -24,7 +24,7 @@ import ( "gorm.io/gorm" ) -func testCarStore() (*CarStore, func(), error) { +func testCarStore() (CarStore, func(), error) { tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { return nil, nil, err @@ -250,7 +250,7 @@ func TestRepeatedCompactions(t *testing.T) { checkRepo(t, cs, buf, recs) } -func checkRepo(t *testing.T, cs *CarStore, r io.Reader, expRecs []cid.Cid) { +func checkRepo(t *testing.T, cs CarStore, r io.Reader, expRecs []cid.Cid) { t.Helper() rep, err := repo.ReadRepoFromCar(context.TODO(), r) if err != nil { diff --git a/carstore/util.go b/carstore/util.go new file mode 100644 index 000000000..56396501f --- /dev/null +++ b/carstore/util.go @@ -0,0 +1,32 @@ +package carstore + +import ( + "encoding/binary" + "io" +) + +// Length-delimited Write +// Writer stream gets Uvarint length then concatenated data +func LdWrite(w io.Writer, d ...[]byte) (int64, error) { + var sum uint64 + for _, s := range d { + sum += uint64(len(s)) + } + + buf := make([]byte, 8) + n := binary.PutUvarint(buf, sum) + nw, err := w.Write(buf[:n]) + if err != nil { + return 0, err + } + + for _, s := range d { + onw, err := w.Write(s) + if err != nil { + return int64(nw), err + } + nw += onw + } + + return int64(nw), nil +} diff --git a/cmd/athome/handlers.go b/cmd/athome/handlers.go index 6c13d9b0d..229cf1eb3 100644 --- a/cmd/athome/handlers.go +++ b/cmd/athome/handlers.go @@ -85,7 +85,7 @@ func (srv *Server) WebProfile(c echo.Context) error { did := pv.Did data["did"] = did - af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "posts_no_replies", 100) + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "posts_no_replies", false, 100) if err != nil { slog.Warn("failed to fetch author feed", "handle", handle, "err", err) // TODO: show some error? @@ -126,7 +126,7 @@ func (srv *Server) WebRepoRSS(c echo.Context) error { //return err } - af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "posts_no_replies", 30) + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "posts_no_replies", false, 30) if err != nil { slog.Warn("failed to fetch author feed", "handle", handle, "err", err) return err diff --git a/cmd/beemo/notify_reports.go b/cmd/beemo/notify_reports.go index 5404619bc..7593bd87c 100644 --- a/cmd/beemo/notify_reports.go +++ b/cmd/beemo/notify_reports.go @@ -68,27 +68,28 @@ func pollNewReports(cctx *cli.Context) error { xrpcc.Auth.RefreshJwt = refresh.RefreshJwt // query just new reports (regardless of resolution state) - // ModerationQueryEvents(ctx context.Context, c *xrpc.Client, createdBy string, cursor string, includeAllUserRecords bool, limit int64, sortDirection string, subject string, types []string) (*ModerationQueryEvents_Output, error) var limit int64 = 50 me, err := toolsozone.ModerationQueryEvents( cctx.Context, xrpcc, - nil, - nil, - "", - "", - "", - "", - "", - false, - true, - limit, - nil, - nil, - nil, - "", - "", - []string{"tools.ozone.moderation.defs#modEventReport"}, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + true, // includeAllUserRecords bool + limit, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string ) if err != nil { return err diff --git a/cmd/bigsky/README.md b/cmd/bigsky/README.md index 25df32ccf..73218e3c5 100644 --- a/cmd/bigsky/README.md +++ b/cmd/bigsky/README.md @@ -150,3 +150,190 @@ Lastly, can monitor progress of any ongoing re-syncs: # check sync progress for all hosts cat hosts.txt | parallel -j1 ./sync_pds.sh {} + + +## Admin API + +The relay has a number of admin HTTP API endpoints. Given a relay setup listening on port 2470 and with a reasonably secure admin secret: + +``` +RELAY_ADMIN_PASSWORD=$(openssl rand --hex 16) +bigsky --api-listen :2470 --admin-key ${RELAY_ADMIN_PASSWORD} ... +``` + +One can, for example, begin compaction of all repos + +``` +curl -H 'Authorization: Bearer '${RELAY_ADMIN_PASSWORD} -H 'Content-Type: application/x-www-form-urlencoded' --data '' http://127.0.0.1:2470/admin/repo/compactAll +``` + +### /admin/subs/getUpstreamConns + +Return list of PDS host names in json array of strings: ["host", ...] + +### /admin/subs/perDayLimit + +Return `{"limit": int}` for the number of new PDS subscriptions that the relay may start in a rolling 24 hour window. + +### /admin/subs/setPerDayLimit + +POST with `?limit={int}` to set the number of new PDS subscriptions that the relay may start in a rolling 24 hour window. + +### /admin/subs/setEnabled + +POST with param `?enabled=true` or `?enabled=false` to enable or disable PDS-requested new-PDS crawling. + +### /admin/subs/getEnabled + +Return `{"enabled": bool}` if non-admin new PDS crawl requests are enabled + +### /admin/subs/killUpstream + +POST with `?host={pds host name}` to disconnect from their firehose. + +Optionally add `&block=true` to prevent connecting to them in the future. + +### /admin/subs/listDomainBans + +Return `{"banned_domains": ["host name", ...]}` + +### /admin/subs/banDomain + +POST `{"Domain": "host name"}` to ban a domain + +### /admin/subs/unbanDomain + +POST `{"Domain": "host name"}` to un-ban a domain + +### /admin/repo/takeDown + +POST `{"did": "did:..."}` to take-down a bad repo; deletes all local data for the repo + +### /admin/repo/reverseTakedown + +POST `?did={did:...}` to reverse a repo take-down + +### /admin/repo/compact + +POST `?did={did:...}` to compact a repo. Optionally `&fast=true`. HTTP blocks until the compaction finishes. + +### /admin/repo/compactAll + +POST to begin compaction of all repos. Optional query params: + + * `fast=true` + * `limit={int}` maximum number of repos to compact (biggest first) (default 50) + * `threhsold={int}` minimum number of shard files a repo must have on disk to merit compaction (default 20) + +### /admin/repo/reset + +POST `?did={did:...}` deletes all local data for the repo + +### /admin/repo/verify + +POST `?did={did:...}` checks that all repo data is accessible. HTTP blocks until done. + +### /admin/pds/requestCrawl + +POST `{"hostname":"pds host"}` to start crawling a PDS + +### /admin/pds/list + +GET returns JSON list of records +```json +[{ + "Host": string, + "Did": string, + "SSL": bool, + "Cursor": int, + "Registered": bool, + "Blocked": bool, + "RateLimit": float, + "CrawlRateLimit": float, + "RepoCount": int, + "RepoLimit": int, + "HourlyEventLimit": int, + "DailyEventLimit": int, + + "HasActiveConnection": bool, + "EventsSeenSinceStartup": int, + "PerSecondEventRate": {"Max": float, "Window": float seconds}, + "PerHourEventRate": {"Max": float, "Window": float seconds}, + "PerDayEventRate": {"Max": float, "Window": float seconds}, + "CrawlRate": {"Max": float, "Window": float seconds}, + "UserCount": int, +}, ...] +``` + +### /admin/pds/resync + +POST `?host={host}` to start a resync of a PDS + +GET `?host={host}` to get status of a PDS resync, return + +```json +{"resync": { + "pds": { + "Host": string, + "Did": string, + "SSL": bool, + "Cursor": int, + "Registered": bool, + "Blocked": bool, + "RateLimit": float, + "CrawlRateLimit": float, + "RepoCount": int, + "RepoLimit": int, + "HourlyEventLimit": int, + "DailyEventLimit": int, + }, + "numRepoPages": int, + "numRepos": int, + "numReposChecked": int, + "numReposToResync": int, + "status": string, + "statusChangedAt": time, +}} +``` + +### /admin/pds/changeLimits + +POST to set the limits for a PDS. body: + +```json +{ + "host": string, + "per_second": int, + "per_hour": int, + "per_day": int, + "crawl_rate": int, + "repo_limit": int, +} +``` + +### /admin/pds/block + +POST `?host={host}` to block a PDS + +### /admin/pds/unblock + +POST `?host={host}` to un-block a PDS + + +### /admin/pds/addTrustedDomain + +POST `?domain={}` to make a domain trusted + +### /admin/consumers/list + +GET returns list json of clients currently reading from the relay firehose + +```json +[{ + "id": int, + "remote_addr": string, + "user_agent": string, + "events_consumed": int, + "connected_at": time, +}, ...] +``` diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 5fb9f73da..540796f51 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -85,7 +85,7 @@ func run(args []string) error { Name: "data-dir", Usage: "path of directory for CAR files and other data", Value: "data/bigsky", - EnvVars: []string{"DATA_DIR"}, + EnvVars: []string{"RELAY_DATA_DIR", "DATA_DIR"}, }, &cli.StringFlag{ Name: "plc-host", @@ -112,8 +112,9 @@ func run(args []string) error { EnvVars: []string{"RELAY_METRICS_LISTEN", "BGS_METRICS_LISTEN"}, }, &cli.StringFlag{ - Name: "disk-persister-dir", - Usage: "set directory for disk persister (implicitly enables disk persister)", + Name: "disk-persister-dir", + Usage: "set directory for disk persister (implicitly enables disk persister)", + EnvVars: []string{"RELAY_PERSISTER_DIR"}, }, &cli.StringFlag{ Name: "admin-key", @@ -188,6 +189,17 @@ func run(args []string) error { EnvVars: []string{"RELAY_DID_CACHE_SIZE"}, Value: 5_000_000, }, + &cli.DurationFlag{ + Name: "event-playback-ttl", + Usage: "time to live for event playback buffering (only applies to disk persister)", + EnvVars: []string{"RELAY_EVENT_PLAYBACK_TTL"}, + Value: 72 * time.Hour, + }, + &cli.IntFlag{ + Name: "num-compaction-workers", + EnvVars: []string{"RELAY_NUM_COMPACTION_WORKERS"}, + Value: 2, + }, } app.Action = runBigsky @@ -327,7 +339,10 @@ func runBigsky(cctx *cli.Context) error { if dpd := cctx.String("disk-persister-dir"); dpd != "" { log.Infow("setting up disk persister") - dp, err := events.NewDiskPersistence(dpd, "", db, events.DefaultDiskPersistOptions()) + + pOpts := events.DefaultDiskPersistOptions() + pOpts.Retention = cctx.Duration("event-playback-ttl") + dp, err := events.NewDiskPersistence(dpd, "", db, pOpts) if err != nil { return fmt.Errorf("setting up disk persister: %w", err) } @@ -403,6 +418,7 @@ func runBigsky(cctx *cli.Context) error { bgsConfig.ConcurrencyPerPDS = cctx.Int64("concurrency-per-pds") bgsConfig.MaxQueuePerPDS = cctx.Int64("max-queue-per-pds") bgsConfig.DefaultRepoLimit = cctx.Int64("default-repo-limit") + bgsConfig.NumCompactionWorkers = cctx.Int("num-compaction-workers") bgs, err := libbgs.NewBGS(db, ix, repoman, evtman, cachedidr, rf, hr, bgsConfig) if err != nil { return err diff --git a/cmd/goat/account.go b/cmd/goat/account.go index 10952f9f1..347ebc02f 100644 --- a/cmd/goat/account.go +++ b/cmd/goat/account.go @@ -2,7 +2,10 @@ package main import ( "context" + "encoding/json" "fmt" + "strings" + "time" comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/syntax" @@ -16,11 +19,6 @@ var cmdAccount = &cli.Command{ Usage: "sub-commands for auth and account management", Flags: []cli.Flag{}, Subcommands: []*cli.Command{ - &cli.Command{ - Name: "check", - Usage: "verifies current auth session is functional", - Action: runAccountCheck, - }, &cli.Command{ Name: "login", Usage: "create session with PDS instance", @@ -39,6 +37,11 @@ var cmdAccount = &cli.Command{ Usage: "password (app password recommended)", EnvVars: []string{"ATP_AUTH_PASSWORD"}, }, + &cli.StringFlag{ + Name: "pds-host", + Usage: "URL of the PDS to create account on (overrides DID doc)", + EnvVars: []string{"ATP_PDS_HOST"}, + }, }, Action: runAccountLogin, }, @@ -48,30 +51,110 @@ var cmdAccount = &cli.Command{ Action: runAccountLogout, }, &cli.Command{ - Name: "status", - Usage: "show account status at PDS", + Name: "activate", + Usage: "(re)activate current account", + Action: runAccountActivate, + }, + &cli.Command{ + Name: "deactivate", + Usage: "deactivate current account", + Action: runAccountDeactivate, + }, + &cli.Command{ + Name: "lookup", + Usage: "show basic account hosting status for any account", ArgsUsage: ``, - Action: runAccountStatus, + Action: runAccountLookup, + }, + &cli.Command{ + Name: "update-handle", + Usage: "change handle for current account", + ArgsUsage: ``, + Action: runAccountUpdateHandle, + }, + &cli.Command{ + Name: "status", + Usage: "show current account status at PDS", + Action: runAccountStatus, + }, + &cli.Command{ + Name: "missing-blobs", + Usage: "list any missing blobs for current account", + Action: runAccountMissingBlobs, + }, + &cli.Command{ + Name: "service-auth", + Usage: "create service auth token", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "endpoint", + Aliases: []string{"lxm"}, + Usage: "restrict token to API endpoint (NSID, optional)", + }, + &cli.StringFlag{ + Name: "audience", + Aliases: []string{"aud"}, + Required: true, + Usage: "DID of service that will receive and validate token", + }, + &cli.IntFlag{ + Name: "duration-sec", + Value: 60, + Usage: "validity time window of token (seconds)", + }, + }, + Action: runAccountServiceAuth, + }, + &cli.Command{ + Name: "create", + Usage: "create a new account on the indicated PDS host", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pds-host", + Usage: "URL of the PDS to create account on", + Required: true, + EnvVars: []string{"ATP_PDS_HOST"}, + }, + &cli.StringFlag{ + Name: "handle", + Usage: "handle for new account", + Required: true, + EnvVars: []string{"ATP_AUTH_HANDLE"}, + }, + &cli.StringFlag{ + Name: "password", + Usage: "initial account password", + Required: true, + EnvVars: []string{"ATP_AUTH_PASSWORD"}, + }, + &cli.StringFlag{ + Name: "invite-code", + Usage: "invite code for account signup", + }, + &cli.StringFlag{ + Name: "email", + Usage: "email address for new account", + }, + &cli.StringFlag{ + Name: "existing-did", + Usage: "an existing DID to use (eg, non-PLC DID, or migration)", + }, + &cli.StringFlag{ + Name: "recovery-key", + Usage: "public cryptographic key (did:key) to add as PLC recovery", + }, + &cli.StringFlag{ + Name: "service-auth", + Usage: "service auth token (for account migration)", + }, + }, + Action: runAccountCreate, }, + cmdAccountMigrate, + cmdAccountPlc, }, } -func runAccountCheck(cctx *cli.Context) error { - ctx := context.Background() - - client, err := loadAuthClient(ctx) - if err == ErrNoAuthSession { - return fmt.Errorf("auth required, but not logged in") - } else if err != nil { - return err - } - // TODO: more explicit check? - fmt.Printf("DID: %s\n", client.Auth.Did) - fmt.Printf("PDS: %s\n", client.Host) - - return nil -} - func runAccountLogin(cctx *cli.Context) error { ctx := context.Background() @@ -80,7 +163,7 @@ func runAccountLogin(cctx *cli.Context) error { return err } - _, err = refreshAuthSession(ctx, *username, cctx.String("app-password")) + _, err = refreshAuthSession(ctx, *username, cctx.String("app-password"), cctx.String("pds-host")) return err } @@ -88,7 +171,7 @@ func runAccountLogout(cctx *cli.Context) error { return wipeAuthSession() } -func runAccountStatus(cctx *cli.Context) error { +func runAccountLookup(cctx *cli.Context) error { ctx := context.Background() username := cctx.Args().First() if username == "" { @@ -122,3 +205,227 @@ func runAccountStatus(cctx *cli.Context) error { } return nil } + +func runAccountStatus(cctx *cli.Context) error { + ctx := context.Background() + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + status, err := comatproto.ServerCheckAccountStatus(ctx, client) + if err != nil { + return fmt.Errorf("failed checking account status: %w", err) + } + + b, err := json.MarshalIndent(status, "", " ") + if err != nil { + return err + } + fmt.Printf("DID: %s\n", client.Auth.Did) + fmt.Printf("Host: %s\n", client.Host) + fmt.Println(string(b)) + + return nil +} + +func runAccountMissingBlobs(cctx *cli.Context) error { + ctx := context.Background() + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + cursor := "" + for { + resp, err := comatproto.RepoListMissingBlobs(ctx, client, cursor, 500) + if err != nil { + return err + } + for _, missing := range resp.Blobs { + fmt.Printf("%s\t%s\n", missing.Cid, missing.RecordUri) + } + if resp.Cursor != nil && *resp.Cursor != "" { + cursor = *resp.Cursor + } else { + break + } + } + return nil +} + +func runAccountActivate(cctx *cli.Context) error { + ctx := context.Background() + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + err = comatproto.ServerActivateAccount(ctx, client) + if err != nil { + return fmt.Errorf("failed activating account: %w", err) + } + + return nil +} + +func runAccountDeactivate(cctx *cli.Context) error { + ctx := context.Background() + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + err = comatproto.ServerDeactivateAccount(ctx, client, &comatproto.ServerDeactivateAccount_Input{}) + if err != nil { + return fmt.Errorf("failed deactivating account: %w", err) + } + + return nil +} + +func runAccountUpdateHandle(cctx *cli.Context) error { + ctx := context.Background() + + raw := cctx.Args().First() + if raw == "" { + return fmt.Errorf("need to provide new handle as argument") + } + handle, err := syntax.ParseHandle(raw) + if err != nil { + return err + } + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + err = comatproto.IdentityUpdateHandle(ctx, client, &comatproto.IdentityUpdateHandle_Input{ + Handle: handle.String(), + }) + if err != nil { + return fmt.Errorf("failed updating handle: %w", err) + } + + return nil +} + +func runAccountServiceAuth(cctx *cli.Context) error { + ctx := context.Background() + + client, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + lxm := cctx.String("endpoint") + if lxm != "" { + _, err := syntax.ParseNSID(lxm) + if err != nil { + return fmt.Errorf("lxm argument must be a valid NSID: %w", err) + } + } + + aud := cctx.String("audience") + // TODO: can aud DID have a fragment? + _, err = syntax.ParseDID(aud) + if err != nil { + return fmt.Errorf("aud argument must be a valid DID: %w", err) + } + + durSec := cctx.Int("duration-sec") + expTimestamp := time.Now().Unix() + int64(durSec) + + resp, err := comatproto.ServerGetServiceAuth(ctx, client, aud, expTimestamp, lxm) + if err != nil { + return fmt.Errorf("failed updating handle: %w", err) + } + + fmt.Println(resp.Token) + + return nil +} + +func runAccountCreate(cctx *cli.Context) error { + ctx := context.Background() + + // validate args + pdsHost := cctx.String("pds-host") + if !strings.Contains(pdsHost, "://") { + return fmt.Errorf("PDS host is not a url: %s", pdsHost) + } + handle := cctx.String("handle") + _, err := syntax.ParseHandle(handle) + if err != nil { + return err + } + password := cctx.String("password") + params := &comatproto.ServerCreateAccount_Input{ + Handle: handle, + Password: &password, + } + raw := cctx.String("existing-did") + if raw != "" { + _, err := syntax.ParseDID(raw) + if err != nil { + return err + } + s := raw + params.Did = &s + } + raw = cctx.String("email") + if raw != "" { + s := raw + params.Email = &s + } + raw = cctx.String("invite-code") + if raw != "" { + s := raw + params.InviteCode = &s + } + raw = cctx.String("recovery-key") + if raw != "" { + s := raw + params.RecoveryKey = &s + } + + // create a new API client to connect to the account's PDS + xrpcc := xrpc.Client{ + Host: pdsHost, + } + + raw = cctx.String("service-auth") + if raw != "" && params.Did != nil { + xrpcc.Auth = &xrpc.AuthInfo{ + Did: *params.Did, + AccessJwt: raw, + } + } + + resp, err := comatproto.ServerCreateAccount(ctx, &xrpcc, params) + if err != nil { + return fmt.Errorf("failed to create account: %w", err) + } + + fmt.Println("Success!") + fmt.Printf("DID: %s\n", resp.Did) + fmt.Printf("Handle: %s\n", resp.Handle) + return nil +} diff --git a/cmd/goat/account_migrate.go b/cmd/goat/account_migrate.go new file mode 100644 index 000000000..fa3d60535 --- /dev/null +++ b/cmd/goat/account_migrate.go @@ -0,0 +1,260 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/urfave/cli/v2" +) + +var cmdAccountMigrate = &cli.Command{ + Name: "migrate", + Usage: "move account to a new PDS. requires full auth.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pds-host", + Usage: "URL of the new PDS to create account on", + Required: true, + EnvVars: []string{"ATP_PDS_HOST"}, + }, + &cli.StringFlag{ + Name: "new-handle", + Required: true, + Usage: "handle on new PDS", + EnvVars: []string{"NEW_ACCOUNT_HANDLE"}, + }, + &cli.StringFlag{ + Name: "new-password", + Required: true, + Usage: "password on new PDS", + EnvVars: []string{"NEW_ACCOUNT_PASSWORD"}, + }, + &cli.StringFlag{ + Name: "plc-token", + Required: true, + Usage: "token from old PDS authorizing token signature", + EnvVars: []string{"PLC_SIGN_TOKEN"}, + }, + &cli.StringFlag{ + Name: "invite-code", + Usage: "invite code for account signup", + }, + &cli.StringFlag{ + Name: "new-email", + Usage: "email address for new account", + }, + }, + Action: runAccountMigrate, +} + +func runAccountMigrate(cctx *cli.Context) error { + // NOTE: this could check rev / commit before and after and ensure last-minute content additions get lost + ctx := context.Background() + + oldClient, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + did := oldClient.Auth.Did + + newHostURL := cctx.String("pds-host") + if !strings.Contains(newHostURL, "://") { + return fmt.Errorf("PDS host is not a url: %s", newHostURL) + } + newHandle := cctx.String("new-handle") + _, err = syntax.ParseHandle(newHandle) + if err != nil { + return err + } + newPassword := cctx.String("new-password") + plcToken := cctx.String("plc-token") + inviteCode := cctx.String("invite-code") + newEmail := cctx.String("new-email") + + newClient := xrpc.Client{ + Host: newHostURL, + } + + // connect to new host to discover service DID + newHostDesc, err := comatproto.ServerDescribeServer(ctx, &newClient) + if err != nil { + return fmt.Errorf("failed connecting to new host: %w", err) + } + newHostDID, err := syntax.ParseDID(newHostDesc.Did) + if err != nil { + return err + } + slog.Info("new host", "serviceDID", newHostDID, "url", newHostURL) + + // 1. Create New Account + slog.Info("creating account on new host", "handle", newHandle, "host", newHostURL) + + // get service auth token from old host + // args: (ctx, client, aud string, exp int64, lxm string) + expTimestamp := time.Now().Unix() + 60 + createAuthResp, err := comatproto.ServerGetServiceAuth(ctx, oldClient, newHostDID.String(), expTimestamp, "com.atproto.server.createAccount") + if err != nil { + return fmt.Errorf("failed getting service auth token from old host: %w", err) + } + + // then create the new account + createParams := comatproto.ServerCreateAccount_Input{ + Did: &did, + Handle: newHandle, + Password: &newPassword, + } + if newEmail != "" { + createParams.Email = &newEmail + } + if inviteCode != "" { + createParams.InviteCode = &inviteCode + } + + // use service auth for access token, temporarily + newClient.Auth = &xrpc.AuthInfo{ + Did: did, + Handle: newHandle, + AccessJwt: createAuthResp.Token, + RefreshJwt: createAuthResp.Token, + } + createAccountResp, err := comatproto.ServerCreateAccount(ctx, &newClient, &createParams) + if err != nil { + return fmt.Errorf("failed creating new account: %w", err) + } + + if createAccountResp.Did != did { + return fmt.Errorf("new account DID not a match: %s != %s", createAccountResp.Did, did) + } + newClient.Auth.AccessJwt = createAccountResp.AccessJwt + newClient.Auth.RefreshJwt = createAccountResp.RefreshJwt + + // login client on the new host + sess, err := comatproto.ServerCreateSession(ctx, &newClient, &comatproto.ServerCreateSession_Input{ + Identifier: did, + Password: newPassword, + }) + if err != nil { + return fmt.Errorf("failed login to newly created account on new host: %w", err) + } + newClient.Auth = &xrpc.AuthInfo{ + Did: did, + AccessJwt: sess.AccessJwt, + RefreshJwt: sess.RefreshJwt, + } + + // 2. Migrate Data + slog.Info("migrating repo") + repoBytes, err := comatproto.SyncGetRepo(ctx, oldClient, did, "") + if err != nil { + return fmt.Errorf("failed exporting repo: %w", err) + } + err = comatproto.RepoImportRepo(ctx, &newClient, bytes.NewReader(repoBytes)) + if err != nil { + return fmt.Errorf("failed importing repo: %w", err) + } + + slog.Info("migrating preferences") + // TODO: service proxy header for AppView? + prefResp, err := ActorGetPreferences(ctx, oldClient) + if err != nil { + return fmt.Errorf("failed fetching old preferences: %w", err) + } + err = ActorPutPreferences(ctx, &newClient, &ActorPutPreferences_Input{ + Preferences: prefResp.Preferences, + }) + if err != nil { + return fmt.Errorf("failed importing preferences: %w", err) + } + + slog.Info("migrating blobs") + blobCursor := "" + for { + listResp, err := comatproto.SyncListBlobs(ctx, oldClient, blobCursor, did, 100, "") + if err != nil { + return fmt.Errorf("failed listing blobs: %w", err) + } + for _, blobCID := range listResp.Cids { + blobBytes, err := comatproto.SyncGetBlob(ctx, oldClient, blobCID, did) + if err != nil { + slog.Warn("failed downloading blob", "cid", blobCID, "err", err) + continue + } + _, err = comatproto.RepoUploadBlob(ctx, &newClient, bytes.NewReader(blobBytes)) + if err != nil { + slog.Warn("failed uploading blob", "cid", blobCID, "err", err, "size", len(blobBytes)) + } + slog.Info("transferred blob", "cid", blobCID, "size", len(blobBytes)) + } + if listResp.Cursor == nil || *listResp.Cursor == "" { + break + } + blobCursor = *listResp.Cursor + } + + // display migration status + // NOTE: this could check between the old PDS and new PDS, polling in a loop showing progress until all records have been indexed + statusResp, err := comatproto.ServerCheckAccountStatus(ctx, &newClient) + if err != nil { + return fmt.Errorf("failed checking account status: %w", err) + } + slog.Info("account migration status", "status", statusResp) + + // 3. Migrate Identity + // NOTE: to work with did:web or non-PDS-managed did:plc, need to do manual migraiton process + slog.Info("updating identity to new host") + + credsResp, err := IdentityGetRecommendedDidCredentials(ctx, &newClient) + if err != nil { + return fmt.Errorf("failed fetching new credentials: %w", err) + } + credsBytes, err := json.Marshal(credsResp) + if err != nil { + return nil + } + + var unsignedOp IdentitySignPlcOperation_Input + if err = json.Unmarshal(credsBytes, &unsignedOp); err != nil { + return fmt.Errorf("failed parsing PLC op: %w", err) + } + unsignedOp.Token = &plcToken + + // NOTE: could add additional sanity checks here that any extra rotation keys were retained, and that old alsoKnownAs and service entries are retained? The stakes aren't super high for the later, as PLC has the full history. PLC and the new PDS already implement some basic sanity checks. + + signedPlcOpResp, err := IdentitySignPlcOperation(ctx, oldClient, &unsignedOp) + if err != nil { + return fmt.Errorf("failed requesting PLC operation signature: %w", err) + } + + err = IdentitySubmitPlcOperation(ctx, &newClient, &IdentitySubmitPlcOperation_Input{ + Operation: signedPlcOpResp.Operation, + }) + if err != nil { + return fmt.Errorf("failed submitting PLC operation: %w", err) + } + + // 4. Finalize Migration + slog.Info("activating new account") + + err = comatproto.ServerActivateAccount(ctx, &newClient) + if err != nil { + return fmt.Errorf("failed activating new host: %w", err) + } + err = comatproto.ServerDeactivateAccount(ctx, oldClient, &comatproto.ServerDeactivateAccount_Input{}) + if err != nil { + return fmt.Errorf("failed deactivating old host: %w", err) + } + + slog.Info("account migration completed") + return nil +} diff --git a/cmd/goat/account_plc.go b/cmd/goat/account_plc.go new file mode 100644 index 000000000..a0dde653b --- /dev/null +++ b/cmd/goat/account_plc.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + + "github.com/urfave/cli/v2" +) + +var cmdAccountPlc = &cli.Command{ + Name: "plc", + Usage: "sub-commands for managing PLC DID via PDS host", + Subcommands: []*cli.Command{ + &cli.Command{ + Name: "recommended", + Usage: "list recommended DID fields for current account", + Action: runAccountPlcRecommended, + }, + &cli.Command{ + Name: "request-token", + Usage: "request a 2FA token (by email) for signing op", + Action: runAccountPlcRequestToken, + }, + &cli.Command{ + Name: "sign", + Usage: "sign a PLC operation", + ArgsUsage: ``, + Action: runAccountPlcSign, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "token", + Usage: "2FA token for signing request", + }, + }, + }, + &cli.Command{ + Name: "submit", + Usage: "submit a PLC operation (via PDS)", + ArgsUsage: ``, + Action: runAccountPlcSubmit, + }, + }, +} + +func runAccountPlcRecommended(cctx *cli.Context) error { + ctx := context.Background() + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + resp, err := IdentityGetRecommendedDidCredentials(ctx, xrpcc) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil +} + +func runAccountPlcRequestToken(cctx *cli.Context) error { + ctx := context.Background() + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + err = comatproto.IdentityRequestPlcOperationSignature(ctx, xrpcc) + if err != nil { + return err + } + + fmt.Println("Success; check email for token.") + return nil +} + +func runAccountPlcSign(cctx *cli.Context) error { + ctx := context.Background() + + opPath := cctx.Args().First() + if opPath == "" { + return fmt.Errorf("need to provide JSON file path as an argument") + } + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + fileBytes, err := os.ReadFile(opPath) + if err != nil { + return err + } + + var body IdentitySignPlcOperation_Input + if err = json.Unmarshal(fileBytes, &body); err != nil { + return fmt.Errorf("failed decoding PLC op JSON: %w", err) + } + + token := cctx.String("token") + if token != "" { + body.Token = &token + } + + resp, err := IdentitySignPlcOperation(ctx, xrpcc, &body) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp.Operation, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil +} + +func runAccountPlcSubmit(cctx *cli.Context) error { + ctx := context.Background() + + opPath := cctx.Args().First() + if opPath == "" { + return fmt.Errorf("need to provide JSON file path as an argument") + } + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + fileBytes, err := os.ReadFile(opPath) + if err != nil { + return err + } + + var op json.RawMessage + if err = json.Unmarshal(fileBytes, &op); err != nil { + return fmt.Errorf("failed decoding PLC op JSON: %w", err) + } + + err = IdentitySubmitPlcOperation(ctx, xrpcc, &IdentitySubmitPlcOperation_Input{ + Operation: &op, + }) + if err != nil { + return fmt.Errorf("failed submitting PLC op via PDS: %w", err) + } + + return nil +} diff --git a/cmd/goat/actorgetPreferences.go b/cmd/goat/actorgetPreferences.go new file mode 100644 index 000000000..bd6e8a18c --- /dev/null +++ b/cmd/goat/actorgetPreferences.go @@ -0,0 +1,28 @@ +// Copied from indigo:api/atproto/actorgetPreferences.go + +package main + +// schema: app.bsky.actor.getPreferences + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ActorGetPreferences_Output is the output of a app.bsky.actor.getPreferences call. +type ActorGetPreferences_Output struct { + Preferences []map[string]any `json:"preferences" cborgen:"preferences"` +} + +// ActorGetPreferences calls the XRPC method "app.bsky.actor.getPreferences". +func ActorGetPreferences(ctx context.Context, c *xrpc.Client) (*ActorGetPreferences_Output, error) { + var out ActorGetPreferences_Output + + params := map[string]interface{}{} + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.getPreferences", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/cmd/goat/actorputPreferences.go b/cmd/goat/actorputPreferences.go new file mode 100644 index 000000000..24042236d --- /dev/null +++ b/cmd/goat/actorputPreferences.go @@ -0,0 +1,25 @@ +// Copied from indigo:api/atproto/actorputPreferences.go + +package main + +// schema: app.bsky.actor.putPreferences + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ActorPutPreferences_Input is the input argument to a app.bsky.actor.putPreferences call. +type ActorPutPreferences_Input struct { + Preferences []map[string]any `json:"preferences" cborgen:"preferences"` +} + +// ActorPutPreferences calls the XRPC method "app.bsky.actor.putPreferences". +func ActorPutPreferences(ctx context.Context, c *xrpc.Client, input *ActorPutPreferences_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.actor.putPreferences", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/cmd/goat/auth.go b/cmd/goat/auth.go index 460117b83..5eaf2b29e 100644 --- a/cmd/goat/auth.go +++ b/cmd/goat/auth.go @@ -79,7 +79,7 @@ func loadAuthClient(ctx context.Context) (*xrpc.Client, error) { if err != nil { // TODO: if failure, try creating a new session from password fmt.Println("trying to refresh auth from password...") - as, err := refreshAuthSession(ctx, sess.DID.AtIdentifier(), sess.Password) + as, err := refreshAuthSession(ctx, sess.DID.AtIdentifier(), sess.Password, sess.PDS) if err != nil { return nil, err } @@ -96,23 +96,32 @@ func loadAuthClient(ctx context.Context) (*xrpc.Client, error) { return &client, nil } -func refreshAuthSession(ctx context.Context, username syntax.AtIdentifier, password string) (*AuthSession, error) { - dir := identity.DefaultDirectory() - ident, err := dir.Lookup(ctx, username) - if err != nil { - return nil, err - } +func refreshAuthSession(ctx context.Context, username syntax.AtIdentifier, password, pdsURL string) (*AuthSession, error) { - pdsURL := ident.PDSEndpoint() + var did syntax.DID if pdsURL == "" { - return nil, fmt.Errorf("empty PDS URL") + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, username) + if err != nil { + return nil, err + } + + pdsURL = ident.PDSEndpoint() + if pdsURL == "" { + return nil, fmt.Errorf("empty PDS URL") + } + did = ident.DID + } + + if did == "" && username.IsDID() { + did, _ = username.AsDID() } client := xrpc.Client{ Host: pdsURL, } sess, err := comatproto.ServerCreateSession(ctx, &client, &comatproto.ServerCreateSession_Input{ - Identifier: ident.DID.String(), + Identifier: username.String(), Password: password, }) if err != nil { @@ -121,9 +130,18 @@ func refreshAuthSession(ctx context.Context, username syntax.AtIdentifier, passw // TODO: check account status? // TODO: warn if email isn't verified? + // TODO: check that sess.Did matches username + if did == "" { + did, err = syntax.ParseDID(sess.Did) + if err != nil { + return nil, err + } + } else if sess.Did != did.String() { + return nil, fmt.Errorf("session DID didn't match expected: %s != %s", sess.Did, did) + } authSession := AuthSession{ - DID: ident.DID, + DID: did, Password: password, PDS: pdsURL, RefreshToken: sess.RefreshJwt, diff --git a/cmd/goat/bsky.go b/cmd/goat/bsky.go index 607e31075..410c70d0c 100644 --- a/cmd/goat/bsky.go +++ b/cmd/goat/bsky.go @@ -23,6 +23,7 @@ var cmdBsky = &cli.Command{ ArgsUsage: ``, Action: runBskyPost, }, + cmdBskyPrefs, }, } diff --git a/cmd/goat/bsky_prefs.go b/cmd/goat/bsky_prefs.go new file mode 100644 index 000000000..725072344 --- /dev/null +++ b/cmd/goat/bsky_prefs.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/urfave/cli/v2" +) + +var cmdBskyPrefs = &cli.Command{ + Name: "prefs", + Usage: "sub-commands for preferences", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + &cli.Command{ + Name: "export", + Usage: "dump preferences out as JSON", + Action: runBskyPrefsExport, + }, + &cli.Command{ + Name: "import", + Usage: "upload preferences from JSON file", + ArgsUsage: ``, + Action: runBskyPrefsImport, + }, + }, +} + +func runBskyPrefsExport(cctx *cli.Context) error { + ctx := context.Background() + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + // TODO: does indigo API code crash with unsupported preference '$type'? Eg "Lexicon decoder" with unsupported type. + resp, err := ActorGetPreferences(ctx, xrpcc) + if err != nil { + return fmt.Errorf("failed fetching old preferences: %w", err) + } + + b, err := json.MarshalIndent(resp.Preferences, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + + return nil +} + +func runBskyPrefsImport(cctx *cli.Context) error { + ctx := context.Background() + prefsPath := cctx.Args().First() + if prefsPath == "" { + return fmt.Errorf("need to provide file path as an argument") + } + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + prefsBytes, err := os.ReadFile(prefsPath) + if err != nil { + return err + } + + var prefsArray []map[string]any + if err = json.Unmarshal(prefsBytes, &prefsArray); err != nil { + return err + } + + err = ActorPutPreferences(ctx, xrpcc, &ActorPutPreferences_Input{ + Preferences: prefsArray, + }) + if err != nil { + return fmt.Errorf("failed fetching old preferences: %w", err) + } + + return nil +} diff --git a/cmd/goat/identitygetRecommendedDidCredentials.go b/cmd/goat/identitygetRecommendedDidCredentials.go new file mode 100644 index 000000000..75abcc522 --- /dev/null +++ b/cmd/goat/identitygetRecommendedDidCredentials.go @@ -0,0 +1,23 @@ +// Copied from indigo:api/atproto/identitygetRecommendedDidCredentials.go + +package main + +// schema: com.atproto.identity.getRecommendedDidCredentials + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/xrpc" +) + +// IdentityGetRecommendedDidCredentials calls the XRPC method "com.atproto.identity.getRecommendedDidCredentials". +func IdentityGetRecommendedDidCredentials(ctx context.Context, c *xrpc.Client) (*json.RawMessage, error) { + var out json.RawMessage + + if err := c.Do(ctx, xrpc.Query, "", "com.atproto.identity.getRecommendedDidCredentials", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/cmd/goat/identitysignPlcOperation.go b/cmd/goat/identitysignPlcOperation.go new file mode 100644 index 000000000..f9c96bea2 --- /dev/null +++ b/cmd/goat/identitysignPlcOperation.go @@ -0,0 +1,38 @@ +// Copied from indigo:api/atproto/identitysignPlcOperation.go + +package main + +// schema: com.atproto.identity.signPlcOperation + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/xrpc" +) + +// IdentitySignPlcOperation_Input is the input argument to a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Input struct { + AlsoKnownAs []string `json:"alsoKnownAs,omitempty" cborgen:"alsoKnownAs,omitempty"` + RotationKeys []string `json:"rotationKeys,omitempty" cborgen:"rotationKeys,omitempty"` + Services *json.RawMessage `json:"services,omitempty" cborgen:"services,omitempty"` + // token: A token received through com.atproto.identity.requestPlcOperationSignature + Token *string `json:"token,omitempty" cborgen:"token,omitempty"` + VerificationMethods *json.RawMessage `json:"verificationMethods,omitempty" cborgen:"verificationMethods,omitempty"` +} + +// IdentitySignPlcOperation_Output is the output of a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Output struct { + // operation: A signed DID PLC operation. + Operation *json.RawMessage `json:"operation" cborgen:"operation"` +} + +// IdentitySignPlcOperation calls the XRPC method "com.atproto.identity.signPlcOperation". +func IdentitySignPlcOperation(ctx context.Context, c *xrpc.Client, input *IdentitySignPlcOperation_Input) (*IdentitySignPlcOperation_Output, error) { + var out IdentitySignPlcOperation_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.identity.signPlcOperation", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/cmd/goat/identitysubmitPlcOperation.go b/cmd/goat/identitysubmitPlcOperation.go new file mode 100644 index 000000000..9f15e8adc --- /dev/null +++ b/cmd/goat/identitysubmitPlcOperation.go @@ -0,0 +1,26 @@ +// Copied from indigo:api/atproto/identitysubmitPlcOperation.go + +package main + +// schema: com.atproto.identity.submitPlcOperation + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/xrpc" +) + +// IdentitySubmitPlcOperation_Input is the input argument to a com.atproto.identity.submitPlcOperation call. +type IdentitySubmitPlcOperation_Input struct { + Operation *json.RawMessage `json:"operation" cborgen:"operation"` +} + +// IdentitySubmitPlcOperation calls the XRPC method "com.atproto.identity.submitPlcOperation". +func IdentitySubmitPlcOperation(ctx context.Context, c *xrpc.Client, input *IdentitySubmitPlcOperation_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.identity.submitPlcOperation", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/cmd/goat/main.go b/cmd/goat/main.go index 538a783e7..c211b698a 100644 --- a/cmd/goat/main.go +++ b/cmd/goat/main.go @@ -37,6 +37,7 @@ func run(args []string) error { cmdRecord, cmdSyntax, cmdCrypto, + cmdPds, } return app.Run(args) } diff --git a/cmd/goat/pds.go b/cmd/goat/pds.go new file mode 100644 index 000000000..9a2b96324 --- /dev/null +++ b/cmd/goat/pds.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/urfave/cli/v2" +) + +var cmdPds = &cli.Command{ + Name: "pds", + Usage: "sub-commands for pds hosts", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + &cli.Command{ + Name: "describe", + Usage: "shows info about a PDS info", + ArgsUsage: ``, + Action: runPdsDescribe, + }, + }, +} + +func runPdsDescribe(cctx *cli.Context) error { + ctx := context.Background() + + pdsHost := cctx.Args().First() + if pdsHost == "" { + return fmt.Errorf("need to provide new handle as argument") + } + if !strings.Contains(pdsHost, "://") { + return fmt.Errorf("PDS host is not a url: %s", pdsHost) + } + client := xrpc.Client{ + Host: pdsHost, + } + + resp, err := comatproto.ServerDescribeServer(ctx, &client) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + + return nil +} diff --git a/cmd/goat/record.go b/cmd/goat/record.go index aced5a737..013913aa2 100644 --- a/cmd/goat/record.go +++ b/cmd/goat/record.go @@ -10,7 +10,6 @@ import ( "github.com/bluesky-social/indigo/atproto/data" "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" - lexutil "github.com/bluesky-social/indigo/lex/util" "github.com/bluesky-social/indigo/xrpc" "github.com/urfave/cli/v2" @@ -231,9 +230,8 @@ func runRecordCreate(cctx *cli.Context) error { return err } - // TODO: replace this with something that allows arbitrary Lexicons, instead of needing registered types - var recordVal lexutil.LexiconTypeDecoder - if err = recordVal.UnmarshalJSON(recordBytes); err != nil { + recordVal, err := data.UnmarshalJSON(recordBytes) + if err != nil { return err } @@ -248,10 +246,10 @@ func runRecordCreate(cctx *cli.Context) error { } validate := !cctx.Bool("no-validate") - resp, err := comatproto.RepoCreateRecord(ctx, xrpcc, &comatproto.RepoCreateRecord_Input{ + resp, err := RepoCreateRecord(ctx, xrpcc, &RepoCreateRecord_Input{ Collection: nsid, Repo: xrpcc.Auth.Did, - Record: &recordVal, + Record: recordVal, Rkey: rkey, Validate: &validate, }) @@ -300,18 +298,17 @@ func runRecordUpdate(cctx *cli.Context) error { return err } - // TODO: replace this with something that allows arbitrary Lexicons, instead of needing registered types - var recordVal lexutil.LexiconTypeDecoder - if err = recordVal.UnmarshalJSON(recordBytes); err != nil { + recordVal, err := data.UnmarshalJSON(recordBytes) + if err != nil { return err } validate := !cctx.Bool("no-validate") - resp, err := comatproto.RepoPutRecord(ctx, xrpcc, &comatproto.RepoPutRecord_Input{ + resp, err := RepoPutRecord(ctx, xrpcc, &RepoPutRecord_Input{ Collection: nsid, Repo: xrpcc.Auth.Did, - Record: &recordVal, + Record: recordVal, Rkey: rkey, Validate: &validate, SwapRecord: existing.Cid, @@ -343,7 +340,7 @@ func runRecordDelete(cctx *cli.Context) error { return err } - err = comatproto.RepoDeleteRecord(ctx, xrpcc, &comatproto.RepoDeleteRecord_Input{ + _, err = comatproto.RepoDeleteRecord(ctx, xrpcc, &comatproto.RepoDeleteRecord_Input{ Collection: collection.String(), Repo: xrpcc.Auth.Did, Rkey: rkey.String(), diff --git a/cmd/goat/repo.go b/cmd/goat/repo.go index df5f85348..97f2b81a2 100644 --- a/cmd/goat/repo.go +++ b/cmd/goat/repo.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -36,6 +37,12 @@ var cmdRepo = &cli.Command{ }, Action: runRepoExport, }, + &cli.Command{ + Name: "import", + Usage: "upload CAR file for current account", + ArgsUsage: ``, + Action: runRepoImport, + }, &cli.Command{ Name: "ls", Aliases: []string{"list"}, @@ -104,6 +111,34 @@ func runRepoExport(cctx *cli.Context) error { return os.WriteFile(carPath, repoBytes, 0666) } +func runRepoImport(cctx *cli.Context) error { + ctx := context.Background() + + carPath := cctx.Args().First() + if carPath == "" { + return fmt.Errorf("need to provide CAR file path as an argument") + } + + xrpcc, err := loadAuthClient(ctx) + if err == ErrNoAuthSession { + return fmt.Errorf("auth required, but not logged in") + } else if err != nil { + return err + } + + fileBytes, err := os.ReadFile(carPath) + if err != nil { + return err + } + + err = comatproto.RepoImportRepo(ctx, xrpcc, bytes.NewReader(fileBytes)) + if err != nil { + return fmt.Errorf("failed to import repo: %w", err) + } + + return nil +} + func runRepoList(cctx *cli.Context) error { ctx := context.Background() carPath := cctx.Args().First() diff --git a/cmd/goat/repocreateRecord.go b/cmd/goat/repocreateRecord.go new file mode 100644 index 000000000..c1fa67a39 --- /dev/null +++ b/cmd/goat/repocreateRecord.go @@ -0,0 +1,51 @@ +// Copied from indigo:api/atproto/repocreateRecords.go + +package main + +// schema: com.atproto.repo.createRecord + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// RepoDefs_CommitMeta is a "commitMeta" in the com.atproto.repo.defs schema. +type RepoDefs_CommitMeta struct { + Cid string `json:"cid" cborgen:"cid"` + Rev string `json:"rev" cborgen:"rev"` +} + +// RepoCreateRecord_Input is the input argument to a com.atproto.repo.createRecord call. +type RepoCreateRecord_Input struct { + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record itself. Must contain a $type field. + Record map[string]any `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` +} + +// RepoCreateRecord_Output is the output of a com.atproto.repo.createRecord call. +type RepoCreateRecord_Output struct { + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoCreateRecord calls the XRPC method "com.atproto.repo.createRecord". +func RepoCreateRecord(ctx context.Context, c *xrpc.Client, input *RepoCreateRecord_Input) (*RepoCreateRecord_Output, error) { + var out RepoCreateRecord_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.createRecord", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/cmd/goat/repoputRecord.go b/cmd/goat/repoputRecord.go new file mode 100644 index 000000000..34011797b --- /dev/null +++ b/cmd/goat/repoputRecord.go @@ -0,0 +1,47 @@ +// Copied from indigo:api/atproto/repoputRecords.go + +package main + +// schema: com.atproto.repo.putRecord + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// RepoPutRecord_Input is the input argument to a com.atproto.repo.putRecord call. +type RepoPutRecord_Input struct { + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record to write. + Record map[string]any `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey string `json:"rkey" cborgen:"rkey"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // swapRecord: Compare and swap with the previous record by CID. WARNING: nullable and optional field; may cause problems with golang implementation + SwapRecord *string `json:"swapRecord" cborgen:"swapRecord"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` +} + +// RepoPutRecord_Output is the output of a com.atproto.repo.putRecord call. +type RepoPutRecord_Output struct { + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoPutRecord calls the XRPC method "com.atproto.repo.putRecord". +func RepoPutRecord(ctx context.Context, c *xrpc.Client, input *RepoPutRecord_Input) (*RepoPutRecord_Output, error) { + var out RepoPutRecord_Output + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.putRecord", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/cmd/gosky/admin.go b/cmd/gosky/admin.go index 769577e5e..9467975da 100644 --- a/cmd/gosky/admin.go +++ b/cmd/gosky/admin.go @@ -389,26 +389,27 @@ var listReportsCmd = &cli.Command{ xrpcc.AdminToken = &adminKey // fetch recent moderation reports - // AdminQueryModerationEvents(ctx context.Context, c *xrpc.Client, createdBy string, cursor string, includeAllUserRecords bool, limit int64, sortDirection string, subject string, types []string) (*AdminQueryModerationEvents_Output, error) resp, err := toolsozone.ModerationQueryEvents( ctx, xrpcc, - nil, - nil, - "", - "", - "", - "", - "", - false, - false, - 100, - nil, - nil, - nil, - "", - "", - []string{"tools.ozone.moderation.defs#modEventReport"}, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 100, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string ) if err != nil { return err @@ -705,22 +706,24 @@ var queryModerationStatusesCmd = &cli.Command{ resp, err := toolsozone.ModerationQueryEvents( ctx, xrpcc, - nil, - nil, - "", - "", - "", - "", - "", - false, - false, - 100, - nil, - nil, - nil, - "", - "", - []string{"tools.ozone.moderation.defs#modEventReport"}, + nil, // addedLabels []string + nil, // addedTags []string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 100, // limit int64 + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string ) if err != nil { return err diff --git a/cmd/gosky/bsky.go b/cmd/gosky/bsky.go index 6a0769e38..afc7cb9df 100644 --- a/cmd/gosky/bsky.go +++ b/cmd/gosky/bsky.go @@ -178,7 +178,7 @@ var bskyGetFeedCmd = &cli.Command{ author = xrpcc.Auth.Did } - tl, err := appbsky.FeedGetAuthorFeed(ctx, xrpcc, author, "", "", 99) + tl, err := appbsky.FeedGetAuthorFeed(ctx, xrpcc, author, "", "", false, 99) if err != nil { return err } @@ -314,11 +314,12 @@ var bskyDeletePostCmd = &cli.Command{ rkey = parts[1] } - return comatproto.RepoDeleteRecord(context.TODO(), xrpcc, &comatproto.RepoDeleteRecord_Input{ + _, err = comatproto.RepoDeleteRecord(context.TODO(), xrpcc, &comatproto.RepoDeleteRecord_Input{ Repo: xrpcc.Auth.Did, Collection: schema, Rkey: rkey, }) + return err }, } diff --git a/cmd/hepa/consumer_ozone.go b/cmd/hepa/consumer_ozone.go deleted file mode 100644 index 406a34a4c..000000000 --- a/cmd/hepa/consumer_ozone.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "context" - "fmt" - "time" - - toolsozone "github.com/bluesky-social/indigo/api/ozone" - "github.com/bluesky-social/indigo/atproto/syntax" -) - -func (s *Server) RunOzoneConsumer(ctx context.Context) error { - - cur, err := s.ReadLastOzoneCursor(ctx) - if err != nil { - return err - } - - if cur == "" { - cur = syntax.DatetimeNow().String() - } - since, err := syntax.ParseDatetime(cur) - if err != nil { - return err - } - - s.logger.Info("subscribing to ozone event log", "upstream", s.engine.OzoneClient.Host, "cursor", cur, "since", since) - var limit int64 = 50 - period := time.Second * 5 - - for { - //func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []string, addedTags []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, types []string) (*ModerationQueryEvents_Output, error) { - me, err := toolsozone.ModerationQueryEvents( - ctx, - s.engine.OzoneClient, - nil, // addedLabels: If specified, only events where all of these labels were added are returned - nil, // addedTags: If specified, only events where all of these tags were added are returned - "", // comment: If specified, only events with comments containing the keyword are returned - since.String(), // createdAfter: Retrieve events created after a given timestamp - "", // createdBefore: Retrieve events created before a given timestamp - "", // createdBy - "", // cursor - false, // hasComment: If true, only events with comments are returned - true, // includeAllUserRecords: If true, events on all record types (posts, lists, profile etc.) owned by the did are returned - limit, - nil, // removedLabels: If specified, only events where all of these labels were removed are returned - nil, // removedTags - nil, // reportTypes - "asc", // sortDirection: Sort direction for the events. Defaults to descending order of created at timestamp. - "", // subject - nil, // types: The types of events (fully qualified string in the format of tools.ozone.moderation.defs#modEvent) to filter by. If not specified, all events are returned. - ) - if err != nil { - s.logger.Warn("ozone query events failed; sleeping then will retrying", "err", err, "period", period.String()) - time.Sleep(period) - continue - } - - // track if the response contained anything new - anyNewEvents := false - for _, evt := range me.Events { - createdAt, err := syntax.ParseDatetime(evt.CreatedAt) - if err != nil { - return fmt.Errorf("invalid time format for ozone 'createdAt': %w", err) - } - // skip if the timestamp is the exact same - if createdAt == since { - continue - } - anyNewEvents = true - // TODO: is there a race condition here? - if !createdAt.Time().After(since.Time()) { - s.logger.Error("out of order ozone event", "createdAt", createdAt, "since", since) - return fmt.Errorf("out of order ozone event") - } - if err = s.HandleOzoneEvent(ctx, evt); err != nil { - s.logger.Error("failed to process ozone event", "event", evt) - } - since = createdAt - s.lastOzoneCursor.Store(since.String()) - } - if !anyNewEvents { - s.logger.Debug("... ozone poller sleeping", "period", period.String()) - time.Sleep(period) - } - } -} - -func (s *Server) HandleOzoneEvent(ctx context.Context, eventView *toolsozone.ModerationDefs_ModEventView) error { - - s.logger.Debug("received ozone event", "eventID", eventView.Id, "createdAt", eventView.CreatedAt) - - if err := s.engine.ProcessOzoneEvent(ctx, eventView); err != nil { - s.logger.Error("engine failed to process ozone event", "err", err) - } - return nil -} diff --git a/cmd/hepa/main.go b/cmd/hepa/main.go index 79d7b648a..527c64877 100644 --- a/cmd/hepa/main.go +++ b/cmd/hepa/main.go @@ -17,6 +17,7 @@ import ( "github.com/bluesky-social/indigo/atproto/identity/redisdir" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/automod/capture" + "github.com/bluesky-social/indigo/automod/consumer" "github.com/carlmjohnson/versioninfo" _ "github.com/joho/godotenv/autoload" @@ -177,7 +178,7 @@ func configDirectory(cctx *cli.Context) (identity.Directory, error) { } var dir identity.Directory if cctx.String("redis-url") != "" { - rdir, err := redisdir.NewRedisDirectory(&baseDir, cctx.String("redis-url"), time.Hour*24, time.Minute*2, 10_000) + rdir, err := redisdir.NewRedisDirectory(&baseDir, cctx.String("redis-url"), time.Hour*24, time.Minute*2, time.Minute*5, 10_000) if err != nil { return nil, err } @@ -241,7 +242,7 @@ var runCmd = &cli.Command{ dir, Config{ Logger: logger, - RelayHost: cctx.String("atp-relay-host"), + RelayHost: cctx.String("atp-relay-host"), // DEPRECATED BskyHost: cctx.String("atp-bsky-host"), OzoneHost: cctx.String("atp-ozone-host"), OzoneDID: cctx.String("ozone-did"), @@ -256,7 +257,7 @@ var runCmd = &cli.Command{ AbyssPassword: cctx.String("abyss-password"), RatelimitBypass: cctx.String("ratelimit-bypass"), RulesetName: cctx.String("ruleset"), - FirehoseParallelism: cctx.Int("firehose-parallelism"), + FirehoseParallelism: cctx.Int("firehose-parallelism"), // DEPRECATED RerouteEvents: cctx.Bool("reroute-events"), PreScreenHost: cctx.String("prescreen-host"), PreScreenToken: cctx.String("prescreen-token"), @@ -266,6 +267,28 @@ var runCmd = &cli.Command{ return fmt.Errorf("failed to construct server: %v", err) } + // ozone event consumer (if configured) + if srv.Engine.OzoneClient != nil { + oc := consumer.OzoneConsumer{ + Logger: logger.With("subsystem", "ozone-consumer"), + RedisClient: srv.RedisClient, + OzoneClient: srv.Engine.OzoneClient, + Engine: srv.Engine, + } + + go func() { + if err := oc.Run(ctx); err != nil { + slog.Error("ozone consumer failed", "err", err) + } + }() + + go func() { + if err := oc.RunPersistCursor(ctx); err != nil { + slog.Error("ozone cursor routine failed", "err", err) + } + }() + } + // prometheus HTTP endpoint: /metrics go func() { runtime.SetBlockProfileRate(10) @@ -276,31 +299,28 @@ var runCmd = &cli.Command{ } }() - go func() { - if err := srv.RunPersistCursor(ctx); err != nil { - slog.Error("cursor routine failed", "err", err) + // firehose event consumer (note this is actually mandatory) + relayHost := cctx.String("atp-relay-host") + if relayHost != "" { + fc := consumer.FirehoseConsumer{ + Engine: srv.Engine, + Logger: logger.With("subsystem", "firehose-consumer"), + Host: cctx.String("atp-relay-host"), + Parallelism: cctx.Int("firehose-parallelism"), + RedisClient: srv.RedisClient, } - }() - // ozone event consumer (if configured) - if srv.engine.OzoneClient != nil { go func() { - if err := srv.RunOzoneConsumer(ctx); err != nil { - slog.Error("ozone consumer failed", "err", err) + if err := fc.RunPersistCursor(ctx); err != nil { + slog.Error("cursor routine failed", "err", err) } }() - go func() { - if err := srv.RunPersistOzoneCursor(ctx); err != nil { - slog.Error("ozone cursor routine failed", "err", err) - } - }() + if err := fc.Run(ctx); err != nil { + return fmt.Errorf("failure consuming and processing firehose: %w", err) + } } - // firehose event consumer (main processor) - if err := srv.RunConsumer(ctx); err != nil { - return fmt.Errorf("failure consuming and processing firehose: %w", err) - } return nil }, } @@ -362,7 +382,7 @@ var processRecordCmd = &cli.Command{ return err } - return capture.FetchAndProcessRecord(ctx, srv.engine, aturi) + return capture.FetchAndProcessRecord(ctx, srv.Engine, aturi) }, } @@ -393,7 +413,7 @@ var processRecentCmd = &cli.Command{ return err } - return capture.FetchAndProcessRecent(ctx, srv.engine, *atid, cctx.Int("limit")) + return capture.FetchAndProcessRecent(ctx, srv.Engine, *atid, cctx.Int("limit")) }, } @@ -424,7 +444,7 @@ var captureRecentCmd = &cli.Command{ return err } - cap, err := capture.CaptureRecent(ctx, srv.engine, *atid, cctx.Int("limit")) + cap, err := capture.CaptureRecent(ctx, srv.Engine, *atid, cctx.Int("limit")) if err != nil { return err } diff --git a/cmd/hepa/server.go b/cmd/hepa/server.go index 4949684e7..5a602c459 100644 --- a/cmd/hepa/server.go +++ b/cmd/hepa/server.go @@ -7,7 +7,6 @@ import ( "net/http" "os" "strings" - "sync/atomic" "time" "github.com/bluesky-social/indigo/atproto/identity" @@ -27,25 +26,17 @@ import ( ) type Server struct { - relayHost string - firehoseParallelism int - logger *slog.Logger - engine *automod.Engine - rdb *redis.Client - - // lastSeq is the most recent event sequence number we've received and begun to handle. - // This number is periodically persisted to redis, if redis is present. - // The value is best-effort (the stream handling itself is concurrent, so event numbers may not be monotonic), - // but nonetheless, you must use atomics when updating or reading this (to avoid data races). - lastSeq int64 + Engine *automod.Engine + RedisClient *redis.Client - // same as lastSeq, but for Ozone timestamp cursor. the value is a string. - lastOzoneCursor atomic.Value + relayHost string // DEPRECATED + firehoseParallelism int // DEPRECATED + logger *slog.Logger } type Config struct { Logger *slog.Logger - RelayHost string + RelayHost string // DEPRECATED BskyHost string OzoneHost string OzoneDID string @@ -60,7 +51,7 @@ type Config struct { AbyssPassword string RulesetName string RatelimitBypass string - FirehoseParallelism int + FirehoseParallelism int // DEPRECATED RerouteEvents bool PreScreenHost string PreScreenToken string @@ -238,8 +229,8 @@ func NewServer(dir identity.Directory, config Config) (*Server, error) { relayHost: config.RelayHost, firehoseParallelism: config.FirehoseParallelism, logger: logger, - engine: &engine, - rdb: rdb, + Engine: &engine, + RedisClient: rdb, } return s, nil @@ -249,132 +240,3 @@ func (s *Server) RunMetrics(listen string) error { http.Handle("/metrics", promhttp.Handler()) return http.ListenAndServe(listen, nil) } - -var cursorKey = "hepa/seq" -var ozoneCursorKey = "hepa/ozoneTimestamp" - -func (s *Server) ReadLastCursor(ctx context.Context) (int64, error) { - // if redis isn't configured, just skip - if s.rdb == nil { - s.logger.Info("redis not configured, skipping cursor read") - return 0, nil - } - - val, err := s.rdb.Get(ctx, cursorKey).Int64() - if err == redis.Nil { - s.logger.Info("no pre-existing cursor in redis") - return 0, nil - } else if err != nil { - return 0, err - } - s.logger.Info("successfully found prior subscription cursor seq in redis", "seq", val) - return val, nil -} - -func (s *Server) ReadLastOzoneCursor(ctx context.Context) (string, error) { - // if redis isn't configured, just skip - if s.rdb == nil { - s.logger.Info("redis not configured, skipping ozone cursor read") - return "", nil - } - - val, err := s.rdb.Get(ctx, ozoneCursorKey).Result() - if err == redis.Nil || val == "" { - s.logger.Info("no pre-existing ozone cursor in redis") - return "", nil - } else if err != nil { - return "", err - } - s.logger.Info("successfully found prior ozone offset timestamp in redis", "cursor", val) - return val, nil -} - -func (s *Server) PersistCursor(ctx context.Context) error { - // if redis isn't configured, just skip - if s.rdb == nil { - return nil - } - lastSeq := atomic.LoadInt64(&s.lastSeq) - if lastSeq <= 0 { - return nil - } - err := s.rdb.Set(ctx, cursorKey, lastSeq, 14*24*time.Hour).Err() - return err -} - -func (s *Server) PersistOzoneCursor(ctx context.Context) error { - // if redis isn't configured, just skip - if s.rdb == nil { - return nil - } - lastCursor := s.lastOzoneCursor.Load() - if lastCursor == nil || lastCursor == "" { - return nil - } - err := s.rdb.Set(ctx, ozoneCursorKey, lastCursor, 14*24*time.Hour).Err() - return err -} - -// this method runs in a loop, persisting the current cursor state every 5 seconds -func (s *Server) RunPersistCursor(ctx context.Context) error { - - // if redis isn't configured, just skip - if s.rdb == nil { - return nil - } - ticker := time.NewTicker(5 * time.Second) - for { - select { - case <-ctx.Done(): - lastSeq := atomic.LoadInt64(&s.lastSeq) - if lastSeq >= 1 { - s.logger.Info("persisting final cursor seq value", "seq", lastSeq) - err := s.PersistCursor(ctx) - if err != nil { - s.logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) - } - } - return nil - case <-ticker.C: - lastSeq := atomic.LoadInt64(&s.lastSeq) - if lastSeq >= 1 { - err := s.PersistCursor(ctx) - if err != nil { - s.logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) - } - } - } - } -} - -// this method runs in a loop, persisting the current cursor state every 5 seconds -func (s *Server) RunPersistOzoneCursor(ctx context.Context) error { - - // if redis isn't configured, just skip - if s.rdb == nil { - return nil - } - ticker := time.NewTicker(5 * time.Second) - for { - select { - case <-ctx.Done(): - lastCursor := s.lastOzoneCursor.Load() - if lastCursor != nil && lastCursor != "" { - s.logger.Info("persisting final ozone cursor timestamp", "cursor", lastCursor) - err := s.PersistOzoneCursor(ctx) - if err != nil { - s.logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) - } - } - return nil - case <-ticker.C: - lastCursor := s.lastOzoneCursor.Load() - if lastCursor != nil && lastCursor != "" { - err := s.PersistOzoneCursor(ctx) - if err != nil { - s.logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) - } - } - } - } -} diff --git a/cmd/sonar/README.md b/cmd/sonar/README.md new file mode 100644 index 000000000..dbd8ccfda --- /dev/null +++ b/cmd/sonar/README.md @@ -0,0 +1,24 @@ +# Sonar +Sonar is an AT Proto Firehose Montioring tool + +Sonar connects to an AT Proto Firehose (either from a PDS or a Relay) following the semantics of `com.atproto.sync.subscribeRepos`. + +Sonar monitors the throughput of events, producing prometheus metrics on the frequency of different kinds of events. + +Sonar additionally walks through repo operations and tracks the frequency of creation/update/deletion of different record collections. + +Sonar's main use is to provide an operational dashboard of activity on the network, allowing us to view changes in event rate over time and understand what kinds of traffic flow through the firehose over time. + +## Running Sonar + +To run sonar in Docker locally, you can run: `make sonar-up` from the root of the `indigo` directory. + +This will start a sonar instance that connects to the Bluesky-operated Relay firehose at `bsky.network` and will expose metrics at `http://localhost:8345` + +Feel free to modify the `docker-compose.yml` in this directory to change any settings via environment variables i.e. to change the firehose host `SONAR_WS_URL` or the listen port `SONAR_PORT`. + +## Dashboard + +Sonar emits Prometheus metrics which you can scrape and then visualize with the Grafana dashboard (JSON template provided in this directory) shown below: + +![A dashboard for Sonar showing event throughput and distribution](./sonar_dash.png) diff --git a/cmd/sonar/grafana-dashboard.json b/cmd/sonar/grafana-dashboard.json new file mode 100644 index 000000000..2ce8c712e --- /dev/null +++ b/cmd/sonar/grafana-dashboard.json @@ -0,0 +1,1541 @@ +{ + "__inputs": [ + { + "name": "DS_MIMIR", + "label": "Mimir", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.1" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 55, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "orange", + "value": 1200 + }, + { + "color": "red", + "value": 1500 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "repo_commit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "identity" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_tombstone" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "avg(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (event_type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Event Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "orange", + "value": 1200 + }, + { + "color": "red", + "value": 1500 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1 Day Ago" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1 Week Ago" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4 Weeks Ago" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Current", + "1 Day Ago" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval]))", + "legendFormat": "Current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 1d))", + "hide": false, + "instant": false, + "legendFormat": "1 Day Ago", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 1w))", + "hide": false, + "instant": false, + "legendFormat": "1 Week Ago", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 4w))", + "hide": false, + "instant": false, + "legendFormat": "4 Weeks Ago", + "range": true, + "refId": "D" + } + ], + "title": "Total Event Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 55, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "orange", + "value": 1200 + }, + { + "color": "red", + "value": 1500 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "avg(rate(sonar_records_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (record_type)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Record Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 15000 + }, + { + "color": "orange", + "value": 60000 + }, + { + "color": "red", + "value": 300000 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Since Event Creation" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sonar_last_evt_created_evt_processed_gap{box=~\"${box}\",socket_url=~\"${socket}\"}", + "legendFormat": "Since Event Creation", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sonar_last_record_created_evt_processed_gap{box=~\"${box}\",socket_url=~\"${socket}\"}", + "hide": false, + "legendFormat": "Since Record Creation", + "range": true, + "refId": "B" + } + ], + "title": "Firehose Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 9, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "avg(increase(sonar_ops_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__range])) by (kind)", + "format": "time_series", + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A" + } + ], + "title": "Operation Kind Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 7, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "avg(increase(sonar_records_processed_total{box=~\"${box}\",socket_url=~\"${socket}\", action=\"create\"}[$__range])) by (record_type)", + "format": "time_series", + "instant": false, + "legendFormat": "{{record_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Record Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(indigo_repo_stream_bytes_total{box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance) * 8", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Socket Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 26 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "sum(rate(process_cpu_seconds_total{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance) / sum(rate(sonar_events_processed_total{box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "CPU Seconds per Event", + "range": true, + "refId": "A" + } + ], + "title": "CPU per Event", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500000 + }, + { + "color": "#EAB839", + "value": 850000 + }, + { + "color": "red", + "value": 1000000 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 27 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "increase(sonar_ops_processed_total{kind=~\"create\",socket_url=~\"wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos\", op_path=\"app.bsky.actor.profile\"}[$__range])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Profiles Created", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 55, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "orange", + "value": 1200 + }, + { + "color": "red", + "value": 1500 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "editorMode": "code", + "expr": "avg(rate(sonar_ops_processed_total{kind=~\"${op_kind}\", box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (op_path, kind) > 0", + "legendFormat": "{{kind}} {{op_path}}", + "range": true, + "refId": "A" + } + ], + "title": "Operation Throughput", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(.*)(app.bsky.)(.*)", + "renamePattern": "$1 $3" + } + } + ], + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "definition": "label_values(sonar_ops_processed_total,kind)", + "hide": 0, + "includeAll": true, + "label": "Operation Kind", + "multi": true, + "name": "op_kind", + "options": [], + "query": { + "query": "label_values(sonar_ops_processed_total,kind)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "definition": "label_values(indigo_repo_stream_bytes_total{job=\"sonar\"},box)", + "hide": 0, + "includeAll": false, + "label": "Sonar Host", + "multi": false, + "name": "box", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(indigo_repo_stream_bytes_total{job=\"sonar\"},box)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "definition": "label_values(socket_url)", + "hide": 0, + "includeAll": false, + "label": "Socket", + "multi": false, + "name": "socket", + "options": [], + "query": { + "query": "label_values(socket_url)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_MIMIR}" + }, + "definition": "label_values(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"},instance)", + "hide": 2, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Sonar", + "uid": "e5c542d0-07d6-44e9-bc30-a4b973bedd86", + "version": 24, + "weekStart": "" +} diff --git a/cmd/sonar/main.go b/cmd/sonar/main.go index 2eb898f95..a867646fe 100644 --- a/cmd/sonar/main.go +++ b/cmd/sonar/main.go @@ -33,19 +33,22 @@ func main() { app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "ws-url", - Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint", - Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos", + Name: "ws-url", + Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint", + Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos", + EnvVars: []string{"SONAR_WS_URL"}, }, &cli.StringFlag{ - Name: "log-level", - Usage: "log level", - Value: "info", + Name: "log-level", + Usage: "log level", + Value: "info", + EnvVars: []string{"SONAR_LOG_LEVEL"}, }, &cli.IntFlag{ - Name: "port", - Usage: "listen port for metrics server", - Value: 8345, + Name: "port", + Usage: "listen port for metrics server", + Value: 8345, + EnvVars: []string{"SONAR_PORT"}, }, &cli.IntFlag{ Name: "max-queue-size", @@ -53,9 +56,10 @@ func main() { Value: 10, }, &cli.StringFlag{ - Name: "cursor-file", - Usage: "path to cursor file", - Value: "sonar_cursor.json", + Name: "cursor-file", + Usage: "path to cursor file", + Value: "sonar_cursor.json", + EnvVars: []string{"SONAR_CURSOR_FILE"}, }, } diff --git a/cmd/sonar/sonar_dash.png b/cmd/sonar/sonar_dash.png new file mode 100644 index 000000000..b4570cbe7 Binary files /dev/null and b/cmd/sonar/sonar_dash.png differ diff --git a/did/web.go b/did/web.go index 322520fdc..baea42e6c 100644 --- a/did/web.go +++ b/did/web.go @@ -6,18 +6,26 @@ import ( "fmt" "net/http" "strings" + "time" "unicode" "github.com/whyrusleeping/go-did" "go.opentelemetry.io/otel" ) +var webDidDefaultTimeout = 5 * time.Second + type WebResolver struct { Insecure bool // TODO: cache? maybe at a different layer + + client http.Client } func (wr *WebResolver) GetDocument(ctx context.Context, didstr string) (*Document, error) { + if wr.client.Timeout == 0 { + wr.client.Timeout = webDidDefaultTimeout + } ctx, span := otel.Tracer("did").Start(ctx, "didWebGetDocument") defer span.End() @@ -36,7 +44,7 @@ func (wr *WebResolver) GetDocument(ctx context.Context, didstr string) (*Documen proto = "http" } - resp, err := http.Get(proto + "://" + val + "/.well-known/did.json") + resp, err := wr.client.Get(proto + "://" + val + "/.well-known/did.json") if err != nil { return nil, err } diff --git a/events/cbor_gen.go b/events/cbor_gen.go index efb873446..8e13f8339 100644 --- a/events/cbor_gen.go +++ b/events/cbor_gen.go @@ -101,21 +101,24 @@ func (t *EventHeader) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("EventHeader: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 2) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.MsgType (string) (string) case "t": @@ -156,7 +159,9 @@ func (t *EventHeader) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -245,21 +250,24 @@ func (t *ErrorFrame) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("ErrorFrame: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Error (string) (string) case "error": @@ -285,7 +293,9 @@ func (t *ErrorFrame) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/events/dbpersist.go b/events/dbpersist.go index c534057d4..a9d6288f2 100644 --- a/events/dbpersist.go +++ b/events/dbpersist.go @@ -51,7 +51,7 @@ func DefaultOptions() *Options { type DbPersistence struct { db *gorm.DB - cs *carstore.CarStore + cs carstore.CarStore lk sync.Mutex @@ -86,7 +86,7 @@ type RepoEventRecord struct { Ops []byte } -func NewDbPersistence(db *gorm.DB, cs *carstore.CarStore, options *Options) (*DbPersistence, error) { +func NewDbPersistence(db *gorm.DB, cs carstore.CarStore, options *Options) (*DbPersistence, error) { if err := db.AutoMigrate(&RepoEventRecord{}); err != nil { return nil, err } diff --git a/events/dbpersist_test.go b/events/dbpersist_test.go index 6954812df..c299569da 100644 --- a/events/dbpersist_test.go +++ b/events/dbpersist_test.go @@ -268,7 +268,7 @@ func BenchmarkPlayback(b *testing.B) { } } -func setupDBs(t testing.TB) (*gorm.DB, *gorm.DB, *carstore.CarStore, string, error) { +func setupDBs(t testing.TB) (*gorm.DB, *gorm.DB, carstore.CarStore, string, error) { dir, err := os.MkdirTemp("", "integtest") if err != nil { return nil, nil, nil, "", err diff --git a/events/diskpersist.go b/events/diskpersist.go index b793ee843..25eb989af 100644 --- a/events/diskpersist.go +++ b/events/diskpersist.go @@ -81,8 +81,8 @@ type DiskPersistOptions struct { func DefaultDiskPersistOptions() *DiskPersistOptions { return &DiskPersistOptions{ EventsPerFile: 10_000, - UIDCacheSize: 100_000, - DIDCacheSize: 100_000, + UIDCacheSize: 1_000_000, + DIDCacheSize: 1_000_000, WriteBufferSize: 50, Retention: time.Hour * 24 * 3, // 3 days } diff --git a/events/diskpersist_test.go b/events/diskpersist_test.go index baefd379d..5d09c0fc2 100644 --- a/events/diskpersist_test.go +++ b/events/diskpersist_test.go @@ -187,7 +187,7 @@ func BenchmarkDiskPersist(b *testing.B) { } -func runPersisterBenchmark(b *testing.B, cs *carstore.CarStore, db *gorm.DB, p events.EventPersistence) { +func runPersisterBenchmark(b *testing.B, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) { ctx := context.Background() db.AutoMigrate(&pds.User{}) @@ -302,7 +302,7 @@ func TestDiskPersister(t *testing.T) { runEventManagerTest(t, cs, db, dp) } -func runEventManagerTest(t *testing.T, cs *carstore.CarStore, db *gorm.DB, p events.EventPersistence) { +func runEventManagerTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) { ctx := context.Background() db.AutoMigrate(&pds.User{}) @@ -409,7 +409,7 @@ func TestDiskPersisterTakedowns(t *testing.T) { runTakedownTest(t, cs, db, dp) } -func runTakedownTest(t *testing.T, cs *carstore.CarStore, db *gorm.DB, p events.EventPersistence) { +func runTakedownTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) { ctx := context.TODO() db.AutoMigrate(&pds.User{}) diff --git a/fakedata/generators.go b/fakedata/generators.go index 9fd99507d..8b13af173 100644 --- a/fakedata/generators.go +++ b/fakedata/generators.go @@ -405,7 +405,7 @@ func BrowseAccount(xrpcc *xrpc.Client, acc *AccountContext) error { if err != nil { return err } - _, err = appbsky.FeedGetAuthorFeed(context.TODO(), xrpcc, notif.Author.Did, "", "", 50) + _, err = appbsky.FeedGetAuthorFeed(context.TODO(), xrpcc, notif.Author.Did, "", "", false, 50) if err != nil { return err } @@ -447,7 +447,7 @@ func BrowseAccount(xrpcc *xrpc.Client, acc *AccountContext) error { if err != nil { return err } - _, err = appbsky.FeedGetAuthorFeed(context.TODO(), xrpcc, post.Post.Author.Did, "", "", 50) + _, err = appbsky.FeedGetAuthorFeed(context.TODO(), xrpcc, post.Post.Author.Did, "", "", false, 50) if err != nil { return err } diff --git a/go.mod b/go.mod index 4b942517b..66391db61 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/flosch/pongo2/v6 v6.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/goccy/go-json v0.10.2 + github.com/gocql/gocql v1.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-retryablehttp v0.7.5 @@ -53,7 +54,7 @@ require ( github.com/samber/slog-echo v1.8.0 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.25.7 - github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c + github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 @@ -79,6 +80,8 @@ require ( require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-redis/redis v6.15.9+incompatible // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/compress v1.17.3 // indirect @@ -91,6 +94,7 @@ require ( github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + gopkg.in/inf.v0 v0.9.1 // indirect ) require ( diff --git a/go.sum b/go.sum index dfc251448..8cd2edd60 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brianvoe/gofakeit/v6 v6.25.0 h1:ZpFjktOpLZUeF8q223o0rUuXtA+m5qW5srjvVi+JkXk= github.com/brianvoe/gofakeit/v6 v6.25.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -152,6 +156,8 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -189,6 +195,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -231,6 +239,8 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -616,8 +626,8 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSD github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= -github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c h1:UsxJNcLPfyLyVaA4iusIrsLAqJn/xh36Qgb8emqtXzk= -github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e h1:28X54ciEwwUxyHn9yrZfl5ojgF4CBNLWX7LR0rvBkf4= +github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 h1:yJ9/LwIGIk/c0CdoavpC9RNSGSruIspSZtxG3Nnldic= @@ -1061,6 +1071,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/lex/gen.go b/lex/gen.go index af8dab70d..29447bb53 100644 --- a/lex/gen.go +++ b/lex/gen.go @@ -232,9 +232,16 @@ func writeMethods(typename string, ts *TypeSchema, w io.Writer) error { case "record": return nil case "query": - return ts.WriteRPC(w, typename) + return ts.WriteRPC(w, typename, fmt.Sprintf("%s_Input", typename)) case "procedure": - return ts.WriteRPC(w, typename) + if ts.Input == nil || ts.Input.Schema == nil || ts.Input.Schema.Type == "object" { + return ts.WriteRPC(w, typename, fmt.Sprintf("%s_Input", typename)) + } else if ts.Input.Schema.Type == "ref" { + inputname, _ := ts.namesFromRef(ts.Input.Schema.Ref) + return ts.WriteRPC(w, typename, inputname) + } else { + return fmt.Errorf("unhandled input type: %s", ts.Input.Schema.Type) + } case "object", "string": return nil case "subscription": diff --git a/lex/type_schema.go b/lex/type_schema.go index aeeb7389d..fcec3575a 100644 --- a/lex/type_schema.go +++ b/lex/type_schema.go @@ -50,7 +50,7 @@ type TypeSchema struct { Maximum any `json:"maximum"` } -func (s *TypeSchema) WriteRPC(w io.Writer, typename string) error { +func (s *TypeSchema) WriteRPC(w io.Writer, typename, inputname string) error { pf := printerf(w) fname := typename @@ -65,7 +65,7 @@ func (s *TypeSchema) WriteRPC(w io.Writer, typename string) error { case EncodingCBOR, EncodingCAR, EncodingANY, EncodingMP4: params = fmt.Sprintf("%s, input io.Reader", params) case EncodingJSON: - params = fmt.Sprintf("%s, input *%s_Input", params, fname) + params = fmt.Sprintf("%s, input *%s", params, inputname) default: return fmt.Errorf("unsupported input encoding (RPC input): %q", s.Input.Encoding) diff --git a/lex/util/cbor_gen.go b/lex/util/cbor_gen.go index dd057f1fd..84e78775e 100644 --- a/lex/util/cbor_gen.go +++ b/lex/util/cbor_gen.go @@ -78,21 +78,24 @@ func (t *CborChecker) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("CborChecker: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Type (string) (string) case "$type": @@ -107,7 +110,9 @@ func (t *CborChecker) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -196,21 +201,24 @@ func (t *LegacyBlob) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("LegacyBlob: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": @@ -236,7 +244,9 @@ func (t *LegacyBlob) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -359,21 +369,24 @@ func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("BlobSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Ref (util.LexLink) (struct) case "ref": @@ -435,7 +448,9 @@ func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/lex/util/cbor_gen_test.go b/lex/util/cbor_gen_test.go index 76bc90ee3..175f2cb00 100644 --- a/lex/util/cbor_gen_test.go +++ b/lex/util/cbor_gen_test.go @@ -254,21 +254,24 @@ func (t *basicSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("basicSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Bool (bool) (bool) case "bool": @@ -430,7 +433,9 @@ func (t *basicSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -567,21 +572,24 @@ func (t *basicSchemaInner) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("basicSchemaInner: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Arr ([]string) (slice) case "arr": @@ -680,7 +688,9 @@ func (t *basicSchemaInner) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -779,21 +789,24 @@ func (t *ipldSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("ipldSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.A (util.LexLink) (struct) case "a": @@ -840,7 +853,9 @@ func (t *ipldSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1059,21 +1074,24 @@ func (t *basicOldSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("basicOldSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.A (string) (string) case "a": @@ -1224,7 +1242,9 @@ func (t *basicOldSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1361,21 +1381,24 @@ func (t *basicOldSchemaInner) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("basicOldSchemaInner: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.H (string) (string) case "h": @@ -1474,7 +1497,9 @@ func (t *basicOldSchemaInner) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -1558,21 +1583,24 @@ func (t *ipldOldSchema) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("ipldOldSchema: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 8192) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.A (util.LexLink) (struct) case "a": @@ -1608,7 +1636,9 @@ func (t *ipldOldSchema) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/mst/cbor_gen.go b/mst/cbor_gen.go index 19dd2f75d..8f7e7dcc9 100644 --- a/mst/cbor_gen.go +++ b/mst/cbor_gen.go @@ -104,21 +104,24 @@ func (t *nodeData) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("nodeData: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Entries ([]mst.treeEntry) (slice) case "e": @@ -184,7 +187,9 @@ func (t *nodeData) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -312,21 +317,24 @@ func (t *treeEntry) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("treeEntry: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 1) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.KeySuffix ([]uint8) (slice) case "k": @@ -415,7 +423,9 @@ func (t *treeEntry) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/mst/mst.go b/mst/mst.go index ee648add4..23810b01e 100644 --- a/mst/mst.go +++ b/mst/mst.go @@ -113,7 +113,7 @@ func CBORTypes() []reflect.Type { // MST tree node as gets serialized to CBOR. Note that the CBOR fields are all // single-character. type nodeData struct { - Left *cid.Cid `cborgen:"l"` // [optional] pointer to lower-level subtree to the "left" of this path/key + Left *cid.Cid `cborgen:"l"` // [nullable] pointer to lower-level subtree to the "left" of this path/key Entries []treeEntry `cborgen:"e"` // ordered list of entries at this node } @@ -122,7 +122,7 @@ type treeEntry struct { PrefixLen int64 `cborgen:"p"` // count of characters shared with previous path/key in tree KeySuffix []byte `cborgen:"k"` // remaining part of path/key (appended to "previous key") Val cid.Cid `cborgen:"v"` // CID pointer at this path/key - Tree *cid.Cid `cborgen:"t"` // [optional] pointer to lower-level subtree to the "right" of this path/key entry + Tree *cid.Cid `cborgen:"t"` // [nullable] pointer to lower-level subtree to the "right" of this path/key entry } // MerkleSearchTree represents an MST tree node (NodeData type). It can be in diff --git a/pds/handlers_test.go b/pds/handlers_test.go index d83c23fb4..fe2bb14b8 100644 --- a/pds/handlers_test.go +++ b/pds/handlers_test.go @@ -17,7 +17,7 @@ import ( "gorm.io/gorm" ) -func testCarStore(t *testing.T, db *gorm.DB) (*carstore.CarStore, func()) { +func testCarStore(t *testing.T, db *gorm.DB) (carstore.CarStore, func()) { t.Helper() tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { diff --git a/pds/server.go b/pds/server.go index 77c58f024..b9d1c903b 100644 --- a/pds/server.go +++ b/pds/server.go @@ -41,7 +41,7 @@ var log = logging.Logger("pds") type Server struct { db *gorm.DB - cs *carstore.CarStore + cs carstore.CarStore repoman *repomgr.RepoManager feedgen *FeedGenerator notifman notifs.NotificationManager @@ -65,7 +65,7 @@ type Server struct { // NewServer. const serverListenerBootTimeout = 5 * time.Second -func NewServer(db *gorm.DB, cs *carstore.CarStore, serkey *did.PrivKey, handleSuffix, serviceUrl string, didr plc.PLCClient, jwtkey []byte) (*Server, error) { +func NewServer(db *gorm.DB, cs carstore.CarStore, serkey *did.PrivKey, handleSuffix, serviceUrl string, didr plc.PLCClient, jwtkey []byte) (*Server, error) { db.AutoMigrate(&User{}) db.AutoMigrate(&Peering{}) diff --git a/repo/cbor_gen.go b/repo/cbor_gen.go index 96bfb2e0d..02594508b 100644 --- a/repo/cbor_gen.go +++ b/repo/cbor_gen.go @@ -194,21 +194,24 @@ func (t *SignedCommit) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("SignedCommit: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -319,7 +322,9 @@ func (t *SignedCommit) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -477,21 +482,24 @@ func (t *UnsignedCommit) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("UnsignedCommit: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 7) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Did (string) (string) case "did": @@ -579,7 +587,9 @@ func (t *UnsignedCommit) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/repomgr/ingest_test.go b/repomgr/ingest_test.go index 4296cd949..dcb9097ac 100644 --- a/repomgr/ingest_test.go +++ b/repomgr/ingest_test.go @@ -69,7 +69,7 @@ func TestLoadNewRepo(t *testing.T) { } } -func testCarstore(t *testing.T, dir string) *carstore.CarStore { +func testCarstore(t *testing.T, dir string) carstore.CarStore { cardb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "car.sqlite"))) if err != nil { t.Fatal(err) @@ -151,7 +151,7 @@ func TestIngestWithGap(t *testing.T) { } } -func doPost(t *testing.T, cs *carstore.CarStore, did string, prev *string, postid int) ([]byte, cid.Cid, string, string) { +func doPost(t *testing.T, cs carstore.CarStore, did string, prev *string, postid int) ([]byte, cid.Cid, string, string) { ctx := context.TODO() ds, err := cs.NewDeltaSession(ctx, 1, prev) if err != nil { diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index 59f90b13d..ad90a43b2 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -33,7 +33,7 @@ import ( var log = logging.Logger("repomgr") -func NewRepoManager(cs *carstore.CarStore, kmgr KeyManager) *RepoManager { +func NewRepoManager(cs carstore.CarStore, kmgr KeyManager) *RepoManager { return &RepoManager{ cs: cs, @@ -53,7 +53,7 @@ func (rm *RepoManager) SetEventHandler(cb func(context.Context, *RepoEvent), hyd } type RepoManager struct { - cs *carstore.CarStore + cs carstore.CarStore kmgr KeyManager lklk sync.Mutex @@ -140,7 +140,7 @@ func (rm *RepoManager) lockUser(ctx context.Context, user models.Uid) func() { } } -func (rm *RepoManager) CarStore() *carstore.CarStore { +func (rm *RepoManager) CarStore() carstore.CarStore { return rm.cs } diff --git a/util/labels/cbor_gen.go b/util/labels/cbor_gen.go index e777d7e45..b70d76aef 100644 --- a/util/labels/cbor_gen.go +++ b/util/labels/cbor_gen.go @@ -285,21 +285,24 @@ func (t *UnsignedLabel) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("UnsignedLabel: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 3) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": @@ -458,7 +461,9 @@ func (t *UnsignedLabel) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } }