Skip to content

Commit

Permalink
search: handle lenient datetimes better
Browse files Browse the repository at this point in the history
Instead of warning in a separate part of data flow, just do a lenient
parse-and-normalize when constructing the search document itself, and
leave field blank if the datetime couldn't parse (similar to existing
behavior).
  • Loading branch information
bnewbold committed Dec 15, 2023
1 parent 8f42363 commit a62f354
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 11 deletions.
6 changes: 0 additions & 6 deletions search/indexing.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,6 @@ func (s *Server) indexPost(ctx context.Context, ident *identity.Identity, rec *a

log = log.With("rkey", rkey)

_, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
if err != nil {
log.Warn("post had invalid timestamp", "createdAt", rec.CreatedAt, "parseErr", err)
rec.CreatedAt = ""
}

doc := TransformPost(rec, ident, rkey, rcid.String())
b, err := json.Marshal(doc)
if err != nil {
Expand Down
15 changes: 10 additions & 5 deletions search/transform.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package search

import (
"strings"
"time"

appbsky "github.com/bluesky-social/indigo/api/bsky"
"github.com/bluesky-social/indigo/atproto/identity"
"github.com/bluesky-social/indigo/util"
"github.com/bluesky-social/indigo/atproto/syntax"

"github.com/rivo/uniseg"
)

Expand Down Expand Up @@ -80,7 +80,7 @@ func TransformProfile(profile *appbsky.ActorProfile, ident *identity.Identity, c
handle = ident.Handle.String()
}
return ProfileDoc{
DocIndexTs: time.Now().UTC().Format(util.ISO8601),
DocIndexTs: syntax.DatetimeNow().String(),
DID: ident.DID.String(),
RecordCID: cid,
Handle: handle,
Expand Down Expand Up @@ -157,7 +157,7 @@ func TransformPost(post *appbsky.FeedPost, ident *identity.Identity, rkey, cid s
}

doc := PostDoc{
DocIndexTs: time.Now().UTC().Format(util.ISO8601),
DocIndexTs: syntax.DatetimeNow().String(),
DID: ident.DID.String(),
RecordRkey: rkey,
RecordCID: cid,
Expand All @@ -177,7 +177,12 @@ func TransformPost(post *appbsky.FeedPost, ident *identity.Identity, rkey, cid s
}

if post.CreatedAt != "" {
doc.CreatedAt = &post.CreatedAt
// there are some old bad timestamps out there!
dt, err := syntax.ParseDatetimeLenient(post.CreatedAt)
if nil == err { // *not* an error
s := dt.String()
doc.CreatedAt = &s
}
}

return doc
Expand Down

0 comments on commit a62f354

Please sign in to comment.