Skip to content

Commit

Permalink
Compare document data between Algolia and PostgreSQL (#341)
Browse files Browse the repository at this point in the history 
* Add logic to compare document data in Algolia and the database

* Compare document data in APIs

* Save ModifiedTime to be consistent with the database

* Save DocumentModifiedAt to be consistent with Algolia
  • Loading branch information
@jfreda
jfreda authored Sep 21, 2023
1 parent 44e8f17 commit a60633f
Show file tree
Hide file tree
Showing 6 changed files with 1,645 additions and 1 deletion.
104 changes: 104 additions & 0 deletions internal/api/approvals.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,58 @@ func ApprovalHandler(
"path", r.URL.Path,
)

// Compare Algolia and database documents to find data inconsistencies.
// Get document object from Algolia.
var algoDoc map[string]any
err = ar.Docs.GetObject(docID, &algoDoc)
if err != nil {
l.Error("error getting Algolia object for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
// Get document from database.
dbDoc := models.Document{
GoogleFileID: docID,
}
if err := dbDoc.Get(db); err != nil {
l.Error("error getting document from database for data comparison",
"error", err,
"path", r.URL.Path,
"method", r.Method,
"doc_id", docID,
)
return
}
// Get all reviews for the document.
var reviews models.DocumentReviews
if err := reviews.Find(db, models.DocumentReview{
Document: models.Document{
GoogleFileID: docID,
},
}); err != nil {
l.Error("error getting all reviews for document for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
if err := compareAlgoliaAndDatabaseDocument(
algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType,
); err != nil {
l.Warn("inconsistencies detected between Algolia and database docs",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
}

case "POST":
// Validate request.
docID, err := parseResourceIDFromURL(r.URL.Path, "approvals")
Expand Down Expand Up @@ -439,6 +491,58 @@ func ApprovalHandler(
"path", r.URL.Path,
)

// Compare Algolia and database documents to find data inconsistencies.
// Get document object from Algolia.
var algoDoc map[string]any
err = ar.Docs.GetObject(docID, &algoDoc)
if err != nil {
l.Error("error getting Algolia object for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
// Get document from database.
dbDoc := models.Document{
GoogleFileID: docID,
}
if err := dbDoc.Get(db); err != nil {
l.Error("error getting document from database for data comparison",
"error", err,
"path", r.URL.Path,
"method", r.Method,
"doc_id", docID,
)
return
}
// Get all reviews for the document.
var reviews models.DocumentReviews
if err := reviews.Find(db, models.DocumentReview{
Document: models.Document{
GoogleFileID: docID,
},
}); err != nil {
l.Error("error getting all reviews for document for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
if err := compareAlgoliaAndDatabaseDocument(
algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType,
); err != nil {
l.Warn("inconsistencies detected between Algolia and database docs",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
}

default:
w.WriteHeader(http.StatusMethodNotAllowed)
return
Expand Down
105 changes: 104 additions & 1 deletion internal/api/documents.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,58 @@ func DocumentHandler(
"doc_id", docID,
)

// Compare Algolia and database documents to find data inconsistencies.
// Get document object from Algolia.
var algoDoc map[string]any
err = ar.Docs.GetObject(docID, &algoDoc)
if err != nil {
l.Error("error getting Algolia object for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
// Get document from database.
dbDoc := models.Document{
GoogleFileID: docID,
}
if err := dbDoc.Get(db); err != nil {
l.Error("error getting document from database for data comparison",
"error", err,
"path", r.URL.Path,
"method", r.Method,
"doc_id", docID,
)
return
}
// Get all reviews for the document.
var reviews models.DocumentReviews
if err := reviews.Find(db, models.DocumentReview{
Document: models.Document{
GoogleFileID: docID,
},
}); err != nil {
l.Error("error getting all reviews for document for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
if err := compareAlgoliaAndDatabaseDocument(
algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType,
); err != nil {
l.Warn("inconsistencies detected between Algolia and database docs",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
}

case "PATCH":
// Authorize request (only the owner can PATCH the doc).
userEmail := r.Context().Value("userEmail").(string)
Expand Down Expand Up @@ -688,7 +740,58 @@ Hermes

w.WriteHeader(http.StatusOK)
l.Info("patched document", "doc_id", docID)
return

// Compare Algolia and database documents to find data inconsistencies.
// Get document object from Algolia.
var algoDoc map[string]any
err = ar.Docs.GetObject(docID, &algoDoc)
if err != nil {
l.Error("error getting Algolia object for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
// Get document from database.
dbDoc := models.Document{
GoogleFileID: docID,
}
if err := dbDoc.Get(db); err != nil {
l.Error("error getting document from database for data comparison",
"error", err,
"path", r.URL.Path,
"method", r.Method,
"doc_id", docID,
)
return
}
// Get all reviews for the document.
var reviews models.DocumentReviews
if err := reviews.Find(db, models.DocumentReview{
Document: models.Document{
GoogleFileID: docID,
},
}); err != nil {
l.Error("error getting all reviews for document for data comparison",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
return
}
if err := compareAlgoliaAndDatabaseDocument(
algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType,
); err != nil {
l.Warn("inconsistencies detected between Algolia and database docs",
"error", err,
"method", r.Method,
"path", r.URL.Path,
"doc_id", docID,
)
}

default:
w.WriteHeader(http.StatusMethodNotAllowed)
Expand Down
Loading

0 comments on commit a60633f

Please sign in to comment.