From ee6ae8f66b60b14c99e467bb1cc429262379ea97 Mon Sep 17 00:00:00 2001 From: Josh Freda Date: Wed, 20 Sep 2023 17:00:16 -0500 Subject: [PATCH 1/4] Add logic to compare document data in Algolia and the database --- internal/api/helpers.go | 443 ++++++++++++++++++++ internal/api/helpers_test.go | 784 +++++++++++++++++++++++++++++++++++ 2 files changed, 1227 insertions(+) diff --git a/internal/api/helpers.go b/internal/api/helpers.go index c4831c50f..dd0d0fa68 100644 --- a/internal/api/helpers.go +++ b/internal/api/helpers.go @@ -5,9 +5,15 @@ import ( "fmt" "io" "net/http" + "reflect" + "regexp" "strings" + "github.com/hashicorp-forge/hermes/internal/config" + "github.com/hashicorp-forge/hermes/pkg/models" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-multierror" + "github.com/iancoleman/strcase" ) // contains returns true if a string is present in a slice of strings. @@ -108,3 +114,440 @@ func respondError( ) http.Error(w, userErrMsg, httpCode) } + +// compareAlgoliaAndDatabaseDocument compares data for a document stored in +// Algolia and the database to determine any inconsistencies, which are returned +// back as a (multierror) error. +func compareAlgoliaAndDatabaseDocument( + algoDoc map[string]any, + dbDoc models.Document, + dbDocReviews models.DocumentReviews, + docTypes []*config.DocumentType, +) error { + + var result *multierror.Error + + // Compare objectID. + algoGoogleFileID, err := getStringValue(algoDoc, "objectID") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting objectID value: %w", err)) + } + if algoGoogleFileID != dbDoc.GoogleFileID { + result = multierror.Append(result, + fmt.Errorf( + "objectID not equal, algolia=%v, db=%v", + algoGoogleFileID, dbDoc.GoogleFileID), + ) + } + + // Compare title. + algoTitle, err := getStringValue(algoDoc, "title") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting title value: %w", err)) + } else { + if algoTitle != dbDoc.Title { + result = multierror.Append(result, + fmt.Errorf( + "title not equal, algolia=%v, db=%v", + algoTitle, dbDoc.Title), + ) + } + } + + // Compare docType. + algoDocType, err := getStringValue(algoDoc, "docType") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting docType value: %w", err)) + } else { + dbDocType := dbDoc.DocumentType.Name + if algoDocType != dbDocType { + result = multierror.Append(result, + fmt.Errorf( + "docType not equal, algolia=%v, db=%v", + algoTitle, dbDocType), + ) + } + } + + // Compare docNumber. + algoDocNumber, err := getStringValue(algoDoc, "docNumber") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting docNumber value: %w", err)) + } else { + // Replace "-???" (how draft doc numbers are defined in Algolia) with a + // zero. + re := regexp.MustCompile(`-\?\?\?$`) + algoDocNumber = re.ReplaceAllString(algoDocNumber, "-000") + + // Note that we pad the database document number to three digits here like + // we do when assigning a document number when a doc review is requested. + dbDocNumber := fmt.Sprintf( + "%s-%03d", dbDoc.Product.Abbreviation, dbDoc.DocumentNumber) + if algoDocNumber != dbDocNumber { + result = multierror.Append(result, + fmt.Errorf( + "docNumber not equal, algolia=%v, db=%v", + algoDocNumber, dbDocNumber), + ) + } + } + + // Compare appCreated. + algoAppCreated, err := getBooleanValue(algoDoc, "appCreated") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting appCreated value: %w", err)) + } else { + dbAppCreated := !dbDoc.Imported + if algoAppCreated != dbAppCreated { + result = multierror.Append(result, + fmt.Errorf( + "appCreated not equal, algolia=%v, db=%v", + algoAppCreated, dbAppCreated), + ) + } + } + + // Compare approvedBy. + algoApprovedBy, err := getStringSliceValue(algoDoc, "approvedBy") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting approvedBy value: %w", err)) + } + dbApprovedBy := []string{} + for _, r := range dbDocReviews { + if r.Status == models.ApprovedDocumentReviewStatus { + dbApprovedBy = append(dbApprovedBy, r.User.EmailAddress) + } + } + if !reflect.DeepEqual(algoApprovedBy, dbApprovedBy) { + result = multierror.Append(result, + fmt.Errorf( + "approvedBy not equal, algolia=%v, db=%v", + algoApprovedBy, dbApprovedBy), + ) + } + + // Compare approvers. + algoApprovers, err := getStringSliceValue(algoDoc, "approvers") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting approvers value: %w", err)) + } + dbApprovers := []string{} + for _, a := range dbDoc.Approvers { + dbApprovers = append(dbApprovers, a.EmailAddress) + } + if !reflect.DeepEqual(algoApprovers, dbApprovers) { + result = multierror.Append(result, + fmt.Errorf( + "approvers not equal, algolia=%v, db=%v", + algoApprovers, dbApprovers), + ) + } + + // Compare changesRequestedBy. + algoChangesRequestedBy, err := getStringSliceValue( + algoDoc, "changesRequestedBy") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting changesRequestedBy value: %w", err)) + } + dbChangesRequestedBy := []string{} + for _, r := range dbDocReviews { + if r.Status == models.ChangesRequestedDocumentReviewStatus { + dbChangesRequestedBy = append(dbChangesRequestedBy, r.User.EmailAddress) + } + } + if !reflect.DeepEqual(algoChangesRequestedBy, dbChangesRequestedBy) { + result = multierror.Append(result, + fmt.Errorf( + "changesRequestedBy not equal, algolia=%v, db=%v", + algoChangesRequestedBy, dbChangesRequestedBy), + ) + } + + // Compare contributors. + algoContributors, err := getStringSliceValue(algoDoc, "contributors") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting contributors value: %w", err)) + } + dbContributors := []string{} + for _, c := range dbDoc.Contributors { + dbContributors = append(dbContributors, c.EmailAddress) + } + if !reflect.DeepEqual(algoContributors, dbContributors) { + result = multierror.Append(result, + fmt.Errorf( + "contributors not equal, algolia=%v, db=%v", + algoContributors, dbContributors), + ) + } + + // Compare createdTime. + algoCreatedTime, err := getInt64Value(algoDoc, "createdTime") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting createdTime value: %w", err)) + } else { + dbCreatedTime := dbDoc.DocumentCreatedAt.Unix() + if algoCreatedTime != dbCreatedTime { + result = multierror.Append(result, + fmt.Errorf( + "createdTime not equal, algolia=%v, db=%v", + algoCreatedTime, dbCreatedTime), + ) + } + } + + // Compare custom fields. + foundDocType := false + for _, dt := range docTypes { + if dt.Name == algoDocType { + foundDocType = true + for _, cf := range dt.CustomFields { + algoCFName := strcase.ToLowerCamel(cf.Name) + + switch cf.Type { + case "string": + algoCFVal, err := getStringValue(algoDoc, algoCFName) + if err != nil { + result = multierror.Append( + result, fmt.Errorf( + "error getting custom field (%s) value: %w", algoCFName, err)) + } else { + for _, c := range dbDoc.CustomFields { + if c.DocumentTypeCustomField.Name == cf.Name { + if algoCFVal != c.Value { + result = multierror.Append(result, + fmt.Errorf( + "custom field %s not equal, algolia=%v, db=%v", + algoCFName, algoCFVal, c.Value), + ) + } + break + } + } + } + case "people": + algoCFVal, err := getStringSliceValue(algoDoc, algoCFName) + if err != nil { + result = multierror.Append( + result, fmt.Errorf( + "error getting custom field (%s) value: %w", algoCFName, err)) + } else { + for _, c := range dbDoc.CustomFields { + if c.DocumentTypeCustomField.Name == cf.Name { + // Unmarshal person custom field value to string slice. + var dbCFVal []string + if err := json.Unmarshal([]byte(c.Value), &dbCFVal); err != nil { + result = multierror.Append(result, + fmt.Errorf( + "error unmarshaling custom field %s to string slice", + algoCFName), + ) + } + + if !reflect.DeepEqual(algoCFVal, dbCFVal) { + result = multierror.Append(result, + fmt.Errorf( + "custom field %s not equal, algolia=%v, db=%v", + algoCFName, algoCFVal, dbCFVal), + ) + } + break + } + } + } + default: + result = multierror.Append(result, + fmt.Errorf( + "unknown type for custom field key %q: %s", dt.Name, cf.Type)) + } + } + break + } + } + if !foundDocType { + result = multierror.Append(result, + fmt.Errorf( + "doc type %q not found", algoDocType)) + } + + // Compare file revisions. + // TODO: need to store this in the database first. + + // Compare modifiedTime. + algoModifiedTime, err := getInt64Value(algoDoc, "modifiedTime") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting modifiedTime value: %w", err)) + } else { + dbModifiedTime := dbDoc.DocumentModifiedAt.Unix() + if algoModifiedTime != dbModifiedTime { + result = multierror.Append(result, + fmt.Errorf( + "modifiedTime not equal, algolia=%v, db=%v", + algoModifiedTime, dbModifiedTime), + ) + } + } + + // Compare owner. + // NOTE: this does not address multiple owners, which can exist for Algolia + // document objects (documents in the database currently only have one owner). + algoOwners, err := getStringSliceValue(algoDoc, "owners") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting owners value: %w", err)) + } else { + var dbOwner string + if dbDoc.Owner != nil { + dbOwner = dbDoc.Owner.EmailAddress + } + if len(algoOwners) > 0 { + if algoOwners[0] != dbOwner { + result = multierror.Append(result, + fmt.Errorf( + "owners not equal, algolia=%#v, db=%#v", + algoOwners, dbOwner), + ) + } + } else { + result = multierror.Append( + result, fmt.Errorf("owners in Algolia was length %d", len(algoOwners))) + } + } + + // Compare product. + algoProduct, err := getStringValue(algoDoc, "product") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting product value: %w", err)) + } else { + dbProduct := dbDoc.Product.Name + if algoProduct != dbProduct { + result = multierror.Append(result, + fmt.Errorf( + "product not equal, algolia=%v, db=%v", + algoProduct, dbProduct), + ) + } + } + + // Compare status. + algoStatus, err := getStringValue(algoDoc, "status") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting status value: %w", err)) + } else { + var dbStatus string + switch dbDoc.Status { + case models.WIPDocumentStatus: + dbStatus = "WIP" + case models.InReviewDocumentStatus: + dbStatus = "In-Review" + case models.ApprovedDocumentStatus: + dbStatus = "Approved" + case models.ObsoleteDocumentStatus: + dbStatus = "Obsolete" + } + if algoStatus != dbStatus { + result = multierror.Append(result, + fmt.Errorf( + "status not equal, algolia=%v, db=%v", + algoStatus, dbStatus), + ) + } + } + + // Compare summary. + algoSummary, err := getStringValue(algoDoc, "summary") + if err != nil { + result = multierror.Append( + result, fmt.Errorf("error getting summary value: %w", err)) + } else { + dbSummary := dbDoc.Summary + if algoSummary != dbSummary { + result = multierror.Append(result, + fmt.Errorf( + "summary not equal, algolia=%v, db=%v", + algoSummary, dbSummary), + ) + } + } + + return result.ErrorOrNil() +} + +func getBooleanValue(in map[string]any, key string) (bool, error) { + var result bool + + if v, ok := in[key]; ok { + if vv, ok := v.(bool); ok { + return vv, nil + } else { + return false, fmt.Errorf( + "invalid type: value is not a boolean, type: %T", v) + } + } + + return result, nil +} + +func getInt64Value(in map[string]any, key string) (int64, error) { + var result int64 + + if v, ok := in[key]; ok { + // These interface{} values are inferred as float64 and need to be converted + // to int64. + if vv, ok := v.(float64); ok { + return int64(vv), nil + } else { + return 0, fmt.Errorf( + "invalid type: value is not an float64 (expected), type: %T", v) + } + } + + return result, nil +} + +func getStringValue(in map[string]any, key string) (string, error) { + var result string + + if v, ok := in[key]; ok { + if vv, ok := v.(string); ok { + return vv, nil + } else { + return "", fmt.Errorf("invalid type: value is not a string, type: %T", v) + } + } + + return result, nil +} + +func getStringSliceValue(in map[string]any, key string) ([]string, error) { + result := []string{} + + if v, ok := in[key]; ok { + if reflect.TypeOf(v).Kind() == reflect.Slice { + for _, vv := range v.([]any) { + if vv, ok := vv.(string); ok { + result = append(result, vv) + } else { + return nil, fmt.Errorf("invalid type: slice element is not a string") + } + } + return result, nil + } else { + return nil, fmt.Errorf("invalid type: value is not a slice") + } + } + + return result, nil +} diff --git a/internal/api/helpers_test.go b/internal/api/helpers_test.go index a011b53b8..66f032bf5 100644 --- a/internal/api/helpers_test.go +++ b/internal/api/helpers_test.go @@ -3,6 +3,12 @@ package api import ( "reflect" "testing" + "time" + + "github.com/hashicorp-forge/hermes/internal/config" + "github.com/hashicorp-forge/hermes/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseResourceIDFromURL(t *testing.T) { @@ -96,3 +102,781 @@ func TestCompareSlices(t *testing.T) { }) } } + +func TestCompareAlgoliaAndDatabaseDocument(t *testing.T) { + cases := map[string]struct { + algoDoc map[string]any + dbDoc models.Document + dbDocReviews models.DocumentReviews + + shouldErr bool + errContains string + }{ + "good": { + algoDoc: map[string]any{ + "objectID": "GoogleFileID1", + "title": "Title1", + "docType": "RFC", + "docNumber": "ABC-123", + "appCreated": true, + "approvedBy": []any{ + "approver1@hashicorp.com", + "approver2@hashicorp.com", + }, + "approvers": []any{ + "approver1@hashicorp.com", + "approver2@hashicorp.com", + }, + "changesRequestedBy": []any{ + "changerequester1@hashicorp.com", + "changerequester2@hashicorp.com", + }, + "contributors": []any{ + "contributor1@hashicorp.com", + "contributor2@hashicorp.com", + }, + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "currentVersion": "1.2.3", + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{ + "owner1@hashicorp.com", + "owner2@hashicorp.com", + }, + "product": "Product1", + "stakeholders": []any{ + "stakeholder1@hashicorp.com", + "stakeholder2@hashicorp.com", + }, + "summary": "Summary1", + "status": "In-Review", + }, + dbDoc: models.Document{ + GoogleFileID: "GoogleFileID1", + Title: "Title1", + DocumentType: models.DocumentType{ + Name: "RFC", + }, + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + Imported: false, + Approvers: []*models.User{ + { + EmailAddress: "approver1@hashicorp.com", + }, + { + EmailAddress: "approver2@hashicorp.com", + }, + }, + Contributors: []*models.User{ + { + EmailAddress: "contributor1@hashicorp.com", + }, + { + EmailAddress: "contributor2@hashicorp.com", + }, + }, + CustomFields: []*models.DocumentCustomField{ + { + DocumentTypeCustomField: models.DocumentTypeCustomField{ + Name: "Current Version", + DocumentType: models.DocumentType{ + Name: "RFC", + }, + }, + Value: "1.2.3", + }, + { + DocumentTypeCustomField: models.DocumentTypeCustomField{ + Name: "Stakeholders", + DocumentType: models.DocumentType{ + Name: "RFC", + }, + }, + Value: `["stakeholder1@hashicorp.com","stakeholder2@hashicorp.com"]`, + }, + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + Summary: "Summary1", + Status: models.InReviewDocumentStatus, + }, + dbDocReviews: models.DocumentReviews{ + { + Status: models.ApprovedDocumentReviewStatus, + User: models.User{ + EmailAddress: "approver1@hashicorp.com", + }, + }, + { + Status: models.ApprovedDocumentReviewStatus, + User: models.User{ + EmailAddress: "approver2@hashicorp.com", + }, + }, + { + Status: models.ChangesRequestedDocumentReviewStatus, + User: models.User{ + EmailAddress: "changerequester1@hashicorp.com", + }, + }, + { + Status: models.ChangesRequestedDocumentReviewStatus, + User: models.User{ + EmailAddress: "changerequester2@hashicorp.com", + }, + }, + }, + shouldErr: false, + }, + + "good draft doc number (test 'ABC-???')": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-???", + "docType": "RFC", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 0, + DocumentType: models.DocumentType{ + Name: "RFC", + }, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + }, + + "bad objectID": { + algoDoc: map[string]any{ + "objectID": "GoogleFileID1", + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + GoogleFileID: "BadGoogleFileID", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "objectID not equal", + }, + + "bad title": { + algoDoc: map[string]any{ + "title": "Title1", + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + Title: "BadTitle", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "title not equal", + }, + + "bad docType": { + algoDoc: map[string]any{ + "docType": "DocType1", + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentType: models.DocumentType{ + Name: "BadDocType", + }, + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "docType not equal", + }, + + "bad appCreated": { + algoDoc: map[string]any{ + "appCreated": false, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "appCreated not equal", + }, + + "bad approvedBy": { + algoDoc: map[string]any{ + "approvedBy": []any{ + "approver1@hashicorp.com", + "approver2@hashicorp.com", + }, + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + GoogleFileID: "BadGoogleFileID", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + dbDocReviews: models.DocumentReviews{ + { + Status: models.ApprovedDocumentReviewStatus, + User: models.User{ + EmailAddress: "badapprover1@hashicorp.com", + }, + }, + { + Status: models.ApprovedDocumentReviewStatus, + User: models.User{ + EmailAddress: "badapprover2@hashicorp.com", + }, + }, + }, + shouldErr: true, + errContains: "approvedBy not equal", + }, + + "bad approvers": { + algoDoc: map[string]any{ + "appCreated": true, + "approvers": []any{ + "approver1@hashicorp.com", + "approver2@hashicorp.com", + }, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + Title: "BadTitle", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + Approvers: []*models.User{ + { + EmailAddress: "badapprover1@hashicorp.com", + }, + { + EmailAddress: "badapprover2@hashicorp.com", + }, + }, + }, + shouldErr: true, + errContains: "approvers not equal", + }, + + "bad changesRequestedBy": { + algoDoc: map[string]any{ + "appCreated": true, + "changesRequestedBy": []any{ + "changerequester1@hashicorp.com", + "changerequester2@hashicorp.com", + }, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + GoogleFileID: "BadGoogleFileID", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + dbDocReviews: models.DocumentReviews{ + { + Status: models.ChangesRequestedDocumentReviewStatus, + User: models.User{ + EmailAddress: "badchangerequester1@hashicorp.com", + }, + }, + { + Status: models.ChangesRequestedDocumentReviewStatus, + User: models.User{ + EmailAddress: "badchangerequester2@hashicorp.com", + }, + }, + }, + shouldErr: true, + errContains: "changesRequestedBy not equal", + }, + + "bad contributors": { + algoDoc: map[string]any{ + "appCreated": true, + "contributors": []any{ + "contributor1@hashicorp.com", + "contributor2@hashicorp.com", + }, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + Title: "BadTitle", + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + Contributors: []*models.User{ + { + EmailAddress: "badcontributor1@hashicorp.com", + }, + { + EmailAddress: "badcontributor2@hashicorp.com", + }, + }, + }, + shouldErr: true, + errContains: "contributors not equal", + }, + + "bad createdTime": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2013, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "createdTime not equal", + }, + + "bad string custom field currentVersion": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "currentVersion": "1", + "docType": "RFC", + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentType: models.DocumentType{ + Name: "RFC", + }, + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + CustomFields: []*models.DocumentCustomField{ + { + DocumentTypeCustomField: models.DocumentTypeCustomField{ + Name: "Current Version", + DocumentType: models.DocumentType{ + Name: "RFC", + }, + }, + Value: "2", + }, + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "custom field currentVersion not equal", + }, + + "bad people custom field stakeholders": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "docType": "RFC", + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + "stakeholders": []any{ + "stakeholder1@hashicorp.com", + "stakeholder2@hashicorp.com", + }, + }, + dbDoc: models.Document{ + DocumentType: models.DocumentType{ + Name: "RFC", + }, + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + CustomFields: []*models.DocumentCustomField{ + { + DocumentTypeCustomField: models.DocumentTypeCustomField{ + Name: "Stakeholders", + DocumentType: models.DocumentType{ + Name: "RFC", + }, + }, + Value: `["stakeholder1@hashicorp.com","badstakeholder2@hashicorp.com"]`, + }, + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "custom field stakeholders not equal", + }, + + "bad modifiedTime": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2013, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "modifiedTime not equal", + }, + + "bad owners": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "badowner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "owners not equal", + }, + + "bad product": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "BadProduct1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + }, + shouldErr: true, + errContains: "product not equal", + }, + + "bad status": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + "status": "Approved", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + Status: models.InReviewDocumentStatus, + }, + shouldErr: true, + errContains: "status not equal", + }, + + "bad summary": { + algoDoc: map[string]any{ + "appCreated": true, + "docNumber": "ABC-123", + "createdTime": float64(time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC).Unix()), + "modifiedTime": float64(time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC).Unix()), + "owners": []any{"owner1@hashicorp.com"}, + "product": "Product1", + "summary": "Summary1", + }, + dbDoc: models.Document{ + DocumentNumber: 123, + Product: models.Product{ + Name: "Product1", + Abbreviation: "ABC", + }, + DocumentCreatedAt: time.Date( + 2023, time.April, 5, 1, 0, 0, 0, time.UTC), + DocumentModifiedAt: time.Date( + 2023, time.April, 5, 23, 0, 0, 0, time.UTC), + Owner: &models.User{ + EmailAddress: "owner1@hashicorp.com", + }, + Summary: "BadSummary1", + }, + shouldErr: true, + errContains: "summary not equal", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + // Define minimum document types configuration for tests. + docTypes := []*config.DocumentType{ + { + Name: "RFC", + CustomFields: []*config.DocumentTypeCustomField{ + { + Name: "Current Version", + Type: "string", + }, + { + Name: "Stakeholders", + Type: "people", + }, + }, + }, + } + + if err := compareAlgoliaAndDatabaseDocument( + c.algoDoc, c.dbDoc, c.dbDocReviews, docTypes, + ); err != nil { + if c.shouldErr { + require.Error(err) + assert.ErrorContains(err, c.errContains) + } else { + require.NoError(err) + } + } + }) + } +} From ed17854a3b2f951addaaf287e971ddadee35bebd Mon Sep 17 00:00:00 2001 From: Josh Freda Date: Wed, 20 Sep 2023 17:02:20 -0500 Subject: [PATCH 2/4] Compare document data in APIs --- internal/api/approvals.go | 104 +++++++++++++++++++++++++ internal/api/documents.go | 105 ++++++++++++++++++++++++- internal/api/drafts.go | 156 ++++++++++++++++++++++++++++++++++++++ internal/api/reviews.go | 52 +++++++++++++ 4 files changed, 416 insertions(+), 1 deletion(-) diff --git a/internal/api/approvals.go b/internal/api/approvals.go index 58890142f..0364be968 100644 --- a/internal/api/approvals.go +++ b/internal/api/approvals.go @@ -231,6 +231,58 @@ func ApprovalHandler( "path", r.URL.Path, ) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Docs.GetObject(docID, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docID, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docID, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docID, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + } + case "POST": // Validate request. docID, err := parseResourceIDFromURL(r.URL.Path, "approvals") @@ -439,6 +491,58 @@ func ApprovalHandler( "path", r.URL.Path, ) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Docs.GetObject(docID, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docID, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docID, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docID, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + } + default: w.WriteHeader(http.StatusMethodNotAllowed) return diff --git a/internal/api/documents.go b/internal/api/documents.go index e3181342a..991da736c 100644 --- a/internal/api/documents.go +++ b/internal/api/documents.go @@ -227,6 +227,58 @@ func DocumentHandler( "doc_id", docID, ) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Docs.GetObject(docID, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docID, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docID, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docID, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + } + case "PATCH": // Authorize request (only the owner can PATCH the doc). userEmail := r.Context().Value("userEmail").(string) @@ -688,7 +740,58 @@ Hermes w.WriteHeader(http.StatusOK) l.Info("patched document", "doc_id", docID) - return + + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Docs.GetObject(docID, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docID, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docID, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docID, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + } default: w.WriteHeader(http.StatusMethodNotAllowed) diff --git a/internal/api/drafts.go b/internal/api/drafts.go index 0c745c725..1f4db6c20 100644 --- a/internal/api/drafts.go +++ b/internal/api/drafts.go @@ -307,6 +307,58 @@ func DraftsHandler( l.Info("created draft", "doc_id", f.Id) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Drafts.GetObject(f.Id, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", f.Id, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: f.Id, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", f.Id, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: f.Id, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", f.Id, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", f.Id, + ) + } + case "GET": // Get OIDC ID id := r.Header.Get("x-amzn-oidc-identity") @@ -584,6 +636,58 @@ func DraftsDocumentHandler( l.Info("retrieved document draft", "doc_id", docId) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Drafts.GetObject(docId, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docId, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docId, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docId, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + } + case "DELETE": // Authorize request. if !isOwner { @@ -1073,6 +1177,58 @@ func DraftsDocumentHandler( w.WriteHeader(http.StatusOK) l.Info("patched draft document", "doc_id", docId) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Drafts.GetObject(docId, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docId, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docId, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docId, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docId, + ) + } + default: w.WriteHeader(http.StatusMethodNotAllowed) return diff --git a/internal/api/reviews.go b/internal/api/reviews.go index d1818b7fd..373027559 100644 --- a/internal/api/reviews.go +++ b/internal/api/reviews.go @@ -570,6 +570,58 @@ func ReviewHandler( "path", r.URL.Path, ) + // Compare Algolia and database documents to find data inconsistencies. + // Get document object from Algolia. + var algoDoc map[string]any + err = ar.Docs.GetObject(docID, &algoDoc) + if err != nil { + l.Error("error getting Algolia object for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + // Get document from database. + dbDoc := models.Document{ + GoogleFileID: docID, + } + if err := dbDoc.Get(db); err != nil { + l.Error("error getting document from database for data comparison", + "error", err, + "path", r.URL.Path, + "method", r.Method, + "doc_id", docID, + ) + return + } + // Get all reviews for the document. + var reviews models.DocumentReviews + if err := reviews.Find(db, models.DocumentReview{ + Document: models.Document{ + GoogleFileID: docID, + }, + }); err != nil { + l.Error("error getting all reviews for document for data comparison", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + return + } + if err := compareAlgoliaAndDatabaseDocument( + algoDoc, dbDoc, reviews, cfg.DocumentTypes.DocumentType, + ); err != nil { + l.Warn("inconsistencies detected between Algolia and database docs", + "error", err, + "method", r.Method, + "path", r.URL.Path, + "doc_id", docID, + ) + } + default: w.WriteHeader(http.StatusMethodNotAllowed) return From e30608de7f600009c777ad959df4423d951b1251 Mon Sep 17 00:00:00 2001 From: Josh Freda Date: Wed, 20 Sep 2023 17:04:34 -0500 Subject: [PATCH 3/4] Save ModifiedTime to be consistent with the database --- internal/api/drafts.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/api/drafts.go b/internal/api/drafts.go index 1f4db6c20..1c5ac4670 100644 --- a/internal/api/drafts.go +++ b/internal/api/drafts.go @@ -181,6 +181,7 @@ func DraftsHandler( DocNumber: fmt.Sprintf("%s-???", req.ProductAbbreviation), DocType: req.DocType, MetaTags: metaTags, + ModifiedTime: ct.Unix(), Owners: []string{userEmail}, OwnerPhotos: op, Product: req.Product, From e8c1d5407a1f51bbae8ef5aaad377675de02ea1b Mon Sep 17 00:00:00 2001 From: Josh Freda Date: Wed, 20 Sep 2023 17:05:15 -0500 Subject: [PATCH 4/4] Save DocumentModifiedAt to be consistent with Algolia --- internal/api/reviews.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/api/reviews.go b/internal/api/reviews.go index 373027559..f765e6523 100644 --- a/internal/api/reviews.go +++ b/internal/api/reviews.go @@ -434,6 +434,7 @@ func ReviewHandler( } d.Status = models.InReviewDocumentStatus d.DocumentNumber = nextDocNum + d.DocumentModifiedAt = modifiedTime if err := d.Upsert(db); err != nil { l.Error("error upserting document in database", "error", err,