From 65ebc5d7fdcbd3f43c7dd7c6e7a1985124c48c31 Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Mon, 11 Nov 2024 12:47:04 -0500 Subject: [PATCH 1/6] Adding file hashing support, adding MongoDB tests --- hash.go | 25 +++++++++++++++++++++++++ hash_test.go | 26 ++++++++++++++++++++++++++ mongodb/mongodb.go | 23 ++++++++++++++++++----- mongodb/mongodb_test.go | 37 +++++++++++++++++++++++++++++++++++++ test/example.txt | 1 + tracker.go | 2 +- 6 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 hash.go create mode 100644 hash_test.go create mode 100644 mongodb/mongodb_test.go create mode 100644 test/example.txt diff --git a/hash.go b/hash.go new file mode 100644 index 0000000..8ec2bb9 --- /dev/null +++ b/hash.go @@ -0,0 +1,25 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "io" + "os" +) + +func GetFileHash(path string) (hash string, err error) { + file, err := os.Open(path) + if err != nil { + return + } + defer file.Close() + + var hashFunction = sha256.New() + _, err = io.Copy(hashFunction, file) + if err != nil { + return + } + + hash = fmt.Sprintf("%x", hashFunction.Sum(nil)) + return +} diff --git a/hash_test.go b/hash_test.go new file mode 100644 index 0000000..a560203 --- /dev/null +++ b/hash_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetFileHashValid(t *testing.T) { + var assert = assert.New(t) + + var hash, err = GetFileHash("hash_test.go") + assert.Equal(nil, err, "Returned an error 1") + assert.Equal(64, len(hash), "Incorrect hash length 1") + + hash, err = GetFileHash("test/example.txt") + assert.Equal(nil, err, "Returned an error 2") + assert.Equal("a9a66978f378456c818fb8a3e7c6ad3d2c83e62724ccbdea7b36253fb8df5edd", hash, "Incorrect hash length 2") +} + +func TestGetFileHashInvalid(t *testing.T) { + var assert = assert.New(t) + + var _, err = GetFileHash("invalid.exe") + assert.NotEqual(nil, err, "Did not return an error 1") +} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index c3127c4..44d2bcf 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -50,8 +50,8 @@ func (m *MongoDB) Disconnect() { func (m *MongoDB) CreateCollection(collection string) (err error) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - // Ensure that we create only missing collection - names, err := m.database.ListCollectionNames(ctx, bson.D{{"name", collection}}) + // Ensure that we create only a missing collection + names, err := m.database.ListCollectionNames(ctx, bson.D{{Key: "name", Value: collection}}) if err != nil { return err } @@ -72,7 +72,7 @@ func (m *MongoDB) Write(collection string, data bson.D) (err error) { return err } -func (m *MongoDB) GetLastDocument(collection string, sortedKey string) (result bson.D, err error) { +func (m *MongoDB) GetLastDocumentFiltered(collection string, sortedKey string, filter bson.D) (result bson.D, err error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -81,9 +81,9 @@ func (m *MongoDB) GetLastDocument(collection string, sortedKey string) (result b if err != nil || count == 0 { return result, err } - opts := options.FindOne().SetSort(bson.D{{sortedKey, 1}}).SetSkip(count - 1) + opts := options.FindOne().SetSort(bson.D{{Key: sortedKey, Value: 1}}).SetSkip(count - 1) - err = mongoCollection.FindOne(ctx, bson.D{}, opts).Decode(&result) + err = mongoCollection.FindOne(ctx, filter, opts).Decode(&result) if err != nil { return result, err } @@ -91,6 +91,10 @@ func (m *MongoDB) GetLastDocument(collection string, sortedKey string) (result b return } +func (m *MongoDB) GetLastDocument(collection string, sortedKey string) (result bson.D, err error) { + return m.GetLastDocumentFiltered(collection, sortedKey, bson.D{}) +} + func (m *MongoDB) GetAllDocuments(collection string) (result []bson.D, err error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -116,3 +120,12 @@ func (m *MongoDB) GetAllDocuments(collection string) (result []bson.D, err error return } + +func (m *MongoDB) DropCollection(collection string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mongoCollection := m.database.Collection(collection) + err = mongoCollection.Drop(ctx) + return +} diff --git a/mongodb/mongodb_test.go b/mongodb/mongodb_test.go new file mode 100644 index 0000000..e4d79ce --- /dev/null +++ b/mongodb/mongodb_test.go @@ -0,0 +1,37 @@ +package mongodb + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewMongoDBValid(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Returned an error 1") + assert.NotEqual(nil, db.client, "Did not return a proper client") + assert.NotEqual(nil, db.database, "Did not return a proper database") + + db2, err := NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Returned an error 2") + assert.NotEqual(nil, db2.client, "Did not return a second proper client") + assert.NotEqual(nil, db2.database, "Did not return a second proper database") + assert.NotEqual(db.client, db2.client, "Did not return a unique client") + assert.NotEqual(db.database, db2.database, "Did not return a unique database") +} + +func TestNewMongoDBInvalid(t *testing.T) { + var assert = assert.New(t) + + var _, err = NewMongoDB("invalid://0.0.0.0:27017", "test") + assert.NotEqual(nil, err, "Did not return an error 1") + + _, err = NewMongoDB("mongodb://0.0.0.0:9999", "test") + assert.NotEqual(nil, err, "Did not return an error 2") + + // At this stage the database is not created, so it is not possible to validate the name + // _, err = NewMongoDB("mongodb://0.0.0.0:27017", strings.Repeat("a", 10000)) + // assert.NotEqual(nil, err, "Did not return an error 3") +} diff --git a/test/example.txt b/test/example.txt new file mode 100644 index 0000000..e5d3534 --- /dev/null +++ b/test/example.txt @@ -0,0 +1 @@ +Lorem ipsum \ No newline at end of file diff --git a/tracker.go b/tracker.go index 009d94c..ccbbfb8 100644 --- a/tracker.go +++ b/tracker.go @@ -114,7 +114,7 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a if err == nil && onlyIfDifferentPassed && onlyIfUniquePassed { var timestamp = time.Now().Unix() - err = mongo.Write(config.Name, bson.D{{"timestamp", timestamp}, {"value", res}}) + err = mongo.Write(config.Name, bson.D{{Key: "timestamp", Value: timestamp}, {Key: "value", Value: res}}) if err != nil { fmt.Printf("Failed to write to MongoDB: %v", err) } else { From 8e010ac7b1083413dd840a178f319f456d0479df Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Mon, 11 Nov 2024 12:49:01 -0500 Subject: [PATCH 2/6] Added MongoDB setup to workflows --- .github/workflows/create-release.yml | 5 +++++ .github/workflows/run-tests.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 9fc5d28..588d9f3 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -28,6 +28,11 @@ jobs: - name: Display Go version run: go version + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.11.0 + with: + mongodb-version: '8.0' + - name: Run tests run: go test diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2875f19..aba2546 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -28,5 +28,10 @@ jobs: - name: Display Go version run: go version + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.11.0 + with: + mongodb-version: '8.0' + - name: Run tests run: go test \ No newline at end of file From 728e52384ba5aa012373a99746d0f96567cb37c9 Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Mon, 11 Nov 2024 12:53:20 -0500 Subject: [PATCH 3/6] Run tests in subdirectories --- .github/workflows/create-release.yml | 2 +- .github/workflows/run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 588d9f3..bf8472a 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -34,7 +34,7 @@ jobs: mongodb-version: '8.0' - name: Run tests - run: go test + run: go test ./... - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index aba2546..a8db2d4 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -34,4 +34,4 @@ jobs: mongodb-version: '8.0' - name: Run tests - run: go test \ No newline at end of file + run: go test ./... \ No newline at end of file From ad02b0ecd658690fe7009339ac3a4d83e60fa58d Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Mon, 11 Nov 2024 13:53:15 -0500 Subject: [PATCH 4/6] Add more MongoDB test cases, prevent using unitialized client or database --- mongodb/mongodb.go | 31 ++++++++++++++--- mongodb/mongodb_test.go | 77 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index 44d2bcf..246a8ff 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -2,6 +2,7 @@ package mongodb import ( "context" + "errors" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -34,20 +35,26 @@ func NewMongoDB(uri string, databaseName string) (result MongoDB, err error) { return result, err } -func Connect(uri string) (client *mongo.Client, err error) { - return mongo.Connect(options.Client().ApplyURI(uri)) -} +func (m *MongoDB) Disconnect() error { + if m.client == nil { + return errors.New("client is nil") + } -func (m *MongoDB) Disconnect() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := m.client.Disconnect(ctx); err != nil { panic(err) } + + return nil } func (m *MongoDB) CreateCollection(collection string) (err error) { + if m.database == nil { + return errors.New("database is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() // Ensure that we create only a missing collection @@ -64,6 +71,10 @@ func (m *MongoDB) CreateCollection(collection string) (err error) { } func (m *MongoDB) Write(collection string, data bson.D) (err error) { + if m.database == nil { + return errors.New("database is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() mongoCollection := m.database.Collection(collection) @@ -73,6 +84,10 @@ func (m *MongoDB) Write(collection string, data bson.D) (err error) { } func (m *MongoDB) GetLastDocumentFiltered(collection string, sortedKey string, filter bson.D) (result bson.D, err error) { + if m.database == nil { + return result, errors.New("database is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -96,6 +111,10 @@ func (m *MongoDB) GetLastDocument(collection string, sortedKey string) (result b } func (m *MongoDB) GetAllDocuments(collection string) (result []bson.D, err error) { + if m.database == nil { + return result, errors.New("database is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -122,6 +141,10 @@ func (m *MongoDB) GetAllDocuments(collection string) (result []bson.D, err error } func (m *MongoDB) DropCollection(collection string) (err error) { + if m.database == nil { + return errors.New("database is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/mongodb/mongodb_test.go b/mongodb/mongodb_test.go index e4d79ce..80fdc20 100644 --- a/mongodb/mongodb_test.go +++ b/mongodb/mongodb_test.go @@ -1,9 +1,11 @@ package mongodb import ( + "strings" "testing" "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/v2/bson" ) func TestNewMongoDBValid(t *testing.T) { @@ -35,3 +37,78 @@ func TestNewMongoDBInvalid(t *testing.T) { // _, err = NewMongoDB("mongodb://0.0.0.0:27017", strings.Repeat("a", 10000)) // assert.NotEqual(nil, err, "Did not return an error 3") } + +func TestDisconnect(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + err = db.Disconnect() + assert.Equal(nil, err, "Returned an error on proper disconnect") + + var db2 = MongoDB{} + err = db2.Disconnect() + assert.NotEqual(nil, err, "Did not return an error for improper db disconnect") +} + +func TestCreateCollection(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + err = db.CreateCollection("a") + assert.Equal(nil, err, "Did not create a collection") + + err = db.CreateCollection("a") + assert.Equal(nil, err, "Did not skip an existing collection") + + err = db.CreateCollection(strings.Repeat("a", 256)) + assert.NotEqual(nil, err, "Was able to use a long collection name") + + err = (&MongoDB{}).CreateCollection("a") + assert.NotEqual(nil, err, "Was able to use an initialized database") +} + +func TestWrite(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + // MongoDB connector for Go allows to write to a collection which was not created using CreateCollection + // This means that it is not really possible to fail a write operation due to non-existing collection + err = db.Write("invalid", bson.D{{Key: "hello", Value: "world"}}) + assert.Equal(nil, err, "Did not write to a collection 1") + + // Empty objects are also accepted + err = db.Write("a", bson.D{}) + assert.Equal(nil, err, "Did not write to a collection 2") + + // Using a custom _id is also accepted + err = db.Write("a", bson.D{{Key: "_id", Value: "z"}}) + assert.Equal(nil, err, "Did not write to a collection 3") + + err = (&MongoDB{}).Write("a", bson.D{}) + assert.NotEqual(nil, err, "Was able to use an initialized database") +} + +func TestDropCollection(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + err = db.Write("a", bson.D{}) + assert.Equal(nil, err, "Did not write to a collection") + err = db.DropCollection("a") + assert.Equal(nil, err, "Did not drop a collection") + documents, err := db.GetAllDocuments("a") + // MongoDB allows to read a dropped collection, but it should be empty + assert.Equal(nil, err, "Was not able to read from a dropped collection") + assert.Equal(0, len(documents), "Was able to find entries in a dropped collection") + + err = (&MongoDB{}).DropCollection("a") + assert.NotEqual(nil, err, "Was able to use an initialized database") +} From 03342eb0104cf8cc87c9490e08725fb2de42ef4e Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Tue, 12 Nov 2024 13:49:06 -0500 Subject: [PATCH 5/6] Implemented MongoDB tests, fixed GetLastDocumentFiltered when using a filter --- mongodb/mongodb.go | 11 +++- mongodb/mongodb_test.go | 114 +++++++++++++++++++++++++++++++++++++++- tracker.go | 13 +---- 3 files changed, 125 insertions(+), 13 deletions(-) diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index 246a8ff..42822b2 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -11,6 +11,15 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/readpref" ) +func BsonToRaw(value bson.D) (document bson.Raw, err error) { + doc, err := bson.Marshal(value) + if err != nil { + return document, err + } + err = bson.Unmarshal(doc, &document) + return +} + type MongoDB struct { client *mongo.Client database *mongo.Database @@ -92,7 +101,7 @@ func (m *MongoDB) GetLastDocumentFiltered(collection string, sortedKey string, f defer cancel() mongoCollection := m.database.Collection(collection) - count, err := mongoCollection.CountDocuments(ctx, bson.D{}) + count, err := mongoCollection.CountDocuments(ctx, filter) if err != nil || count == 0 { return result, err } diff --git a/mongodb/mongodb_test.go b/mongodb/mongodb_test.go index 80fdc20..fc7d6f2 100644 --- a/mongodb/mongodb_test.go +++ b/mongodb/mongodb_test.go @@ -77,9 +77,13 @@ func TestWrite(t *testing.T) { var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") assert.Equal(nil, err, "Did not connect to a database") + // Drop the test collection before validating + err = db.DropCollection("a") + assert.Equal(nil, err, "Did not drop a collection") + // MongoDB connector for Go allows to write to a collection which was not created using CreateCollection // This means that it is not really possible to fail a write operation due to non-existing collection - err = db.Write("invalid", bson.D{{Key: "hello", Value: "world"}}) + err = db.Write("a", bson.D{{Key: "hello", Value: "world"}}) assert.Equal(nil, err, "Did not write to a collection 1") // Empty objects are also accepted @@ -94,6 +98,114 @@ func TestWrite(t *testing.T) { assert.NotEqual(nil, err, "Was able to use an initialized database") } +func TestGetLastDocument(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + // Drop the test collection before validating + err = db.DropCollection("test") + assert.Equal(nil, err, "Did not drop a collection") + + document, err := db.GetLastDocument("test", "none") + assert.Equal(nil, err, "Did not return a document 1") + assert.Equal(0, len(document), "Returned a non-empty document") + + err = db.Write("test", bson.D{{Key: "hello", Value: "a"}}) + assert.Equal(nil, err, "Did not write to a collection 1") + + document, err = db.GetLastDocument("test", "hello") + assert.Equal(nil, err, "Did not return a document 2") + assert.True(len(document) > 0, "Returned an empty document") + rawDocument, err := BsonToRaw(document) + assert.Equal(nil, err, "Failed to convert a document 1") + assert.Equal("a", rawDocument.Lookup("hello").StringValue(), "Incorrect document value 1") + + err = db.Write("test", bson.D{{Key: "hello", Value: "c"}}) + assert.Equal(nil, err, "Did not write to a collection 2") + + err = db.Write("test", bson.D{{Key: "hello", Value: "b"}}) + assert.Equal(nil, err, "Did not write to a collection 3") + + document, err = db.GetLastDocument("test", "hello") + assert.Equal(nil, err, "Did not return a document 3") + assert.True(len(document) > 0, "Returned an empty document 2") + rawDocument, err = BsonToRaw(document) + assert.Equal(nil, err, "Failed to convert a document 2") + assert.Equal("c", rawDocument.Lookup("hello").StringValue(), "Incorrect document value 2") + + document, err = db.GetLastDocument("test", "incorrect") + assert.Equal(nil, err, "Did not return a document 4") + assert.True(len(document) > 0, "Returned an empty document 3") + rawDocument, err = BsonToRaw(document) + assert.Equal(nil, err, "Failed to convert a document 3") + assert.Equal("b", rawDocument.Lookup("hello").StringValue(), "Incorrect document value 3") +} + +func TestGetLastDocumentFiltered(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + // Drop the test collection before validating + err = db.DropCollection("test") + assert.Equal(nil, err, "Did not drop a collection") + + db.Write("test", bson.D{{Key: "filter", Value: "ok"}, {Key: "hello", Value: "a"}}) + db.Write("test", bson.D{{Key: "filter", Value: "ok"}, {Key: "hello", Value: "b"}}) + db.Write("test", bson.D{{Key: "filter", Value: "notok"}, {Key: "hello", Value: "c"}}) + + document, err := db.GetLastDocumentFiltered("test", "hello", bson.D{{Key: "filter", Value: "invalid"}}) + assert.Equal(nil, err, "Did not return a document for an invalid request") + assert.Equal(0, len(document), "Returned a non-empty document") + + document, err = db.GetLastDocumentFiltered("test", "hello", bson.D{{Key: "filter", Value: "ok"}}) + assert.Equal(nil, err, "Did not return a document 1") + assert.True(len(document) > 0, "Returned an empty document 1") + rawDocument, err := BsonToRaw(document) + assert.Equal(nil, err, "Failed to convert a document 1") + assert.Equal("b", rawDocument.Lookup("hello").StringValue(), "Incorrect document value 1") + + document, err = db.GetLastDocumentFiltered("test", "hello", bson.D{{Key: "filter", Value: "notok"}}) + assert.Equal(nil, err, "Did not return a document 2") + assert.True(len(document) > 0, "Returned an empty document 2") + rawDocument, err = BsonToRaw(document) + assert.Equal(nil, err, "Failed to convert a document 2") + assert.Equal("c", rawDocument.Lookup("hello").StringValue(), "Incorrect document value 2") +} + +func TestGetAllDocuments(t *testing.T) { + var assert = assert.New(t) + + var db, err = NewMongoDB("mongodb://0.0.0.0:27017", "test") + assert.Equal(nil, err, "Did not connect to a database") + + // Drop the test collection before validating + err = db.DropCollection("test") + assert.Equal(nil, err, "Did not drop a collection") + + documents, err := db.GetAllDocuments("test") + assert.Equal(nil, err, "Did not return documents 1") + assert.Equal(0, len(documents), "Returned a non-empty document list") + + db.Write("test", bson.D{{Key: "filter", Value: "ok"}, {Key: "hello", Value: "a"}}) + db.Write("test", bson.D{{Key: "filter", Value: "notok"}, {Key: "hello", Value: "b"}}) + + documents, err = db.GetAllDocuments("test") + assert.Equal(nil, err, "Did not return documents 2") + assert.Equal(2, len(documents), "Incorrect documents count") + rawDocument1, err := BsonToRaw(documents[0]) + assert.Equal(nil, err, "Failed to convert a document 1") + rawDocument2, err := BsonToRaw(documents[1]) + assert.Equal(nil, err, "Failed to convert a document 2") + assert.Equal("a", rawDocument1.Lookup("hello").StringValue(), "Incorrect document value 1-1") + assert.Equal("ok", rawDocument1.Lookup("filter").StringValue(), "Incorrect document value 1-2") + assert.Equal("b", rawDocument2.Lookup("hello").StringValue(), "Incorrect document value 2-1") + assert.Equal("notok", rawDocument2.Lookup("filter").StringValue(), "Incorrect document value 2-2") +} + func TestDropCollection(t *testing.T) { var assert = assert.New(t) diff --git a/tracker.go b/tracker.go index ccbbfb8..cb11cc7 100644 --- a/tracker.go +++ b/tracker.go @@ -12,15 +12,6 @@ import ( "go.mongodb.org/mongo-driver/v2/bson" ) -func bsonToRaw(value bson.D) (document bson.Raw, err error) { - doc, err := bson.Marshal(value) - if err != nil { - return document, err - } - err = bson.Unmarshal(doc, &document) - return -} - func floatToNiceString(value float64) (res string) { res = fmt.Sprintf("%f", value) // Trim 0s past the decimal point @@ -52,7 +43,7 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a log.Fatal(err) } - lastDocument, err := bsonToRaw(lastDocumentBson) + lastDocument, err := mongodb.BsonToRaw(lastDocumentBson) if err != nil { log.Fatal(err) } @@ -67,7 +58,7 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a log.Fatal(err) } for _, documentBson := range documentBsons { - document, err := bsonToRaw(documentBson) + document, err := mongodb.BsonToRaw(documentBson) if err != nil { log.Fatal(err) } From 3cd9a31a0753a33d1fff4a49f629592769ac52be Mon Sep 17 00:00:00 2001 From: Ihor Sikachyna Date: Wed, 13 Nov 2024 11:39:33 -0500 Subject: [PATCH 6/6] Implemented versionning support for queries --- README.md | 2 +- autoini/autoini.go | 2 ++ main.go | 4 +-- mongodb/mongodb.go | 1 + query.go | 5 ++- tracker.go | 76 ++++++++++++++++++++++++++++++++++++++++++---- 6 files changed, 80 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 75b07ac..7d95a7b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Application written in Go to collect the information from webpages or API endpoi ## Prerequisites - Tested with Go 1.23.2 -- [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/) +- [MongoDB 8.0](https://www.mongodb.com/docs/manual/administration/install-community/) ## Installation diff --git a/autoini/autoini.go b/autoini/autoini.go index 16d7b46..9aae8e7 100644 --- a/autoini/autoini.go +++ b/autoini/autoini.go @@ -92,6 +92,8 @@ func ReadIni[T Configurable](path string) (result T) { setStringKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg) case "int": setIntKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg) + case "int64": + setIntKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg) case "bool": setBoolKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg) default: diff --git a/main.go b/main.go index a8a9a26..b66842d 100644 --- a/main.go +++ b/main.go @@ -22,7 +22,7 @@ func main() { defer mongo.Disconnect() // Create the default versions collection - err = mongo.CreateCollection("versions") + err = mongo.CreateCollection(config.VersionCollectionName) if err != nil { log.Fatal(err) } @@ -30,7 +30,7 @@ func main() { var dir = "./queries" var stopRequest = make(chan any) var stopResponse = make(chan any) - err = StartTrackers(ListIniFiles(dir), mongo, stopRequest, stopResponse) + err = StartTrackers(ListIniFiles(dir), config, mongo, stopRequest, stopResponse) if err != nil { log.Fatal(err) } diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index 42822b2..27ebb2f 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -107,6 +107,7 @@ func (m *MongoDB) GetLastDocumentFiltered(collection string, sortedKey string, f } opts := options.FindOne().SetSort(bson.D{{Key: sortedKey, Value: 1}}).SetSkip(count - 1) + // TODO: Return un-decoded result err = mongoCollection.FindOne(ctx, filter, opts).Decode(&result) if err != nil { return result, err diff --git a/query.go b/query.go index 32b7925..fc4bacd 100644 --- a/query.go +++ b/query.go @@ -3,7 +3,8 @@ package main import "log" type QueryConfig struct { - Name string + Name string // Internal only + Version int64 // Internal only Url string AnyTag string Before string @@ -19,6 +20,8 @@ func (q QueryConfig) Optional(key string) bool { switch key { case "Name": return true + case "Version": + return true case "Url": return false case "AnyTag": diff --git a/tracker.go b/tracker.go index cb11cc7..fe9bb5e 100644 --- a/tracker.go +++ b/tracker.go @@ -105,7 +105,7 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a if err == nil && onlyIfDifferentPassed && onlyIfUniquePassed { var timestamp = time.Now().Unix() - err = mongo.Write(config.Name, bson.D{{Key: "timestamp", Value: timestamp}, {Key: "value", Value: res}}) + err = mongo.Write(config.Name, bson.D{{Key: "timestamp", Value: timestamp}, {Key: "value", Value: res}, {Key: "version", Value: config.Version}}) if err != nil { fmt.Printf("Failed to write to MongoDB: %v", err) } else { @@ -129,18 +129,78 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a } } -func StartTrackers(configs []string, mongo mongodb.MongoDB, stopRequest chan any, stopResponse chan any) (err error) { +func writeQueryVersion(mongo mongodb.MongoDB, globalConfig Config, queryName string, queryHash string, version int64) error { + return mongo.Write(globalConfig.VersionCollectionName, bson.D{{Key: "version", Value: version}, {Key: "name", Value: queryName}, {Key: "hash", Value: queryHash}}) +} + +func checkQueryVersion(mongo mongodb.MongoDB, globalConfig Config, queryName string, configPath string) (int64, error) { + document, err := mongo.GetLastDocumentFiltered(globalConfig.VersionCollectionName, "version", bson.D{{Key: "name", Value: queryName}}) + if err != nil { + return 0, err + } + + queryHash, err := GetFileHash(configPath) + if err != nil { + return 0, err + } + fmt.Println(queryName) + fmt.Println(queryHash) + + // Document exists and we potentially need to increment the version + if len(document) > 0 { + rawDocument, err := mongodb.BsonToRaw(document) + if err != nil { + return 0, err + } + var hash = rawDocument.Lookup("hash").StringValue() + var version, ok = rawDocument.Lookup("version").AsInt64OK() + if !ok { + return 0, errors.New("failed to read version number from MongoDB") + } + if hash != queryHash { + // Check if some older query version is used + oldVersionDocument, err := mongo.GetLastDocumentFiltered(globalConfig.VersionCollectionName, "version", bson.D{{Key: "name", Value: queryName}, {Key: "hash", Value: queryHash}}) + if err != nil { + return 0, err + } + + if len(oldVersionDocument) > 0 { + // Found a matching old version + rawDocument, err := mongodb.BsonToRaw(oldVersionDocument) + if err != nil { + return 0, err + } + version, ok = rawDocument.Lookup("version").AsInt64OK() + if !ok { + return 0, errors.New("failed to read version number from MongoDB") + } + return version, nil + } else { + // It is a new query which requires a version increment + version++ + return version, writeQueryVersion(mongo, globalConfig, queryName, queryHash, version) + } + } + // If the hash matches then no action is required, simply return the latest version + return version, nil + } else { + // It is a new query without a version entry + return 0, writeQueryVersion(mongo, globalConfig, queryName, queryHash, 0) + } +} + +func StartTrackers(queries []string, globalConfig Config, mongo mongodb.MongoDB, stopRequest chan any, stopResponse chan any) (err error) { // Reserve the "versions" name since it is used for query versioning - for _, configPath := range configs { + for _, configPath := range queries { var fileName = GetFileNameWithoutExtension(configPath) - if fileName == "versions" { - return errors.New("versions name is reserved") + if fileName == globalConfig.VersionCollectionName { + return errors.New("version collection name is reserved") } } go func() { var stopChannels = []chan any{} - for _, configPath := range configs { + for _, configPath := range queries { var threadStopResponse = make(chan any) stopChannels = append(stopChannels, threadStopResponse) var config = autoini.ReadIni[QueryConfig](configPath) @@ -149,6 +209,10 @@ func StartTrackers(configs []string, mongo mongodb.MongoDB, stopRequest chan any if err != nil { log.Fatal(err) } + config.Version, err = checkQueryVersion(mongo, globalConfig, config.Name, configPath) + if err != nil { + log.Fatal(err) + } go trackerThread(config, mongo, stopRequest, threadStopResponse) } // Await all channels to terminate