Implemented versionning support for queries

README.md

@@ -7,7 +7,7 @@ Application written in Go to collect the information from webpages or API endpoi
 ## Prerequisites
 
 - Tested with Go 1.23.2
-- [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/)
+- [MongoDB 8.0](https://www.mongodb.com/docs/manual/administration/install-community/)
 
 ## Installation
 

autoini/autoini.go

@@ -92,6 +92,8 @@ func ReadIni[T Configurable](path string) (result T) {
 			setStringKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg)
 		case "int":
 			setIntKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg)
+		case "int64":
+			setIntKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg)
 		case "bool":
 			setBoolKey(result, fieldName, reflect.Indirect(configReflection).Field(i), cfg)
 		default:

main.go

@@ -22,15 +22,15 @@ func main() {
 	defer mongo.Disconnect()
 
 	// Create the default versions collection
-	err = mongo.CreateCollection("versions")
+	err = mongo.CreateCollection(config.VersionCollectionName)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	var dir = "./queries"
 	var stopRequest = make(chan any)
 	var stopResponse = make(chan any)
-	err = StartTrackers(ListIniFiles(dir), mongo, stopRequest, stopResponse)
+	err = StartTrackers(ListIniFiles(dir), config, mongo, stopRequest, stopResponse)
 	if err != nil {
 		log.Fatal(err)
 	}

mongodb/mongodb.go

@@ -107,6 +107,7 @@ func (m *MongoDB) GetLastDocumentFiltered(collection string, sortedKey string, f
 	}
 	opts := options.FindOne().SetSort(bson.D{{Key: sortedKey, Value: 1}}).SetSkip(count - 1)
 
+	// TODO: Return un-decoded result
 	err = mongoCollection.FindOne(ctx, filter, opts).Decode(&result)
 	if err != nil {
 		return result, err

query.go

@@ -3,7 +3,8 @@ package main
 import "log"
 
 type QueryConfig struct {
-	Name                   string
+	Name                   string // Internal only
+	Version                int64  // Internal only
 	Url                    string
 	AnyTag                 string
 	Before                 string
@@ -19,6 +20,8 @@ func (q QueryConfig) Optional(key string) bool {
 	switch key {
 	case "Name":
 		return true
+	case "Version":
+		return true
 	case "Url":
 		return false
 	case "AnyTag":

tracker.go

@@ -105,7 +105,7 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a
 
 					if err == nil && onlyIfDifferentPassed && onlyIfUniquePassed {
 						var timestamp = time.Now().Unix()
-						err = mongo.Write(config.Name, bson.D{{Key: "timestamp", Value: timestamp}, {Key: "value", Value: res}})
+						err = mongo.Write(config.Name, bson.D{{Key: "timestamp", Value: timestamp}, {Key: "value", Value: res}, {Key: "version", Value: config.Version}})
 						if err != nil {
 							fmt.Printf("Failed to write to MongoDB: %v", err)
 						} else {
@@ -129,18 +129,78 @@ func trackerThread(config QueryConfig, mongo mongodb.MongoDB, stopRequest chan a
 	}
 }
 
-func StartTrackers(configs []string, mongo mongodb.MongoDB, stopRequest chan any, stopResponse chan any) (err error) {
+func writeQueryVersion(mongo mongodb.MongoDB, globalConfig Config, queryName string, queryHash string, version int64) error {
+	return mongo.Write(globalConfig.VersionCollectionName, bson.D{{Key: "version", Value: version}, {Key: "name", Value: queryName}, {Key: "hash", Value: queryHash}})
+}
+
+func checkQueryVersion(mongo mongodb.MongoDB, globalConfig Config, queryName string, configPath string) (int64, error) {
+	document, err := mongo.GetLastDocumentFiltered(globalConfig.VersionCollectionName, "version", bson.D{{Key: "name", Value: queryName}})
+	if err != nil {
+		return 0, err
+	}
+
+	queryHash, err := GetFileHash(configPath)
+	if err != nil {
+		return 0, err
+	}
+	fmt.Println(queryName)
+	fmt.Println(queryHash)
+
+	// Document exists and we potentially need to increment the version
+	if len(document) > 0 {
+		rawDocument, err := mongodb.BsonToRaw(document)
+		if err != nil {
+			return 0, err
+		}
+		var hash = rawDocument.Lookup("hash").StringValue()
+		var version, ok = rawDocument.Lookup("version").AsInt64OK()
+		if !ok {
+			return 0, errors.New("failed to read version number from MongoDB")
+		}
+		if hash != queryHash {
+			// Check if some older query version is used
+			oldVersionDocument, err := mongo.GetLastDocumentFiltered(globalConfig.VersionCollectionName, "version", bson.D{{Key: "name", Value: queryName}, {Key: "hash", Value: queryHash}})
+			if err != nil {
+				return 0, err
+			}
+
+			if len(oldVersionDocument) > 0 {
+				// Found a matching old version
+				rawDocument, err := mongodb.BsonToRaw(oldVersionDocument)
+				if err != nil {
+					return 0, err
+				}
+				version, ok = rawDocument.Lookup("version").AsInt64OK()
+				if !ok {
+					return 0, errors.New("failed to read version number from MongoDB")
+				}
+				return version, nil
+			} else {
+				// It is a new query which requires a version increment
+				version++
+				return version, writeQueryVersion(mongo, globalConfig, queryName, queryHash, version)
+			}
+		}
+		// If the hash matches then no action is required, simply return the latest version
+		return version, nil
+	} else {
+		// It is a new query without a version entry
+		return 0, writeQueryVersion(mongo, globalConfig, queryName, queryHash, 0)
+	}
+}
+
+func StartTrackers(queries []string, globalConfig Config, mongo mongodb.MongoDB, stopRequest chan any, stopResponse chan any) (err error) {
 	// Reserve the "versions" name since it is used for query versioning
-	for _, configPath := range configs {
+	for _, configPath := range queries {
 		var fileName = GetFileNameWithoutExtension(configPath)
-		if fileName == "versions" {
-			return errors.New("versions name is reserved")
+		if fileName == globalConfig.VersionCollectionName {
+			return errors.New("version collection name is reserved")
 		}
 	}
 
 	go func() {
 		var stopChannels = []chan any{}
-		for _, configPath := range configs {
+		for _, configPath := range queries {
 			var threadStopResponse = make(chan any)
 			stopChannels = append(stopChannels, threadStopResponse)
 			var config = autoini.ReadIni[QueryConfig](configPath)
@@ -149,6 +209,10 @@ func StartTrackers(configs []string, mongo mongodb.MongoDB, stopRequest chan any
 			if err != nil {
 				log.Fatal(err)
 			}
+			config.Version, err = checkQueryVersion(mongo, globalConfig, config.Name, configPath)
+			if err != nil {
+				log.Fatal(err)
+			}
 			go trackerThread(config, mongo, stopRequest, threadStopResponse)
 		}
 		// Await all channels to terminate