Merge branch 'main' into encountertime

.github/workflows/publish.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: '1.21.x'
       - name: Install dependencies

config.toml.example

@@ -14,7 +14,7 @@ pokemon = true          # Keep pokemon table is kept nice and short
 incidents = true        # Remove incidents after expiry
 quests = true           # Remove quests after expiry
 stats = true            # Enable/Disable stats history
-stats_days = 7          # Remove entries from ["pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats" after x days
+stats_days = 7          # Remove entries from "pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "quest_stats", "raid_stats" after x days
 device_hours = 24       # Remove devices from in memory after not seen for x hours
 
 [logging]
@@ -31,7 +31,6 @@ password = ""
 address = "127.0.0.1:3306"
 db = ""
 
-
 [pvp]
 enabled = true
 include_hundos_under_cap = false
@@ -58,3 +57,8 @@ url = "http://localhost:4201"
 #url = "http://localhost:4202"
 #types = ["raid"]
 #areas = ["London/*", "*/Harrow", "Harrow"]
+
+[tuning]
+max_pokemon_distance = 100  # Maximum distance in kilometers for searching pokemon
+max_pokemon_results = 3000  # Maximum number of pokemon to return
+extended_timeout = false

config/config.go

@@ -111,8 +111,9 @@ type database struct {
 }
 
 type tuning struct {
-	ExtendedTimeout   bool `koanf:"extended_timeout"`
-	MaxPokemonResults int  `koanf:"max_pokemon_results"`
+	ExtendedTimeout    bool    `koanf:"extended_timeout"`
+	MaxPokemonResults  int     `koanf:"max_pokemon_results"`
+	MaxPokemonDistance float64 `koanf:"max_pokemon_distance"`
 }
 
 type scanRule struct {

config/reader.go

@@ -47,7 +47,8 @@ func ReadConfig() (configDefinition, error) {
 			MaxPool: 100,
 		},
 		Tuning: tuning{
-			MaxPokemonResults: 3000,
+			MaxPokemonResults:  3000,
+			MaxPokemonDistance: 100,
 		},
 		Pvp: pvp{
 			LevelCaps: []int{50, 51},

db/pokestop.go

@@ -3,9 +3,10 @@ package db
 import (
 	"context"
 	"database/sql"
-	"golbat/geo"
+	"errors"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/paulmach/orb/geojson"
 )
 
 type QuestLocation struct {
@@ -24,14 +25,17 @@ type QuestStatus struct {
 	TotalStops uint32 `db:"total" json:"total"`
 }
 
-func GetPokestopPositions(db DbDetails, fence geo.Geofence) ([]QuestLocation, error) {
-	bbox := fence.GetBoundingBox()
-
+func GetPokestopPositions(db DbDetails, fence *geojson.Feature) ([]QuestLocation, error) {
+	bbox := fence.Geometry.Bound()
+	bytes, err := fence.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
 	areas := []QuestLocation{}
-	err := db.GeneralDb.Select(&areas, "SELECT id, lat, lon FROM pokestop "+
+	err = db.GeneralDb.Select(&areas, "SELECT id, lat, lon FROM pokestop "+
 		"WHERE lat > ? and lon > ? and lat < ? and lon < ? and enabled = 1 "+
-		"and ST_CONTAINS(ST_GEOMFROMTEXT('POLYGON(("+fence.ToPolygonString()+"))'), point(lat,lon))",
-		bbox.MinimumLatitude, bbox.MinimumLongitude, bbox.MaximumLatitude, bbox.MaximumLongitude)
+		"and ST_CONTAINS(ST_GeomFromGeoJSON('"+string(bytes)+"', 2, 0), POINT(lon, lat))",
+		bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon())
 
 	statsCollector.IncDbQuery("select pokestop-positions", err)
 	if err == sql.ErrNoRows {
@@ -45,33 +49,92 @@ func GetPokestopPositions(db DbDetails, fence geo.Geofence) ([]QuestLocation, er
 	return areas, nil
 }
 
-func RemoveQuests(ctx context.Context, db DbDetails, fence geo.Geofence) (sql.Result, error) {
-	bbox := fence.GetBoundingBox()
-
-	query := "UPDATE pokestop " +
-		"SET " +
-		"quest_type = NULL," +
-		"quest_timestamp = NULL," +
-		"quest_target = NULL," +
-		"quest_conditions = NULL," +
-		"quest_rewards = NULL," +
-		"quest_template = NULL," +
-		"quest_title = NULL, " +
-		"quest_expiry = NULL, " +
-		"alternative_quest_type = NULL," +
-		"alternative_quest_timestamp = NULL," +
-		"alternative_quest_target = NULL," +
-		"alternative_quest_conditions = NULL," +
-		"alternative_quest_rewards = NULL," +
-		"alternative_quest_template = NULL," +
-		"alternative_quest_title = NULL, " +
-		"alternative_quest_expiry = NULL " +
-		"WHERE lat > ? and lon > ? and lat < ? and lon < ? and enabled = 1 " +
-		"and ST_CONTAINS(ST_GEOMFROMTEXT('POLYGON((" + fence.ToPolygonString() + "))'), point(lat,lon))"
-	res, err := db.GeneralDb.ExecContext(ctx, query,
-		bbox.MinimumLatitude, bbox.MinimumLongitude, bbox.MaximumLatitude, bbox.MaximumLongitude)
+func RemoveQuests(ctx context.Context, db DbDetails, fence *geojson.Feature) (int64, error) {
+	const updateChunkSize = 500
+
+	//goland:noinspection GoPreferNilSlice
+	allIdsToUpdate := []string{}
+	var removedQuestsCount int64
+
+	bbox := fence.Geometry.Bound()
+	bytes, err := fence.MarshalJSON()
+	if err != nil {
+		statsCollector.IncDbQuery("remove quests", err)
+		return removedQuestsCount, err
+	}
+
+	idQueryString := "SELECT `id` FROM `pokestop` " +
+		"WHERE lat >= ? and lon >= ? and lat <= ? and lon <= ? and enabled = 1 " +
+		"AND ST_CONTAINS(ST_GeomFromGeoJSON('" + string(bytes) + "', 2, 0), POINT(lon, lat))"
+
+	//log.Debugf("Clear quests query: %s", idQueryString)
+
+	// collect allIdsToUpdate
+	err = db.GeneralDb.Select(&allIdsToUpdate, idQueryString,
+		bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(),
+	)
+
+	if errors.Is(err, sql.ErrNoRows) {
+		statsCollector.IncDbQuery("remove quests", err)
+		return removedQuestsCount, nil
+	}
+
+	if err != nil {
+		statsCollector.IncDbQuery("remove quests", err)
+		return removedQuestsCount, err
+	}
+
+	for {
+		// take at most updateChunkSize elements from allIdsToUpdate
+		updateIdsCount := len(allIdsToUpdate)
+
+		if updateIdsCount == 0 {
+			break
+		}
+
+		if updateIdsCount > updateChunkSize {
+			updateIdsCount = updateChunkSize
+		}
+
+		updateIds := allIdsToUpdate[:updateIdsCount]
+
+		// remove processed elements from allIdsToUpdate
+		allIdsToUpdate = allIdsToUpdate[updateIdsCount:]
+
+		query, args, _ := sqlx.In("UPDATE pokestop "+
+			"SET "+
+			"quest_type = NULL,"+
+			"quest_timestamp = NULL,"+
+			"quest_target = NULL,"+
+			"quest_conditions = NULL,"+
+			"quest_rewards = NULL,"+
+			"quest_template = NULL,"+
+			"quest_title = NULL, "+
+			"quest_expiry = NULL, "+
+			"alternative_quest_type = NULL,"+
+			"alternative_quest_timestamp = NULL,"+
+			"alternative_quest_target = NULL,"+
+			"alternative_quest_conditions = NULL,"+
+			"alternative_quest_rewards = NULL,"+
+			"alternative_quest_template = NULL,"+
+			"alternative_quest_title = NULL, "+
+			"alternative_quest_expiry = NULL "+
+			"WHERE id IN (?)", updateIds)
+
+		query = db.GeneralDb.Rebind(query)
+		res, err := db.GeneralDb.ExecContext(ctx, query, args...)
+
+		if err != nil {
+			statsCollector.IncDbQuery("remove quests", err)
+			return removedQuestsCount, err
+		}
+
+		rowsAffected, _ := res.RowsAffected()
+		removedQuestsCount += rowsAffected
+	}
+
 	statsCollector.IncDbQuery("remove quests", err)
-	return res, err
+	return removedQuestsCount, err
 }
 
 func FindOldPokestops(ctx context.Context, db DbDetails, cellId int64) ([]string, error) {
@@ -106,30 +169,32 @@ func ClearOldPokestops(ctx context.Context, db DbDetails, stopIds []string) erro
 	return nil
 }
 
-func GetQuestStatus(db DbDetails, fence geo.Geofence) (QuestStatus, error) {
-	if len(fence.Fence) == 0 {
-		return QuestStatus{}, nil
+func GetQuestStatus(db DbDetails, fence *geojson.Feature) (QuestStatus, error) {
+	bbox := fence.Geometry.Bound()
+	status := QuestStatus{}
+
+	bytes, err := fence.MarshalJSON()
+	if err != nil {
+		return status, err
 	}
-	bbox := fence.GetBoundingBox()
 
-	areas := QuestStatus{}
-	err := db.GeneralDb.Get(&areas,
+	err = db.GeneralDb.Get(&status,
 		"SELECT COUNT(*) AS total, "+
 			"COUNT(CASE WHEN quest_type IS NOT NULL THEN 1 END) AS ar_quests, "+
 			"COUNT(CASE WHEN alternative_quest_type IS NOT NULL THEN 1 END) AS no_ar_quests FROM pokestop "+
 			"WHERE lat > ? AND lon > ? AND lat < ? AND lon < ? AND enabled = 1 AND deleted = 0 "+
-			"AND ST_CONTAINS(ST_GEOMFROMTEXT('POLYGON(("+fence.ToPolygonString()+"))'), point(lat,lon)) ",
-		bbox.MinimumLatitude, bbox.MinimumLongitude, bbox.MaximumLatitude, bbox.MaximumLongitude,
+			"AND ST_CONTAINS(ST_GeomFromGeoJSON('"+string(bytes)+"', 2, 0), POINT(lon, lat)) ",
+		bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(),
 	)
 
 	statsCollector.IncDbQuery("select quest-status", err)
 	if err == sql.ErrNoRows {
-		return areas, nil
+		return status, nil
 	}
 
 	if err != nil {
-		return areas, err
+		return status, err
 	}
 
-	return areas, nil
+	return status, nil
 }

decoder/api_pokemon.go

@@ -1,15 +1,20 @@
 package decoder
 
 import (
-	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/rtree"
+	"fmt"
 	"golbat/config"
 	"golbat/geo"
+	"math"
 	"slices"
 	"strconv"
 	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/rtree"
 )
 
+const earthRadiusKm = 6371
+
 type ApiPokemonAvailableResult struct {
 	PokemonId int16 `json:"id"`
 	Form      int16 `json:"form"`
@@ -56,13 +61,38 @@ type ApiPokemonSearch struct {
 	SearchIds []int16      `json:"searchIds"`
 }
 
-func SearchPokemon(request ApiPokemonSearch) []*Pokemon {
+func calculateHypotenuse(a, b float64) float64 {
+	return math.Sqrt(a*a + b*b)
+}
+
+func toRadians(deg float64) float64 {
+	return deg * math.Pi / 180
+}
+
+func haversine(start, end geo.Location) float64 {
+	lat1Rad := toRadians(start.Latitude)
+	lat2Rad := toRadians(end.Latitude)
+	deltaLat := toRadians(end.Latitude - start.Latitude)
+	deltaLon := toRadians(end.Longitude - start.Longitude)
+
+	a := math.Sin(deltaLat/2)*math.Sin(deltaLat/2) +
+		math.Cos(lat1Rad)*math.Cos(lat2Rad)*
+			math.Sin(deltaLon/2)*math.Sin(deltaLon/2)
+	c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
+
+	return earthRadiusKm * c
+}
+
+func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) {
 	start := time.Now()
 	results := make([]*Pokemon, 0, request.Limit)
 	pokemonMatched := 0
 
 	if request.SearchIds == nil {
-		return nil
+		return nil, fmt.Errorf("SearchPokemon - no search ids provided")
+	}
+	if haversine(request.Min, request.Max) > config.Config.Tuning.MaxPokemonDistance {
+		return nil, fmt.Errorf("SearchPokemon - the distance between max and min points is greater than the configurable max distance")
 	}
 
 	pokemonTreeMutex.RLock()
@@ -75,8 +105,10 @@ func SearchPokemon(request ApiPokemonSearch) []*Pokemon {
 	}
 	pokemonSkipped := 0
 	pokemonScanned := 0
-	maxDistance := float64(1000) // This should come from the request?
-
+	maxDistance := calculateHypotenuse(request.Max.Longitude-request.Min.Longitude, request.Max.Latitude-request.Min.Latitude) / 2
+	if maxDistance == 0 {
+		maxDistance = 10
+	}
 	pokemonTree2.Nearby(
 		rtree.BoxDist[float64, uint64]([2]float64{request.Center.Longitude, request.Center.Latitude}, [2]float64{request.Center.Longitude, request.Center.Latitude}, nil),
 		func(min, max [2]float64, pokemonId uint64, dist float64) bool {
@@ -113,7 +145,7 @@ func SearchPokemon(request ApiPokemonSearch) []*Pokemon {
 	)
 
 	log.Infof("SearchPokemon - scanned %d pokemon, total time %s, %d returned", pokemonScanned, time.Since(start), len(results))
-	return results
+	return results, nil
 }
 
 // Get one result

decoder/geography.go

@@ -31,7 +31,10 @@ const kojiCacheFilename = "cache/koji_geofence.json"
 // const nestFilename = "geojson/nests.json"
 
 func SetKojiUrl(geofenceUrl string, bearerToken string) {
-	log.Print("Setting Koji Info "+geofenceUrl+"with bearer token:"+bearerToken != "")
+	if geofenceUrl == "" {
+		return
+	}
+	log.Infof("Setting koji url to %s with bearer token: %t", geofenceUrl, bearerToken != "")
 	kojiUrl = geofenceUrl
 	kojiBearerToken = bearerToken
 }

decoder/gym.go

@@ -229,7 +229,7 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64
 			gym.RaidPokemonEvolution = null.IntFrom(0)
 		}
 
-		gym.RaidIsExclusive = null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive))
+		gym.RaidIsExclusive = null.IntFrom(0) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive))
 	}
 
 	gym.CellId = null.IntFrom(int64(cellId))
@@ -536,6 +536,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) {
 	gymCache.Set(gym.Id, *gym, ttlcache.DefaultTTL)
 	createGymWebhooks(oldGym, gym)
 	createGymFortWebhooks(oldGym, gym)
+
+	areas := MatchStatsGeofence(gym.Lat, gym.Lon)
+	updateRaidStats(oldGym, gym, areas)
 }
 
 func updateGymGetMapFortCache(gym *Gym, skipName bool) {

decoder/incident.go

@@ -143,6 +143,14 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident
 
 	incidentCache.Set(incident.Id, *incident, ttlcache.DefaultTTL)
 	createIncidentWebhooks(ctx, db, oldIncident, incident)
+
+	stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId)
+	if stop == nil {
+		stop = &Pokestop{}
+	}
+
+	areas := MatchStatsGeofence(stop.Lat, stop.Lon)
+	updateIncidentStats(oldIncident, incident, areas)
 }
 
 func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *Incident, incident *Incident) {