Dynamic debug logging and spawnpoint age config (#49)

* Add endpoints to twiddle debug logging

This adds 2 new endpoints so that debug logging can
be turned on and off without restarting Fletchling.

* GET /debug/logging/on
* GET /debug/logging/off

This also fixes logging of the min_spawnpoints and area
config that was not cleaned up during a previous re-factor

* Add max_spawnpoint_age_days config option

This was hardcoded at 7 before and defaults to 7 now. Spawnpoints
older than this many days (not seen by Golbat) will not be
counted when computing spawnpoint counts for min_spawnpoints.

app_config/config.go

@@ -30,12 +30,7 @@ import (
 const (
 	DEFAULT_OVERPASS_URL = "https://overpass-api.de/api/interpreter"
 
-	DEFAULT_FILTER_CONCURRENCY  = 4
-	DEFAULT_MIN_SPAWNPOINTS     = 10
-	DEFAULT_MIN_AREA_M2         = float64(100)
-	DEFAULT_MAX_AREA_M2         = float64(10000000)
-	DEFAULT_MAX_OVERLAP_PERCENT = float64(60)
-	DEFAULT_NEST_NAME           = "Unknown Nest"
+	DEFAULT_NEST_NAME = "Unknown Nest"
 )
 
 type KojiConfig struct {
@@ -72,7 +67,7 @@ type Config struct {
 	NestsDb         db_store.DBConfig                `koanf:"nests_db"`
 	GolbatDb        *db_store.DBConfig               `koanf:"golbat_db"`
 	Overpass        overpass.Config                  `koanf:"overpass"`
-	Filters         filters.Config                   `koanf:"filters"`
+	Filters         filters.FiltersConfig            `koanf:"filters"`
 	Importer        importer.Config                  `koanf:"importer"`
 	Areas           areas.Config                     `koanf:"areas"`
 	WebhookSettings webhook_sender.SettingsConfig    `koanf:"webhook_settings"`
@@ -148,6 +143,8 @@ func (cfg *Config) Validate() error {
 }
 
 func GetDefaultConfig() Config {
+	defaultFilters := filters.DefaultFiltersConfig()
+
 	return Config{
 		Areas: areas.GetDefaultConfig(),
 
@@ -159,17 +156,11 @@ func GetDefaultConfig() Config {
 
 		Importer: importer.Config{
 			DefaultName: DEFAULT_NEST_NAME,
-			MinAreaM2:   DEFAULT_MIN_AREA_M2,
-			MaxAreaM2:   DEFAULT_MAX_AREA_M2,
+			MinAreaM2:   defaultFilters.MinAreaM2,
+			MaxAreaM2:   defaultFilters.MaxAreaM2,
 		},
 
-		Filters: filters.Config{
-			Concurrency:       DEFAULT_FILTER_CONCURRENCY,
-			MinSpawnpoints:    DEFAULT_MIN_SPAWNPOINTS,
-			MinAreaM2:         DEFAULT_MIN_AREA_M2,
-			MaxAreaM2:         DEFAULT_MAX_AREA_M2,
-			MaxOverlapPercent: DEFAULT_MAX_OVERLAP_PERCENT,
-		},
+		Filters: defaultFilters,
 
 		WebhookSettings: webhook_sender.SettingsConfig{
 			FlushIntervalSeconds: 1,

bin/fletchling-osm-importer/fletchling-osm-importer.go

@@ -277,11 +277,8 @@ func main() {
 	if areasProcessed > 0 && dbRefresher != nil {
 		logger.Infof("Gathering missing spawnpoints, running filters, and activating/deactivating nests...")
 		refreshConfig := filters.RefreshNestConfig{
-			Concurrency:       cfg.Filters.Concurrency,
-			MinAreaM2:         cfg.Filters.MinAreaM2,
-			MaxAreaM2:         cfg.Filters.MaxAreaM2,
-			MinSpawnpoints:    cfg.Filters.MinSpawnpoints,
-			MaxOverlapPercent: cfg.Filters.MaxOverlapPercent,
+			FiltersConfig: cfg.Filters,
+			Concurrency:   cfg.Filters.Concurrency,
 		}
 		err := dbRefresher.RefreshAllNests(ctx, refreshConfig)
 		if err == nil {

bin/fletchling/fletchling.go

@@ -192,12 +192,14 @@ func main() {
 	var filtersConfigMutex sync.Mutex
 	filtersConfig := cfg.Filters
 
-	getFiltersConfigFn := func() filters.Config {
+	getFiltersConfigFn := func() filters.FiltersConfig {
 		filtersConfigMutex.Lock()
 		defer filtersConfigMutex.Unlock()
 		return filtersConfig
 	}
 
+	cfg.Filters.Log(logger, "STARTUP: Filters config loaded: ")
+
 	reloadFn := func() error {
 		cfg, err := app_config.LoadConfig(configFilename, defaultConfig)
 		if err != nil {
@@ -207,6 +209,7 @@ func main() {
 		if err != nil {
 			return fmt.Errorf("failed to reload processor manager: %w", err)
 		}
+		cfg.Filters.Log(logger, "Filters config reloaded: ")
 		filtersConfigMutex.Lock()
 		defer filtersConfigMutex.Unlock()
 		filtersConfig = cfg.Filters

configs/fletchling.toml.example

@@ -55,6 +55,9 @@ max_area_m2 = 10000000.0
 ## via an API call as long as golbat_db is configured above.
 min_spawnpoints = 10
 
+## Ignore spawnpoints older than this age when computing spawnpoint counts.
+max_spawnpoint_age_days = 7
+
 ## How much overlap with a larger nest to allow. If any nest overlaps
 ## with a larger nest by more than this percent, then the nest will be
 ## deactivated upon filtering. (default 60)

db_store/golbat.go

@@ -2,7 +2,6 @@ package db_store
 
 import (
 	"context"
-	"database/sql"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/paulmach/orb/geojson"
@@ -14,50 +13,12 @@ type GolbatDBStore struct {
 	db     *sqlx.DB
 }
 
-func (st *GolbatDBStore) GetContainedSpawnpoints(ctx context.Context, geom *geojson.Geometry) (spawnpointIds []uint64, err error) {
-	const getContainedSpawnpointsQuery = `
-SELECT id FROM spawnpoint
-    WHERE lat > ? AND lon > ?
-		AND lat < ? AND lon < ?
-		AND last_seen > UNIX_TIMESTAMP(NOW() - INTERVAL 2 DAY)
-		AND ST_CONTAINS(ST_GeomFromGeoJSON(?, 2, 0), POINT(lon, lat))`
-
-	bbox := geom.Geometry().Bound()
-	bytes, err := geom.MarshalJSON()
-	if err != nil {
-		return nil, err
-	}
-
-	rows, err := st.db.QueryxContext(ctx, getContainedSpawnpointsQuery, bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(), bytes)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() { err = closeRows(rows, err) }()
-
-	spawnpointIds = make([]uint64, 0, 128)
-	var spawnpointId uint64
-
-	for rows.Next() {
-		if err = rows.Scan(&spawnpointId); err != nil {
-			if err == sql.ErrNoRows {
-				err = nil
-			}
-			spawnpointIds = nil
-			return
-		}
-		spawnpointIds = append(spawnpointIds, spawnpointId)
-	}
-
-	return
-}
-
-func (st *GolbatDBStore) GetSpawnpointsCount(ctx context.Context, geom *geojson.Geometry) (int64, error) {
+func (st *GolbatDBStore) GetSpawnpointsCount(ctx context.Context, geom *geojson.Geometry, maxDays int) (int64, error) {
 	const getContainedSpawnpointsQuery = `
 SELECT COUNT(*) FROM spawnpoint
     WHERE lat > ? AND lon > ?
 		AND lat < ? AND lon < ?
-		AND last_seen > UNIX_TIMESTAMP(NOW() - INTERVAL 7 DAY)
+		AND last_seen > UNIX_TIMESTAMP(NOW() - INTERVAL ? DAY)
 		AND ST_CONTAINS(ST_GeomFromGeoJSON(?, 2, 0), POINT(lon, lat))`
 
 	bbox := geom.Geometry().Bound()
@@ -66,7 +27,7 @@ SELECT COUNT(*) FROM spawnpoint
 		return 0, err
 	}
 
-	row := st.db.QueryRowxContext(ctx, getContainedSpawnpointsQuery, bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(), bytes)
+	row := st.db.QueryRowxContext(ctx, getContainedSpawnpointsQuery, bbox.Min.Lat(), bbox.Min.Lon(), bbox.Max.Lat(), bbox.Max.Lon(), maxDays, bytes)
 	if err != nil {
 		return 0, err
 	}

docs/API.md

@@ -25,6 +25,14 @@
 ## Get single nest and its stats history
 `curl http://localhost:9042/api/nests/_/:nest_id`
 
+## Enable debug logging
+
+`curl http://localhost:9042/debug/logging/on`
+
+## Disable debug logging
+
+`curl http://localhost:9042/debug/logging/off`
+
 Untested:
 
 ## Purge all stats

filters/config.go

@@ -1,12 +1,27 @@
 package filters
 
-import "fmt"
+import (
+	"fmt"
 
-type Config struct {
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	DEFAULT_FILTER_CONCURRENCY      = 4
+	DEFAULT_MIN_SPAWNPOINTS         = 10
+	DEFAULT_MAX_SPAWNPOINT_AGE_DAYS = 7
+	DEFAULT_MIN_AREA_M2             = float64(100)
+	DEFAULT_MAX_AREA_M2             = float64(10000000)
+	DEFAULT_MAX_OVERLAP_PERCENT     = float64(60)
+)
+
+type FiltersConfig struct {
 	// how many threads to use for filtering.
 	Concurrency int `koanf:"concurrency" json:"concurrency"`
 	// how many spawnpoints required in the geofence in order to track.
 	MinSpawnpoints int64 `koanf:"min_spawnpoints" json:"min_spawnpoints"`
+	// when querying spawnpoints, ignore spawnpoints older than this may days.
+	MaxSpawnpointAgeDays int `koanf:"max_spawnpoint_age_days" json:"max_spawnpoint_age_days"`
 	// minimum area required in order to track.
 	MinAreaM2 float64 `koanf:"min_area_m2" json:"min_area_m2"`
 	// maximum area that cannot be exceeded in order to track.
@@ -15,10 +30,16 @@ type Config struct {
 	MaxOverlapPercent float64 `koanf:"max_overlap_percent" json:"max_overlap_percent"`
 }
 
-func (cfg *Config) Validate() error {
+func (cfg *FiltersConfig) Validate() error {
 	if val := cfg.Concurrency; val < 1 {
 		return fmt.Errorf("concurrency should probably be at least 1, not %d", val)
 	}
+	if val := cfg.MinSpawnpoints; val < 0 {
+		return fmt.Errorf("min_spawnpoints should probably be at least 0, not %d", val)
+	}
+	if val := cfg.MaxSpawnpointAgeDays; val < 1 {
+		return fmt.Errorf("max_spawnpoint_age_days should probably be at least 1, not %d", val)
+	}
 	if val := cfg.MinSpawnpoints; val < 1 {
 		return fmt.Errorf("min_spawnpoints should probably be at least 1, not %d", val)
 	}
@@ -33,3 +54,25 @@ func (cfg *Config) Validate() error {
 	}
 	return nil
 }
+
+func (cfg *FiltersConfig) Log(logger *logrus.Logger, prefix string) {
+	logger.Infof("%sconcurrency=%d, min_spawnpoints=%d, max_spawnpoint_age_days=%d, min_area_m2=%0.3f max_area_m2=%0.3f",
+		prefix,
+		cfg.Concurrency,
+		cfg.MinSpawnpoints,
+		cfg.MaxSpawnpointAgeDays,
+		cfg.MinAreaM2,
+		cfg.MaxAreaM2,
+	)
+}
+
+func DefaultFiltersConfig() FiltersConfig {
+	return FiltersConfig{
+		Concurrency:          DEFAULT_FILTER_CONCURRENCY,
+		MinSpawnpoints:       DEFAULT_MIN_SPAWNPOINTS,
+		MaxSpawnpointAgeDays: DEFAULT_MAX_SPAWNPOINT_AGE_DAYS,
+		MinAreaM2:            DEFAULT_MIN_AREA_M2,
+		MaxAreaM2:            DEFAULT_MAX_AREA_M2,
+		MaxOverlapPercent:    DEFAULT_MAX_OVERLAP_PERCENT,
+	}
+}

filters/db_refresher.go

@@ -14,13 +14,9 @@ import (
 )
 
 type RefreshNestConfig struct {
+	FiltersConfig
 	Concurrency             int
 	ForceSpawnpointsRefresh bool
-
-	MinAreaM2         float64
-	MaxAreaM2         float64
-	MinSpawnpoints    int64
-	MaxOverlapPercent float64
 }
 
 type DBRefresher struct {
@@ -93,7 +89,7 @@ func (refresher *DBRefresher) refreshNest(ctx context.Context, config RefreshNes
 		if !spawnpoints.Valid {
 			refresher.logger.Infof("DB-REFRESHER[%s]: number of spawnpoints is unknown. Will query golbat for them.", fullName)
 		}
-		numSpawnpoints, err := refresher.golbatDBStore.GetSpawnpointsCount(ctx, jsonGeometry)
+		numSpawnpoints, err := refresher.golbatDBStore.GetSpawnpointsCount(ctx, jsonGeometry, config.MaxSpawnpointAgeDays)
 		if err == nil {
 			if spawnpoints.Valid {
 				if spawnpoints.Int64 != numSpawnpoints {

httpserver/api_config.go

@@ -31,12 +31,9 @@ func (srv *HTTPServer) doDBRefresh(c *gin.Context, allSpawnpoints bool) error {
 	ctx := c.Request.Context()
 
 	refreshConfig := filters.RefreshNestConfig{
+		FiltersConfig:           filtersConfig,
 		Concurrency:             concurrency,
 		ForceSpawnpointsRefresh: allSpawnpoints,
-		MinAreaM2:               filtersConfig.MinAreaM2,
-		MaxAreaM2:               filtersConfig.MaxAreaM2,
-		MinSpawnpoints:          filtersConfig.MinSpawnpoints,
-		MaxOverlapPercent:       filtersConfig.MaxOverlapPercent,
 	}
 
 	srv.logger.Infof("starting nest refresh")

httpserver/routes.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
 )
 
 func (srv *HTTPServer) authorizeAPI(c *gin.Context) {
@@ -48,6 +49,22 @@ func (srv *HTTPServer) setupRoutes() {
 
 	debugGroup := r.Group("/debug")
 
+	debugGroup.GET("/debug/logging/on", func(c *gin.Context) {
+		srv.logger.SetLevel(logrus.DebugLevel)
+		resp := struct {
+			Message string `json:"message"`
+		}{"debug logging is on"}
+		c.JSON(http.StatusOK, &resp)
+	})
+
+	debugGroup.GET("/debug/logging/off", func(c *gin.Context) {
+		srv.logger.SetLevel(logrus.InfoLevel)
+		resp := struct {
+			Message string `json:"message"`
+		}{"debug logging is off"}
+		c.JSON(http.StatusOK, &resp)
+	})
+
 	// run the GC. I guess this is also available via '/debug/pprof/heap?gc=1"
 	debugGroup.PUT("/gc/flush", func(c *gin.Context) {
 		now := time.Now()

httpserver/server.go

@@ -26,7 +26,7 @@ type HTTPServer struct {
 	statsCollector       stats_collector.StatsCollector
 	dbRefresher          *filters.DBRefresher
 	reloadFn             func() error
-	filtersConfigFn      func() filters.Config
+	filtersConfigFn      func() filters.FiltersConfig
 }
 
 // Run starts and runs the HTTP server until 'ctx' is cancelled or the server fails to start.
@@ -70,7 +70,7 @@ func (srv *HTTPServer) Run(ctx context.Context, address string, shutdownWaitTime
 	}
 }
 
-func NewHTTPServer(logger *logrus.Logger, nestProcessorManager *processor.NestProcessorManager, statsCollector stats_collector.StatsCollector, dbRefresher *filters.DBRefresher, reloadFn func() error, filtersConfigFn func() filters.Config) (*HTTPServer, error) {
+func NewHTTPServer(logger *logrus.Logger, nestProcessorManager *processor.NestProcessorManager, statsCollector stats_collector.StatsCollector, dbRefresher *filters.DBRefresher, reloadFn func() error, filtersConfigFn func() filters.FiltersConfig) (*HTTPServer, error) {
 	// Create the web server.
 	r := gin.New()
 	r.Use(gin.RecoveryWithWriter(logger.Writer()))

processor/config.go

@@ -23,12 +23,6 @@ const (
 type Config struct {
 	// Whether to log the last stats period when processing
 	LogLastStatsPeriod bool `koanf:"log_last_stats_period" json:"log_last_stats_period"`
-	// how many spawnpoints required in the geofence in order to track.
-	MinSpawnpoints int `koanf:"min_spawnpoints" json:"min_spawnpoints"`
-	// minimum area required in order to track.
-	MinAreaM2 float64 `koanf:"min_area_m2" json:"min_area_m2"`
-	// maximum area that cannot be exceeded in order to track.
-	MaxAreaM2 float64 `koanf:"max_area_m2" json:"max_area_m2"`
 	// how often to rotate stats
 	RotationIntervalMinutes int `koanf:"rotation_interval_minutes" json:"rotation_interval_minutes"`
 	// Require this many horus of stats in order to produce the nesting pokemon and update the DB.
@@ -53,9 +47,6 @@ type Config struct {
 
 func (cfg *Config) writeConfiguration(buf *bytes.Buffer) {
 	buf.WriteString(fmt.Sprintf("log_last_stats_period: %t, ", cfg.LogLastStatsPeriod))
-	buf.WriteString(fmt.Sprintf("min_spawnpoints: %d, ", cfg.MinSpawnpoints))
-	buf.WriteString(fmt.Sprintf("min_area_m2: %0.3f, ", cfg.MinAreaM2))
-	buf.WriteString(fmt.Sprintf("max_area_m2: %0.3f, ", cfg.MaxAreaM2))
 	buf.WriteString(fmt.Sprintf("rotation_interval_minutes: %d(%s), ", cfg.RotationIntervalMinutes, cfg.RotationInterval()))
 	buf.WriteString(fmt.Sprintf("min_history_duration_hours: %d(%s), ", cfg.MinHistoryDurationHours, cfg.MinHistoryDuration()))
 	buf.WriteString(fmt.Sprintf("max_history_duration_hours: %d(%s), ", cfg.MaxHistoryDurationHours, cfg.MaxHistoryDuration()))