config file: log_max_entries

cmd/system-api/main.go

@@ -53,9 +53,9 @@ func runCli(cCtx *cli.Context) (err error) {
 	pipeFile := cCtx.String("pipe-file")
 
 	// Create configuration file (load from file if requested)
-	config := systemapi.NewSystemAPIConfig()
+	config := systemapi.NewConfig()
 	if configFile != "" {
-		config, err = systemapi.LoadConfigFromFile(configFile)
+		config, err = systemapi.NewConfigFromFile(configFile)
 		if err != nil {
 			fmt.Println("Error loading config", err)
 			return err

systemapi-config.toml

@@ -5,6 +5,9 @@ pprof = true
 log_json = false
 log_debug = true
 
+# Maximum number of entries in the log
+log_max_entries = 1000
+
 # HTTP Basic Auth
 basic_auth_secret_path = "basic-auth-secret.txt" # basic auth is supported if a path is provided
 basic_auth_secret_salt = "D;%yL9TS:5PalS/d"      # use a random string for the salt

systemapi/config.go

@@ -3,15 +3,19 @@ package systemapi
 import (
 	"os"
 
+	"github.com/flashbots/system-api/common"
 	toml "github.com/pelletier/go-toml/v2"
 )
 
+var DefaultLogMaxEntries = common.GetEnvInt("MAX_EVENTS", 1000)
+
 type systemAPIConfigGeneral struct {
-	ListenAddr  string `toml:"listen_addr"`
-	PipeFile    string `toml:"pipe_file"`
-	LogJSON     bool   `toml:"log_json"`
-	LogDebug    bool   `toml:"log_debug"`
-	EnablePprof bool   `toml:"pprof"` // Enables pprof endpoints
+	ListenAddr    string `toml:"listen_addr"`
+	PipeFile      string `toml:"pipe_file"`
+	LogJSON       bool   `toml:"log_json"`
+	LogDebug      bool   `toml:"log_debug"`
+	EnablePprof   bool   `toml:"pprof"`           // Enables pprof endpoints
+	LogMaxEntries int    `toml:"log_max_entries"` // Maximum number of log entries
 
 	BasicAuthSecretPath string `toml:"basic_auth_secret_path"`
 	BasicAuthSecretSalt string `toml:"basic_auth_secret_salt"`
@@ -27,27 +31,36 @@ type SystemAPIConfig struct {
 	FileUploads map[string]string `toml:"file_uploads"`
 }
 
-func LoadConfigFromFile(path string) (*SystemAPIConfig, error) {
+func NewConfig() *SystemAPIConfig {
+	return &SystemAPIConfig{
+		General: systemAPIConfigGeneral{
+			LogMaxEntries: DefaultLogMaxEntries,
+		},
+		Actions:     make(map[string]string),
+		FileUploads: make(map[string]string),
+	}
+}
+
+func NewConfigFromFile(path string) (*SystemAPIConfig, error) {
 	content, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	return LoadConfig(content)
+	return NewConfigFromTOML(content)
 }
 
-func LoadConfig(content []byte) (*SystemAPIConfig, error) {
+func NewConfigFromTOML(content []byte) (*SystemAPIConfig, error) {
 	cfg := &SystemAPIConfig{}
 	err := toml.Unmarshal(content, cfg)
 	if err != nil {
 		return nil, err
 	}
+	cfg.loadDefaults()
 	return cfg, nil
 }
 
-func NewSystemAPIConfig() *SystemAPIConfig {
-	return &SystemAPIConfig{
-		General:     systemAPIConfigGeneral{},
-		Actions:     make(map[string]string),
-		FileUploads: make(map[string]string),
+func (cfg *SystemAPIConfig) loadDefaults() {
+	if cfg.General.LogMaxEntries == 0 {
+		cfg.General.LogMaxEntries = DefaultLogMaxEntries
 	}
 }

systemapi/config_test.go

@@ -8,9 +8,16 @@ import (
 
 func TestLoadConfig(t *testing.T) {
 	path := "../systemapi-config.toml"
-	cfg, err := LoadConfigFromFile(path)
+	cfg, err := NewConfigFromFile(path)
 	require.NoError(t, err)
 	require.NotNil(t, cfg)
 	require.NotEmpty(t, cfg.Actions)
 	require.Equal(t, "echo test", cfg.Actions["echo_test"])
 }
+
+func TestEmptyConfig(t *testing.T) {
+	cfg, err := NewConfigFromTOML([]byte{})
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+	require.Equal(t, DefaultLogMaxEntries, cfg.General.LogMaxEntries)
+}

systemapi/server.go

@@ -206,7 +206,7 @@ func (s *Server) addEvent(event Event) {
 	// Add event to the list and prune if necessary
 	s.eventsLock.Lock()
 	s.events = append(s.events, event)
-	if len(s.events) > MaxEvents {
+	if len(s.events) > s.cfg.General.LogMaxEntries {
 		s.events = s.events[1:]
 	}
 	s.eventsLock.Unlock()

systemapi/server_test.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/flashbots/system-api/common"
 	"github.com/go-chi/httplog/v2"
@@ -24,7 +26,7 @@ func getTestLogger() *httplog.Logger {
 
 func newTestServer(t *testing.T) *Server {
 	t.Helper()
-	srv, err := NewServer(getTestLogger(), NewSystemAPIConfig())
+	srv, err := NewServer(getTestLogger(), NewConfig())
 	require.NoError(t, err)
 	return srv
 }
@@ -102,7 +104,7 @@ func TestBasicAuth(t *testing.T) {
 	basicAuthSecretHash := hex.EncodeToString(h.Sum(nil))
 
 	// Create the config
-	cfg := NewSystemAPIConfig()
+	cfg := NewConfig()
 	cfg.General.BasicAuthSecretPath = tempDir + "/basic_auth_secret"
 	cfg.General.BasicAuthSecretSalt = basicAuthSalt
 
@@ -152,3 +154,26 @@ func TestBasicAuth(t *testing.T) {
 	code, _ = reqGetLiveZ("admin", "foo", nil)
 	require.Equal(t, http.StatusUnauthorized, code)
 }
+
+func TestMaxEntries(t *testing.T) {
+	// Verify maximum number of log entries is working correctly
+	maxEntries := 5
+
+	cfg := NewConfig()
+	cfg.General.LogMaxEntries = maxEntries
+	srv, err := NewServer(getTestLogger(), cfg)
+	require.NoError(t, err)
+
+	// Add 6 events, only last 5 should be stored
+	for i := range 6 {
+		srv.addEvent(Event{ReceivedAt: time.Now(), Message: fmt.Sprint(i)}) //nolint:perfsprint
+	}
+
+	// Ensure only 5 events are stored
+	require.Len(t, srv.events, 5)
+	require.Equal(t, "1", srv.events[0].Message) // originally, 0 was written to this position, but has been overwritten
+	require.Equal(t, "2", srv.events[1].Message)
+	require.Equal(t, "3", srv.events[2].Message)
+	require.Equal(t, "4", srv.events[3].Message)
+	require.Equal(t, "5", srv.events[4].Message)
+}