From 55748daa156ffba9ac24a2441dd87e312bfeedba Mon Sep 17 00:00:00 2001 From: Prashant Shubham Date: Mon, 7 Oct 2024 18:43:36 +0530 Subject: [PATCH] Adding default config file to be used when dice server starts (#908) --- config/config.go | 15 ++++++--- dice.toml | 31 +++++++++++++++++++ go.mod | 2 +- integration_tests/commands/async/setup.go | 2 +- integration_tests/commands/http/setup.go | 2 +- integration_tests/commands/websocket/setup.go | 2 +- internal/eval/eval.go | 5 ++- main.go | 2 +- 8 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 dice.toml diff --git a/config/config.go b/config/config.go index 83c7f25b64..8c85f5fea5 100644 --- a/config/config.go +++ b/config/config.go @@ -65,13 +65,13 @@ type Config struct { EvictionRatio float64 `mapstructure:"evictionratio"` KeysLimit int `mapstructure:"keyslimit"` AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` LFULogFactor int `mapstructure:"lfulogfactor"` LogLevel string `mapstructure:"loglevel"` PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` EnableMultiThreading bool `mapstructure:"enablemultithreading"` StoreMapInitSize int `mapstructure:"storemapinitsize"` + WatchChanBufSize int `mapstructure:"watchchanbufsize"` } `mapstructure:"server"` Auth struct { UserName string `mapstructure:"username"` @@ -99,20 +99,20 @@ var baseConfig = Config{ EvictionRatio float64 `mapstructure:"evictionratio"` KeysLimit int `mapstructure:"keyslimit"` AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` LFULogFactor int `mapstructure:"lfulogfactor"` LogLevel string `mapstructure:"loglevel"` PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` EnableMultiThreading bool `mapstructure:"enablemultithreading"` StoreMapInitSize int `mapstructure:"storemapinitsize"` + WatchChanBufSize int `mapstructure:"watchchanbufsize"` }{ Addr: DefaultHost, Port: DefaultPort, KeepAlive: int32(300), Timeout: int32(300), MaxConn: int32(0), - ShardCronFrequency: 1 * time.Second, + ShardCronFrequency: 30 * time.Second, MultiplexerPollTimeout: 100 * time.Millisecond, MaxClients: int32(20000), MaxMemory: 0, @@ -120,13 +120,13 @@ var baseConfig = Config{ EvictionRatio: 0.9, KeysLimit: DefaultKeysLimit, AOFFile: "./dice-master.aof", - PersistenceEnabled: true, WriteAOFOnCleanup: false, LFULogFactor: 10, LogLevel: "info", PrettyPrintLogs: false, EnableMultiThreading: false, StoreMapInitSize: 1024000, + WatchChanBufSize: 20000, }, Auth: struct { UserName string `mapstructure:"username"` @@ -305,7 +305,14 @@ func mergeFlagsWithConfig() { // This function checks if the config file is present or not at ConfigFileLocation func isConfigFilePresent() bool { + // If config file present in current directory use it + if _, err := os.Stat(filepath.Join(".", DefaultConfigName)); err == nil { + FileLocation = filepath.Join(".", DefaultConfigName) + return true + } + _, err := os.Stat(FileLocation) + return err == nil } diff --git a/dice.toml b/dice.toml new file mode 100644 index 0000000000..3bd1f2c666 --- /dev/null +++ b/dice.toml @@ -0,0 +1,31 @@ +[Server] +Addr = '0.0.0.0' +Port = 7379 +KeepAlive = 300 +Timeout = 300 +MaxConn = 0 +# Value in nanoseconds (30 seconds) +ShardCronFrequency = 30000000000 +# Value in nanoseconds (100 ms/0.1 seconds) +MultiplexerPollTimeout = 100000000 +MaxClients = 20000 +MaxMemory = 0 +EvictionPolicy = 'allkeys-lfu' +EvictionRatio = 0.4 +KeysLimit = 100000 +AOFFile = './dice-master.aof' +WriteAOFOnCleanup = false +LFULogFactor = 10 +LogLevel = 'info' +PrettyPrintLogs = false +EnableMultiThreading = false +StoreMapInitSize = 10240 +WatchChanBufSize = 20000 + +[Auth] +UserName = 'dice' +Password = '' + +[Network] +IOBufferLength = 512 +IOBufferLengthMAX = 51200 diff --git a/go.mod b/go.mod index 9ebacd77be..df97f9fb06 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -44,6 +43,7 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da github.com/dicedb/go-dice v0.0.0-20240820180649-d97f15fca831 + github.com/gobwas/glob v0.2.3 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 github.com/gorilla/websocket v1.5.3 diff --git a/integration_tests/commands/async/setup.go b/integration_tests/commands/async/setup.go index c922c3eaa7..939db0bf10 100644 --- a/integration_tests/commands/async/setup.go +++ b/integration_tests/commands/async/setup.go @@ -120,7 +120,7 @@ func RunTestServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption const totalRetries = 100 var err error - watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.KeysLimit) + watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.WatchChanBufSize) gec := make(chan error) shardManager := shard.NewShardManager(1, watchChan, gec, opt.Logger) // Initialize the AsyncServer diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index b0c20bf288..e0ff654a37 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -102,7 +102,7 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption config.DiceConfig.Server.WriteAOFOnCleanup = false globalErrChannel := make(chan error) - watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.KeysLimit) + watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, globalErrChannel, opt.Logger) queryWatcherLocal := querymanager.NewQueryManager(opt.Logger) config.HTTPPort = opt.Port diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index 406f5fd525..d706a14c7b 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -98,7 +98,7 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO // Initialize the WebsocketServer globalErrChannel := make(chan error) - watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.KeysLimit) + watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, globalErrChannel, opt.Logger) config.WebsocketPort = opt.Port testServer := server.NewWebSocketServer(shardManager, watchChan, opt.Logger) diff --git a/internal/eval/eval.go b/internal/eval/eval.go index e045188bc2..820908f0c6 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -171,8 +171,11 @@ func evalDBSIZE(args []string, store *dstore.Store) []byte { return diceerrors.NewErrArity("DBSIZE") } + // Expired keys must be explicitly deleted since the cronFrequency for cleanup is configurable. + // A longer delay may prevent timely cleanup, leading to incorrect DBSIZE results. + dstore.DeleteExpiredKeys(store) // return the RESP encoded value - return clientio.Encode(store.GetKeyCount(), false) + return clientio.Encode(store.GetDBSize(), false) } // evalGETDEL returns the value for the queried key in args diff --git a/main.go b/main.go index 50bc02886d..ed08af8549 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,7 @@ func main() { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) - watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.KeysLimit) + watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Server.WatchChanBufSize) var serverErrCh chan error // Get the number of available CPU cores on the machine using runtime.NumCPU().