From fb59ab3daec8c91aaa2d9b21cde0ebc10901fed9 Mon Sep 17 00:00:00 2001 From: Prashant Shubham Date: Mon, 30 Sep 2024 15:21:21 +0530 Subject: [PATCH] Fixing master build by making map store size and eviction configurable for test server (#876) --- config/config.go | 3 +++ go.mod | 4 ++-- integration_tests/commands/async/setup.go | 5 +++++ integration_tests/server/max_conn_test.go | 11 +++++++++++ integration_tests/server/server_abort_test.go | 4 +++- internal/eval/hmap.go | 2 +- internal/server/cmd_meta.go | 2 +- internal/store/store.go | 8 ++++---- 8 files changed, 30 insertions(+), 9 deletions(-) diff --git a/config/config.go b/config/config.go index c8d18ab11..540532754 100644 --- a/config/config.go +++ b/config/config.go @@ -67,6 +67,7 @@ type Config struct { LogLevel string `mapstructure:"loglevel"` PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` EnableMultiThreading bool `mapstructure:"enablemultithreading"` + StoreMapInitSize int `mapstructure:"storemapinitsize"` } `mapstructure:"server"` Auth struct { UserName string `mapstructure:"username"` @@ -100,6 +101,7 @@ var baseConfig = Config{ LogLevel string `mapstructure:"loglevel"` PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` EnableMultiThreading bool `mapstructure:"enablemultithreading"` + StoreMapInitSize int `mapstructure:"storemapinitsize"` }{ Addr: DefaultHost, Port: DefaultPort, @@ -120,6 +122,7 @@ var baseConfig = Config{ LogLevel: "info", PrettyPrintLogs: false, EnableMultiThreading: false, + StoreMapInitSize: 1024000, }, Auth: struct { UserName string `mapstructure:"username"` diff --git a/go.mod b/go.mod index bf4955e47..92823267f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/google/btree v1.1.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -45,8 +44,9 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 github.com/dicedb/go-dice v0.0.0-20240820180649-d97f15fca831 - github.com/gorilla/websocket v1.5.3 + github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 + github.com/gorilla/websocket v1.5.3 github.com/ohler55/ojg v1.24.0 github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/xid v1.6.0 diff --git a/integration_tests/commands/async/setup.go b/integration_tests/commands/async/setup.go index 65ef46b02..2b59f643b 100644 --- a/integration_tests/commands/async/setup.go +++ b/integration_tests/commands/async/setup.go @@ -100,8 +100,13 @@ func fireCommandAndGetRESPParser(conn net.Conn, cmd string) *clientio.RESPParser } func RunTestServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOptions) { + // Override configs to test server config, this is enabled to handle test env runs + // as those envs are resource constrained config.DiceConfig.Network.IOBufferLength = 16 config.DiceConfig.Server.WriteAOFOnCleanup = false + config.DiceConfig.Server.StoreMapInitSize = 1024 + config.DiceConfig.Server.EvictionRatio = 0.4 + config.DiceConfig.Server.KeysLimit = 2000000 if opt.Port != 0 { config.DiceConfig.Server.Port = opt.Port diff --git a/integration_tests/server/max_conn_test.go b/integration_tests/server/max_conn_test.go index ca0e5cf7f..72467d7ff 100644 --- a/integration_tests/server/max_conn_test.go +++ b/integration_tests/server/max_conn_test.go @@ -35,6 +35,15 @@ func TestMaxConnection(t *testing.T) { var maxConnLimit = maxConnTestOptions.MaxClients + 2 connections := make([]net.Conn, maxConnLimit) + defer func() { + // Ensure all connections are closed at the end of the test + for _, conn := range connections { + if conn != nil { + conn.Close() + } + } + }() + for i := 0; i < maxConnLimit; i++ { conn, err := getConnection(maxConnTestOptions.Port) if err == nil { @@ -51,6 +60,8 @@ func TestMaxConnection(t *testing.T) { result := commands.FireCommand(connections[0], "ABORT") if result != "OK" { t.Fatalf("Unexpected response to ABORT command: %v", result) + } else { + slog.Info("Closed server for max_conn_test") } wg.Wait() } diff --git a/integration_tests/server/server_abort_test.go b/integration_tests/server/server_abort_test.go index 4a66dd46e..c1010b81d 100644 --- a/integration_tests/server/server_abort_test.go +++ b/integration_tests/server/server_abort_test.go @@ -88,7 +88,7 @@ func TestServerRestartAfterAbort(t *testing.T) { conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", config.DiceConfig.Server.Port)) if err != nil { - t.Fatalf("Server should be running after restart: %v", err) + t.Fatalf("Server should be running at start: %v", err) } // Send ABORT command to shut down server @@ -100,9 +100,11 @@ func TestServerRestartAfterAbort(t *testing.T) { // wait for the server to shut down time.Sleep(2 * time.Second) + testServerOptions.Logger.Info("Wait completed for server shutdown") wg.Wait() + testServerOptions.Logger.Info("Restarting server after abort for server_abort_test") // restart server ctx2, cancel2 := context.WithCancel(context.Background()) t.Cleanup(cancel2) diff --git a/internal/eval/hmap.go b/internal/eval/hmap.go index 4958086f3..67ac0df72 100644 --- a/internal/eval/hmap.go +++ b/internal/eval/hmap.go @@ -130,4 +130,4 @@ func (h HashMap) incrementFloatValue(field string, incr float64) (string, error) h[field] = fmt.Sprintf("%v", total) return strValue, nil -} \ No newline at end of file +} diff --git a/internal/server/cmd_meta.go b/internal/server/cmd_meta.go index a9ea992b0..e7f2fe87a 100644 --- a/internal/server/cmd_meta.go +++ b/internal/server/cmd_meta.go @@ -37,7 +37,7 @@ var ( WorkerCmdsMeta = map[string]CmdsMeta{} // Metadata for global commands that don't interact with shards. - // INFO and PING are examples of global commands. + // PING is an example of global command. pingCmdMeta = CmdsMeta{ Cmd: "PING", CmdType: Global, diff --git a/internal/store/store.go b/internal/store/store.go index 5682707cf..159203a67 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -31,16 +31,16 @@ type Store struct { func NewStore(watchChan chan WatchEvent) *Store { return &Store{ - store: swiss.New[string, *object.Obj](1024000), - expires: swiss.New[*object.Obj, uint64](1024000), + store: swiss.New[string, *object.Obj](config.DiceConfig.Server.StoreMapInitSize), + expires: swiss.New[*object.Obj, uint64](config.DiceConfig.Server.StoreMapInitSize), watchChan: watchChan, } } func ResetStore(store *Store) *Store { store.numKeys = 0 - store.store = swiss.New[string, *object.Obj](1024000) - store.expires = swiss.New[*object.Obj, uint64](1024000) + store.store = swiss.New[string, *object.Obj](config.DiceConfig.Server.StoreMapInitSize) + store.expires = swiss.New[*object.Obj, uint64](config.DiceConfig.Server.StoreMapInitSize) return store }