From 2ad29d8092801a7d058433da6c5649ed8f8443a5 Mon Sep 17 00:00:00 2001 From: Sredny M Date: Mon, 22 Jan 2024 15:29:56 -0300 Subject: [PATCH 1/3] TT-9158 prefetch org expiry when loading the apis (#5798) Previously, when operating in a slave configuration, the Tyk Gateway fetched session expiry information from the master layer the first time an API was accessed for a given organization. This approach led to a significant issue: if the MDCB connection was lost, the next API consumption attempt would incur a long response time. This delay, typically around 30 seconds, was caused by the gateway waiting for the session fetching operation to time out, as it tried to communicate with the now-inaccessible master layer. Implemented Solution: To mitigate this issue, the PR introduces a proactive fetching strategy. Now, the gateway fetches the session expiry information beforehand, while there is an active connection to MDCB. By doing so, it ensures that this data is already available locally in the event of an MDCB disconnection. Outcome: This change significantly improves the API response time under MDCB disconnection scenarios. It eliminates the need for the gateway to wait for a timeout when attempting to fetch session information from the master layer, thus avoiding the previous 30-second delay. This optimization enhances the resilience and efficiency of the Tyk Gateway in distributed environments. This option comes to complement the config option `slave_options.call_timeout` which defaults to 30sec, users can set a lower value. TT-9158 - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Refactoring or add test (improvements in base code or adds test coverage to functionality) - [ ] I ensured that the documentation is up to date - [ ] I explained why this PR updates go.mod in detail with reasoning why it's required - [ ] I would like a code coverage CI quality gate exception and have explained why (cherry picked from commit 76b4847da1a6588961997742c6a6b7086a02ad14) --- certs/manager_test.go | 206 +---------- gateway/api_loader.go | 62 ++-- gateway/api_loader_test.go | 178 ++++++++++ gateway/middleware.go | 4 +- storage/dummy.go | 255 ++++++++++++++ storage/dummy_test.go | 640 +++++++++++++++++++++++++++++++++++ storage/mdcb_storage.go | 15 +- storage/mdcb_storage_test.go | 69 +++- 8 files changed, 1208 insertions(+), 221 deletions(-) create mode 100644 storage/dummy.go create mode 100644 storage/dummy_test.go diff --git a/certs/manager_test.go b/certs/manager_test.go index 649308a84ad..fa6449ceff8 100644 --- a/certs/manager_test.go +++ b/certs/manager_test.go @@ -8,7 +8,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" - "errors" "io/ioutil" "math/big" "os" @@ -16,200 +15,15 @@ import ( "testing" "time" + "github.com/TykTechnologies/tyk/storage" + "github.com/stretchr/testify/assert" tykcrypto "github.com/TykTechnologies/tyk/internal/crypto" ) -type dummyStorage struct { - data map[string]string - indexList map[string][]string -} - -func (s *dummyStorage) GetMultiKey([]string) ([]string, error) { - panic("implement me") -} - -func (s *dummyStorage) GetRawKey(string) (string, error) { - panic("implement me") -} - -func (s *dummyStorage) SetRawKey(string, string, int64) error { - panic("implement me") -} - -func (s *dummyStorage) SetExp(string, int64) error { - panic("implement me") -} - -func (s *dummyStorage) GetExp(string) (int64, error) { - panic("implement me") -} - -func (s *dummyStorage) DeleteAllKeys() bool { - panic("implement me") -} - -func (s *dummyStorage) DeleteRawKey(string) bool { - panic("implement me") -} - -func (s *dummyStorage) Connect() bool { - panic("implement me") -} - -func (s *dummyStorage) GetKeysAndValues() map[string]string { - panic("implement me") -} - -func (s *dummyStorage) GetKeysAndValuesWithFilter(string) map[string]string { - panic("implement me") -} - -func (s *dummyStorage) DeleteKeys([]string) bool { - panic("implement me") -} - -func (s *dummyStorage) Decrement(string) { - panic("implement me") -} - -func (s *dummyStorage) IncrememntWithExpire(string, int64) int64 { - panic("implement me") -} - -func (s *dummyStorage) SetRollingWindow(key string, per int64, val string, pipeline bool) (int, []interface{}) { - panic("implement me") -} - -func (s *dummyStorage) GetRollingWindow(key string, per int64, pipeline bool) (int, []interface{}) { - panic("implement me") -} - -func (s *dummyStorage) GetSet(string) (map[string]string, error) { - panic("implement me") -} - -func (s *dummyStorage) AddToSet(string, string) { - panic("implement me") -} - -func (s *dummyStorage) GetAndDeleteSet(string) []interface{} { - panic("implement me") -} - -func (s *dummyStorage) RemoveFromSet(string, string) { - panic("implement me") -} - -func (s *dummyStorage) GetKeyPrefix() string { - panic("implement me") -} - -func (s *dummyStorage) AddToSortedSet(string, string, float64) { - panic("implement me") -} - -func (s *dummyStorage) GetSortedSetRange(string, string, string) ([]string, []float64, error) { - panic("implement me") -} - -func (s *dummyStorage) RemoveSortedSetRange(string, string, string) error { - panic("implement me") -} - -func newDummyStorage() *dummyStorage { - return &dummyStorage{ - data: make(map[string]string), - indexList: make(map[string][]string), - } -} - -func (s *dummyStorage) GetKey(key string) (string, error) { - if value, ok := s.data[key]; ok { - return value, nil - } - - return "", errors.New("Not found") -} - -func (s *dummyStorage) SetKey(key, value string, exp int64) error { - s.data[key] = value - return nil -} - -func (s *dummyStorage) DeleteKey(key string) bool { - if _, ok := s.data[key]; !ok { - return false - } - - delete(s.data, key) - return true -} - -func (s *dummyStorage) DeleteScanMatch(pattern string) bool { - if pattern == "*" { - s.data = make(map[string]string) - return true - } - - return false -} - -func (s *dummyStorage) RemoveFromList(keyName, value string) error { - for key, keyList := range s.indexList { - if key == keyName { - new := keyList[:] - newL := 0 - for _, e := range new { - if e == value { - continue - } - - new[newL] = e - newL++ - } - new = new[:newL] - s.indexList[key] = new - } - } - - return nil -} - -func (s *dummyStorage) GetListRange(keyName string, from, to int64) ([]string, error) { - for key := range s.indexList { - if key == keyName { - return s.indexList[key], nil - } - } - return []string{}, nil -} - -func (s *dummyStorage) Exists(keyName string) (bool, error) { - _, existIndex := s.indexList[keyName] - _, existRaw := s.data[keyName] - return existIndex || existRaw, nil -} - -func (s *dummyStorage) AppendToSet(keyName string, value string) { - s.indexList[keyName] = append(s.indexList[keyName], value) -} - -func (s *dummyStorage) GetKeys(pattern string) (keys []string) { - if pattern != "*" { - return nil - } - - for k := range s.data { - keys = append(keys, k) - } - - return keys -} - func newManager() *certificateManager { - return NewCertificateManager(newDummyStorage(), "test", nil, false) + return NewCertificateManager(storage.NewDummyStorage(), "test", nil, false) } func genCertificate(template *x509.Certificate, isExpired bool) ([]byte, []byte) { @@ -379,9 +193,13 @@ func TestCertificateStorage(t *testing.T) { func TestStorageIndex(t *testing.T) { m := newManager() storageCert, _ := genCertificateFromCommonName("dummy", false) - storage := m.storage.(*dummyStorage) + storage, ok := m.storage.(*storage.DummyStorage) + + if !ok { + t.Error("cannot make storage.DummyStorage of type storage.Handler") + } - if len(storage.indexList) != 0 { + if len(storage.IndexList) != 0 { t.Error("Storage index list should have 0 certificates and indexes after creation") } if _, err := storage.GetKey("orgid-1-index-migrated"); err == nil { @@ -393,16 +211,16 @@ func TestStorageIndex(t *testing.T) { t.Error("Migrated flag should be set after first listing", err) } // Set recound outside of collection. It should not be visible if migration was applied. - storage.data["raw-raw-orgid-1dummy"] = "test" + storage.Data["raw-raw-orgid-1dummy"] = "test" certID, _ := m.Add(storageCert, "orgid-1") - if len(storage.indexList["orgid-1-index"]) != 1 { + if len(storage.IndexList["orgid-1-index"]) != 1 { t.Error("Storage index list should have 1 certificates after adding a certificate") } m.Delete(certID, "orgid-1") - if len(storage.indexList["orgid-1-index"]) != 0 { + if len(storage.IndexList["orgid-1-index"]) != 0 { t.Error("Storage index list should have 0 certificates after deleting a certificate") } } diff --git a/gateway/api_loader.go b/gateway/api_loader.go index ac7a5039425..be31dec2423 100644 --- a/gateway/api_loader.go +++ b/gateway/api_loader.go @@ -15,6 +15,8 @@ import ( "sync" textTemplate "text/template" + "github.com/TykTechnologies/tyk/rpc" + "github.com/gorilla/mux" "github.com/justinas/alice" "github.com/rs/cors" @@ -43,7 +45,8 @@ func (gw *Gateway) prepareStorage() generalStores { gs.redisOrgStore = &storage.RedisCluster{KeyPrefix: "orgkey.", RedisController: gw.RedisController} gs.healthStore = &storage.RedisCluster{KeyPrefix: "apihealth.", RedisController: gw.RedisController} gs.rpcAuthStore = &RPCStorageHandler{KeyPrefix: "apikey-", HashKeys: gw.GetConfig().HashKeys, Gw: gw} - gs.rpcOrgStore = &RPCStorageHandler{KeyPrefix: "orgkey.", Gw: gw} + gs.rpcOrgStore = gw.getGlobalMDCBStorageHandler("orgkey.", false) + gw.GlobalSessionManager.Init(gs.redisStore) return gs } @@ -213,27 +216,7 @@ func (gw *Gateway) processSpec(spec *APISpec, apisByListen map[string]int, } // Initialise the auth and session managers (use Redis for now) - authStore := gs.redisStore - orgStore := gs.redisOrgStore - switch spec.AuthProvider.StorageEngine { - case LDAPStorageEngine: - storageEngine := LDAPStorageHandler{} - storageEngine.LoadConfFromMeta(spec.AuthProvider.Meta) - authStore = &storageEngine - case RPCStorageEngine: - authStore = gs.rpcAuthStore - orgStore = gs.rpcOrgStore - spec.GlobalConfig.EnforceOrgDataAge = true - globalConf := gw.GetConfig() - globalConf.EnforceOrgDataAge = true - gw.SetConfig(globalConf) - } - - sessionStore := gs.redisStore - switch spec.SessionProvider.StorageEngine { - case RPCStorageEngine: - sessionStore = gs.rpcAuthStore - } + authStore, orgStore, sessionStore := gw.configureAuthAndOrgStores(gs, spec) // Health checkers are initialised per spec so that each API handler has it's own connection and redis storage pool spec.Init(authStore, sessionStore, gs.healthStore, orgStore) @@ -409,6 +392,14 @@ func (gw *Gateway) processSpec(spec *APISpec, apisByListen map[string]int, chainArray = append(chainArray, authArray...) + // if gw is edge, then prefetch any existent org session expiry + if gw.GetConfig().SlaveOptions.UseRPC { + // if not in emergency so load from backup is not blocked + if !rpc.IsEmergencyMode() { + baseMid.OrgSessionExpiry(spec.OrgID) + } + } + for _, obj := range mwPostAuthCheckFuncs { if mwDriver == apidef.GoPluginDriver { gw.mwAppendEnabled( @@ -525,6 +516,33 @@ func (gw *Gateway) processSpec(spec *APISpec, apisByListen map[string]int, return &chainDef } +func (gw *Gateway) configureAuthAndOrgStores(gs *generalStores, spec *APISpec) (storage.Handler, storage.Handler, storage.Handler) { + authStore := gs.redisStore + orgStore := gs.redisOrgStore + + switch spec.AuthProvider.StorageEngine { + case LDAPStorageEngine: + storageEngine := LDAPStorageHandler{} + storageEngine.LoadConfFromMeta(spec.AuthProvider.Meta) + authStore = &storageEngine + case RPCStorageEngine: + authStore = gs.rpcAuthStore + orgStore = gs.rpcOrgStore + spec.GlobalConfig.EnforceOrgDataAge = true + globalConf := gw.GetConfig() + globalConf.EnforceOrgDataAge = true + gw.SetConfig(globalConf) + } + + sessionStore := gs.redisStore + switch spec.SessionProvider.StorageEngine { + case RPCStorageEngine: + sessionStore = gs.rpcAuthStore + } + + return authStore, orgStore, sessionStore +} + // Check for recursion const defaultLoopLevelLimit = 5 diff --git a/gateway/api_loader_test.go b/gateway/api_loader_test.go index d64acf4f6fc..74fd78daa36 100644 --- a/gateway/api_loader_test.go +++ b/gateway/api_loader_test.go @@ -1,10 +1,12 @@ package gateway import ( + "context" "fmt" "net/http" "path" _ "path" + "reflect" "sync/atomic" "testing" @@ -426,3 +428,179 @@ func TestAllApisAreMTLS(t *testing.T) { t.Errorf("Expected AllApisAreMTLS to return %v, but got %v", expected, result) } } +<<<<<<< HEAD +======= + +func TestOpenTelemetry(t *testing.T) { + t.Run("Opentelemetry enabled - check if we are sending traces", func(t *testing.T) { + otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) { + //check the body + body, err := io.ReadAll(r.Body) + assert.Nil(t, err) + + assert.NotEmpty(t, body) + + // check the user agent + agent, ok := r.Header["User-Agent"] + assert.True(t, ok) + assert.Len(t, agent, 1) + assert.Contains(t, agent[0], "OTLP") + + //check if we are sending the traces to the right endpoint + assert.Contains(t, r.URL.Path, "/v1/traces") + + // Here you can check the request and return a response + w.WriteHeader(http.StatusOK) + }, ":0") + + // Start the server. + otelCollectorMock.Start() + // Stop the server on return from the function. + defer otelCollectorMock.Close() + + ts := StartTest(func(globalConf *config.Config) { + globalConf.OpenTelemetry.Enabled = true + globalConf.OpenTelemetry.Exporter = "http" + globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL + globalConf.OpenTelemetry.SpanProcessorType = "simple" + }) + defer ts.Close() + detailedTracing := []bool{true, false} + for _, detailed := range detailedTracing { + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.APIID = "test" + spec.Proxy.ListenPath = "/my-api/" + spec.UseKeylessAccess = true + spec.DetailedTracing = detailed + }) + + response, _ := ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK}) + assert.NotEmpty(t, response.Header.Get("X-Tyk-Trace-Id")) + assert.Equal(t, "otel", ts.Gw.TracerProvider.Type()) + } + + }) + + t.Run("Opentelemetry disabled - check if we are not sending traces", func(t *testing.T) { + + otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) { + t.Fail() + }, ":0") + + // Start the server. + otelCollectorMock.Start() + // Stop the server on return from the function. + defer otelCollectorMock.Close() + + ts := StartTest(func(globalConf *config.Config) { + globalConf.OpenTelemetry.Enabled = false + globalConf.OpenTelemetry.Exporter = "http" + globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL + globalConf.OpenTelemetry.SpanProcessorType = "simple" + }) + defer ts.Close() + + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.APIID = "test" + spec.Proxy.ListenPath = "/my-api/" + spec.UseKeylessAccess = true + }) + + response, _ := ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK}) + assert.Empty(t, response.Header.Get("X-Tyk-Trace-Id")) + assert.Equal(t, "noop", ts.Gw.TracerProvider.Type()) + }) +} + +func httpCollectorMock(t *testing.T, fn http.HandlerFunc, address string) *httptest.Server { + t.Helper() + + l, err := net.Listen("tcp", address) + if err != nil { + t.Fatalf("error setting up collector mock: %s", err.Error()) + } + + otelCollectorMock := httptest.NewUnstartedServer(fn) + // NewUnstartedServer creates a listener. Close that listener and replace + // with the one we created. + otelCollectorMock.Listener.Close() + otelCollectorMock.Listener = l + + return otelCollectorMock +} + +func TestConfigureAuthAndOrgStores(t *testing.T) { + + testCases := []struct { + name string + storageEngine apidef.StorageEngineCode + expectedAuthStore string + expectedOrgStore string + expectedSessionStore string + configureGateway func(gw *Gateway) + }{ + { + name: "LDAP Storage Engine", + storageEngine: LDAPStorageEngine, + expectedAuthStore: "*gateway.LDAPStorageHandler", + expectedOrgStore: "*storage.RedisCluster", + expectedSessionStore: "*storage.RedisCluster", + configureGateway: func(gw *Gateway) { + }, + }, { + name: "RPC Storage engine", + storageEngine: RPCStorageEngine, + expectedAuthStore: "*gateway.RPCStorageHandler", + expectedOrgStore: "*storage.MdcbStorage", + expectedSessionStore: "*gateway.RPCStorageHandler", + configureGateway: func(gw *Gateway) { + conf := gw.GetConfig() + conf.SlaveOptions.UseRPC = true + gw.SetConfig(conf, true) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gw := NewGateway(config.Config{}, context.Background()) + tc.configureGateway(gw) + gs := gw.prepareStorage() + + spec := BuildAPI(func(spec *APISpec) { + spec.AuthProvider.StorageEngine = tc.storageEngine + + if tc.storageEngine == RPCStorageEngine { + spec.SessionProvider.StorageEngine = RPCStorageEngine + } + + if tc.storageEngine == LDAPStorageEngine { + // populate ldap meta + meta := map[string]interface{}{ + "ldap_server": "dummy-ldap-server", + "ldap_port": 389.0, + "base_dn": "base-dn", + "attributes": []interface{}{"attr1", "attr2", "attr3"}, + "session_attribute_name": "attr-name", + "search_string": "the-search", + } + spec.AuthProvider.Meta = meta + } + }) + + // Call configureAuthAndOrgStores + authStore, orgStore, sessionStore := gw.configureAuthAndOrgStores(&gs, spec[0]) + + if reflect.TypeOf(authStore).String() != tc.expectedAuthStore { + t.Errorf("Expected authStore type %s, got %s", tc.expectedAuthStore, reflect.TypeOf(authStore).String()) + } + if reflect.TypeOf(orgStore).String() != tc.expectedOrgStore { + t.Errorf("Expected orgStore type %s, got %s", tc.expectedOrgStore, reflect.TypeOf(orgStore).String()) + } + if reflect.TypeOf(sessionStore).String() != tc.expectedSessionStore { + t.Errorf("Expected sessionStore type %s, got %s", tc.expectedSessionStore, reflect.TypeOf(sessionStore).String()) + } + }) + } +} +>>>>>>> 76b4847da... TT-9158 prefetch org expiry when loading the apis (#5798) diff --git a/gateway/middleware.go b/gateway/middleware.go index 28221a46649..a307c3f6961 100644 --- a/gateway/middleware.go +++ b/gateway/middleware.go @@ -280,6 +280,7 @@ func (t BaseMiddleware) SetOrgExpiry(orgid string, expiry int64) { func (t BaseMiddleware) OrgSessionExpiry(orgid string) int64 { t.Logger().Debug("Checking: ", orgid) + // Cache failed attempt id, err, _ := orgSessionExpiryCache.Do(orgid, func() (interface{}, error) { cachedVal, found := t.Gw.ExpiryCache.Get(orgid) @@ -291,11 +292,10 @@ func (t BaseMiddleware) OrgSessionExpiry(orgid string) int64 { if found && t.Spec.GlobalConfig.EnforceOrgDataAge { return s.DataExpires, nil } - return 0, errors.New("missing session") }) - if err != nil { + if err != nil { t.Logger().Debug("no cached entry found, returning 7 days") t.SetOrgExpiry(orgid, DEFAULT_ORG_SESSION_EXPIRATION) return DEFAULT_ORG_SESSION_EXPIRATION diff --git a/storage/dummy.go b/storage/dummy.go new file mode 100644 index 00000000000..a2d4ff6083d --- /dev/null +++ b/storage/dummy.go @@ -0,0 +1,255 @@ +package storage + +import ( + "errors" + "fmt" +) + +// DummyStorage is a simple in-memory storage structure used for testing or +// demonstration purposes. It simulates a storage system. +type DummyStorage struct { + Data map[string]string + IndexList map[string][]string +} + +// NewDummyStorage creates and returns a new instance of DummyStorage. +func NewDummyStorage() *DummyStorage { + return &DummyStorage{ + Data: make(map[string]string), + IndexList: make(map[string][]string), + } +} + +// GetMultiKey retrieves multiple values from the DummyStorage based on a slice of keys. +// It returns a slice of strings containing the values corresponding to each provided key, +// and an error if the operation cannot be completed. +func (s *DummyStorage) GetMultiKey(keys []string) ([]string, error) { + var values []string + for _, key := range keys { + value, ok := s.Data[key] + if !ok { + return nil, fmt.Errorf("key not found: %s", key) + } + values = append(values, value) + } + return values, nil +} + +// GetRawKey retrieves the value associated with a given key from the DummyStorage. +// The method accepts a single string as the key and returns the corresponding string value. +// An error is also returned to indicate whether the retrieval was successful. +// Currently, this method is not implemented and will cause a panic if invoked. +func (s *DummyStorage) GetRawKey(key string) (string, error) { + value, ok := s.Data[key] + if !ok { + return "", fmt.Errorf("key not found: %s", key) + } + return value, nil +} + +// SetRawKey stores a value with a specified key in the DummyStorage. +// It takes three parameters: the key and value as strings, and an expiry time as int64. +// The expiry time could be used to simulate time-sensitive data storage or caching behavior. +// Currently, this method is not implemented and will trigger a panic if it is called. +// TODO: Proper implementation is needed for this method to handle data storage, or manage +func (s *DummyStorage) SetRawKey(string, string, int64) error { + panic("implement me") +} + +// SetExp updates the expiration time of a specific key in the DummyStorage. +// This method accepts two parameters: a string representing the key, and an int64 +// indicating the new expiration time. +func (s *DummyStorage) SetExp(string, int64) error { + panic("implement me") +} + +// GetExp retrieves the expiration time of a specific key from the DummyStorage. +// This method accepts a string parameter representing the key and returns an int64 +// which is the expiration time associated with that key, along with an error. +func (s *DummyStorage) GetExp(string) (int64, error) { + panic("implement me") +} + +// DeleteAllKeys removes all keys and their associated data from the DummyStorage. +// This method is intended to provide a way to clear the entire storage, which can +// be particularly useful in testing scenarios to ensure a clean state before tests. +func (s *DummyStorage) DeleteAllKeys() bool { + panic("implement me") +} + +// DeleteRawKey removes a specified key from DummyStorage, returning success status; not yet implemented. +func (s *DummyStorage) DeleteRawKey(string) bool { + panic("implement me") +} + +// Connect establishes a connection to the storage backend; not currently implemented. +func (s *DummyStorage) Connect() bool { + return true +} + +// GetKeysAndValues retrieves all key-value pairs from DummyStorage; currently not implemented. +func (s *DummyStorage) GetKeysAndValues() map[string]string { + panic("implement me") +} + +// GetKeysAndValuesWithFilter fetches key-value pairs matching a filter from DummyStorage; not implemented. +func (s *DummyStorage) GetKeysAndValuesWithFilter(string) map[string]string { + panic("implement me") +} + +// DeleteKeys removes a list of keys from DummyStorage, returning a success status; not yet implemented. +func (s *DummyStorage) DeleteKeys([]string) bool { + panic("implement me") +} + +// Decrement reduces the value of a specified key in DummyStorage; implementation pending. +func (s *DummyStorage) Decrement(string) { + panic("implement me") +} + +// IncrememntWithExpire increments the value of a key and sets an expiry; not yet implemented. +func (s *DummyStorage) IncrememntWithExpire(string, int64) int64 { + panic("implement me") +} + +// SetRollingWindow sets a rolling window for a key with specified parameters; implementation pending. +func (s *DummyStorage) SetRollingWindow(string, int64, string, bool) (int, []interface{}) { + panic("implement me") +} + +// GetRollingWindow retrieves data for a specified rolling window; currently not implemented. +func (s *DummyStorage) GetRollingWindow(string, int64, bool) (int, []interface{}) { + panic("implement me") +} + +// GetSet retrieves a set of values associated with a key in DummyStorage; not yet implemented. +func (s *DummyStorage) GetSet(string) (map[string]string, error) { + panic("implement me") +} + +// AddToSet adds a value to a set associated with a key in DummyStorage; implementation pending. +func (s *DummyStorage) AddToSet(string, string) { + panic("implement me") +} + +// GetAndDeleteSet retrieves and then deletes a set associated with a key in DummyStorage; not implemented. +func (s *DummyStorage) GetAndDeleteSet(string) []interface{} { + panic("implement me") +} + +// RemoveFromSet deletes a specific value from a set in DummyStorage; currently not implemented. +func (s *DummyStorage) RemoveFromSet(string, string) { + panic("implement me") +} + +// GetKeyPrefix returns the prefix used for keys in DummyStorage; not yet implemented. +func (s *DummyStorage) GetKeyPrefix() string { + panic("implement me") +} + +// AddToSortedSet inserts a value with a score into a sorted set in DummyStorage; implementation pending. +func (s *DummyStorage) AddToSortedSet(string, string, float64) { + panic("implement me") +} + +// GetSortedSetRange retrieves a range of values and scores from a sorted set in DummyStorage; not implemented. +func (s *DummyStorage) GetSortedSetRange(string, string, string) ([]string, []float64, error) { + panic("implement me") +} + +// RemoveSortedSetRange deletes a range of values from a sorted set in DummyStorage; yet to be implemented. +func (s *DummyStorage) RemoveSortedSetRange(string, string, string) error { + panic("implement me") +} + +// GetKey retrieves the value for a given key from DummyStorage, or an error if not found. +func (s *DummyStorage) GetKey(key string) (string, error) { + if value, ok := s.Data[key]; ok { + return value, nil + } + + return "", errors.New("Not found") +} + +// SetKey assigns a value to a key in DummyStorage with an expiration time; returns nil for success. +func (s *DummyStorage) SetKey(key, value string, _ int64) error { + s.Data[key] = value + return nil +} + +// DeleteKey removes a specified key from DummyStorage, returning true if successful. +func (s *DummyStorage) DeleteKey(key string) bool { + if _, ok := s.Data[key]; !ok { + return false + } + + delete(s.Data, key) + return true +} + +// DeleteScanMatch deletes keys matching a pattern from DummyStorage, returning true if successful. +func (s *DummyStorage) DeleteScanMatch(pattern string) bool { + if pattern == "*" { + s.Data = make(map[string]string) + return true + } + + return false +} + +// RemoveFromList eliminates a specific value from a list within DummyStorage; always returns nil. +func (s *DummyStorage) RemoveFromList(keyName, value string) error { + for key, keyList := range s.IndexList { + if key == keyName { + new := keyList[:] + newL := 0 + for _, e := range new { + if e == value { + continue + } + + new[newL] = e + newL++ + } + new = new[:newL] + s.IndexList[key] = new + } + } + + return nil +} + +// GetListRange retrieves a range of list elements from DummyStorage for a specified key; returns an error if not found. +func (s *DummyStorage) GetListRange(keyName string, _, _ int64) ([]string, error) { + for key := range s.IndexList { + if key == keyName { + return s.IndexList[key], nil + } + } + return []string{}, nil +} + +// Exists checks if a key exists in either the IndexList or Data in DummyStorage; returns true if found. +func (s *DummyStorage) Exists(keyName string) (bool, error) { + _, existIndex := s.IndexList[keyName] + _, existRaw := s.Data[keyName] + return existIndex || existRaw, nil +} + +// AppendToSet adds a new value to the end of a list associated with a key in DummyStorage. +func (s *DummyStorage) AppendToSet(keyName string, value string) { + s.IndexList[keyName] = append(s.IndexList[keyName], value) +} + +// GetKeys retrieves all keys matching a specified pattern from DummyStorage; currently supports only '*'. +func (s *DummyStorage) GetKeys(pattern string) (keys []string) { + if pattern != "*" { + return nil + } + + for k := range s.Data { + keys = append(keys, k) + } + + return keys +} diff --git a/storage/dummy_test.go b/storage/dummy_test.go new file mode 100644 index 00000000000..8814de1a598 --- /dev/null +++ b/storage/dummy_test.go @@ -0,0 +1,640 @@ +package storage + +import ( + "reflect" + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +// assertPanic checks if the provided function f panics. It fails the test if no panic occurs. +func assertPanic(t *testing.T, f func()) { + t.Helper() + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + f() // Call the provided function, expecting a panic +} + +func TestDummyStorage_GetMultiKey(t *testing.T) { + ds := NewDummyStorage() + ds.Data["key1"] = "value1" + ds.Data["key2"] = "value2" + + tests := []struct { + name string + keys []string + want []string + wantErr bool + }{ + { + name: "Valid keys", + keys: []string{"key1", "key2"}, + want: []string{"value1", "value2"}, + wantErr: false, + }, + { + name: "Invalid key", + keys: []string{"unknown"}, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ds.GetMultiKey(tt.keys) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.GetMultiKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DummyStorage.GetMultiKey() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDummyStorage_GetRawKey(t *testing.T) { + ds := NewDummyStorage() + ds.Data["key1"] = "value1" + + tests := []struct { + name string + key string + want string + wantErr bool + }{ + { + name: "Key exists", + key: "key1", + want: "value1", + wantErr: false, + }, + { + name: "Key does not exist", + key: "key2", + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ds.GetRawKey(tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.GetRawKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DummyStorage.GetRawKey() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDummyStorage_SetRawKey(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + err := ds.SetRawKey("key", "val", 0) + if err != nil { + return + } + }) +} + +func TestDummyStorage_SetExp(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + err := ds.SetExp("key", 0) + if err != nil { + return + } + }) +} + +func TestDummyStorage_GetExp(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + _, err := ds.GetExp("key") + if err != nil { + return + } + }) +} + +func TestDummyStorage_DeleteAllKeys(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.DeleteAllKeys() + }) +} + +func TestDummyStorage_DeleteRawKey(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.DeleteRawKey("key") + }) +} + +func TestDummyStorage_Connect(t *testing.T) { + ds := NewDummyStorage() + assert.True(t, ds.Connect()) +} + +func TestDummyStorage_GetKeysAndValues(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.GetKeysAndValues() + }) +} + +func TestDummyStorage_GetKeysAndValuesWithFilter(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.GetKeysAndValuesWithFilter("*") + }) +} + +func TestDummyStorage_DeleteKeys(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.DeleteKeys([]string{"key"}) + }) +} + +func TestDummyStorage_Decrement(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.Decrement("key") + }) +} + +func TestDummyStorage_IncrememntWithExpire(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.IncrememntWithExpire("key", 0) + }) +} + +func TestDummyStorage_SetRollingWindow(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.SetRollingWindow("key", 1, "val", false) + }) +} + +func TestDummyStorage_GetRollingWindow(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.GetRollingWindow("key", 1, false) + }) +} + +func TestDummyStorage_GetSet(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + _, err := ds.GetSet("Set") + if err != nil { + return + } + }) +} + +func TestDummyStorage_AddToSet(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.AddToSet("Set", "key") + }) +} + +func TestDummyStorage_GetAndDeleteSet(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.GetAndDeleteSet("Set") + }) +} + +func TestDummyStorage_RemoveFromSet(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.RemoveFromSet("set", "key") + }) +} + +func TestDummyStorage_GetKeyPrefix(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.GetKeyPrefix() + }) +} + +func TestDummyStorage_AddToSortedSet(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + ds.AddToSortedSet("Set", "key", 1) + }) +} + +func TestDummyStorage_GetSortedSetRange(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + _, _, err := ds.GetSortedSetRange("set", "from", "to") + if err != nil { + return + } + }) +} + +func TestDummyStorage_RemoveSortedSetRange(t *testing.T) { + ds := NewDummyStorage() + assertPanic(t, func() { + err := ds.RemoveSortedSetRange("set", "from", "to") + if err != nil { + return + } + }) +} + +func TestDummyStorage_GetKey(t *testing.T) { + ds := NewDummyStorage() + ds.Data["existingKey"] = "value1" + + tests := []struct { + name string + key string + want string + wantErr bool + errMsg string + }{ + { + name: "Key exists", + key: "existingKey", + want: "value1", + wantErr: false, + errMsg: "", + }, + { + name: "Key does not exist", + key: "nonExistingKey", + want: "", + wantErr: true, + errMsg: "Not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ds.GetKey(tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.GetKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DummyStorage.GetKey() = %v, want %v", got, tt.want) + } + if tt.wantErr && err.Error() != tt.errMsg { + t.Errorf("DummyStorage.GetKey() error = %v, wantErrMsg %v", err, tt.errMsg) + } + }) + } +} + +func TestDummyStorage_SetKey(t *testing.T) { + ds := NewDummyStorage() + + tests := []struct { + name string + key string + value string + exp int64 + }{ + { + name: "Set key-value pair", + key: "testKey", + value: "testValue", + exp: 0, // Since expiration is not implemented, it can be set to a default value like 0 + }, + // Add more test cases if necessary + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ds.SetKey(tt.key, tt.value, tt.exp) + if err != nil { + t.Errorf("DummyStorage.SetKey() error = %v", err) + } + + // Verify that the key-value pair is set correctly + if got, exists := ds.Data[tt.key]; !exists || got != tt.value { + t.Errorf("DummyStorage.SetKey() did not set the value correctly, got = %v, want = %v", got, tt.value) + } + }) + } +} + +func TestDummyStorage_DeleteKey(t *testing.T) { + ds := NewDummyStorage() + ds.Data["key1"] = "value1" + + tests := []struct { + name string + key string + want bool + expectKey bool + }{ + { + name: "Delete existing key", + key: "key1", + want: true, + expectKey: false, // After deletion, the key should not exist + }, + { + name: "Delete non-existing key", + key: "nonExistingKey", + want: false, + expectKey: false, // The key does not exist in the first place + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ds.DeleteKey(tt.key) + if got != tt.want { + t.Errorf("DummyStorage.DeleteKey() = %v, want %v", got, tt.want) + } + + // Check if the key still exists in the map + _, exists := ds.Data[tt.key] + if exists != tt.expectKey { + t.Errorf("After DummyStorage.DeleteKey(), key existence = %v, expectKey %v", exists, tt.expectKey) + } + }) + } +} + +func TestDummyStorage_DeleteScanMatch(t *testing.T) { + ds := NewDummyStorage() + ds.Data["key1"] = "value1" + ds.Data["key2"] = "value2" + + tests := []struct { + name string + pattern string + want bool + expectDataEmpty bool + }{ + { + name: "Delete all with '*' pattern", + pattern: "*", + want: true, + expectDataEmpty: true, // Expect data to be empty after deletion + }, + { + name: "Delete with non-matching pattern", + pattern: "nonMatchingPattern", + want: false, + expectDataEmpty: true, // Data should remain unchanged + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ds.DeleteScanMatch(tt.pattern) + if got != tt.want { + t.Errorf("DummyStorage.DeleteScanMatch() = %v, want %v", got, tt.want) + } + + // Check if the data map is empty or not based on the test expectation + if (len(ds.Data) == 0) != tt.expectDataEmpty { + t.Errorf("After DummyStorage.DeleteScanMatch(), data empty = %v, expectDataEmpty %v", len(ds.Data) == 0, tt.expectDataEmpty) + } + }) + } +} + +func TestDummyStorage_RemoveFromList(t *testing.T) { + ds := NewDummyStorage() + + tests := []struct { + name string + keyName string + value string + wantList []string + wantErr bool + }{ + { + name: "Remove existing value", + keyName: "key1", + value: "value2", + wantList: []string{"value1", "value3"}, + wantErr: false, + }, + { + name: "Remove non-existing value", + keyName: "key1", + value: "nonExistingValue", + wantList: []string{"value1", "value2", "value3"}, // List remains unchanged + wantErr: false, + }, + { + name: "Remove from non-existing key", + keyName: "nonExistingKey", + value: "value1", + wantList: nil, // Key does not exist + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ds.IndexList["key1"] = []string{"value1", "value2", "value3"} + + err := ds.RemoveFromList(tt.keyName, tt.value) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.RemoveFromList() error = %v, wantErr %v", err, tt.wantErr) + } + + gotList := ds.IndexList[tt.keyName] + if !reflect.DeepEqual(gotList, tt.wantList) { + t.Errorf("DummyStorage.RemoveFromList() gotList = %v, want %v", gotList, tt.wantList) + } + }) + } +} + +func TestDummyStorage_GetListRange(t *testing.T) { + ds := NewDummyStorage() + ds.IndexList["key1"] = []string{"value1", "value2", "value3"} + + tests := []struct { + name string + keyName string + from int64 + to int64 + want []string + wantErr bool + }{ + { + name: "Existing key", + keyName: "key1", + from: 0, + to: 2, + want: []string{"value1", "value2", "value3"}, // Expect full list (current implementation) + wantErr: false, + }, + { + name: "Non-existing key", + keyName: "nonExistingKey", + from: 0, + to: 2, + want: []string{}, // Expect empty list + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ds.GetListRange(tt.keyName, tt.from, tt.to) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.GetListRange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DummyStorage.GetListRange() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDummyStorage_Exists(t *testing.T) { + ds := NewDummyStorage() + ds.Data["dataKey"] = "value1" + ds.IndexList["indexKey"] = []string{"value1", "value2"} + + tests := []struct { + name string + keyName string + want bool + wantErr bool + }{ + { + name: "Key exists in Data", + keyName: "dataKey", + want: true, + wantErr: false, + }, + { + name: "Key exists in IndexList", + keyName: "indexKey", + want: true, + wantErr: false, + }, + { + name: "Key exists in both Data and IndexList", + keyName: "sharedKey", + want: true, + wantErr: false, + }, + { + name: "Key does not exist", + keyName: "nonExistingKey", + want: false, + wantErr: false, + }, + } + + // Add a key that exists in both Data and IndexList + ds.Data["sharedKey"] = "sharedValue" + ds.IndexList["sharedKey"] = []string{"sharedValue1", "sharedValue2"} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ds.Exists(tt.keyName) + if (err != nil) != tt.wantErr { + t.Errorf("DummyStorage.Exists() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DummyStorage.Exists() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDummyStorage_AppendToSet(t *testing.T) { + ds := NewDummyStorage() + ds.IndexList["existingKey"] = []string{"value1", "value2"} + + tests := []struct { + name string + keyName string + value string + want []string + }{ + { + name: "Append to existing key", + keyName: "existingKey", + value: "value3", + want: []string{"value1", "value2", "value3"}, + }, + { + name: "Append to new key", + keyName: "newKey", + value: "newValue", + want: []string{"newValue"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ds.AppendToSet(tt.keyName, tt.value) + + got := ds.IndexList[tt.keyName] + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DummyStorage.AppendToSet() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDummyStorage_GetKeys(t *testing.T) { + ds := NewDummyStorage() + ds.Data["key1"] = "value1" + ds.Data["key2"] = "value2" + ds.Data["key3"] = "value3" + + tests := []struct { + name string + pattern string + want []string + }{ + { + name: "Valid pattern '*'", + pattern: "*", + want: []string{"key1", "key2", "key3"}, + }, + { + name: "Invalid pattern", + pattern: "non-*", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ds.GetKeys(tt.pattern) + + // Sort slices for consistent comparison + sort.Strings(got) + sort.Strings(tt.want) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DummyStorage.GetKeys() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/storage/mdcb_storage.go b/storage/mdcb_storage.go index dc8355098b2..b135bfcdec4 100644 --- a/storage/mdcb_storage.go +++ b/storage/mdcb_storage.go @@ -65,8 +65,19 @@ func getResourceType(key string) string { } } -func (m MdcbStorage) GetMultiKey([]string) ([]string, error) { - panic("implement me") +// GetMultiKey gets multiple keys from the MDCB layer +func (m MdcbStorage) GetMultiKey(keyNames []string) ([]string, error) { + var err error + var value string + + for _, key := range keyNames { + value, err = m.GetKey(key) + if err == nil { + return []string{value}, nil + } + } + + return nil, err } func (m MdcbStorage) GetRawKey(string) (string, error) { diff --git a/storage/mdcb_storage_test.go b/storage/mdcb_storage_test.go index cd87ecc490d..0d12bc29940 100644 --- a/storage/mdcb_storage_test.go +++ b/storage/mdcb_storage_test.go @@ -1,6 +1,13 @@ package storage -import "testing" +import ( + "context" + "io" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) func TestGetResourceType(t *testing.T) { tests := []struct { @@ -22,3 +29,63 @@ func TestGetResourceType(t *testing.T) { }) } } + +func TestMdcbStorage_GetMultiKey(t *testing.T) { + rpcHandler := NewDummyStorage() + err := rpcHandler.SetKey("key1", "1", 0) + if err != nil { + t.Error(err.Error()) + } + + localHandler := NewDummyStorage() + err = localHandler.SetKey("key2", "1", 0) + if err != nil { + t.Error(err.Error()) + } + err = localHandler.SetKey("key3", "1", 0) + if err != nil { + t.Error(err.Error()) + } + + logger := logrus.New() + logger.Out = io.Discard + log := logger.WithContext(context.Background()) + + mdcb := NewMdcbStorage(localHandler, rpcHandler, log) + + testsCases := []struct { + name string + keyNames []string + want []string + wantErr bool + }{ + { + name: "First key exists, pulled from RPC", + keyNames: []string{"key1", "nonExistingKey"}, + want: []string{"1"}, + wantErr: false, + }, + { + name: "First key exist, pulled from local storage", + keyNames: []string{"key3", "nonExistingKey"}, + want: []string{"1"}, + wantErr: false, + }, + { + name: "No keys exist", + keyNames: []string{"nonExistingKey1", "nonExistingKey2"}, + want: nil, + wantErr: true, + }, + } + + for _, tc := range testsCases { + t.Run(tc.name, func(t *testing.T) { + keys, err := mdcb.GetMultiKey(tc.keyNames) + + didErr := err != nil + assert.Equal(t, tc.wantErr, didErr) + assert.Equal(t, tc.want, keys) + }) + } +} From 946a501ed881c034525f732bd167c3e4d0895227 Mon Sep 17 00:00:00 2001 From: sredny buitrago Date: Thu, 1 Feb 2024 07:38:03 -0300 Subject: [PATCH 2/3] fix conflict --- gateway/api_loader_test.go | 178 ------------------------------------- 1 file changed, 178 deletions(-) diff --git a/gateway/api_loader_test.go b/gateway/api_loader_test.go index 74fd78daa36..d64acf4f6fc 100644 --- a/gateway/api_loader_test.go +++ b/gateway/api_loader_test.go @@ -1,12 +1,10 @@ package gateway import ( - "context" "fmt" "net/http" "path" _ "path" - "reflect" "sync/atomic" "testing" @@ -428,179 +426,3 @@ func TestAllApisAreMTLS(t *testing.T) { t.Errorf("Expected AllApisAreMTLS to return %v, but got %v", expected, result) } } -<<<<<<< HEAD -======= - -func TestOpenTelemetry(t *testing.T) { - t.Run("Opentelemetry enabled - check if we are sending traces", func(t *testing.T) { - otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) { - //check the body - body, err := io.ReadAll(r.Body) - assert.Nil(t, err) - - assert.NotEmpty(t, body) - - // check the user agent - agent, ok := r.Header["User-Agent"] - assert.True(t, ok) - assert.Len(t, agent, 1) - assert.Contains(t, agent[0], "OTLP") - - //check if we are sending the traces to the right endpoint - assert.Contains(t, r.URL.Path, "/v1/traces") - - // Here you can check the request and return a response - w.WriteHeader(http.StatusOK) - }, ":0") - - // Start the server. - otelCollectorMock.Start() - // Stop the server on return from the function. - defer otelCollectorMock.Close() - - ts := StartTest(func(globalConf *config.Config) { - globalConf.OpenTelemetry.Enabled = true - globalConf.OpenTelemetry.Exporter = "http" - globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL - globalConf.OpenTelemetry.SpanProcessorType = "simple" - }) - defer ts.Close() - detailedTracing := []bool{true, false} - for _, detailed := range detailedTracing { - ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { - spec.APIID = "test" - spec.Proxy.ListenPath = "/my-api/" - spec.UseKeylessAccess = true - spec.DetailedTracing = detailed - }) - - response, _ := ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK}) - assert.NotEmpty(t, response.Header.Get("X-Tyk-Trace-Id")) - assert.Equal(t, "otel", ts.Gw.TracerProvider.Type()) - } - - }) - - t.Run("Opentelemetry disabled - check if we are not sending traces", func(t *testing.T) { - - otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) { - t.Fail() - }, ":0") - - // Start the server. - otelCollectorMock.Start() - // Stop the server on return from the function. - defer otelCollectorMock.Close() - - ts := StartTest(func(globalConf *config.Config) { - globalConf.OpenTelemetry.Enabled = false - globalConf.OpenTelemetry.Exporter = "http" - globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL - globalConf.OpenTelemetry.SpanProcessorType = "simple" - }) - defer ts.Close() - - ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { - spec.APIID = "test" - spec.Proxy.ListenPath = "/my-api/" - spec.UseKeylessAccess = true - }) - - response, _ := ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK}) - assert.Empty(t, response.Header.Get("X-Tyk-Trace-Id")) - assert.Equal(t, "noop", ts.Gw.TracerProvider.Type()) - }) -} - -func httpCollectorMock(t *testing.T, fn http.HandlerFunc, address string) *httptest.Server { - t.Helper() - - l, err := net.Listen("tcp", address) - if err != nil { - t.Fatalf("error setting up collector mock: %s", err.Error()) - } - - otelCollectorMock := httptest.NewUnstartedServer(fn) - // NewUnstartedServer creates a listener. Close that listener and replace - // with the one we created. - otelCollectorMock.Listener.Close() - otelCollectorMock.Listener = l - - return otelCollectorMock -} - -func TestConfigureAuthAndOrgStores(t *testing.T) { - - testCases := []struct { - name string - storageEngine apidef.StorageEngineCode - expectedAuthStore string - expectedOrgStore string - expectedSessionStore string - configureGateway func(gw *Gateway) - }{ - { - name: "LDAP Storage Engine", - storageEngine: LDAPStorageEngine, - expectedAuthStore: "*gateway.LDAPStorageHandler", - expectedOrgStore: "*storage.RedisCluster", - expectedSessionStore: "*storage.RedisCluster", - configureGateway: func(gw *Gateway) { - }, - }, { - name: "RPC Storage engine", - storageEngine: RPCStorageEngine, - expectedAuthStore: "*gateway.RPCStorageHandler", - expectedOrgStore: "*storage.MdcbStorage", - expectedSessionStore: "*gateway.RPCStorageHandler", - configureGateway: func(gw *Gateway) { - conf := gw.GetConfig() - conf.SlaveOptions.UseRPC = true - gw.SetConfig(conf, true) - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - gw := NewGateway(config.Config{}, context.Background()) - tc.configureGateway(gw) - gs := gw.prepareStorage() - - spec := BuildAPI(func(spec *APISpec) { - spec.AuthProvider.StorageEngine = tc.storageEngine - - if tc.storageEngine == RPCStorageEngine { - spec.SessionProvider.StorageEngine = RPCStorageEngine - } - - if tc.storageEngine == LDAPStorageEngine { - // populate ldap meta - meta := map[string]interface{}{ - "ldap_server": "dummy-ldap-server", - "ldap_port": 389.0, - "base_dn": "base-dn", - "attributes": []interface{}{"attr1", "attr2", "attr3"}, - "session_attribute_name": "attr-name", - "search_string": "the-search", - } - spec.AuthProvider.Meta = meta - } - }) - - // Call configureAuthAndOrgStores - authStore, orgStore, sessionStore := gw.configureAuthAndOrgStores(&gs, spec[0]) - - if reflect.TypeOf(authStore).String() != tc.expectedAuthStore { - t.Errorf("Expected authStore type %s, got %s", tc.expectedAuthStore, reflect.TypeOf(authStore).String()) - } - if reflect.TypeOf(orgStore).String() != tc.expectedOrgStore { - t.Errorf("Expected orgStore type %s, got %s", tc.expectedOrgStore, reflect.TypeOf(orgStore).String()) - } - if reflect.TypeOf(sessionStore).String() != tc.expectedSessionStore { - t.Errorf("Expected sessionStore type %s, got %s", tc.expectedSessionStore, reflect.TypeOf(sessionStore).String()) - } - }) - } -} ->>>>>>> 76b4847da... TT-9158 prefetch org expiry when loading the apis (#5798) From f4f6bfc4e82d0fe0ad1339aee342d5f98ef5c968 Mon Sep 17 00:00:00 2001 From: Esteban Ricardo Mirizio Date: Fri, 2 Feb 2024 13:58:59 -0300 Subject: [PATCH 3/3] remove v1 pipeline --- .github/workflows/api-tests.yml | 178 -------------------------------- 1 file changed, 178 deletions(-) delete mode 100644 .github/workflows/api-tests.yml diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml deleted file mode 100644 index 2e5da3ad73c..00000000000 --- a/.github/workflows/api-tests.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: API integration Tests - -on: - pull_request: - branches: - - master - - release-** - -env: - GOPRIVATE: github.com/TykTechnologies - -jobs: - test: - strategy: - matrix: - go-version: [1.16.x] - platform: [ubuntu-latest] - arch: [amd64] - node-version: [15.x] - runs-on: ${{ matrix.platform }} - - steps: - - name: Set up Python 3.7 - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Fix private module deps - env: - TOKEN: '${{ secrets.ORG_GH_TOKEN }}' - run: > - git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com" - - - name: Checkout - uses: actions/checkout@v2 - with: - path: tyk - token: ${{ secrets.ORG_GH_TOKEN }} - submodules: true - - - name: Check if test framework branch exists - id: check_test_branch - env: - TOKEN: '${{ secrets.ORG_GH_TOKEN }}' - run: | - echo "branch=master" >> $GITHUB_OUTPUT - if [ ! -z "${{ github.head_ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-automated-tests ${{ github.head_ref }}; then - echo "branch=${{ github.head_ref }}" >> $GITHUB_OUTPUT - fi - if [ ! -z "${{ github.base_ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-automated-tests ${{ github.base_ref }}; then - echo "branch=${{ github.base_ref }}" >> $GITHUB_OUTPUT - fi - if [ ! -z "${{ github.ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-automated-tests ${{ github.ref }}; then - echo "branch=${{ github.ref }}" >> $GITHUB_OUTPUT - fi - - - name: Checkout test repository - uses: actions/checkout@v2 - with: - repository: TykTechnologies/tyk-automated-tests - token: ${{ secrets.ORG_GH_TOKEN }} - path: tyk-automated-tests - ref: ${{ steps.check_test_branch.outputs.branch }} - - - name: Check if dashboard branch exists - id: check_dashboard_branch - env: - TOKEN: '${{ secrets.ORG_GH_TOKEN }}' - run: | - echo "branch=master" >> $GITHUB_OUTPUT - if [ ! -z "${{ github.head_ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-analytics ${{ github.head_ref }}; then - echo "branch=${{ github.head_ref }}" >> $GITHUB_OUTPUT - fi - if [ ! -z "${{ github.base_ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-analytics ${{ github.base_ref }}; then - echo "branch=${{ github.base_ref }}" >> $GITHUB_OUTPUT - fi - if [ ! -z "${{ github.ref }}" ] && git ls-remote --exit-code --heads https://${TOKEN}@github.com/TykTechnologies/tyk-analytics ${{ github.ref }}; then - echo "branch=${{ github.ref }}" >> $GITHUB_OUTPUT - fi - - - name: Checkout dashboard - uses: actions/checkout@v2 - with: - repository: TykTechnologies/tyk-analytics - token: ${{ secrets.ORG_GH_TOKEN }} - submodules: true - path: tyk-analytics - ref: ${{ steps.check_dashboard_branch.outputs.branch }} - - - name: start docker compose - run: docker-compose -f ci/ci_testing_env.yml up -d - env: - TYK_DB_LICENSEKEY: ${{secrets.DASH_LICENSE}} - DASH_REPO_PATH: /home/runner/work/tyk/tyk/tyk-analytics - GW_REPO_PATH: /home/runner/work/tyk/tyk/tyk - GOPATH: /home/runner/work/tyk/tyk - GOPRIVATE: github.com/TykTechnologies - TOKEN: ${{ secrets.ORG_GH_TOKEN }} - working-directory: tyk-automated-tests - - - name: Install test dependecies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - working-directory: tyk-automated-tests - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - working-directory: tyk-automated-tests - - - name: Waiting for dashboard - run: | - while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:3000/hello/)" != "200" ]]; do sleep 60 && echo "waiting for dashboard '$(date +"%T")'"; done - timeout-minutes: 15 - - - name: Test with pytest - id: test_execution - env: - TYK_TEST_BASE_URL: "http://localhost:3000/" - TYK_TEST_GW_URL: "https://localhost:8080/" - TYK_TEST_MONGODB: "tyk-mongo:27017" - TYK_TEST_REDIS: "tyk-redis" - TYK_TEST_DB_ADMIN: "12345" - TYK_TEST_GW_SECRET: "352d20ee67be67f6340b4c0605b044b7" - TYK_TEST_FEDERATION_HOST: federation - run: | - pytest - working-directory: tyk-automated-tests - timeout-minutes: 30 - - - name: Archive Integration tests report - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: api-test-report - path: ./tyk-automated-tests/reports/ - - - name: Notify slack - if: ${{ always() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GW_SLACK_WEBHOOK }} - SLACK_COLOR: ${{ job.status }} - SLACK_TITLE: "Result: ${{ steps.test_execution.outcome }}" - SLACK_USERNAME: API GW test automation - SLACK_FOOTER: "" - - - name: Comment on PR - if: ${{ always() }} && github.event.issue.pull_request - uses: mshick/add-pr-comment@v1 - with: - message: | - **API tests result: ${{ steps.test_execution.outcome }}** ${{ env.STATUS }} - Branch used: ${{ github.ref }} - Commit: ${{ github.event.after }} ${{ github.event.commits[0].message }} - Triggered by: ${{ github.event_name }} (@${{ github.actor }}) - [Execution page](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) - repo-token: ${{ secrets.ORG_GH_TOKEN }} - allow-repeats: true - env: - STATUS: "${{ steps.test_execution.outcome == 'success' && ':white_check_mark:' || ':no_entry_sign:' }}" - - - name: Xray update - if: ${{ always() }} && github.event_name != 'pull_request' - run: | - ./update_xray.sh - working-directory: tyk-automated-tests - env: - TEST: "QA-901" - STATUS: "${{ steps.test_execution.outcome }}" - CLIENT_ID: ${{secrets.XRAY_CLIENT_ID}} - CLIENT_SECRET: ${{secrets.XRAY_CLIENT_SECRET}} - BRANCH: ${{ github.ref }}