From 895f6990ccb9a3d8e165f1a4e29b97471ba976bd Mon Sep 17 00:00:00 2001 From: Ritik Jain <60597329+re-Tick@users.noreply.github.com> Date: Tue, 19 Mar 2024 22:39:12 +0530 Subject: [PATCH] feat(test): adds clearUnusedMocks flag to remove the unused mocks of a test-set (#1713) * feat(test): adds clearUnusedMocks flag to remove the unused mocks of test-set Signed-off-by: re-Tick --------- Signed-off-by: re-Tick --- cli/provider/cmd.go | 3 + config/config.go | 1 + config/default.go | 1 + pkg/core/core.go | 16 ++-- pkg/core/proxy/integrations/integrations.go | 2 + pkg/core/proxy/integrations/mongo/decode.go | 7 ++ pkg/core/proxy/integrations/mongo/encode.go | 3 +- pkg/core/proxy/integrations/mysql/decode.go | 8 +- pkg/core/proxy/integrations/mysql/match.go | 19 ++++- .../proxy/integrations/postgres/v1/match.go | 4 +- pkg/core/proxy/mockmanager.go | 74 +++++++++++++++++ pkg/core/proxy/proxy.go | 19 ++++- pkg/core/record.go | 6 +- pkg/core/replay.go | 14 +--- pkg/core/service.go | 2 + pkg/platform/yaml/mockdb/db.go | 82 +++++++++++++++++++ pkg/service/replay/replay.go | 48 +++++++++-- pkg/service/replay/service.go | 5 ++ 18 files changed, 279 insertions(+), 35 deletions(-) diff --git a/cli/provider/cmd.go b/cli/provider/cmd.go index 0166c8960..44a1a288d 100644 --- a/cli/provider/cmd.go +++ b/cli/provider/cmd.go @@ -196,6 +196,7 @@ func (c *CmdConfigurator) AddFlags(cmd *cobra.Command, cfg *config.Config) error cmd.Flags().StringP("language", "l", cfg.Test.Language, "application programming language") cmd.Flags().Bool("ignoreOrdering", cfg.Test.IgnoreOrdering, "Ignore ordering of array in response") cmd.Flags().Bool("coverage", cfg.Test.Coverage, "Enable coverage reporting for the testcases. for golang please set language flag to golang, ref https://keploy.io/docs/server/sdk-installation/go/") + cmd.Flags().Bool("removeUnusedMocks", false, "Clear the unused mocks for the passed test-sets") } else { cmd.Flags().Uint64("recordTimer", 0, "User provided time to record its application") } @@ -260,6 +261,8 @@ func (c CmdConfigurator) ValidateFlags(ctx context.Context, cmd *cobra.Command, return errors.New(errMsg) } } + c.logger.Debug("config has been initialised", zap.Any("for cmd", cmd.Name()), zap.Any("config", cfg)) + switch cmd.Name() { case "record", "test": bypassPorts, err := cmd.Flags().GetUintSlice("passThroughPorts") diff --git a/config/config.go b/config/config.go index fb66f9478..eb8f195c1 100644 --- a/config/config.go +++ b/config/config.go @@ -50,6 +50,7 @@ type Test struct { IgnoreOrdering bool `json:"ignoreOrdering" yaml:"ignoreOrdering" mapstructure:"ignoreOrdering"` MongoPassword string `json:"mongoPassword" yaml:"mongoPassword" mapstructure:"mongoPassword"` Language string `json:"language" yaml:"language" mapstructure:"language"` + RemoveUnusedMocks bool `json:"removeUnusedMocks" yaml:"removeUnusedMocks" mapstructure:"removeUnusedMocks"` } type Globalnoise struct { diff --git a/config/default.go b/config/default.go index 95968e9b6..17d599b0d 100644 --- a/config/default.go +++ b/config/default.go @@ -31,6 +31,7 @@ test: ignoreOrdering: true mongoPassword: "default@123" language: "" + removeUnusedMocks: false record: recordTimer: 0s filters: [] diff --git a/pkg/core/core.go b/pkg/core/core.go index b0803d9c0..4de96fa00 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -18,11 +18,11 @@ import ( ) type Core struct { + Proxy // embedding the Proxy interface to transfer the proxy methods to the core object + Hooks // embedding the Hooks interface to transfer the hooks methods to the core object logger *zap.Logger id utils.AutoInc apps sync.Map - hook Hooks - proxy Proxy proxyStarted bool hostConfigStr string // hosts string in the nsswitch.conf of linux system. To restore the system hosts configuration after completion of test } @@ -30,8 +30,8 @@ type Core struct { func New(logger *zap.Logger, hook Hooks, proxy Proxy) *Core { return &Core{ logger: logger, - hook: hook, - proxy: proxy, + Hooks: hook, + Proxy: proxy, } } @@ -131,7 +131,7 @@ func (c *Core) Hook(ctx context.Context, id uint64, opts models.HookOptions) err }) //load hooks - err = c.hook.Load(hookCtx, id, HookCfg{ + err = c.Hooks.Load(hookCtx, id, HookCfg{ AppID: id, Pid: 0, IsDocker: isDocker, @@ -156,8 +156,8 @@ func (c *Core) Hook(ctx context.Context, id uint64, opts models.HookOptions) err // TODO: Hooks can be loaded multiple times but proxy should be started only once // if there is another containerized app, then we need to pass new (ip:port) of proxy to the eBPF // as the network namespace is different for each container and so is the keploy/proxy IP to communicate with the app. - //start proxy - err = c.proxy.StartProxy(proxyCtx, ProxyOptions{ + // start proxy + err = c.Proxy.StartProxy(proxyCtx, ProxyOptions{ DNSIPv4Addr: a.KeployIPv4Addr(), //DnsIPv6Addr: "" }) @@ -207,7 +207,7 @@ func (c *Core) Run(ctx context.Context, id uint64, opts models.RunOptions) model return nil } inode := <-inodeChan - err := c.hook.SendInode(ctx, id, inode) + err := c.Hooks.SendInode(ctx, id, inode) if err != nil { utils.LogError(c.logger, err, "") inodeErrCh <- errors.New("failed to send inode to the kernel") diff --git a/pkg/core/proxy/integrations/integrations.go b/pkg/core/proxy/integrations/integrations.go index a48834692..feb5146b0 100644 --- a/pkg/core/proxy/integrations/integrations.go +++ b/pkg/core/proxy/integrations/integrations.go @@ -49,4 +49,6 @@ type MockMemDb interface { UpdateUnFilteredMock(old *models.Mock, new *models.Mock) bool DeleteFilteredMock(mock *models.Mock) bool DeleteUnFilteredMock(mock *models.Mock) bool + // Flag the mock as used which matches the external request from application in test mode + FlagMockAsUsed(mock *models.Mock) error } diff --git a/pkg/core/proxy/integrations/mongo/decode.go b/pkg/core/proxy/integrations/mongo/decode.go index 81756e1c8..5f43f115f 100644 --- a/pkg/core/proxy/integrations/mongo/decode.go +++ b/pkg/core/proxy/integrations/mongo/decode.go @@ -183,6 +183,13 @@ func decodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientC logger.Debug("the mongo request do not matches with any config mocks", zap.Any("request", mongoRequests)) continue } + // set the config as used in the mockManager + err = mockDb.FlagMockAsUsed(configMocks[bestMatchIndex]) + if err != nil { + utils.LogError(logger, err, "failed to flag mock as used in mongo parser", zap.Any("for mock", configMocks[bestMatchIndex].Name)) + errCh <- err + return + } for _, mongoResponse := range configMocks[bestMatchIndex].Spec.MongoResponses { switch mongoResponse.Header.Opcode { case wiremessage.OpReply: diff --git a/pkg/core/proxy/integrations/mongo/encode.go b/pkg/core/proxy/integrations/mongo/encode.go index b862d1a01..f965e0528 100644 --- a/pkg/core/proxy/integrations/mongo/encode.go +++ b/pkg/core/proxy/integrations/mongo/encode.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "golang.org/x/sync/errgroup" "io" "net" "time" + "golang.org/x/sync/errgroup" + "go.keploy.io/server/v2/pkg/core/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/decode.go b/pkg/core/proxy/integrations/mysql/decode.go index 8c9d01a91..821510113 100644 --- a/pkg/core/proxy/integrations/mysql/decode.go +++ b/pkg/core/proxy/integrations/mysql/decode.go @@ -74,6 +74,12 @@ func decodeMySQL(ctx context.Context, logger *zap.Logger, clientConn net.Conn, d configMocks[matchedIndex].Spec.MySQLResponses = append(configMocks[matchedIndex].Spec.MySQLResponses[:matchedReqIndex], configMocks[matchedIndex].Spec.MySQLResponses[matchedReqIndex+1:]...) if len(configMocks[matchedIndex].Spec.MySQLResponses) == 0 { configMocks = append(configMocks[:matchedIndex], configMocks[matchedIndex+1:]...) + err = mockDb.FlagMockAsUsed(configMocks[matchedIndex]) + if err != nil { + utils.LogError(logger, err, "Failed to flag mock as used") + errCh <- err + return + } } //h.SetConfigMocks(configMocks) firstLoop = false @@ -162,7 +168,7 @@ func decodeMySQL(ctx context.Context, logger *zap.Logger, clientConn net.Conn, d } //TODO: both in case of no match or some other error, we are receiving the error. // Due to this, there will be no passthrough in case of no match. - matchedResponse, matchedIndex, _, err := matchRequestWithMock(ctx, mysqlRequest, configMocks, tcsMocks) + matchedResponse, matchedIndex, _, err := matchRequestWithMock(ctx, mysqlRequest, configMocks, tcsMocks, mockDb) if err != nil { utils.LogError(logger, err, "Failed to match request with mock") errCh <- err diff --git a/pkg/core/proxy/integrations/mysql/match.go b/pkg/core/proxy/integrations/mysql/match.go index ceaf8ca11..5db69c736 100644 --- a/pkg/core/proxy/integrations/mysql/match.go +++ b/pkg/core/proxy/integrations/mysql/match.go @@ -4,10 +4,11 @@ import ( "context" "fmt" + "go.keploy.io/server/v2/pkg/core/proxy/integrations" "go.keploy.io/server/v2/pkg/models" ) -func matchRequestWithMock(ctx context.Context, mysqlRequest models.MySQLRequest, configMocks, tcsMocks []*models.Mock) (*models.MySQLResponse, int, string, error) { +func matchRequestWithMock(ctx context.Context, mysqlRequest models.MySQLRequest, configMocks, tcsMocks []*models.Mock, mockDb integrations.MockMemDb) (*models.MySQLResponse, int, string, error) { //TODO: any reason to write the similar code twice? allMocks := append([]*models.Mock(nil), configMocks...) allMocks = append(allMocks, tcsMocks...) @@ -53,6 +54,14 @@ func matchRequestWithMock(ctx context.Context, mysqlRequest models.MySQLRequest, } configMocks[matchedIndex].Spec.MySQLRequests = append(configMocks[matchedIndex].Spec.MySQLRequests[:matchedReqIndex], configMocks[matchedIndex].Spec.MySQLRequests[matchedReqIndex+1:]...) configMocks[matchedIndex].Spec.MySQLResponses = append(configMocks[matchedIndex].Spec.MySQLResponses[:matchedReqIndex], configMocks[matchedIndex].Spec.MySQLResponses[matchedReqIndex+1:]...) + if len(configMocks[matchedIndex].Spec.MySQLResponses) == 0 { + configMocks = append(configMocks[:matchedIndex], configMocks[matchedIndex+1:]...) + err := mockDb.FlagMockAsUsed(configMocks[matchedIndex]) + if err != nil { + return nil, -1, "", fmt.Errorf("failed to flag mock as used: %v", err.Error()) + } + // deleteConfigMock + } //h.SetConfigMocks(configMocks) } else { realIndex := matchedIndex - len(configMocks) @@ -61,6 +70,14 @@ func matchRequestWithMock(ctx context.Context, mysqlRequest models.MySQLRequest, } tcsMocks[realIndex].Spec.MySQLRequests = append(tcsMocks[realIndex].Spec.MySQLRequests[:matchedReqIndex], tcsMocks[realIndex].Spec.MySQLRequests[matchedReqIndex+1:]...) tcsMocks[realIndex].Spec.MySQLResponses = append(tcsMocks[realIndex].Spec.MySQLResponses[:matchedReqIndex], tcsMocks[realIndex].Spec.MySQLResponses[matchedReqIndex+1:]...) + if len(tcsMocks[realIndex].Spec.MySQLResponses) == 0 { + tcsMocks = append(tcsMocks[:realIndex], tcsMocks[realIndex+1:]...) + err := mockDb.FlagMockAsUsed(tcsMocks[realIndex]) + if err != nil { + return nil, -1, "", fmt.Errorf("failed to flag mock as used: %v", err.Error()) + } + // deleteTcsMock + } //h.SetTcsMocks(tcsMocks) } diff --git a/pkg/core/proxy/integrations/postgres/v1/match.go b/pkg/core/proxy/integrations/postgres/v1/match.go index fd5811062..9f95164fe 100644 --- a/pkg/core/proxy/integrations/postgres/v1/match.go +++ b/pkg/core/proxy/integrations/postgres/v1/match.go @@ -157,10 +157,10 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, requestBuffers if matched { logger.Debug("Matched mock", zap.String("mock", matchedMock.Name)) if matchedMock.TestModeInfo.IsFiltered { - originalMatchedMock := matchedMock + originalMatchedMock := *matchedMock matchedMock.TestModeInfo.IsFiltered = false matchedMock.TestModeInfo.SortOrder = math.MaxInt - updated := mockDb.UpdateUnFilteredMock(originalMatchedMock, matchedMock) + updated := mockDb.UpdateUnFilteredMock(&originalMatchedMock, matchedMock) if !updated { continue } diff --git a/pkg/core/proxy/mockmanager.go b/pkg/core/proxy/mockmanager.go index 025164dc3..c58776947 100644 --- a/pkg/core/proxy/mockmanager.go +++ b/pkg/core/proxy/mockmanager.go @@ -2,19 +2,33 @@ package proxy import ( "fmt" + "sort" + "strconv" + "strings" "go.keploy.io/server/v2/pkg/models" ) +const ( + filteredMock = "filtered" + unFilteredMock = "unfiltered" + totalMock = "total" +) + type MockManager struct { filtered *TreeDb unfiltered *TreeDb + // usedMocks contains the name of the mocks as key which were used by the parsers during the test execution. + // + // value is an array that will contain the type of mock + usedMocks map[string][]string } func NewMockManager(filtered, unfiltered *TreeDb) *MockManager { return &MockManager{ filtered: filtered, unfiltered: unfiltered, + usedMocks: make(map[string][]string), } } @@ -64,11 +78,34 @@ func (m *MockManager) GetUnFilteredMocks() ([]*models.Mock, error) { func (m *MockManager) UpdateUnFilteredMock(old *models.Mock, new *models.Mock) bool { updated := m.unfiltered.update(old.TestModeInfo, new.TestModeInfo, new) + if updated { + // mark the unfiltered mock as used for the current simulated test-case + m.usedMocks[old.Name] = []string{unFilteredMock, totalMock} + } return updated } +func (m *MockManager) FlagMockAsUsed(mock *models.Mock) error { + if mock == nil { + return fmt.Errorf("mock is empty") + } + + if mockType, ok := mock.Spec.Metadata["type"]; ok && mockType == "config" { + // mark the unfiltered mock as used for the current simulated test-case + m.usedMocks[mock.Name] = []string{unFilteredMock, totalMock} + } else { + // mark the filtered mock as used for the current simulated test-case + m.usedMocks[mock.Name] = []string{filteredMock, totalMock} + } + return nil +} + func (m *MockManager) DeleteFilteredMock(mock *models.Mock) bool { isDeleted := m.filtered.delete(mock.TestModeInfo) + if isDeleted { + // mark the unfiltered mock as used for the current simulated test-case + m.usedMocks[mock.Name] = []string{filteredMock, totalMock} + } return isDeleted } @@ -76,3 +113,40 @@ func (m *MockManager) DeleteUnFilteredMock(mock *models.Mock) bool { isDeleted := m.unfiltered.delete(mock.TestModeInfo) return isDeleted } + +func (m *MockManager) GetConsumedFilteredMocks() []string { + var allNames []string + // Extract all names from the map + for mockName, typeList := range m.usedMocks { + for _, mockType := range typeList { + // add mock name which are consumed by the parsers during the test-case simulation. + // Since, test-case are simulated synchronously, so the order of the mock consumption is preserved. + if mockType == filteredMock || mockType == unFilteredMock { + allNames = append(allNames, mockName) + } + } + } + + // Custom sorting function to sort names by sequence number + sort.Slice(allNames, func(i, j int) bool { + seqNo1, _ := strconv.Atoi(strings.Split(allNames[i], "-")[1]) + seqNo2, _ := strconv.Atoi(strings.Split(allNames[j], "-")[1]) + return seqNo1 < seqNo2 + }) + + // add the consumed filtered mocks into the total consumed mocks + for mockName, typeList := range m.usedMocks { + for indx, mockType := range typeList { + // reset the consumed unfiltered slice for the test-case simulation. + if mockType == unFilteredMock || mockType == filteredMock { + m.usedMocks[mockName] = append(typeList[:indx], typeList[indx+1:]...) + } + } + } + + return allNames +} + +func (m *MockManager) GetConsumedMocks() map[string][]string { + return m.usedMocks +} diff --git a/pkg/core/proxy/proxy.go b/pkg/core/proxy/proxy.go index bef574c63..3d74b698b 100755 --- a/pkg/core/proxy/proxy.go +++ b/pkg/core/proxy/proxy.go @@ -406,7 +406,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { addr := fmt.Sprintf("%v:%v", dstURL, destInfo.Port) if rule.Mode != models.MODE_TEST { dialer := &net.Dialer{ - Timeout: 1 * time.Second, + Timeout: 4 * time.Second, } dstConn, err = tls.DialWithDialer(dialer, "tcp", addr, cfg) if err != nil { @@ -556,3 +556,20 @@ func (p *Proxy) SetMocks(_ context.Context, id uint64, filtered []*models.Mock, return nil } + +// GetConsumedFilteredMocks returns the consumed filtered mocks for a given app id +func (p *Proxy) GetConsumedFilteredMocks(_ context.Context, id uint64) ([]string, error) { + m, ok := p.MockManagers.Load(id) + if !ok { + return nil, fmt.Errorf("mock manager not found to get consumed filtered mocks") + } + return m.(*MockManager).GetConsumedFilteredMocks(), nil +} + +func (p *Proxy) GetConsumedMocks(_ context.Context, id uint64) (map[string][]string, error) { + m, ok := p.MockManagers.Load(id) + if !ok { + return nil, fmt.Errorf("mock manager not found to get consumed mocks") + } + return m.(*MockManager).GetConsumedMocks(), nil +} diff --git a/pkg/core/record.go b/pkg/core/record.go index 42b516e97..906aacb88 100644 --- a/pkg/core/record.go +++ b/pkg/core/record.go @@ -7,7 +7,7 @@ import ( ) func (c *Core) GetIncoming(ctx context.Context, id uint64, _ models.IncomingOptions) (<-chan *models.TestCase, error) { - return c.hook.Record(ctx, id) + return c.Hooks.Record(ctx, id) } func (c *Core) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { @@ -15,13 +15,13 @@ func (c *Core) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingO ports := GetPortToSendToKernel(ctx, opts.Rules) if len(ports) > 0 { - err := c.hook.PassThroughPortsInKernel(ctx, id, ports) + err := c.Hooks.PassThroughPortsInKernel(ctx, id, ports) if err != nil { return nil, err } } - err := c.proxy.Record(ctx, id, m, opts) + err := c.Proxy.Record(ctx, id, m, opts) if err != nil { return nil, err } diff --git a/pkg/core/replay.go b/pkg/core/replay.go index f1f97202f..c83bd7eca 100644 --- a/pkg/core/replay.go +++ b/pkg/core/replay.go @@ -4,31 +4,21 @@ import ( "context" "go.keploy.io/server/v2/pkg/models" - "go.keploy.io/server/v2/utils" ) func (c *Core) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { ports := GetPortToSendToKernel(ctx, opts.Rules) if len(ports) > 0 { - err := c.hook.PassThroughPortsInKernel(ctx, id, ports) + err := c.Hooks.PassThroughPortsInKernel(ctx, id, ports) if err != nil { return err } } - err := c.proxy.Mock(ctx, id, opts) + err := c.Proxy.Mock(ctx, id, opts) if err != nil { return err } return nil } - -func (c *Core) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { - err := c.proxy.SetMocks(ctx, id, filtered, unFiltered) - if err != nil { - utils.LogError(c.logger, nil, "failed to set mocks") - return err - } - return nil -} diff --git a/pkg/core/service.go b/pkg/core/service.go index 1c635aa28..66cfca18d 100644 --- a/pkg/core/service.go +++ b/pkg/core/service.go @@ -38,6 +38,8 @@ type Proxy interface { Record(ctx context.Context, id uint64, mocks chan<- *models.Mock, opts models.OutgoingOptions) error Mock(ctx context.Context, id uint64, opts models.OutgoingOptions) error SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error + GetConsumedFilteredMocks(ctx context.Context, id uint64) ([]string, error) + GetConsumedMocks(ctx context.Context, id uint64) (map[string][]string, error) } type ProxyOptions struct { diff --git a/pkg/platform/yaml/mockdb/db.go b/pkg/platform/yaml/mockdb/db.go index eb9c11abb..ff3ed8248 100644 --- a/pkg/platform/yaml/mockdb/db.go +++ b/pkg/platform/yaml/mockdb/db.go @@ -36,6 +36,88 @@ func New(Logger *zap.Logger, mockPath string, mockName string) *MockYaml { } } +// DeleteMocks deletes the mocks from the mock file with given names +// +// mockNames is a map which contains the name of the mocks as key and a isConfig boolean as value +func (ys *MockYaml) DeleteMocks(ctx context.Context, testSetID string, mockNames map[string]bool) error { + mockFileName := "mocks" + if ys.MockName != "" { + mockFileName = ys.MockName + } + path := filepath.Join(ys.MockPath, testSetID) + ys.Logger.Debug("logging the names of the unused mocks to be removed", zap.Any("mockNames", mockNames), zap.Any("for testset", testSetID), zap.Any("at path", filepath.Join(path, mockFileName+".yaml"))) + + // Read the mocks from the yaml file + mockPath, err := yaml.ValidatePath(filepath.Join(path, mockFileName+".yaml")) + if err != nil { + utils.LogError(ys.Logger, err, "failed to read mocks due to inaccessible path", zap.Any("at path", filepath.Join(path, mockFileName+".yaml"))) + return err + } + if _, err := os.Stat(mockPath); err != nil { + utils.LogError(ys.Logger, err, "failed to find the mocks yaml file") + return err + } + data, err := yaml.ReadFile(ctx, ys.Logger, path, mockFileName) + if err != nil { + utils.LogError(ys.Logger, err, "failed to read the mocks from yaml file", zap.Any("at path", filepath.Join(path, mockFileName+".yaml"))) + return err + } + + // decode the mocks read from the yaml file + dec := yamlLib.NewDecoder(bytes.NewReader(data)) + var mockYamls []*yaml.NetworkTrafficDoc + for { + var doc *yaml.NetworkTrafficDoc + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + utils.LogError(ys.Logger, err, "failed to decode the yaml file documents", zap.Any("at path", filepath.Join(path, mockFileName+".yaml"))) + return fmt.Errorf("failed to decode the yaml file documents. error: %v", err.Error()) + } + mockYamls = append(mockYamls, doc) + } + mocks, err := decodeMocks(mockYamls, ys.Logger) + if err != nil { + return err + } + var newMocks []*models.Mock + for _, mock := range mocks { + if _, ok := mockNames[mock.Name]; !ok { + newMocks = append(newMocks, mock) + continue + } + } + ys.Logger.Debug("logging the names of the used mocks", zap.Any("mockNames", newMocks), zap.Any("for testset", testSetID)) + + // remove the old mock yaml file + err = os.Remove(filepath.Join(path, mockFileName+".yaml")) + if err != nil { + return err + } + + // write the new mocks to the new yaml file + for _, newMock := range newMocks { + mockYaml, err := EncodeMock(newMock, ys.Logger) + if err != nil { + utils.LogError(ys.Logger, err, "failed to encode the mock to yaml", zap.Any("mock", newMock.Name), zap.Any("for testset", testSetID)) + return err + } + data, err = yamlLib.Marshal(&mockYaml) + if err != nil { + utils.LogError(ys.Logger, err, "failed to marshal the mock to yaml", zap.Any("mock", newMock.Name), zap.Any("for testset", testSetID)) + return err + } + err = yaml.WriteFile(ctx, ys.Logger, path, mockFileName, data, true) + if err != nil { + utils.LogError(ys.Logger, err, "failed to write the mock to yaml", zap.Any("mock", newMock.Name), zap.Any("for testset", testSetID)) + return err + } + } + return nil +} + func (ys *MockYaml) InsertMock(ctx context.Context, mock *models.Mock, testSetID string) error { mock.Name = fmt.Sprint("mock-", ys.getNextID()) mockYaml, err := EncodeMock(mock, ys.Logger) diff --git a/pkg/service/replay/replay.go b/pkg/service/replay/replay.go index 7ffb2e349..9da75c844 100644 --- a/pkg/service/replay/replay.go +++ b/pkg/service/replay/replay.go @@ -246,7 +246,11 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s return models.TestSetStatusFailed, err } - err = r.instrumentation.MockOutgoing(runTestSetCtx, appID, models.OutgoingOptions{}) + err = r.instrumentation.MockOutgoing(runTestSetCtx, appID, models.OutgoingOptions{ + Rules: r.config.BypassRules, + MongoPassword: r.config.Test.MongoPassword, + SQLDelay: time.Duration(r.config.Test.Delay), + }) if err != nil { utils.LogError(r.logger, err, "failed to mock outgoing") return models.TestSetStatusFailed, err @@ -378,7 +382,12 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } testPass, testResult = r.compareResp(testCase, resp, testSetID) if !testPass { - r.logger.Info("result", zap.Any("testcase id", models.HighlightFailingString(testCase.Name)), zap.Any("testset id", models.HighlightFailingString(testSetID)), zap.Any("passed", models.HighlightFailingString(testPass))) + // log the consumed mocks during the test run of the test case for test set + consumedFilteredMocks, err := r.instrumentation.GetConsumedFilteredMocks(runTestSetCtx, appID) + if err != nil { + utils.LogError(r.logger, err, "failed to get consumed filtered mocks") + } + r.logger.Info("result", zap.Any("testcase id", models.HighlightFailingString(testCase.Name)), zap.Any("testset id", models.HighlightFailingString(testSetID)), zap.Any("passed", models.HighlightFailingString(testPass)), zap.Any("consumed mocks", consumedFilteredMocks)) } else { r.logger.Info("result", zap.Any("testcase id", models.HighlightPassingString(testCase.Name)), zap.Any("testset id", models.HighlightPassingString(testSetID)), zap.Any("passed", models.HighlightPassingString(testPass))) } @@ -388,6 +397,7 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } else { testStatus = models.TestStatusFailed failure++ + testSetStatus = models.TestSetStatusFailed } if testResult != nil { @@ -430,9 +440,6 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s utils.LogError(r.logger, err, "failed to insert test case result") break } - if !testPass { - testSetStatus = models.TestSetStatusFailed - } } else { utils.LogError(r.logger, nil, "test result is nil") break @@ -447,7 +454,7 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } } - // Checking errors for fina iteration + // Checking errors for final iteration // Checking for errors in the loop if loopErr != nil && !errors.Is(loopErr, context.Canceled) { testSetStatus = models.TestSetStatusInternalErr @@ -478,6 +485,35 @@ func (r *replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s return models.TestSetStatusInternalErr, fmt.Errorf("failed to insert report") } + // remove the unused mocks by the test cases of a testset + if r.config.Test.RemoveUnusedMocks { + + // fetch the consumed mocks by the testcases of the testset + consumedMocks, err := r.instrumentation.GetConsumedMocks(runTestSetCtx, appID) + if err != nil { + utils.LogError(r.logger, err, "failed to get consumed mocks", zap.Any("for test-set", testSetID)) + } + r.logger.Debug("consumed mocks from the completed testset", zap.Any("for test-set", testSetID), zap.Any("consumed mocks", consumedMocks)) + // if the mock is not consumed by the testset then it is unused + unusedMocks := map[string]bool{} + for _, filteredMock := range filteredMocks { + if _, ok := consumedMocks[filteredMock.Name]; !ok { + unusedMocks[filteredMock.Name] = false + } + } + for _, unfilteredMock := range unfilteredMocks { + if _, ok := consumedMocks[unfilteredMock.Name]; !ok { + unusedMocks[unfilteredMock.Name] = true + } + } + + // delete the unused mocks from the data store + err = r.mockDB.DeleteMocks(runTestSetCtx, testSetID, unusedMocks) + if err != nil { + utils.LogError(r.logger, err, "failed to delete unused mocks") + } + } + // TODO Need to decide on whether to use global variable or not verdict := TestReportVerdict{ total: testReport.Total, diff --git a/pkg/service/replay/service.go b/pkg/service/replay/service.go index 6d63f5a40..b8a7ce51f 100644 --- a/pkg/service/replay/service.go +++ b/pkg/service/replay/service.go @@ -15,6 +15,10 @@ type Instrumentation interface { MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error // SetMocks Allows for setting mocks between test runs for better filtering and matching SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error + // GetConsumedFilteredMocks to log the names of the mocks that were consumed during the test run of failed test cases + GetConsumedFilteredMocks(ctx context.Context, id uint64) ([]string, error) + // GetConsumedMocks returns all the names of mock which are used in the test run of a test set + GetConsumedMocks(ctx context.Context, id uint64) (map[string][]string, error) // Run is blocking call and will execute until error Run(ctx context.Context, id uint64, opts models.RunOptions) models.AppError @@ -39,6 +43,7 @@ type TestDB interface { type MockDB interface { GetFilteredMocks(ctx context.Context, testSetID string, afterTime time.Time, beforeTime time.Time) ([]*models.Mock, error) GetUnFilteredMocks(ctx context.Context, testSetID string, afterTime time.Time, beforeTime time.Time) ([]*models.Mock, error) + DeleteMocks(ctx context.Context, testSetID string, mockNames map[string]bool) error } type ReportDB interface {