From 260d86ec6707160c62c4127ca543edb140689ca8 Mon Sep 17 00:00:00 2001 From: mye956 Date: Wed, 23 Oct 2024 18:25:51 +0000 Subject: [PATCH] Adding unit tests for fault request ordering --- .../fault/v1/handlers/handlers_test.go | 267 +++++++++++++++++- 1 file changed, 264 insertions(+), 3 deletions(-) diff --git a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go index 6ba1242d33d..35a52c4c349 100644 --- a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go +++ b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go @@ -58,6 +58,8 @@ const ( tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]` tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]` tcCommandEmptyOutput = `[]` + startEndpoint = "/api/%s/fault/v1/%s/start" + stopEndpoint = "/api/%s/fault/v1/%s/stop" ) var ( @@ -1899,9 +1901,13 @@ func generateStopNetworkPacketLossTestCases() []networkFaultInjectionTestCase { setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) mockCMD := mock_execwrapper.NewMockCmd(ctrl) - exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel) - exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD) - mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil) + gomock.InOrder( + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD), + mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil), + ) + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).Return(mockCMD) + mockCMD.EXPECT().CombinedOutput().Times(2).Return([]byte(""), nil) }, }, { @@ -2047,3 +2053,258 @@ func TestCheckNetworkPacketLoss(t *testing.T) { tcs := generateCheckNetworkPacketLossTestCases() testNetworkFaultInjectionCommon(t, tcs, NetworkFaultPath(types.PacketLossFaultType, types.CheckNetworkFaultPostfix)) } + +func TestNetworkFaultRequestOrdering(t *testing.T) { + tcs := []struct { + name string + faultType string + requestBody interface{} + setAgentStateExpectations func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) + setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) + }{ + { + name: types.BlackHolePortFaultType + "ordering", + faultType: types.BlackHolePortFaultType, + requestBody: happyBlackHolePortReqBody, + setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) { + agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient). + Return(happyTaskResponse, nil). + Times(2) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + // We want to ensure that the start fault request is being executed first + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) { + // Sleep for 2 seconds to mock that the request is taking some time + time.Sleep(2 * time.Second) + }).Times(1).Return(startCtx, startCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit status 1")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true), + exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + + // We want to ensure that the stop fault request is being executed second + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + }, + { + name: types.LatencyFaultType + "ordering", + faultType: types.LatencyFaultType, + requestBody: happyNetworkLatencyReqBody, + setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) { + agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient). + Return(happyTaskResponse, nil). + Times(2) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + // We want to ensure that the start fault request is being executed first + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) { + // Sleep for 2 seconds to mock that the request is taking some time + time.Sleep(2 * time.Second) + }).Times(1).Return(startCtx, startCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + + // We want to ensure that the stop fault request is being executed second + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil), + ) + }, + }, + { + name: types.PacketLossFaultType + "ordering", + faultType: types.PacketLossFaultType, + requestBody: happyNetworkPacketLossReqBody, + setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) { + agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient). + Return(happyTaskResponse, nil). + Times(2) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration) + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) { + // Sleep for 2 seconds to mock that the request is taking some time + time.Sleep(2 * time.Second) + }).Times(1).Return(startCtx, startCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil), + + // We want to ensure that the stop fault request is being executed second + exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil), + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil), + ) + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + // Mocks + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + agentState := mock_state.NewMockAgentState(ctrl) + metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + + router := mux.NewRouter() + mockExec := mock_execwrapper.NewMockExec(ctrl) + handler := New(agentState, metricsFactory, mockExec) + networkConfigClient := netconfig.NewNetworkConfigClient() + + if tc.setAgentStateExpectations != nil { + tc.setAgentStateExpectations(agentState, networkConfigClient) + } + if tc.setExecExpectations != nil { + tc.setExecExpectations(mockExec, ctrl) + } + var startHandleMethod, stopHandleMethod func(http.ResponseWriter, *http.Request) + switch tc.faultType { + case types.BlackHolePortFaultType: + startHandleMethod = handler.StartNetworkBlackholePort() + stopHandleMethod = handler.StopNetworkBlackHolePort() + case types.LatencyFaultType: + startHandleMethod = handler.StartNetworkLatency() + stopHandleMethod = handler.StopNetworkLatency() + case types.PacketLossFaultType: + startHandleMethod = handler.StartNetworkPacketLoss() + stopHandleMethod = handler.StopNetworkPacketLoss() + default: + t.Error("Unrecognized network fault type") + } + router.HandleFunc( + NetworkFaultPath(tc.faultType, types.StartNetworkFaultPostfix), + startHandleMethod, + ).Methods(http.MethodPost) + + router.HandleFunc( + NetworkFaultPath(tc.faultType, types.StopNetworkFaultPostfix), + stopHandleMethod, + ).Methods(http.MethodPost) + + var requestBody io.Reader + if tc.requestBody != nil { + reqBodyBytes, err := json.Marshal(tc.requestBody) + require.NoError(t, err) + requestBody = bytes.NewReader(reqBodyBytes) + } + startReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(startEndpoint, endpointId, tc.faultType), requestBody) + require.NoError(t, err) + + ch1 := make(chan struct { + int + error + }) + + if tc.requestBody != nil { + reqBodyBytes, err := json.Marshal(tc.requestBody) + require.NoError(t, err) + requestBody = bytes.NewReader(reqBodyBytes) + } + + stopReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(stopEndpoint, endpointId, tc.faultType), requestBody) + require.NoError(t, err) + + ch2 := make(chan struct { + int + error + }) + + // Make Start request first + go makeAsyncRequest(router, startReq, ch1) + + // Waiting a bit before sending the next request + time.Sleep(1 * time.Second) + + // Make Stop request after the router has received the start request first + go makeAsyncRequest(router, stopReq, ch2) + + resp1 := <-ch1 + require.NoError(t, resp1.error) + assert.Equal(t, 200, resp1.int) + + resp2 := <-ch2 + require.NoError(t, resp2.error) + assert.Equal(t, 200, resp2.int) + }) + } +} + +func makeAsyncRequest(router *mux.Router, req *http.Request, ch chan<- struct { + int + error +}) { + defer close(ch) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, req) + + var actualResponseBody types.NetworkFaultInjectionResponse + err := json.Unmarshal(recorder.Body.Bytes(), &actualResponseBody) + if err != nil { + ch <- struct { + int + error + }{-1, err} + } else { + ch <- struct { + int + error + }{recorder.Code, nil} + } +}