package recover
import (
"context"
"errors"
"testing"
"time"
"github.com/agiledragon/gomonkey/v2"
"github.com/smartystreets/goconvey/convey"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"ascend-common/api"
"clusterd/pkg/common/constant"
"clusterd/pkg/domain/common"
"clusterd/pkg/domain/pod"
"clusterd/pkg/interface/grpc/recover"
"clusterd/pkg/interface/kube"
)
type hotSwitchTestCase struct {
name string
controller *EventController
mockFaultRanks []*pb.FaultRank
mockSignalEnqueueResp string
mockSignalEnqueueCode common.RespCode
mockSignalEnqueueErr error
expectedEvent string
expectedCode common.RespCode
expectedError error
}
type signalTestConfig struct {
signalType string
actions []string
strategy string
}
func buildHotSwitchTestCases() []hotSwitchTestCase {
return []hotSwitchTestCase{
{
name: "successful hot switch preparation",
controller: &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-1"}, uuid: "test-uuid-1"},
mockFaultRanks: []*pb.FaultRank{{RankId: "0"}},
mockSignalEnqueueResp: "success",
mockSignalEnqueueCode: common.OK,
mockSignalEnqueueErr: nil,
expectedEvent: "success",
expectedCode: common.OK,
expectedError: nil,
},
{
name: "hot switch preparation with error",
controller: &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-2"}, uuid: "test-uuid-2"},
mockFaultRanks: []*pb.FaultRank{{RankId: "1"}},
mockSignalEnqueueResp: "",
mockSignalEnqueueCode: common.ServerInnerError,
mockSignalEnqueueErr: nil,
expectedEvent: "",
expectedCode: common.ServerInnerError,
expectedError: nil,
},
}
}
func buildSignalTestCases(signalType string, actions []string, strategy string) []hotSwitchTestCase {
return []hotSwitchTestCase{
{
name: "successful " + signalType + " handling",
controller: &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-1"}, uuid: "test-uuid-1"},
mockSignalEnqueueResp: "success",
mockSignalEnqueueCode: common.OK,
mockSignalEnqueueErr: nil,
expectedEvent: "success",
expectedCode: common.OK,
expectedError: nil,
},
{
name: signalType + " handling with error",
controller: &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-2"}, uuid: "test-uuid-2"},
mockSignalEnqueueResp: "",
mockSignalEnqueueCode: common.ServerInnerError,
mockSignalEnqueueErr: nil,
expectedEvent: "",
expectedCode: common.ServerInnerError,
expectedError: nil,
},
}
}
func runSignalTest(t *testing.T, tests []hotSwitchTestCase, testFunc func(*EventController) (string, common.RespCode, error), config signalTestConfig) {
for _, tt := range tests {
convey.Convey(tt.name, t, func() {
patch := gomonkey.ApplyPrivateMethod((*EventController)(nil), "signalEnqueue",
func(_ *EventController, signal *pb.ProcessManageSignal) (string, common.RespCode, error) {
convey.So(signal.Uuid, convey.ShouldEqual, tt.controller.uuid)
convey.So(signal.JobId, convey.ShouldEqual, tt.controller.jobInfo.JobId)
convey.So(signal.SignalType, convey.ShouldEqual, config.signalType)
convey.So(signal.Actions, convey.ShouldResemble, config.actions)
convey.So(signal.ChangeStrategy, convey.ShouldEqual, config.strategy)
return tt.mockSignalEnqueueResp, tt.mockSignalEnqueueCode, tt.mockSignalEnqueueErr
})
defer patch.Reset()
event, code, err := testFunc(tt.controller)
convey.So(event, convey.ShouldEqual, tt.expectedEvent)
convey.So(code, convey.ShouldEqual, tt.expectedCode)
convey.So(err, convey.ShouldEqual, tt.expectedError)
})
}
}
func TestNotifyPrepareHotSwitch(t *testing.T) {
tests := buildHotSwitchTestCases()
for _, tt := range tests {
convey.Convey(tt.name, t, func() {
patch := gomonkey.NewPatches()
defer patch.Reset()
patch.ApplyPrivateMethod(tt.controller, "normalFaultAssociateSameNodeRank",
func(_ *EventController) ([]*pb.FaultRank, map[string]string) {
return tt.mockFaultRanks, nil
}).ApplyPrivateMethod(tt.controller, "signalEnqueue",
func(_ *EventController, signal *pb.ProcessManageSignal) (string, common.RespCode, error) {
convey.So(signal.Uuid, convey.ShouldEqual, tt.controller.uuid)
convey.So(signal.JobId, convey.ShouldEqual, tt.controller.jobInfo.JobId)
convey.So(signal.SignalType, convey.ShouldEqual, constant.HotSwitchSignalType)
convey.So(signal.Actions, convey.ShouldResemble, hotSwitchActions)
convey.So(signal.FaultRanks, convey.ShouldResemble, tt.mockFaultRanks)
convey.So(signal.ChangeStrategy, convey.ShouldEqual, "")
return tt.mockSignalEnqueueResp, tt.mockSignalEnqueueCode, tt.mockSignalEnqueueErr
})
event, code, err := tt.controller.notifyPrepareHotSwitch()
convey.So(event, convey.ShouldEqual, tt.expectedEvent)
convey.So(code, convey.ShouldEqual, tt.expectedCode)
convey.So(err, convey.ShouldEqual, tt.expectedError)
})
}
}
func TestNotifyNewPodFailedHandler(t *testing.T) {
tests := buildSignalTestCases("new pod failed", stopHotSwitchActions, "")
config := signalTestConfig{
signalType: constant.HotSwitchSignalType,
actions: stopHotSwitchActions,
strategy: "",
}
testFunc := func(controller *EventController) (string, common.RespCode, error) {
return controller.notifyNewPodFailedHandler()
}
runSignalTest(t, tests, testFunc, config)
}
func TestNotifyNewPodRunningHandler(t *testing.T) {
tests := buildSignalTestCases("new pod running", newPodRunningActions, "")
config := signalTestConfig{
signalType: constant.HotSwitchSignalType,
actions: newPodRunningActions,
strategy: "",
}
testFunc := func(controller *EventController) (string, common.RespCode, error) {
return controller.notifyNewPodRunningHandler()
}
runSignalTest(t, tests, testFunc, config)
}
func TestNotifyRestartTrain(t *testing.T) {
tests := buildSignalTestCases("restart train", changeStrategyActions, constant.ProcessMigration)
config := signalTestConfig{
signalType: constant.ChangeStrategySignalType,
actions: changeStrategyActions,
strategy: constant.ProcessMigration,
}
testFunc := func(controller *EventController) (string, common.RespCode, error) {
return controller.notifyRestartTrain()
}
runSignalTest(t, tests, testFunc, config)
}
type hotSwitchTestCaseB struct {
name string
podName string
podExist bool
annotationExist bool
deleteErr error
newPodExist bool
patchErr error
expectedCode common.RespCode
expectError bool
expectedEvent string
labelErr error
faultPods map[string]string
}
func TestCleanStateWhenFailed(t *testing.T) {
ctl := buildBaseController()
testCases := buildHotSwitchTestCase1()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
doPatches(patches, ctl, tc)
event, code, err := ctl.cleanStateWhenFailed()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, "")
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
})
}
}
func doPatches(patches *gomonkey.Patches, ctl *EventController, tc hotSwitchTestCaseB) {
patchReset(patches, ctl)
if !tc.podExist {
patchGetPodByPodId(patches, v1.Pod{}, false)
} else {
testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fault-pod", Namespace: "default"}}
if tc.annotationExist {
testPod.Annotations = map[string]string{api.InHotSwitchFlowKey: "true", api.BackupNewPodNameKey: "new-pod"}
} else {
testPod.Annotations = make(map[string]string)
}
patchGetPodByPodId(patches, testPod, true)
patches.ApplyFuncReturn(kube.DeletePodAnnotation, tc.deleteErr)
if tc.newPodExist {
patches.ApplyFuncReturn(pod.GetPodByJobIdAndPodName,
v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new-pod", Namespace: "default"}}, true)
} else {
patches.ApplyFuncReturn(pod.GetPodByJobIdAndPodName, v1.Pod{}, false)
}
patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
}
}
func patchGetPodByPodId(patches *gomonkey.Patches, p v1.Pod, exist bool) *gomonkey.Patches {
return patches.ApplyFuncReturn(pod.GetPodByPodId, p, exist)
}
func buildBaseController() *EventController {
ctl := &EventController{
currentHotSwitchFaultPodId: "test-pod-id",
jobInfo: common.JobBaseInfo{JobId: "test-job-id"},
}
return ctl
}
func buildHotSwitchTestCase1() []hotSwitchTestCaseB {
testCases := []hotSwitchTestCaseB{
{name: "pod not exist",
podExist: false,
expectedCode: common.OK,
expectError: false},
{name: "annotation not exist",
podExist: true,
annotationExist: false,
expectedCode: common.OK,
expectError: false},
{name: "delete annotation failed",
podExist: true,
annotationExist: true,
deleteErr: errors.New("delete failed"),
expectedCode: common.ServerInnerError,
expectError: true},
{name: "new pod not exist",
podExist: true,
annotationExist: true,
deleteErr: nil,
newPodExist: false,
expectedCode: common.ServerInnerError,
expectError: true},
{name: "patch new pod failed",
podExist: true,
annotationExist: true,
deleteErr: nil,
newPodExist: true,
patchErr: errors.New("patch failed"),
expectedCode: common.ServerInnerError,
expectError: true},
{name: "success",
podExist: true,
annotationExist: true,
deleteErr: nil,
newPodExist: true,
patchErr: nil,
expectedCode: common.OK,
expectError: false},
}
return testCases
}
func TestCleanStateWhenSuccess(t *testing.T) {
ctl := &EventController{
currentHotSwitchBackupPodId: "test-backup-pod-id",
jobInfo: common.JobBaseInfo{JobId: "test-job-id"},
}
testCases := buildHotSwitchTestCases2()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
patchReset(patches, ctl)
if !tc.podExist {
patchGetPodByPodId(patches, v1.Pod{}, false)
} else {
testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "backup-pod", Namespace: "default"}}
patchGetPodByPodId(patches, testPod, true)
patches.ApplyFuncReturn(kube.DeletePodAnnotation, tc.deleteErr)
}
event, code, err := ctl.cleanStateWhenSuccess()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, "")
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
})
}
}
func buildHotSwitchTestCases2() []hotSwitchTestCaseB {
testCases := []hotSwitchTestCaseB{
{
name: "pod not exist",
podExist: false,
expectedCode: common.OK,
expectError: false,
}, {
name: "delete annotation failed",
podExist: true,
deleteErr: errors.New("delete failed"),
expectedCode: common.ServerInnerError,
expectError: true,
}, {
name: "success",
podExist: true,
deleteErr: nil,
expectedCode: common.OK,
expectError: false,
},
}
return testCases
}
func TestNotifyStopJob(t *testing.T) {
ctl := &EventController{
jobInfo: common.JobBaseInfo{
JobId: "test-job-id",
JobName: "test-job-name",
},
}
testCases := []hotSwitchTestCaseB{
{
name: "pod not found",
podName: "",
expectError: false,
expectedCode: common.OK,
expectedEvent: common.NotifySuccessEvent,
},
{
name: "pod found and stop job",
podName: "test-pod",
expectError: false,
expectedCode: common.OK,
expectedEvent: common.NotifySuccessEvent,
},
}
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
patchReset(patches, ctl)
patches.ApplyFunc(pod.GetPodByRankIndex, func(jobId string, rankIndex string) v1.Pod {
return v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: tc.podName,
Namespace: "default",
},
}
})
patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, nil)
patches.ApplyFuncReturn(kube.RetryPatchPodLabels, nil)
event, code, err := ctl.notifyStopJob()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, tc.expectedEvent)
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
})
}
}
func patchReset(patches *gomonkey.Patches, ctl *EventController) *gomonkey.Patches {
return patches.ApplyPrivateMethod(ctl, "reset", func(bool) {})
}
func TestNotifyDeleteOldPod(t *testing.T) {
controller := &EventController{
currentHotSwitchFaultPodId: "test-pod-id",
jobInfo: common.JobBaseInfo{JobId: "test-job-id"},
}
testCases := buildHotSwitchTestCase3()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
patchReset(patches, controller)
if !tc.podExist {
patchGetPodByPodId(patches, v1.Pod{}, false)
} else {
testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}}
patchGetPodByPodId(patches, testPod, true)
patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
patches.ApplyFuncReturn(kube.RetryPatchPodLabels, tc.labelErr)
}
event, code, err := controller.notifyDeleteOldPod()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, "")
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
})
}
}
func buildHotSwitchTestCase3() []hotSwitchTestCaseB {
testCases := []hotSwitchTestCaseB{
{
name: "pod not exist",
podExist: false,
expectedCode: common.OK,
expectError: false,
}, {
name: "patch annotation failed",
podExist: true,
patchErr: errors.New("patch annotation failed"),
expectedCode: common.OK,
expectError: false,
}, {
name: "patch label failed",
podExist: true,
labelErr: errors.New("patch label failed"),
expectedCode: common.OK,
expectError: false,
}, {
name: "success",
podExist: true,
patchErr: nil,
labelErr: nil,
expectedCode: common.OK,
expectError: false},
}
return testCases
}
func TestNotifyCreateNewPod(t *testing.T) {
controller := &EventController{
jobInfo: common.JobBaseInfo{JobId: "test-job-id"},
}
testCases := buildHotSwitchTestCases3()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethodReturn(controller, "GetFaultPod", tc.faultPods)
if len(tc.faultPods) == 0 {
} else if !tc.podExist {
patchGetPodByPodId(patches, v1.Pod{}, false)
} else {
testPod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
UID: "test-pod-uid",
},
}
patchGetPodByPodId(patches, testPod, true)
patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
}
event, code, err := controller.notifyCreateNewPod()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, "")
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
if tc.patchErr == nil && len(tc.faultPods) > 0 && tc.podExist {
convey.So(controller.currentHotSwitchFaultPodId, convey.ShouldEqual, "test-pod-uid")
}
})
}
}
func buildHotSwitchTestCases3() []hotSwitchTestCaseB {
testCases := []hotSwitchTestCaseB{
{
name: "should return OK when no fault pods",
faultPods: map[string]string{},
expectedCode: common.OK,
expectError: false,
},
{
name: "should return OK when pod not exist",
faultPods: map[string]string{"0": "test-pod-id"},
podExist: false,
expectedCode: common.OK,
expectError: false,
},
{
name: "should return OK when patch annotation failed",
faultPods: map[string]string{"0": "test-pod-id"},
podExist: true,
patchErr: errors.New("patch failed"),
expectedCode: common.OK,
expectError: false,
},
{
name: "should return OK when success",
faultPods: map[string]string{"0": "test-pod-id"},
podExist: true,
patchErr: nil,
expectedCode: common.OK,
expectError: false,
},
}
return testCases
}
type hotSwitchTestCaseC struct {
name string
chanNil bool
channelClosed bool
contextCanceled bool
timeout bool
strategies []string
expectedCode common.RespCode
expectedCtlEvent string
expectedRtnEvent string
expectError bool
responseCode int32
podStatus v1.PodPhase
}
func TestWaitReportPauseTrainResult(t *testing.T) {
controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
testCases := buildHotSwitchTestCases4()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
strategyChan := make(chan *pb.RecoverStrategyRequest, 1)
ctx, cancel := context.WithCancel(context.Background())
if tc.chanNil {
patchGetCtxAndReportRecoverStrategyChan(patches, controller, ctx, nil)
} else {
patchGetCtxAndReportRecoverStrategyChan(patches, controller, ctx, strategyChan)
}
if tc.channelClosed {
close(strategyChan)
} else if tc.contextCanceled {
cancel()
} else if tc.timeout {
patchTimeOut(patches)
} else {
safeWriteToChannel(t, strategyChan, &pb.RecoverStrategyRequest{Strategies: tc.strategies})
}
eventAdded := ""
patches.ApplyPrivateMethod(controller, "addEvent", func(_ *EventController, event string) {
eventAdded = event
})
event, code, err := controller.waitReportPauseTrainResult()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, tc.expectedRtnEvent)
convey.So(eventAdded, convey.ShouldEqual, tc.expectedCtlEvent)
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
if !tc.contextCanceled && ctx.Err() == nil {
cancel()
}
if !tc.channelClosed {
close(strategyChan)
}
})
}
}
func patchTimeOut(patches *gomonkey.Patches) *gomonkey.Patches {
return patches.ApplyFunc(time.After, func(d time.Duration) <-chan time.Time {
ch := make(chan time.Time, 1)
ch <- time.Time{}
return ch
})
}
func patchGetCtxAndReportRecoverStrategyChan(patches *gomonkey.Patches, controller *EventController,
ctx context.Context, strategyChan chan *pb.RecoverStrategyRequest) *gomonkey.Patches {
return patches.ApplyPrivateMethod(controller, "getCtxAndReportRecoverStrategyChan",
func(*EventController) (context.Context, chan *pb.RecoverStrategyRequest) {
return ctx, strategyChan
})
}
func buildHotSwitchTestCases4() []hotSwitchTestCaseC {
testCases := []hotSwitchTestCaseC{
{name: "should return server inner error when strategy channel is nil",
chanNil: true,
expectedCode: common.ServerInnerError,
expectError: true,
}, {
name: "should return OK when channel closed",
channelClosed: true,
expectedCode: common.OK,
expectError: false,
}, {
name: "should return controller event cancel when context canceled",
contextCanceled: true,
expectedCode: common.ControllerEventCancel,
expectError: false,
}, {
name: "should return wait report timeout when timeout occurs",
timeout: true,
expectedCode: common.WaitReportTimeout,
expectedRtnEvent: common.ReportTimeoutEvent,
expectError: false,
}, {
name: "should add exit event when strategies is empty",
strategies: []string{},
expectedCode: common.OK,
expectedCtlEvent: common.ExitEvent,
expectedRtnEvent: "",
expectError: false,
}, {
name: "should add exit event when exit strategy is received",
strategies: []string{constant.ProcessExitStrategyName},
expectedCode: common.OK,
expectedCtlEvent: common.ExitEvent,
expectedRtnEvent: "",
expectError: false,
}, {
name: "should add migration event when migration strategy is received",
strategies: []string{"migration"},
expectedCode: common.OK,
expectedCtlEvent: common.MigrationEvent,
expectedRtnEvent: "",
expectError: false,
},
}
return testCases
}
func safeWriteToChannel[T any](t *testing.T, ch chan T, value T) {
select {
case ch <- value:
t.Logf("write to channel success")
default:
t.Errorf("write to channel failed: channel is full or nil")
}
}
func TestHandleWaitReportRestartTrainStatus(t *testing.T) {
controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
testCases := buildHotSwitchTestCases5()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
reportChan := make(chan *pb.RecoverStatusRequest, 1)
ctx, cancel := context.WithCancel(context.Background())
if tc.chanNil {
patchGetCtxAndResultChan(patches, controller, ctx, nil)
} else {
patchGetCtxAndResultChan(patches, controller, ctx, reportChan)
}
if tc.contextCanceled {
cancel()
} else if tc.timeout {
patchTimeOut(patches)
} else {
reportChan <- &pb.RecoverStatusRequest{Status: &pb.Status{Code: tc.responseCode}}
}
event, code, err := controller.handleWaitReportRestartTrainStatus()
convey.So(code, convey.ShouldEqual, tc.expectedCode)
convey.So(event, convey.ShouldEqual, tc.expectedRtnEvent)
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
if !tc.contextCanceled && ctx.Err() == nil {
cancel()
}
close(reportChan)
})
}
}
func buildHotSwitchTestCases5() []hotSwitchTestCaseC {
testCases := []hotSwitchTestCaseC{
{
name: "should return OK and restart failed event when report channel is nil",
chanNil: true,
expectedCode: common.OK,
expectedRtnEvent: common.RestartFaildEvent,
expectError: false,
},
{
name: "should return controller event cancel when context canceled",
contextCanceled: true,
expectedCode: common.ControllerEventCancel,
expectError: false,
},
{
name: "should return wait report timeout when timeout occurs",
timeout: true,
expectedCode: common.WaitReportTimeout,
expectedRtnEvent: common.ReportTimeoutEvent,
expectError: false,
},
{
name: "should return OK and restart failed event when unrecoverable retry error received",
responseCode: int32(common.UnRecoverableRetryError),
expectedCode: common.OK,
expectedRtnEvent: common.RestartFaildEvent,
expectError: false,
},
{
name: "should return OK and restart success event when success response received",
responseCode: int32(common.OK),
expectedCode: common.OK,
expectedRtnEvent: common.RestartSuccessEvent,
expectError: false,
},
{
name: "should return OK when unexpected response code received",
responseCode: int32(999),
expectedCode: common.OK,
expectedRtnEvent: "",
expectError: false,
},
}
return testCases
}
func patchGetCtxAndResultChan(patches *gomonkey.Patches, controller *EventController, ctx context.Context, reportChan chan *pb.RecoverStatusRequest) *gomonkey.Patches {
return patches.ApplyPrivateMethod(controller, "getCtxAndResultChan", func(*EventController) (context.Context, chan *pb.RecoverStatusRequest) {
return ctx, reportChan
})
}
func patchGetCtxAndNewStatusMonitorChan(patches *gomonkey.Patches, controller *EventController, ctx context.Context, podPhase chan v1.PodPhase) *gomonkey.Patches {
return patches.ApplyPrivateMethod(controller, "getCtxAndNewStatusMonitorChan", func(*EventController) (context.Context, chan v1.PodPhase) {
return ctx, podPhase
})
}
func TestMonitorNewPodStatus(t *testing.T) {
controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
testCases := buildHotSwitchTestCases6()
for _, tc := range testCases {
convey.Convey(tc.name, t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
statusChan := make(chan v1.PodPhase, 1)
ctx, cancel := context.WithCancel(context.Background())
eventAdded := ""
patches.ApplyPrivateMethod(controller, "addEvent", func(_ *EventController, event string) {
eventAdded = event
})
if tc.chanNil {
patchGetCtxAndNewStatusMonitorChan(patches, controller, ctx, nil)
} else {
patchGetCtxAndNewStatusMonitorChan(patches, controller, ctx, statusChan)
}
if tc.channelClosed {
close(statusChan)
} else if tc.contextCanceled {
cancel()
} else if tc.timeout {
patchTimeOut(patches)
} else if tc.podStatus != "" {
safeWriteToChannel(t, statusChan, tc.podStatus)
}
monitorNewPodStatus(controller)
if tc.expectedCtlEvent != "" {
convey.So(eventAdded, convey.ShouldEqual, tc.expectedCtlEvent)
}
if !tc.contextCanceled && ctx.Err() == nil {
cancel()
}
if !tc.channelClosed && !tc.chanNil {
close(statusChan)
}
})
}
}
func buildHotSwitchTestCases6() []hotSwitchTestCaseC {
testCases := []hotSwitchTestCaseC{
{
name: "should not panic when channel is nil",
chanNil: true,
},
{
name: "should not panic when channel closed",
channelClosed: true,
},
{
name: "should not panic when context canceled",
contextCanceled: true,
},
{
name: "should add new pod timeout event when timeout occurs",
timeout: true,
expectedCtlEvent: common.NewPodTimeoutEvent,
},
{
name: "should add new pod running event when pod running success",
podStatus: v1.PodRunning,
expectedCtlEvent: common.NewPodRunningEvent,
},
{
name: "should add new pod running event when pod in other status",
podStatus: v1.PodPending,
expectedCtlEvent: common.NewPodRunningEvent,
},
}
return testCases
}
