Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package common
import (
"encoding/json"
"errors"
"strconv"
"strings"
"testing"
"time"
"github.com/agiledragon/gomonkey/v2"
"github.com/smartystreets/goconvey/convey"
"k8s.io/apimachinery/pkg/util/sets"
"ascend-common/common-utils/utils"
"ascend-common/devmanager/common"
)
func TestLoadFaultCodeFromFile(t *testing.T) {
convey.Convey("test LoadFaultCodeFromFile", t, func() {
convey.Convey("utils.LoadFile err", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, errors.New("failed"))
defer mockLoadFile.Reset()
convey.So(LoadFaultCodeFromFile(), convey.ShouldNotBeNil)
})
convey.Convey("test LoadFaultCodeFromFile", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, nil)
defer mockLoadFile.Reset()
mockUnmarshal := gomonkey.ApplyFuncReturn(json.Unmarshal, errors.New("failed"))
defer mockUnmarshal.Reset()
convey.So(LoadFaultCodeFromFile(), convey.ShouldNotBeNil)
})
convey.Convey("utils.LoadFile success", func() {
fileInfo := faultFileInfo{
NotHandleFaultCodes: []string{"80E21007"},
SeparateNPUCodes: []string{"80E3A201"},
PreSeparateNPUCodes: []string{"81078603"},
}
codeBytes, err := json.Marshal(fileInfo)
convey.So(err, convey.ShouldBeNil)
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, codeBytes, nil)
defer mockLoadFile.Reset()
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{})
defer mockFaultTypeCode.Reset()
convey.So(LoadFaultCodeFromFile(), convey.ShouldBeNil)
convey.So(faultTypeCode.NotHandleFaultCodes, convey.ShouldResemble, []int64{2162298887})
convey.So(faultTypeCode.SeparateNPUCodes, convey.ShouldResemble, []int64{2162401793})
convey.So(len(faultTypeCode.RestartBusinessCodes), convey.ShouldEqual, 0)
})
})
}
func TestMappingChipFaultToNetworkFaultCodesNotSupport(t *testing.T) {
convey.Convey("TestMappingChipFaultToNetworkFaultCodesNotSupport", t, func() {
convey.Convey("01-RestartRequestCodes has networkFaultCodes, NotHandleFaultNetworkCodes should be updated", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
RestartRequestCodes: []int64{2164753923},
})
defer mockFaultTypeCode.Reset()
mappingChipFaultToNetworkFaultCodesNotSupport()
convey.So(faultTypeCode.NotHandleFaultNetworkCodes, convey.ShouldResemble, []int64{2164753923})
})
convey.Convey("02-RestartBusinessCodes has networkFaultCodes, NotHandleFaultNetworkCodes should be updated", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
RestartBusinessCodes: []int64{2164753923},
})
defer mockFaultTypeCode.Reset()
mappingChipFaultToNetworkFaultCodesNotSupport()
convey.So(faultTypeCode.NotHandleFaultNetworkCodes, convey.ShouldResemble, []int64{2164753923})
})
convey.Convey("RestartNPUCodes has networkFaultCodes, NotHandleFaultNetworkCodes should be updated", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
RestartNPUCodes: []int64{2164753923},
})
defer mockFaultTypeCode.Reset()
mappingChipFaultToNetworkFaultCodesNotSupport()
convey.So(faultTypeCode.NotHandleFaultNetworkCodes, convey.ShouldResemble, []int64{2164753923})
})
convey.Convey("FreeRestartNPUCodes has networkFaultCodes, NotHandleFaultNetworkCodes should be updated", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
FreeRestartNPUCodes: []int64{2164753923},
})
defer mockFaultTypeCode.Reset()
mappingChipFaultToNetworkFaultCodesNotSupport()
convey.So(faultTypeCode.NotHandleFaultNetworkCodes, convey.ShouldResemble, []int64{2164753923})
})
})
}
func TestLoadSwitchFaultCodeFromFile(t *testing.T) {
convey.Convey("TestLoadSwitchFaultCodeFromFile", t, func() {
convey.Convey("01-utils.LoadFile err, should return error", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, errors.New("failed"))
defer mockLoadFile.Reset()
convey.So(LoadSwitchFaultCodeFromFile(), convey.ShouldNotBeNil)
})
convey.Convey("02-utils.LoadFile success, should return error", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, nil)
defer mockLoadFile.Reset()
mockUnmarshal := gomonkey.ApplyFuncReturn(json.Unmarshal, errors.New("failed"))
defer mockUnmarshal.Reset()
convey.So(LoadSwitchFaultCodeFromFile(), convey.ShouldNotBeNil)
})
})
}
func TestLoadFaultCustomizationFromFile(t *testing.T) {
convey.Convey("TestLoadFaultCustomizationFromFile", t, func() {
convey.Convey("01-utils.LoadFile err, should return error", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, errors.New("failed"))
defer mockLoadFile.Reset()
convey.So(LoadFaultCustomizationFromFile(), convey.ShouldNotBeNil)
})
convey.Convey("02-utils.LoadFile success, should return error", func() {
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, nil)
defer mockLoadFile.Reset()
mockUnmarshal := gomonkey.ApplyFuncReturn(json.Unmarshal, errors.New("failed"))
defer mockUnmarshal.Reset()
convey.So(LoadFaultCustomizationFromFile(), convey.ShouldNotBeNil)
})
})
}
func TestLoadFaultCustomization(t *testing.T) {
convey.Convey("TestLoadFaultCustomization", t, func() {
mockRestTime := gomonkey.ApplyGlobalVar(&WaitDeviceResetTime, time.Duration(0))
defer mockRestTime.Reset()
mockReadCMTime := gomonkey.ApplyGlobalVar(&WaitProcessReadCMTime, time.Duration(0))
defer mockReadCMTime.Reset()
mockHealTime := gomonkey.ApplyGlobalVar(&WaitFaultSelfHealingTime, time.Duration(0))
defer mockHealTime.Reset()
faultCustomization := FaultCustomization{
GraceTolerance: GraceToleranceCustomization{
WaitProcessReadCMTime: 30,
WaitDeviceResetTime: 150,
WaitFaultSelfHealingTime: 15,
},
}
codeBytes, err := json.Marshal(faultCustomization)
convey.So(err, convey.ShouldBeNil)
err = LoadFaultCustomization(codeBytes)
convey.So(err, convey.ShouldBeNil)
convey.So(WaitDeviceResetTime, convey.ShouldEqual, time.Duration(150))
convey.So(WaitProcessReadCMTime, convey.ShouldEqual, time.Duration(30))
convey.So(WaitFaultSelfHealingTime, convey.ShouldEqual, time.Duration(15))
})
}
func TestGetFaultTypeByCode(t *testing.T) {
convey.Convey("test GetFaultTypeByCode", t, func() {
faultCodes := []int64{1}
convey.Convey("fault type NormalNPU", func() {
convey.So(GetFaultTypeByCode(nil), convey.ShouldEqual, NormalNPU)
})
convey.Convey("fault type NotHandleFault", func() {
faultTypeCode = FaultTypeCode{NotHandleFaultCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, NotHandleFault)
})
convey.Convey("fault type SeparateNPU", func() {
faultTypeCode = FaultTypeCode{SeparateNPUCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, SeparateNPU)
faultTypeCode = FaultTypeCode{}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, SeparateNPU)
})
convey.Convey("fault type RestartNPU", func() {
faultTypeCode = FaultTypeCode{RestartNPUCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, RestartNPU)
})
convey.Convey("fault type FreeRestartNPU", func() {
faultTypeCode = FaultTypeCode{FreeRestartNPUCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, FreeRestartNPU)
})
convey.Convey("fault type RestartBusiness", func() {
faultTypeCode = FaultTypeCode{RestartBusinessCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, RestartBusiness)
})
convey.Convey("fault type RestartRequest", func() {
faultTypeCode = FaultTypeCode{RestartRequestCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, RestartRequest)
})
convey.Convey("fault type PreSeparateNPU", func() {
faultTypeCode = FaultTypeCode{PreSeparateNPUCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, PreSeparateNPU)
})
convey.Convey("fault type SubHealthFault", func() {
faultTypeCode = FaultTypeCode{SubHealthFaultCodes: faultCodes}
convey.So(GetFaultTypeByCode(faultCodes), convey.ShouldEqual, SubHealthFault)
})
})
}
func TestSetDeviceInit(t *testing.T) {
initId := int32(0)
initLogicIDsLen := 1
convey.Convey("test SetDeviceInit", t, func() {
convey.Convey("SetDeviceInit success", func() {
initLogicIDs = nil
SetDeviceInit(initId)
convey.So(len(initLogicIDs), convey.ShouldEqual, initLogicIDsLen)
})
})
}
func TestGetAndCleanLogicID(t *testing.T) {
convey.Convey("test GetAndCleanLogicID", t, func() {
convey.Convey("initLogicIDs is empty", func() {
initLogicIDs = nil
convey.So(GetAndCleanLogicID(), convey.ShouldBeNil)
})
convey.Convey("initLogicIDs is not empty", func() {
testIDs := []int32{1}
initLogicIDs = testIDs
convey.So(GetAndCleanLogicID(), convey.ShouldResemble, testIDs)
})
})
}
func TestSetNewFaultAndCacheOnceRecoverFault(t *testing.T) {
convey.Convey("test SetNewFaultAndCacheOnceRecoverFault", t, func() {
convey.Convey("device is nil, should return directly", func() {
SetNewFaultAndCacheOnceRecoverFault(0, nil, nil, sets.NewInt64())
convey.So(len(recoverFaultMap), convey.ShouldEqual, 0)
})
convey.Convey("SetNewFaultAndCacheOnceRecoverFault success", func() {
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover},
{Assertion: common.FaultRecover, EventID: 1},
{Assertion: common.FaultOnce, EventID: 0},
{Assertion: common.FaultOccur, EventID: LinkDownFaultCode},
{Assertion: common.FaultRecover, EventID: LinkDownFaultCode},
}
device := &NpuDevice{FaultCodes: []int64{1}}
expectedFaultCodes, expectedFaultMapLen := []int64{0}, 2
originNetworkFaultCodes := NetworkFaultCodes
defer func() { NetworkFaultCodes = originNetworkFaultCodes }()
NetworkFaultCodes = sets.NewInt64()
NetworkFaultCodes.Insert(LinkDownFaultCode)
classified := ClassifyFaultInfos(faultInfos)
SetNewFaultAndCacheOnceRecoverFault(logicID, classified[ChipFaultKey], device, sets.NewInt64())
convey.So(device.FaultCodes, convey.ShouldResemble, expectedFaultCodes)
convey.So(len(recoverFaultMap[logicID]), convey.ShouldEqual, expectedFaultMapLen)
})
convey.Convey("A950 card type, a950 chip fault steps should be applied", func() {
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultOccur, EventID: CardDropFaultCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
SetNewFaultAndCacheOnceRecoverFault(logicID, classified[ChipFaultKey], device, sets.NewInt64())
convey.So(device.FaultCodes, convey.ShouldContain, CardDropFaultCode)
convey.So(device.AlarmRaisedTime, convey.ShouldBeGreaterThan, 0)
})
})
}
func TestSetNetworkNewFaultAndCacheOnceRecoverFault(t *testing.T) {
convey.Convey("test SetNetworkNewFaultAndCacheOnceRecoverFault", t, func() {
convey.Convey("device is nil", func() {
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover},
}
SetNetworkNewFaultAndCacheOnceRecoverFault(0, faultInfos, nil)
convey.So(faultInfos, convey.ShouldNotBeNil)
})
convey.Convey("SetNetworkNewFaultAndCacheOnceRecoverFault success", func() {
recoverNetworkFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
eventId0 := int64(0)
eventId1 := int64(1)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover},
{Assertion: common.FaultRecover, EventID: eventId1},
{Assertion: common.FaultOnce, EventID: eventId0},
{Assertion: common.FaultOccur, EventID: LinkDownFaultCode},
{Assertion: common.FaultRecover, EventID: LinkDownFaultCode},
{Assertion: common.FaultOnce, EventID: LinkDownFaultCode},
}
device := &NpuDevice{NetworkFaultCodes: []int64{LinkDownFaultCode}}
expectedNetworkFaultCodes := []int64{LinkDownFaultCode, LinkDownFaultCode}
expectedRecoverNetworkFaultMapLen := 1
originNetworkFaultCodes := NetworkFaultCodes
defer func() { NetworkFaultCodes = originNetworkFaultCodes }()
NetworkFaultCodes = sets.NewInt64()
NetworkFaultCodes.Insert(LinkDownFaultCode)
classified := ClassifyFaultInfos(faultInfos)
SetNetworkNewFaultAndCacheOnceRecoverFault(logicID, classified[ParameterPlaneFaultKey], device)
convey.So(device.NetworkFaultCodes, convey.ShouldResemble, expectedNetworkFaultCodes)
convey.So(len(recoverNetworkFaultMap[logicID]), convey.ShouldEqual, expectedRecoverNetworkFaultMapLen)
})
convey.Convey("A950 card type, a950 parameter plane fault steps should be applied", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverNetworkFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultOccur, EventID: UBOEPortDownCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBOEDownCnt, 1)
defer patchDownCnt.Reset()
SetNetworkNewFaultAndCacheOnceRecoverFault(logicID, classified[ParameterPlaneFaultKey], device)
convey.So(device.NetworkFaultCodes, convey.ShouldContain, UBOESubHealFaultCode)
convey.So(device.NetworkFaultCodes, convey.ShouldContain, UBOEPortDownCode)
convey.So(device.NetworkAlarmRaisedTime, convey.ShouldBeGreaterThan, 0)
})
convey.Convey("A950 card type recover, should remove precise fault codes when no port down", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverNetworkFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBOEPortDownCode},
}
device := &NpuDevice{
LogicID: logicID,
NetworkFaultCodes: []int64{UBOEPortDownCode, UBOEPreSeparateFaultCode, UBOESubHealFaultCode},
}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBOEDownCnt, common.PortNoDownCount)
defer patchDownCnt.Reset()
SetNetworkNewFaultAndCacheOnceRecoverFault(logicID, classified[ParameterPlaneFaultKey], device)
convey.So(device.NetworkFaultCodes, convey.ShouldBeEmpty)
convey.So(len(recoverNetworkFaultMap[logicID]), convey.ShouldEqual, 0)
})
convey.Convey("A950 card type recover, event not in NetworkFaultCodes should be cached", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverNetworkFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBOEPortDownCode},
{Assertion: common.FaultOnce, EventID: UBOEPortDownCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBOEDownCnt, common.PortNoDownCount)
defer patchDownCnt.Reset()
SetNetworkNewFaultAndCacheOnceRecoverFault(logicID, classified[ParameterPlaneFaultKey], device)
convey.So(recoverNetworkFaultMap[logicID], convey.ShouldResemble,
[]int64{UBOEPortDownCode, UBOEPortDownCode})
convey.So(device.NetworkFaultCodes, convey.ShouldBeEmpty)
})
convey.Convey("A950 card type recover with port still down, should re-occur precise fault", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverNetworkFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBOEPortDownCode},
}
device := &NpuDevice{
LogicID: logicID,
NetworkFaultCodes: []int64{UBOEPortDownCode, UBOEPreSeparateFaultCode, UBOESubHealFaultCode},
}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBOEDownCnt, 1)
defer patchDownCnt.Reset()
SetNetworkNewFaultAndCacheOnceRecoverFault(logicID, classified[ParameterPlaneFaultKey], device)
convey.So(device.NetworkFaultCodes, convey.ShouldContain, UBOESubHealFaultCode)
convey.So(device.NetworkFaultCodes, convey.ShouldContain, UBOEPortDownCode)
convey.So(device.NetworkFaultCodes, convey.ShouldNotContain, UBOEPreSeparateFaultCode)
})
})
}
func TestSetHyperPlaneNewFaultAndCacheOnceRecoverFault(t *testing.T) {
convey.Convey("test SetHyperPlaneNewFaultAndCacheOnceRecoverFault", t, func() {
convey.Convey("device is nil, should return directly", func() {
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(0, nil, nil)
convey.So(len(recoverFaultMap), convey.ShouldEqual, 0)
})
convey.Convey("base card type with empty steps, should not modify device", func() {
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultOccur, EventID: UBPortDownCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(logicID, classified[HyperPlaneFaultKey], device)
convey.So(device.FaultCodes, convey.ShouldBeEmpty)
})
convey.Convey("A950 card type occur, should add precise fault code", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultOccur, EventID: UBPortDownCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBDownCnt, 1)
defer patchDownCnt.Reset()
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(logicID, classified[HyperPlaneFaultKey], device)
convey.So(device.FaultCodes, convey.ShouldContain, UBSeparateFaultCode)
convey.So(device.FaultCodes, convey.ShouldContain, UBPortDownCode)
})
convey.Convey("A950 card type recover, should remove precise fault codes when no port down", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBPortDownCode},
}
device := &NpuDevice{
LogicID: logicID,
FaultCodes: []int64{UBPortDownCode, UBSeparateFaultCode, UBSubHealFaultCode},
}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBDownCnt, common.PortNoDownCount)
defer patchDownCnt.Reset()
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(logicID, classified[HyperPlaneFaultKey], device)
convey.So(device.FaultCodes, convey.ShouldBeEmpty)
convey.So(len(recoverFaultMap[logicID]), convey.ShouldEqual, 0)
})
convey.Convey("A950 card type recover, event not in FaultCodes should be cached", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBPortDownCode},
{Assertion: common.FaultOnce, EventID: UBPortDownCode},
}
device := &NpuDevice{LogicID: logicID}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBDownCnt, common.PortNoDownCount)
defer patchDownCnt.Reset()
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(logicID, classified[HyperPlaneFaultKey], device)
convey.So(recoverFaultMap[logicID], convey.ShouldResemble,
[]int64{UBPortDownCode, UBPortDownCode})
convey.So(device.FaultCodes, convey.ShouldBeEmpty)
})
convey.Convey("A950 card type recover with port still down, should re-occur precise fault", func() {
ParamOption.RealCardType = Ascend910A5
defer func() { ParamOption = Option{} }()
recoverFaultMap = make(map[int32][]int64, GeneralMapSize)
logicID := int32(0)
faultInfos := []common.DevFaultInfo{
{Assertion: common.FaultRecover, EventID: UBPortDownCode},
}
device := &NpuDevice{
LogicID: logicID,
FaultCodes: []int64{UBPortDownCode, UBSeparateFaultCode, UBSubHealFaultCode},
}
classified := ClassifyFaultInfos(faultInfos)
patchCache := gomonkey.ApplyFuncReturn(cacheUBports, nil)
defer patchCache.Reset()
patchDownCnt := gomonkey.ApplyFuncReturn(getUBDownCnt, 1)
defer patchDownCnt.Reset()
SetHyperPlaneNewFaultAndCacheOnceRecoverFault(logicID, classified[HyperPlaneFaultKey], device)
convey.So(device.FaultCodes, convey.ShouldContain, UBSeparateFaultCode)
convey.So(device.FaultCodes, convey.ShouldContain, UBPortDownCode)
})
})
}
func TestDelOnceRecoverFault(t *testing.T) {
convey.Convey("test DelOnceRecoverFault", t, func() {
convey.Convey("DelOnceRecoverFault success", func() {
faultCodes := []int64{1}
networkFaultCodes := []int64{LinkDownFaultCode}
logicId := int32(0)
expectedNum := 0
device := &NpuDevice{LogicID: logicId, FaultCodes: faultCodes, NetworkFaultCodes: networkFaultCodes}
recoverFaultMap = map[int32][]int64{
logicId: faultCodes,
}
recoverNetworkFaultMap = map[int32][]int64{
logicId: networkFaultCodes,
}
groupDevice := map[string][]*NpuDevice{
"test": {device},
}
DelOnceRecoverFault(groupDevice)
convey.So(len(device.FaultCodes), convey.ShouldEqual, expectedNum)
convey.So(len(recoverFaultMap), convey.ShouldEqual, expectedNum)
convey.So(len(device.NetworkFaultCodes), convey.ShouldEqual, expectedNum)
convey.So(len(recoverNetworkFaultMap), convey.ShouldEqual, expectedNum)
})
})
}
func TestDelOnceFrequencyFault(t *testing.T) {
convey.Convey("test DelOnceFrequencyFault", t, func() {
convey.Convey("DelOnceFrequencyFault success", func() {
logicId := int32(0)
frequencyValue := []int64{3, 5, 6}
eventId := "80C98008"
result := 0
faultFrequencyCache := &FaultFrequencyCache{
Frequency: map[int32][]int64{
logicId: frequencyValue,
},
}
faultFrequencyMap = map[string]*FaultFrequencyCache{
eventId: faultFrequencyCache,
}
recoverFaultFrequencyMap = map[int32]string{
logicId: eventId,
}
DelOnceFrequencyFault()
convey.So(len(faultFrequencyMap[eventId].Frequency[logicId]), convey.ShouldEqual, result)
convey.So(len(recoverFaultFrequencyMap), convey.ShouldEqual, result)
})
})
}
func TestDoSaveDevFaultInfo(t *testing.T) {
convey.Convey("test DoSaveDevFaultInfo", t, func() {
convey.Convey("DoSaveDevFaultInfo success", func() {
expectedNum0 := 0
expectedNum1 := 1
eventId := int64(1)
patch := gomonkey.ApplyGlobalVar(&devFaultInfoMap, make(map[int32][]common.DevFaultInfo, GeneralMapSize))
defer patch.Reset()
DoSaveDevFaultInfo(common.DevFaultInfo{}, false)
convey.So(len(devFaultInfoMap), convey.ShouldEqual, expectedNum0)
DoSaveDevFaultInfo(common.DevFaultInfo{EventID: eventId}, false)
convey.So(len(devFaultInfoMap), convey.ShouldEqual, expectedNum1)
})
})
}
func TestTakeOutDevFaultInfo(t *testing.T) {
convey.Convey("test TakeOutDevFaultInfo", t, func() {
convey.Convey("TakeOutDevFaultInfo success", func() {
expectedNum := 0
eventId := int64(1)
logicId := int32(0)
patch := gomonkey.ApplyGlobalVar(&devFaultInfoMap, make(map[int32][]common.DevFaultInfo, GeneralMapSize))
defer patch.Reset()
convey.So(len(GetAndCleanFaultInfo()), convey.ShouldEqual, expectedNum)
testInfo := []common.DevFaultInfo{{EventID: eventId}}
devFaultInfoMap[logicId] = testInfo
convey.So(GetAndCleanFaultInfo()[logicId], convey.ShouldResemble, testInfo)
convey.So(len(devFaultInfoMap[logicId]), convey.ShouldEqual, expectedNum)
})
})
}
func TestGetNetworkFaultTypeByCode(t *testing.T) {
convey.Convey("test GetNetworkFaultTypeByCode", t, func() {
faultCodes := []int64{LinkDownFaultCode}
testFaultCode := int64(1)
convey.Convey("fault type NormalNetwork", func() {
convey.So(GetNetworkFaultTypeByCode(nil), convey.ShouldEqual, NormalNetwork)
})
convey.Convey("fault type NotHandleFault", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
NotHandleFaultNetworkCodes: faultCodes,
NotHandleFaultCodes: []int64{testFaultCode},
})
defer mockFaultTypeCode.Reset()
convey.So(GetNetworkFaultTypeByCode(faultCodes), convey.ShouldEqual, NotHandleFault)
})
convey.Convey("fault type SeparateNPU", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
SeparateNPUNetworkCodes: faultCodes,
NotHandleFaultCodes: []int64{testFaultCode},
})
defer mockFaultTypeCode.Reset()
convey.So(GetNetworkFaultTypeByCode(faultCodes), convey.ShouldEqual, SeparateNPU)
})
convey.Convey("fault type PreSeparateNPU", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{
PreSeparateNPUNetworkCodes: faultCodes,
NotHandleFaultCodes: []int64{testFaultCode},
})
defer mockFaultTypeCode.Reset()
convey.So(GetNetworkFaultTypeByCode(faultCodes), convey.ShouldEqual, PreSeparateNPU)
})
convey.Convey("read json failed", func() {
mockFaultTypeCode := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{})
defer mockFaultTypeCode.Reset()
mockLoadFile := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, errors.New("failed"))
defer mockLoadFile.Reset()
convey.So(GetNetworkFaultTypeByCode(faultCodes), convey.ShouldEqual, PreSeparateNPU)
})
})
}
func TestDevFaultInfoBasedTimeAscendLen(t *testing.T) {
convey.Convey("test DevFaultInfoBasedTimeAscend.Len success", t, func() {
devFault := []common.DevFaultInfo{{}}
convey.So(DevFaultInfoBasedTimeAscend(devFault).Len(), convey.ShouldEqual, len(devFault))
})
}
func TestDevFaultInfoBasedTimeAscendSwap(t *testing.T) {
convey.Convey("test DevFaultInfoBasedTimeAscend.Swap success", t, func() {
devFault := DevFaultInfoBasedTimeAscend([]common.DevFaultInfo{{EventID: 0}, {EventID: 1}})
iKey, jKey := 0, 1
if len(devFault) > iKey && len(devFault) > jKey {
expectIVal, expectJVal := devFault[jKey], devFault[iKey]
devFault.Swap(iKey, jKey)
convey.So(devFault[iKey], convey.ShouldResemble, expectIVal)
convey.So(devFault[jKey], convey.ShouldResemble, expectJVal)
}
})
}
func TestDevFaultInfoBasedTimeAscendLess(t *testing.T) {
convey.Convey("test DevFaultInfoBasedTimeAscend.Less success", t, func() {
devFault := DevFaultInfoBasedTimeAscend([]common.DevFaultInfo{{AlarmRaisedTime: 0}, {AlarmRaisedTime: 1}})
iKey, jKey := 0, 1
convey.So(devFault.Less(iKey, jKey), convey.ShouldBeTrue)
})
}
func TestQueryManuallyFaultInfoByLogicID(t *testing.T) {
convey.Convey("test QueryManuallyFaultInfoByLogicID", t, func() {
convey.Convey("test valid logicID", func() {
logicID := int32(10)
_, ok := manuallySeparateNpuMap[logicID]
convey.So(QueryManuallyFaultInfoByLogicID(logicID), convey.ShouldEqual, ok)
})
convey.Convey("test invalid logicID", func() {
logicID := int32(20)
convey.So(QueryManuallyFaultInfoByLogicID(logicID), convey.ShouldBeFalse)
})
})
}
func TestSetManuallyFaultNPUHandled(t *testing.T) {
convey.Convey("test SetManuallyFaultNPUHandled success", t, func() {
logicId := int32(0)
manuallySeparateNpuMap = map[int32]ManuallyFaultInfo{logicId: {FirstHandle: true}}
expectVal := map[int32]ManuallyFaultInfo{logicId: {FirstHandle: false}}
SetManuallyFaultNPUHandled()
convey.So(manuallySeparateNpuMap, convey.ShouldResemble, expectVal)
})
}
func TestCollectEachFaultEvent(t *testing.T) {
convey.Convey("test collectEachFaultEvent success", t, func() {
logicID := int32(0)
linkDownFaultTimeout := int64(30)
linkDownRecoverTimeout := int64(60)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: linkDownFaultTimeout,
RecoverTimeout: linkDownRecoverTimeout,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur},
{EventID: CardDropFaultCode, Assertion: common.FaultOccur},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover},
}
collectEachFaultEvent(logicID, faultInfos)
expected1 := 1
expected2 := 2
convey.So(len(faultDurationMap), convey.ShouldEqual, expected1)
convey.So(len(faultDurationMap[linkDownFaultCodeStr].Duration[logicID].FaultEventQueue),
convey.ShouldEqual, expected2)
})
}
func TestSortFaultEventsInAscendingOrder(t *testing.T) {
convey.Convey("test sortFaultEventsInAscendingOrder success", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
CardDropFaultCodeStr := strings.ToLower(strconv.FormatInt(CardDropFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
CardDropFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: SeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 4},
{EventID: CardDropFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 3},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 1},
{EventID: CardDropFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 2},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 9},
}
linkDownFaultExpectVal := []common.DevFaultInfo{{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: 1}, {EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 4}, {
EventID: LinkDownFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 9}}
cardDropFaultExpectVal := []common.DevFaultInfo{{EventID: CardDropFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: 2}, {EventID: CardDropFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 3}}
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
sortFaultEventsInAscendingOrder(logicID, CardDropFaultCodeStr)
convey.So(faultDurationMap[linkDownFaultCodeStr].Duration[logicID].FaultEventQueue, convey.ShouldResemble,
linkDownFaultExpectVal)
convey.So(faultDurationMap[CardDropFaultCodeStr].Duration[logicID].FaultEventQueue, convey.ShouldResemble,
cardDropFaultExpectVal)
})
}
func TestMergeContinuousElementBasedAssertion(t *testing.T) {
convey.Convey("test deFaultInfo is empty ,merge fault occur continuous assertion failed ,return", t, func() {
devFaultInfo := &[]common.DevFaultInfo{}
mergeContinuousElementBasedAssertion(devFaultInfo)
convey.So(*devFaultInfo, convey.ShouldResemble, []common.DevFaultInfo{})
})
convey.Convey("test merge fault occur continuous assertion success", t, func() {
devFaultInfo := []common.DevFaultInfo{{Assertion: common.FaultOccur}, {Assertion: common.FaultOccur}}
expectVal := []common.DevFaultInfo{{Assertion: common.FaultOccur}}
mergeContinuousElementBasedAssertion(&devFaultInfo)
convey.So(devFaultInfo, convey.ShouldResemble, expectVal)
})
convey.Convey("test merge fault recover continuous assertion success", t, func() {
devFaultInfo := []common.DevFaultInfo{{Assertion: common.FaultRecover}, {Assertion: common.FaultRecover},
{Assertion: common.FaultRecover}}
expectVal := []common.DevFaultInfo{{Assertion: common.FaultRecover}}
mergeContinuousElementBasedAssertion(&devFaultInfo)
convey.So(devFaultInfo, convey.ShouldResemble, expectVal)
})
convey.Convey("test merge mix fault continuous assertion success", t, func() {
devFaultInfo := []common.DevFaultInfo{{Assertion: common.FaultRecover}, {Assertion: common.FaultRecover},
{Assertion: common.FaultOccur}, {Assertion: common.FaultOccur}}
expectVal := []common.DevFaultInfo{{Assertion: common.FaultRecover}, {Assertion: common.FaultOccur}}
mergeContinuousElementBasedAssertion(&devFaultInfo)
convey.So(devFaultInfo, convey.ShouldResemble, expectVal)
})
}
func TestClearFirstEventBasedOnFaultStatus(t *testing.T) {
convey.Convey("test clearFirstEventBasedOnFaultStatus timeout success", t, func() {
faultDurationData := FaultDurationData{
TimeoutStatus: false,
FaultEventQueue: []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultRecover,
AlarmRaisedTime: 2}, {
EventID: LinkDownFaultCode,
Assertion: common.FaultOccur,
AlarmRaisedTime: 4,
}}}
expectVal := []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultOccur,
AlarmRaisedTime: 4}}
clearFirstEventBasedOnFaultStatus(&faultDurationData)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
})
convey.Convey("test clearFirstEventBasedOnFaultStatus recover success", t, func() {
faultDurationData := FaultDurationData{
TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultOccur,
AlarmRaisedTime: 2}, {
EventID: LinkDownFaultCode,
Assertion: common.FaultRecover,
AlarmRaisedTime: 4,
}}}
expectVal := []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultRecover,
AlarmRaisedTime: 4}}
clearFirstEventBasedOnFaultStatus(&faultDurationData)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
})
}
func TestCleanFaultQueueWhenFaultTimeStatusFalse(t *testing.T) {
convey.Convey("test CleanFaultQueue when fault time status is false", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
CardDropFaultCodeStr := strings.ToLower(strconv.FormatInt(CardDropFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: false}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
CardDropFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: SeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 165},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 100},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 150},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 150},
}
expectVal := []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultOccur,
AlarmRaisedTime: 150}}
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
convey.So(faultDurationMap[linkDownFaultCodeStr].Duration[logicID].FaultEventQueue,
convey.ShouldResemble, expectVal)
})
}
func TestCleanFaultQueueWhenFaultTimeStatusTrue(t *testing.T) {
convey.Convey("test CleanFaultQueue when fault time status is true", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{}}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 165},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover, AlarmRaisedTime: 100},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur, AlarmRaisedTime: 150},
}
expectVal := []common.DevFaultInfo{{
EventID: LinkDownFaultCode,
Assertion: common.FaultRecover,
AlarmRaisedTime: 100}, {
EventID: LinkDownFaultCode,
Assertion: common.FaultOccur,
AlarmRaisedTime: 150,
}}
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
convey.So(faultDurationMap[linkDownFaultCodeStr].Duration[logicID].FaultEventQueue,
convey.ShouldResemble, expectVal)
})
}
func TestHandleFaultQueueCase01(t *testing.T) {
convey.Convey("test handleFaultQueue case 01", t, func() {
logicID := int32(0)
alarmRaisedTime50, alarmRaisedTime81, alarmRaisedTime82 := int64(50), int64(81), int64(82)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime81 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime82 * SecondMagnification},
}
expectVal := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime81 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime82 * SecondMagnification},
}
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
faultDurationTime := int64(31)
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, true)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual, faultDurationTime*SecondMagnification)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual, 0)
})
}
func TestHandleFaultQueueCase02(t *testing.T) {
convey.Convey("test handleFaultQueue case 02", t, func() {
logicID := int32(0)
alarmRaisedTime30, alarmRaisedTime50, alarmRaisedTime80 := int64(30), int64(50), int64(80)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {FaultEventQueue: []common.DevFaultInfo{}}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime80 * SecondMagnification},
}
expectVal := make([]common.DevFaultInfo, 0)
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, false)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual, alarmRaisedTime30*SecondMagnification)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual, 0)
})
}
func TestHandleFaultQueueCase03(t *testing.T) {
convey.Convey("test handleFaultQueue case 03", t, func() {
logicID := int32(0)
alarmRaisedTime50, alarmRaisedTime80, alarmRaisedTime82, alarmRaisedTime112 :=
int64(50), int64(80), int64(82), int64(112)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime80 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: alarmRaisedTime82 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime112 * SecondMagnification},
}
expectVal := make([]common.DevFaultInfo, 0)
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
alarmRaisedTime30 := int64(30)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, false)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual, alarmRaisedTime30*SecondMagnification)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual, 0)
})
}
func TestHandleFaultQueueCase04(t *testing.T) {
convey.Convey("test handleFaultQueue case 04", t, func() {
logicID := int32(0)
alarmRaisedTime50, AlarmRaisedTime110 := int64(50), int64(110)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: alarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: AlarmRaisedTime110 * SecondMagnification},
}
expectVal := make([]common.DevFaultInfo, 0)
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
alarmRaisedTime60 := int64(60)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, true)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual, 0)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual, alarmRaisedTime60*SecondMagnification)
})
}
func TestHandleFaultQueueCase05(t *testing.T) {
convey.Convey("test handleFaultQueue case 05", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
AlarmRaisedTime50, AlarmRaisedTime111, AlarmRaisedTime112 := int64(50), int64(111), int64(112)
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: AlarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: AlarmRaisedTime111 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: AlarmRaisedTime112 * SecondMagnification},
}
expectVal := make([]common.DevFaultInfo, 0)
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
AlarmRaisedTime1, AlarmRaisedTime61 := int64(1), int64(61)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, false)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual,
AlarmRaisedTime1*SecondMagnification)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual,
AlarmRaisedTime61*SecondMagnification)
})
}
func TestHandleFaultQueueCase06(t *testing.T) {
convey.Convey("test handleFaultQueue case 06", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
faultDurationMap = map[string]*FaultDurationCache{
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{}}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
AlarmRaisedTime50, AlarmRaisedTime111, AlarmRaisedTime142 := int64(50), int64(111), int64(142)
faultInfos := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: AlarmRaisedTime50 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: AlarmRaisedTime111 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: AlarmRaisedTime142 * SecondMagnification},
}
expectVal := []common.DevFaultInfo{
{EventID: LinkDownFaultCode, Assertion: common.FaultOccur,
AlarmRaisedTime: AlarmRaisedTime111 * SecondMagnification},
{EventID: LinkDownFaultCode, Assertion: common.FaultRecover,
AlarmRaisedTime: AlarmRaisedTime142 * SecondMagnification},
}
collectEachFaultEvent(logicID, faultInfos)
sortFaultEventsInAscendingOrder(logicID, linkDownFaultCodeStr)
cleanFaultQueue(logicID, linkDownFaultCodeStr)
handleFaultQueue(logicID, linkDownFaultCodeStr)
faultDurationData := faultDurationMap[linkDownFaultCodeStr].Duration[logicID]
AlarmRaisedTime31, AlarmRaisedTime61 := int64(31), int64(61)
convey.So(faultDurationData.TimeoutStatus, convey.ShouldEqual, true)
convey.So(faultDurationData.FaultEventQueue, convey.ShouldResemble, expectVal)
convey.So(faultDurationData.FaultDurationTime, convey.ShouldEqual,
AlarmRaisedTime31*SecondMagnification)
convey.So(faultDurationData.FaultRecoverDurationTime, convey.ShouldEqual,
AlarmRaisedTime61*SecondMagnification)
})
}
func TestResetFaultCustomizationCache(t *testing.T) {
convey.Convey("test ResetFaultCustomizationCache success", t, func() {
faultFrequencyMap = map[string]*FaultFrequencyCache{
"80E18005": {
Frequency: make(map[int32][]int64, GeneralMapSize),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
FaultHandling: ManuallySeparateNPU,
},
},
}
faultDurationMap = map[string]*FaultDurationCache{
"81078603": {
Duration: make(map[int32]FaultDurationData, GeneralMapSize),
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
expectVal := 0
ResetFaultCustomizationCache()
convey.So(len(faultFrequencyMap), convey.ShouldEqual, expectVal)
convey.So(len(faultDurationMap), convey.ShouldEqual, expectVal)
})
}
func TestSaveManuallyFaultInfo(t *testing.T) {
convey.Convey("test SaveManuallyFaultInfo", t, func() {
convey.Convey("test valid logicID", func() {
manuallySeparateNpuMap = make(map[int32]ManuallyFaultInfo, GeneralMapSize)
logicID, expectVal := int32(10), 1
SaveManuallyFaultInfo(logicID)
convey.So(len(manuallySeparateNpuMap), convey.ShouldEqual, expectVal)
})
convey.Convey("test invalid logicID", func() {
manuallySeparateNpuMap = make(map[int32]ManuallyFaultInfo, GeneralMapSize)
logicID, expectVal := int32(20), 0
SaveManuallyFaultInfo(logicID)
convey.So(len(manuallySeparateNpuMap), convey.ShouldEqual, expectVal)
})
})
}
func TestDeleteManuallyFaultInfo(t *testing.T) {
convey.Convey("test DeleteManuallyFaultInfo", t, func() {
convey.Convey("test valid logicID", func() {
mockMap := gomonkey.ApplyGlobalVar(&manuallySeparateNpuMap, map[int32]ManuallyFaultInfo{
int32(10): {},
})
defer mockMap.Reset()
logicID, expectVal := int32(10), 0
DeleteManuallyFaultInfo(logicID)
convey.So(len(manuallySeparateNpuMap), convey.ShouldEqual, expectVal)
})
convey.Convey("test invalid logicID", func() {
mockMap := gomonkey.ApplyGlobalVar(&manuallySeparateNpuMap, map[int32]ManuallyFaultInfo{
int32(10): {},
})
defer mockMap.Reset()
logicID, expectVal := int32(20), 1
DeleteManuallyFaultInfo(logicID)
convey.So(len(manuallySeparateNpuMap), convey.ShouldEqual, expectVal)
})
convey.Convey("test not exist logicID", func() {
mockMap := gomonkey.ApplyGlobalVar(&manuallySeparateNpuMap, map[int32]ManuallyFaultInfo{
int32(10): {},
})
defer mockMap.Reset()
logicID, expectVal := int32(11), 1
DeleteManuallyFaultInfo(logicID)
convey.So(len(manuallySeparateNpuMap), convey.ShouldEqual, expectVal)
})
})
}
func TestLoadGraceToleranceCustomization(t *testing.T) {
convey.Convey("test loadGraceToleranceCustomization success", t, func() {
graceToleranceCustomization := GraceToleranceCustomization{
WaitDeviceResetTime: 150,
WaitProcessReadCMTime: 30,
WaitFaultSelfHealingTime: 15,
}
WaitDeviceResetTime = time.Duration(0)
WaitProcessReadCMTime = time.Duration(0)
WaitFaultSelfHealingTime = time.Duration(0)
loadGraceToleranceCustomization(graceToleranceCustomization)
expectResetTime, expectReadCMTime, expectSelfHealingTime := 150, 30, 15
convey.So(WaitDeviceResetTime, convey.ShouldEqual, expectResetTime)
convey.So(WaitProcessReadCMTime, convey.ShouldEqual, expectReadCMTime)
convey.So(WaitFaultSelfHealingTime, convey.ShouldEqual, expectSelfHealingTime)
})
convey.Convey("test loadGraceToleranceCustomization abnormal condition success", t, func() {
graceToleranceCustomization := GraceToleranceCustomization{
WaitDeviceResetTime: 59,
WaitProcessReadCMTime: 91,
WaitFaultSelfHealingTime: 0,
}
WaitDeviceResetTime = time.Duration(0)
WaitProcessReadCMTime = time.Duration(0)
WaitFaultSelfHealingTime = time.Duration(0)
loadGraceToleranceCustomization(graceToleranceCustomization)
expectResetTime, expectReadCMTime, expectSelfHealingTime := 150, 30, 15
convey.So(WaitDeviceResetTime, convey.ShouldEqual, expectResetTime)
convey.So(WaitProcessReadCMTime, convey.ShouldEqual, expectReadCMTime)
convey.So(WaitFaultSelfHealingTime, convey.ShouldEqual, expectSelfHealingTime)
})
}
func TestValidateFaultFrequencyCustomizationPart1(t *testing.T) {
convey.Convey("test validateFaultFrequencyCustomization success", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{"80C98000", "80B78000"},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, true)
})
}
func TestValidateFaultFrequencyCustomizationPart2(t *testing.T) {
convey.Convey("test validateFaultFrequencyCustomization failed case1", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case2", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{"80C98000", "80B78000"},
FaultFrequency: FaultFrequency{
TimeWindow: 59,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case3", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{"80C98000", "80B78000"},
FaultFrequency: FaultFrequency{
TimeWindow: 60,
Times: 0,
FaultHandling: ManuallySeparateNPU,
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case4", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{"80C98000", "80B78000"},
FaultFrequency: FaultFrequency{
TimeWindow: 60,
Times: 2,
FaultHandling: "separatesNPU",
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
}
func TestValidateFaultFrequencyCustomizationPart3(t *testing.T) {
convey.Convey("test validateFaultFrequencyCustomization success", t, func() {
faultFrequencyCustomization := FaultFrequencyCustomization{
EventId: []string{"80C98000", "80B78000"},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: SeparateNPU,
},
}
result := validateFaultFrequencyCustomization(&faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
}
func TestLoadFaultFrequencyCustomizationCase1(t *testing.T) {
convey.Convey("test loadFaultFrequencyCustomization success case1", t, func() {
faultCode1 := "80C98000"
faultCode2 := "80E18005"
faultFrequencyCustomizations := []FaultFrequencyCustomization{{
EventId: []string{faultCode1}, FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
FaultHandling: ManuallySeparateNPU,
}}, {
EventId: []string{faultCode2, faultCode1}, FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU}}}
expectVal := map[string]*FaultFrequencyCache{
strings.ToLower(faultCode1): {Frequency: make(map[int32][]int64, common.MaxErrorCodeCount),
LastFaultTime: map[int32]int64{},
LastFaultRecoverTime: make(map[int32]int64),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
ReleaseTimeWindow: MaxReleaseTimeWindow,
FaultHandling: ManuallySeparateNPU}},
strings.ToLower(faultCode2): {Frequency: make(map[int32][]int64, common.MaxErrorCodeCount),
LastFaultTime: map[int32]int64{},
LastFaultRecoverTime: make(map[int32]int64),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
ReleaseTimeWindow: MaxReleaseTimeWindow,
FaultHandling: ManuallySeparateNPU}}}
faultFrequencyMap = make(map[string]*FaultFrequencyCache, common.MaxErrorCodeCount)
loadFaultFrequencyCustomization(faultFrequencyCustomizations)
convey.So(faultFrequencyMap, convey.ShouldResemble, expectVal)
ResetFaultCustomizationCache()
})
}
func TestLoadFaultFrequencyCustomizationCase2(t *testing.T) {
convey.Convey("test loadFaultFrequencyCustomization success case2", t, func() {
faultCode1 := "80C98000"
faultCode2 := "80E18005"
faultFrequencyCustomizations := []FaultFrequencyCustomization{{
EventId: []string{faultCode1}, FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 0,
FaultHandling: ManuallySeparateNPU,
}}, {
EventId: []string{faultCode2}, FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU}}}
expectVal := map[string]*FaultFrequencyCache{
strings.ToLower(faultCode2): {Frequency: make(map[int32][]int64, common.MaxErrorCodeCount),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
ReleaseTimeWindow: MaxReleaseTimeWindow}}}
faultFrequencyMap = map[string]*FaultFrequencyCache{
strings.ToLower(faultCode1): {Frequency: make(map[int32][]int64, common.MaxErrorCodeCount),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
FaultHandling: ManuallySeparateNPU}},
strings.ToLower(faultCode2): {Frequency: make(map[int32][]int64, common.MaxErrorCodeCount),
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU}}}
loadFaultFrequencyCustomization(faultFrequencyCustomizations)
convey.So(faultFrequencyMap, convey.ShouldResemble, expectVal)
ResetFaultCustomizationCache()
})
}
func TestValidateFaultDurationCustomizationPart1(t *testing.T) {
convey.Convey("test validateFaultFrequencyCustomization success", t, func() {
faultFrequencyCustomization := FaultDurationCustomization{
EventId: []string{"81078603"},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
}
result := validateFaultDurationCustomization(faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, true)
})
}
func TestValidateFaultDurationCustomizationPart2(t *testing.T) {
convey.Convey("test validateFaultFrequencyCustomization failed case1", t, func() {
faultFrequencyCustomization := FaultDurationCustomization{
EventId: []string{},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
}
result := validateFaultDurationCustomization(faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case2", t, func() {
faultFrequencyCustomization := FaultDurationCustomization{
EventId: []string{"81078603"},
FaultDuration: FaultDuration{
FaultTimeout: -1,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
}
result := validateFaultDurationCustomization(faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case3", t, func() {
faultFrequencyCustomization := FaultDurationCustomization{
EventId: []string{"81078603"},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: -1,
FaultHandling: PreSeparateNPU,
},
}
result := validateFaultDurationCustomization(faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
convey.Convey("test validateFaultFrequencyCustomization failed case4", t, func() {
faultFrequencyCustomization := FaultDurationCustomization{
EventId: []string{"81078603"},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: ManuallySeparateNPU,
},
}
result := validateFaultDurationCustomization(faultFrequencyCustomization)
convey.So(result, convey.ShouldEqual, false)
})
}
func TestLoadFaultDurationCustomizationCase1(t *testing.T) {
convey.Convey("test loadFaultDurationCustomizationCase1 success case1", t, func() {
faultCode1 := "81078603"
faultCode2 := "80E0180F"
faultDurationCustomization := []FaultDurationCustomization{{
EventId: []string{faultCode1}, FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
}}, {
EventId: []string{faultCode2, faultCode1}, FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness}}}
expectVal := map[string]*FaultDurationCache{
strings.ToLower(faultCode1): {Duration: make(map[int32]FaultDurationData, common.MaxErrorCodeCount),
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU}},
strings.ToLower(faultCode2): {Duration: make(map[int32]FaultDurationData, common.MaxErrorCodeCount),
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness}}}
faultDurationMap = make(map[string]*FaultDurationCache, common.MaxErrorCodeCount)
loadFaultDurationCustomization(faultDurationCustomization)
convey.So(faultDurationMap, convey.ShouldResemble, expectVal)
})
}
func TestLoadFaultDurationCustomizationCase2(t *testing.T) {
convey.Convey("test loadFaultDurationCustomizationCase1 success case2", t, func() {
faultCode1 := "81078603"
faultCode2 := "80E0180F"
faultDurationCustomization := []FaultDurationCustomization{{
EventId: []string{faultCode1}, FaultDuration: FaultDuration{
FaultTimeout: -1,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
}}, {
EventId: []string{faultCode2}, FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness}}}
expectVal := map[string]*FaultDurationCache{
strings.ToLower(faultCode2): {Duration: make(map[int32]FaultDurationData, common.MaxErrorCodeCount),
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness}}}
faultDurationMap = map[string]*FaultDurationCache{
strings.ToLower(faultCode1): {Duration: make(map[int32]FaultDurationData, common.MaxErrorCodeCount),
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU}},
strings.ToLower(faultCode2): {Duration: make(map[int32]FaultDurationData, common.MaxErrorCodeCount),
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness}}}
loadFaultDurationCustomization(faultDurationCustomization)
convey.So(faultDurationMap, convey.ShouldResemble, expectVal)
})
}
func TestGetMostSeriousFaultType(t *testing.T) {
convey.Convey("test getMostSeriousFaultType success case1", t, func() {
fautTypes := []string{NotHandleFault, ManuallySeparateNPU}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, ManuallySeparateNPU)
})
convey.Convey("test getMostSeriousFaultType success case2", t, func() {
fautTypes := []string{NotHandleFault, SeparateNPU}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, SeparateNPU)
})
convey.Convey("test getMostSeriousFaultType success case3", t, func() {
fautTypes := []string{NotHandleFault, PreSeparateNPU}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, PreSeparateNPU)
})
convey.Convey("test getMostSeriousFaultType success case4", t, func() {
fautTypes := []string{NotHandleFault, RestartNPU}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, RestartNPU)
})
convey.Convey("test getMostSeriousFaultType success case5", t, func() {
fautTypes := []string{NotHandleFault, FreeRestartNPU}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, FreeRestartNPU)
})
convey.Convey("test getMostSeriousFaultType success case6", t, func() {
fautTypes := []string{NotHandleFault, RestartBusiness}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, RestartBusiness)
})
convey.Convey("test getMostSeriousFaultType success case7", t, func() {
fautTypes := []string{NotHandleFault, RestartRequest}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, RestartRequest)
})
convey.Convey("test getMostSeriousFaultType success case8", t, func() {
fautTypes := []string{NotHandleFault, SubHealthFault}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, SubHealthFault)
})
convey.Convey("test getMostSeriousFaultType success case9", t, func() {
fautTypes := []string{NotHandleFault, NotHandleFault}
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, NotHandleFault)
})
convey.Convey("test getMostSeriousFaultType success case10", t, func() {
fautTypes := make([]string, 0)
convey.So(getMostSeriousFaultType(fautTypes), convey.ShouldEqual, NormalNPU)
})
}
func TestGetFaultTypeFromFaultFrequency(t *testing.T) {
convey.Convey("test GetFaultTypeFromFaultFrequency success case1", t, func() {
logicId := int32(0)
faultCode := "80E18005"
timeDiff := int64(2000)
faultFrequencyMap = map[string]*FaultFrequencyCache{
strings.ToLower(faultCode): {
Frequency: map[int32][]int64{
logicId: {time.Now().UnixMilli() - timeDiff},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
},
}
convey.So(GetFaultTypeFromFaultFrequency(logicId, NetworkFaultMode), convey.ShouldEqual, NormalNPU)
})
convey.Convey("test GetFaultTypeFromFaultFrequency success case2", t, func() {
logicId := int32(0)
faultCode := "80E18005"
firstTime, secondTime, thirdTime := int64(10000), int64(8000), int64(2000)
faultFrequencyMap = map[string]*FaultFrequencyCache{
strings.ToLower(faultCode): {
Frequency: map[int32][]int64{
logicId: {time.Now().UnixMilli() - firstTime, time.Now().UnixMilli() - secondTime,
time.Now().UnixMilli() - thirdTime},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
},
}
manuallySeparateNpuMap = make(map[int32]ManuallyFaultInfo, GeneralMapSize)
recoverFaultFrequencyMap = make(map[int32]string, GeneralMapSize)
convey.So(GetFaultTypeFromFaultFrequency(logicId, ChipFaultMode), convey.ShouldEqual, ManuallySeparateNPU)
convey.So(manuallySeparateNpuMap[logicId].FirstHandle, convey.ShouldEqual, true)
convey.So(recoverFaultFrequencyMap[logicId], convey.ShouldEqual, strings.ToLower("80E18005"))
ResetFaultCustomizationCache()
})
}
func TestGetFaultTypeFromFaultDurationPart1(t *testing.T) {
convey.Convey("test GetFaultTypeFromFaultDuration success part1", t, func() {
logicId := int32(0)
faultCode := "80E0180F"
faultDurationTime := int64(121)
faultDurationMap = map[string]*FaultDurationCache{
strings.ToLower(faultCode): {
Duration: map[int32]FaultDurationData{
logicId: {
TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{},
FaultDurationTime: faultDurationTime * SecondMagnification,
FaultRecoverDurationTime: 0,
},
},
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness,
},
},
}
convey.So(GetFaultTypeFromFaultDuration(logicId, ChipFaultMode), convey.ShouldEqual, RestartBusiness)
})
}
func TestGetFaultTypeFromFaultDurationPart2(t *testing.T) {
convey.Convey("test GetFaultTypeFromFaultDuration success part2", t, func() {
logicId := int32(0)
faultCode := "81078603"
faultDurationTime := int64(25)
faultDurationMap = map[string]*FaultDurationCache{
strings.ToLower(faultCode): {
Duration: map[int32]FaultDurationData{
logicId: {
TimeoutStatus: false,
FaultEventQueue: []common.DevFaultInfo{},
FaultDurationTime: faultDurationTime * SecondMagnification,
FaultRecoverDurationTime: 0,
},
},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}
convey.So(GetFaultTypeFromFaultDuration(logicId, NetworkFaultMode), convey.ShouldEqual, NormalNPU)
})
}
func TestGetFaultType(t *testing.T) {
convey.Convey("test GetFaultType success", t, func() {
logicId := int32(0)
faultCode := "80E0180F"
faultCodes := []int64{0x80E0180F, 0x80C98002}
firstTime, secondTime, thirdTime := int64(10000), int64(8000), int64(2000)
faultFrequencyMap = map[string]*FaultFrequencyCache{
strings.ToLower(faultCode): {
Frequency: map[int32][]int64{
logicId: {time.Now().UnixMilli() - firstTime, time.Now().UnixMilli() - secondTime,
time.Now().UnixMilli() - thirdTime},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 3,
FaultHandling: ManuallySeparateNPU,
},
},
}
faultDurationTime := int64(121)
faultDurationMap = map[string]*FaultDurationCache{
strings.ToLower(faultCode): {
Duration: map[int32]FaultDurationData{
logicId: {
TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{},
FaultDurationTime: faultDurationTime * SecondMagnification,
FaultRecoverDurationTime: 0,
},
},
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: RestartBusiness,
},
},
}
manuallySeparateNpuMap = make(map[int32]ManuallyFaultInfo, GeneralMapSize)
recoverFaultFrequencyMap = make(map[int32]string, GeneralMapSize)
convey.So(GetFaultType(faultCodes, logicId), convey.ShouldEqual, ManuallySeparateNPU)
convey.So(manuallySeparateNpuMap[logicId].FirstHandle, convey.ShouldEqual, true)
convey.So(recoverFaultFrequencyMap[logicId], convey.ShouldEqual, strings.ToLower(faultCode))
})
}
func TestGetFaultTypeFromUpgradeFault(t *testing.T) {
convey.Convey("test TestGetFaultTypeFromUpgradeFault success", t, func() {
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyFunc(GetUpgradeFaultLevelAndTime, func(logicId int32, mode string) map[int64]FaultTimeAndLevel {
res := make(map[int64]FaultTimeAndLevel)
res[AivBusFaultCode] = FaultTimeAndLevel{FaultLevel: PreSeparateNPU}
res[AicBusFaultCode] = FaultTimeAndLevel{FaultLevel: ManuallySeparateNPU}
return res
})
faultType := GetFaultType([]int64{}, 0)
convey.So(ManuallySeparateNPU, convey.ShouldEqual, faultType)
})
}
func TestGetNetworkFaultType(t *testing.T) {
convey.Convey("test GetNetworkFaultType success", t, func() {
logicId := int32(0)
faultCode := "81078603"
faultCodes := []int64{0x81078603}
faultDurationTime := int64(31)
mockFaultVar := gomonkey.ApplyGlobalVar(&faultTypeCode, FaultTypeCode{}).
ApplyGlobalVar(&faultDurationMap, map[string]*FaultDurationCache{
strings.ToLower(faultCode): {
Duration: map[int32]FaultDurationData{
logicId: {
TimeoutStatus: true,
FaultEventQueue: []common.DevFaultInfo{},
FaultDurationTime: faultDurationTime * SecondMagnification,
FaultRecoverDurationTime: 0,
},
},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
}).
ApplyGlobalVar(&faultFrequencyMap, map[string]*FaultFrequencyCache{}).
ApplyGlobalVar(&recoverFaultFrequencyMap, map[int32]string{})
defer mockFaultVar.Reset()
convey.So(GetNetworkFaultType(faultCodes, logicId), convey.ShouldEqual, PreSeparateNPU)
})
}
func TestSetAlarmRaisedTime(t *testing.T) {
alarmRaisedTime := int64(123)
convey.Convey("test setAlarmRaisedTime success case1", t, func() {
device := NpuDevice{
FaultCodes: []int64{0x80C98002, 0x80C98003},
AlarmRaisedTime: alarmRaisedTime,
}
setAlarmRaisedTime(&device)
convey.So(device.AlarmRaisedTime, convey.ShouldEqual, alarmRaisedTime)
})
convey.Convey("test setAlarmRaisedTime success case2", t, func() {
device := NpuDevice{
FaultCodes: []int64{},
AlarmRaisedTime: alarmRaisedTime,
}
setAlarmRaisedTime(&device)
convey.So(device.AlarmRaisedTime, convey.ShouldEqual, 0)
})
convey.Convey("test setAlarmRaisedTime success case3", t, func() {
device := NpuDevice{
FaultCodes: []int64{0x80C98002},
AlarmRaisedTime: 0,
}
setAlarmRaisedTime(&device)
convey.So(device.AlarmRaisedTime, convey.ShouldBeGreaterThan, 0)
})
}
func TestSetNetworkAlarmRaisedTime(t *testing.T) {
networkAlarmRaisedTime := int64(125)
convey.Convey("test setNetworkAlarmRaisedTime success case1", t, func() {
device := NpuDevice{
NetworkFaultCodes: []int64{0x81078603},
NetworkAlarmRaisedTime: networkAlarmRaisedTime,
}
setNetworkAlarmRaisedTime(&device)
convey.So(device.NetworkAlarmRaisedTime, convey.ShouldEqual, networkAlarmRaisedTime)
})
convey.Convey("test setNetworkAlarmRaisedTime success case2", t, func() {
device := NpuDevice{
NetworkFaultCodes: []int64{},
NetworkAlarmRaisedTime: networkAlarmRaisedTime,
}
setNetworkAlarmRaisedTime(&device)
convey.So(device.NetworkAlarmRaisedTime, convey.ShouldEqual, 0)
})
convey.Convey("test setNetworkAlarmRaisedTime success case3", t, func() {
device := NpuDevice{
NetworkFaultCodes: []int64{0x81078603},
NetworkAlarmRaisedTime: 0,
}
setNetworkAlarmRaisedTime(&device)
convey.So(device.NetworkAlarmRaisedTime, convey.ShouldBeGreaterThan, 0)
})
}
func TestGetChangedDevFaultInfo(t *testing.T) {
convey.Convey("test GetChangedDevFaultInfo when device is nil", t, func() {
oldErrCodes := make([]int64, 0)
newErrCodes := []int64{0x81078603}
devFaultInfos := GetChangedDevFaultInfo(nil, oldErrCodes, newErrCodes)
convey.So(len(devFaultInfos), convey.ShouldEqual, 0)
})
convey.Convey("test GetChangedDevFaultInfo success case1", t, func() {
device := NpuDevice{LogicID: 0}
oldErrCodes := make([]int64, 0)
newErrCodes := []int64{0x81078603}
devFaultInfos := GetChangedDevFaultInfo(&device, oldErrCodes, newErrCodes)
convey.So(len(devFaultInfos), convey.ShouldEqual, 1)
convey.So(devFaultInfos[0].Assertion, convey.ShouldEqual, common.FaultOccur)
convey.So(devFaultInfos[0].EventID, convey.ShouldEqual, 0x81078603)
})
convey.Convey("test GetChangedDevFaultInfo success case2", t, func() {
device := NpuDevice{LogicID: 0}
oldErrCodes := []int64{0x81078603}
newErrCodes := make([]int64, 0)
devFaultInfos := GetChangedDevFaultInfo(&device, oldErrCodes, newErrCodes)
convey.So(len(devFaultInfos), convey.ShouldEqual, 1)
convey.So(devFaultInfos[0].Assertion, convey.ShouldEqual, common.FaultRecover)
convey.So(devFaultInfos[0].EventID, convey.ShouldEqual, 0x81078603)
})
convey.Convey("test GetChangedDevFaultInfo success case3", t, func() {
device := NpuDevice{LogicID: 0}
oldErrCodes := []int64{0x81078603}
newErrCodes := []int64{0x81078603}
devFaultInfos := GetChangedDevFaultInfo(&device, oldErrCodes, newErrCodes)
convey.So(len(devFaultInfos), convey.ShouldEqual, 0)
})
}
func TestGetTimeoutFaultCodes(t *testing.T) {
convey.Convey("test GetTimeoutFaultCodes success", t, func() {
logicID := int32(0)
linkDownFaultCodeStr := strings.ToLower(strconv.FormatInt(LinkDownFaultCode, Hex))
CardDropFaultCodeStr := strings.ToLower(strconv.FormatInt(CardDropFaultCode, Hex))
ResetFinishFaultCodeStr := strings.ToLower(strconv.FormatInt(ResetFinishFaultCode, Hex))
NetworkFaultCodes = sets.NewInt64(LinkDownFaultCode)
faultDurationMap = map[string]*FaultDurationCache{
CardDropFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true}},
FaultDuration: FaultDuration{
FaultTimeout: 120,
RecoverTimeout: 0,
FaultHandling: SeparateNPU,
},
},
linkDownFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: false}},
FaultDuration: FaultDuration{
FaultTimeout: 30,
RecoverTimeout: 60,
FaultHandling: PreSeparateNPU,
},
},
ResetFinishFaultCodeStr: {
Duration: map[int32]FaultDurationData{logicID: {TimeoutStatus: true}},
FaultDuration: FaultDuration{
FaultTimeout: 240,
RecoverTimeout: 0,
FaultHandling: SeparateNPU,
},
},
}
expectedChipFaultCodesLen := 2
expectedNetworkFaultCodes := make(map[int64]FaultTimeAndLevel)
convey.So(len(GetTimeoutFaultLevelAndCodes(ChipFaultMode, logicID)), convey.ShouldResemble, expectedChipFaultCodesLen)
convey.So(GetTimeoutFaultLevelAndCodes(NetworkFaultMode, logicID), convey.ShouldResemble, expectedNetworkFaultCodes)
})
}
func TestGetFrequencyFaultCodes(t *testing.T) {
convey.Convey("test GetTimeoutFaultCodes success", t, func() {
logicID := int32(0)
faultCode1 := "80C98000"
faultCode2 := "80C98008"
faultCode3 := "80E18005"
firstTime, secondTime, thirdTime := int64(10), int64(8), int64(2)
NetworkFaultCodes = sets.NewInt64(LinkDownFaultCode)
faultFrequencyMap = map[string]*FaultFrequencyCache{
strings.ToLower(faultCode1): {
Frequency: map[int32][]int64{
logicID: {time.Now().Unix() - firstTime},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
FaultHandling: ManuallySeparateNPU,
},
},
strings.ToLower(faultCode2): {
Frequency: map[int32][]int64{
logicID: {time.Now().Unix() - firstTime, time.Now().Unix() - secondTime,
time.Now().Unix() - thirdTime},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 2,
FaultHandling: ManuallySeparateNPU,
},
},
strings.ToLower(faultCode3): {
Frequency: map[int32][]int64{
logicID: {time.Now().Unix() - firstTime},
},
FaultFrequency: FaultFrequency{
TimeWindow: 86400,
Times: 0,
FaultHandling: ManuallySeparateNPU,
},
},
}
expectedChipFaultCodesLen := 2
expectedNetworkFaultCodes := make(map[int64]FaultTimeAndLevel)
convey.So(len(GetFrequencyFaultLevelAndCodes(ChipFaultMode, logicID)), convey.ShouldResemble, expectedChipFaultCodesLen)
convey.So(GetFrequencyFaultLevelAndCodes(NetworkFaultMode, logicID), convey.ShouldResemble, expectedNetworkFaultCodes)
ResetFaultCustomizationCache()
})
}
func TestLoadVaildSwitchFaultCode(t *testing.T) {
convey.Convey("test loadValidSwitchFaultCode", t, func() {
switchFileInfo := SwitchFaultFileInfo{
NotHandleFaultCodes: []string{generalFaultCode},
}
convey.Convey("when switch fault is invalid, should not load switch fault code", func() {
mockFunc := gomonkey.ApplyFuncReturn(isValidSwitchFaultCode, false)
defer mockFunc.Reset()
loadValidSwitchFaultCode(switchFileInfo.NotHandleFaultCodes, &NotHandleFaultCodes,
"NotHandleFaultCodes")
convey.So(len(NotHandleFaultCodes) == 0, convey.ShouldBeTrue)
})
convey.Convey("when switch fault is valid, should load switch fault code", func() {
loadValidSwitchFaultCode(switchFileInfo.NotHandleFaultCodes, &NotHandleFaultCodes,
"NotHandleFaultCodes")
convey.So(len(NotHandleFaultCodes) > 0, convey.ShouldBeTrue)
})
})
}
func TestLoadSwitchFaultCode(t *testing.T) {
convey.Convey("test LoadSwitchFaultCode", t, func() {
switchFileInfo := SwitchFaultFileInfo{
NotHandleFaultCodes: []string{generalFaultCode},
RestartRequestFaultCodes: []string{generalFaultCode},
SubHealthFaultCodes: []string{generalFaultCode},
SeparateFaultCodes: []string{generalFaultCode},
PreSeparateFaultCodes: []string{generalFaultCode},
}
bytes, err := json.Marshal(switchFileInfo)
convey.So(err, convey.ShouldBeNil)
convey.Convey("when switch fault unmarchal failed, should return error", func() {
mockFunc := gomonkey.ApplyFuncReturn(json.Unmarshal, errors.New("failed"))
defer mockFunc.Reset()
err = LoadSwitchFaultCode(bytes)
convey.So(err, convey.ShouldNotBeNil)
})
convey.Convey("when switch fault is not valid, should filter this fault", func() {
mockFunc := gomonkey.ApplyFuncReturn(isValidSwitchFaultCode, false)
defer mockFunc.Reset()
err = LoadSwitchFaultCode(bytes)
convey.So(err, convey.ShouldBeNil)
convey.So(len(NotHandleFaultCodes) == 0, convey.ShouldBeTrue)
convey.So(len(RestartRequestFaultCodes) == 0, convey.ShouldBeTrue)
convey.So(len(SubHealthFaultCodes) == 0, convey.ShouldBeTrue)
convey.So(len(SeparateFaultCodes) == 0, convey.ShouldBeTrue)
convey.So(len(PreSeparateFaultCodes) == 0, convey.ShouldBeTrue)
})
convey.Convey("when switch fault is valid, should update fault codes slice", func() {
err = LoadSwitchFaultCode(bytes)
convey.So(err, convey.ShouldBeNil)
convey.So(len(NotHandleFaultCodes) > 0, convey.ShouldBeTrue)
convey.So(len(RestartRequestFaultCodes) > 0, convey.ShouldBeTrue)
convey.So(len(SubHealthFaultCodes) > 0, convey.ShouldBeTrue)
convey.So(len(SeparateFaultCodes) > 0, convey.ShouldBeTrue)
convey.So(len(PreSeparateFaultCodes) > 0, convey.ShouldBeTrue)
convey.So(NotHandleFaultCodes[firstFaultIdx] == generalFaultCode, convey.ShouldBeTrue)
convey.So(RestartRequestFaultCodes[firstFaultIdx] == generalFaultCode, convey.ShouldBeTrue)
convey.So(SubHealthFaultCodes[firstFaultIdx] == generalFaultCode, convey.ShouldBeTrue)
convey.So(SeparateFaultCodes[firstFaultIdx] == generalFaultCode, convey.ShouldBeTrue)
convey.So(PreSeparateFaultCodes[firstFaultIdx] == generalFaultCode, convey.ShouldBeTrue)
})
})
}
func TestHbmFaultManager(t *testing.T) {
var fakeAlarmRaisedTime int64 = 123456789
convey.Convey("test new fault manager", t, func() {
hbmFaultManager := NewHbmFaultManager()
convey.So(hbmFaultManager, convey.ShouldNotBeNil)
})
convey.Convey("test update hbm occur time ", t, func() {
hbmFaultManager := NewHbmFaultManager()
faultInfo := common.DevFaultInfo{
LogicID: 1,
EventID: HbmDoubleBitFaultCode,
AlarmRaisedTime: fakeAlarmRaisedTime,
}
hbmFaultManager.updateHbmOccurTime(faultInfo)
hbmOccurTime, ok := hbmFaultManager.HbmOccurTimeCache[1]
convey.So(ok, convey.ShouldBeTrue)
convey.So(hbmOccurTime, convey.ShouldEqual, fakeAlarmRaisedTime)
})
convey.Convey("test fault event in que ", t, func() {
hbmFaultManager := NewHbmFaultManager()
faultInfo := common.DevFaultInfo{
LogicID: 1,
EventID: AivBusFaultCode,
AlarmRaisedTime: fakeAlarmRaisedTime,
}
hbmFaultManager.aicFaultEventInQue(faultInfo)
faultInfoList, ok := hbmFaultManager.AicFaultEventQue[1]
convey.So(ok, convey.ShouldBeTrue)
convey.So(len(faultInfoList), convey.ShouldEqual, 1)
convey.So(faultInfoList[0].AlarmRaisedTime, convey.ShouldEqual, fakeAlarmRaisedTime)
})
convey.Convey("test fault event in que ", t, func() {
hbmFaultManager := NewHbmFaultManager()
faultInfo := common.DevFaultInfo{
LogicID: 1,
EventID: AicBusFaultCode,
AlarmRaisedTime: 100000000,
}
hbmFaultManager.aicFaultEventInQue(faultInfo)
faultInfoList := hbmFaultManager.aicFaultEventOutQue(1)
convey.So(len(faultInfoList), convey.ShouldEqual, 1)
convey.So(faultInfoList[0].AlarmRaisedTime, convey.ShouldEqual, 100000000)
convey.So(faultInfoList[0].EventID, convey.ShouldEqual, AicBusFaultCode)
})
convey.Convey("test fault event in que ", t, func() {
hbmFaultManager := NewHbmFaultManager()
faultInfo := common.DevFaultInfo{
LogicID: 1,
EventID: AicBusFaultCode,
AlarmRaisedTime: 300,
}
hbmFaultManager.aicFaultEventInQue(faultInfo)
faultInfoList := hbmFaultManager.aicFaultEventOutQue(1)
convey.So(len(faultInfoList), convey.ShouldEqual, 0)
})
}
func TestUpdateDeviceFaultTimeMap(t *testing.T) {
t.Run("Test_updateDeviceFaultTimeMap", func(t *testing.T) {
npuDevice := &NpuDevice{
FaultTimeMap: make(map[int64]int64),
}
eventId := int64(4326743278)
faultInfo := common.DevFaultInfo{
EventID: eventId,
AlarmRaisedTime: time.Now().UnixMilli(),
}
isAdd := true
updateDeviceFaultTimeMap(npuDevice, faultInfo, isAdd)
faultTime, found := npuDevice.FaultTimeMap[eventId]
if !found {
t.Errorf("cannot found fault time")
return
}
if faultTime != faultInfo.AlarmRaisedTime {
t.Errorf("cannot found fault error")
return
}
isAdd = false
updateDeviceFaultTimeMap(npuDevice, faultInfo, isAdd)
_, found = npuDevice.FaultTimeMap[eventId]
if found {
t.Errorf("found fault time")
return
}
})
}
func TestNewFaultInfosForHBMErr(t *testing.T) {
convey.Convey("test newFaultInfosForHBMErr", t, func() {
faultInfo := []common.DevFaultInfo{
{
LogicID: 1,
EventID: AicBusFaultCode,
AlarmRaisedTime: 3000,
},
{
LogicID: 1,
EventID: HbmDoubleBitFaultCode,
AlarmRaisedTime: 3400,
Assertion: common.FaultOccur,
},
}
newFaultInfos := newFaultInfosForHBMErr(1, faultInfo)
convey.So(len(newFaultInfos), convey.ShouldEqual, 1)
convey.So(newFaultInfos[0], convey.ShouldResemble, faultInfo[1])
})
}
func TestQueryManuallyFaultNPULogicIDsByHandleStatus(t *testing.T) {
convey.Convey("test queryManuallyFaultNPULogicIDsByHandleStatus", t, func() {
convey.Convey("01-handleStatus is FirstHandle and FirstHandle is true in map, should contain logic id", func() {
manuallySeparateNpuMap = map[int32]ManuallyFaultInfo{1: {LogicID: 1, FirstHandle: true}}
convey.So(QueryManuallyFaultNPULogicIDsByHandleStatus(ManuallySeparateNpuFirstHandle), convey.ShouldResemble, []int32{1})
})
convey.Convey("02-handleStatus is handled and FirstHandle is false, should contain logic id", func() {
manuallySeparateNpuMap = map[int32]ManuallyFaultInfo{1: {LogicID: 1, FirstHandle: false}}
convey.So(QueryManuallyFaultNPULogicIDsByHandleStatus(ManuallySeparateNpuHandled), convey.ShouldResemble, []int32{1})
})
convey.Convey("03-handleStatus is all, should contain logic id", func() {
manuallySeparateNpuMap = map[int32]ManuallyFaultInfo{1: {LogicID: 1}}
convey.So(QueryManuallyFaultNPULogicIDsByHandleStatus(ManuallySeparateNpuAll), convey.ShouldResemble, []int32{1})
})
convey.Convey("04-handleStatus is other value, should return empty list", func() {
manuallySeparateNpuMap = map[int32]ManuallyFaultInfo{1: {LogicID: 1}}
convey.So(QueryManuallyFaultNPULogicIDsByHandleStatus("other"), convey.ShouldResemble, []int32{})
})
})
}
func TestGetFaultAssertionName(t *testing.T) {
convey.Convey("test getFaultAssertionName", t, func() {
convey.So(GetFaultAssertionName(common.FaultRecover), convey.ShouldEqual, AssertionRecovery)
convey.So(GetFaultAssertionName(common.FaultOccur), convey.ShouldEqual, AssertionOccur)
convey.So(GetFaultAssertionName(common.FaultOnce), convey.ShouldEqual, AssertionNotice)
convey.So(GetFaultAssertionName(3), convey.ShouldEqual, "")
})
}
func TestCheckErrorMessage(t *testing.T) {
convey.Convey("test checkErrorMessage", t, func() {
convey.So(CheckErrorMessage(errors.New("contain error code: 8102"), "8102"), convey.ShouldBeTrue)
convey.So(CheckErrorMessage(errors.New("contain error code: 8102"), "8103"), convey.ShouldBeFalse)
})
}
func TestIsValidSwitchFaultCode(t *testing.T) {
convey.Convey("test isValidSwitchFaultCode max length", t, func() {
code := strings.Repeat("a", MaxLengthOfFaultCode) + "]"
ret := isValidSwitchFaultCode(code)
convey.So(ret, convey.ShouldEqual, false)
})
convey.Convey("test isValidSwitchFaultCode don't match format", t, func() {
ret := isValidSwitchFaultCode("aaaa")
convey.So(ret, convey.ShouldEqual, false)
})
}
func TestGetUpgradeFaultLevelAndTime(t *testing.T) {
convey.Convey("test TestGetUpgradeFaultLevelAndTime should succeed", t, func() {
InsertUpgradeFaultCache(LogicId(0), time.Now().UnixMilli(), HbmDoubleBitFaultCodeStr,
ManuallySeparateNPU, FrequencyUpgradeType)
faultLevelAndTime := GetUpgradeFaultLevelAndTime(0, ChipFaultMode)
num, err := strconv.ParseInt(HbmDoubleBitFaultCodeStr, Hex, 0)
convey.So(err, convey.ShouldEqual, err)
convey.So(faultLevelAndTime[num].FaultLevel, convey.ShouldEqual, ManuallySeparateNPU)
RemoveTimeoutReasonCache(LogicId(0))
})
}
