Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package device
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/agiledragon/gomonkey/v2"
"github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/assert"
"golang.org/x/time/rate"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
"Ascend-device-plugin/pkg/common"
"Ascend-device-plugin/pkg/device/hangdetection"
"Ascend-device-plugin/pkg/kubeclient"
"ascend-common/api"
"ascend-common/devmanager"
npuCommon "ascend-common/devmanager/common"
)
const (
phyIDNum = 1
logicIDNum = 2
vDevIDNum = 3
aiCoreNum = 4
aiCoreCount = 8
vDevChipID = 100
FaultOnce = 1
NoneFault = 0
routineWaitTime = 50 * time.Millisecond
atlas300VPro = "Atlas 300V Pro"
numTwo = 2
numFifty = 50
numOneHundred = 100
)
func TestAsyncReleaseAutoFill(t *testing.T) {
convey.Convey("should release auto fill when all ids convert successfully", t, func() {
tool := mockAscendTools()
patches := gomonkey.NewPatches()
defer patches.Reset()
var removeCacheCalled int32 = 0
patches.ApplyFunc(common.RemoveTimeoutReasonCache,
func(id common.LogicId, matchers ...common.ReasonKeyMatcher) {
atomic.StoreInt32(&removeCacheCalled, 1)
})
patches.ApplyFunc(time.After, func(d time.Duration) <-chan time.Time {
ch := make(chan time.Time, 1)
ch <- time.Now()
return ch
})
tool.asyncReleaseAutoFill(context.Background(), []common.PhyId{phyIDNum}, 0)
time.Sleep(routineWaitTime)
called := atomic.LoadInt32(&removeCacheCalled)
convey.So(called, convey.ShouldEqual, 1)
})
}
func deepCopyGroupDevice(groupDevice map[string][]*common.NpuDevice) map[string][]*common.NpuDevice {
newGroupDevice := make(map[string][]*common.NpuDevice, len(groupDevice))
for deviceType, npuDevices := range groupDevice {
newNpuDevices := make([]*common.NpuDevice, 0, len(npuDevices))
for _, npuDevice := range npuDevices {
newNpuDevice := &common.NpuDevice{
FaultCodes: npuDevice.FaultCodes,
AlarmRaisedTime: npuDevice.AlarmRaisedTime,
NetworkFaultCodes: npuDevice.NetworkFaultCodes,
NetworkAlarmRaisedTime: npuDevice.NetworkAlarmRaisedTime,
DevType: npuDevice.DevType,
DeviceName: npuDevice.DeviceName,
Health: npuDevice.Health,
NetworkHealth: npuDevice.NetworkHealth,
IP: npuDevice.IP,
LogicID: npuDevice.LogicID,
PhyID: npuDevice.PhyID,
CardID: npuDevice.CardID,
}
newNpuDevices = append(newNpuDevices, newNpuDevice)
}
newGroupDevice[deviceType] = newNpuDevices
}
return newGroupDevice
}
func TestGetChipAICore(t *testing.T) {
convey.Convey("test GetChipAICore", t, func() {
coreCnt := int32(8)
tool := mockAscendTools()
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{AiCoreCount: coreCnt})
defer mockOption.Reset()
convey.So(tool.GetChipAICore() == coreCnt, convey.ShouldBeTrue)
})
}
func TestGetName(t *testing.T) {
convey.Convey("test GetName", t, func() {
tool := mockAscendTools()
toolName := "mock tool"
tool.name = toolName
convey.So(tool.GetName() == toolName, convey.ShouldBeTrue)
})
}
func TestConvertLogicIDsToDeviceNames(t *testing.T) {
convey.Convey("test convertLogicIDsToDeviceNames", t, func() {
convey.Convey("01-get device run mode failed, should return empty string", func() {
tool := mockAscendTools()
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: ""})
defer mockOption.Reset()
logicIds := []int32{3}
convey.So(tool.convertLogicIDsToDeviceNames(logicIds), convey.ShouldEqual, "")
})
convey.Convey("02-convert logic ids to device names success, should return Ascend910-1", func() {
tool := mockAscendTools()
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910A3})
defer mockOption.Reset()
logicIds := []int32{3}
convey.So(tool.convertLogicIDsToDeviceNames(logicIds), convey.ShouldEqual, api.Ascend910MinuxPrefix+"1")
})
})
}
func TestHandleManuallySeparateNPUFaultInfo(t *testing.T) {
convey.Convey("test handleManuallySeparateNPUFaultInfo", t, func() {
convey.Convey("01-get device run mode fail, should return empty string", func() {
tool := mockAscendTools()
info, ids := tool.handleManuallySeparateNPUFaultInfo()
convey.So(info, convey.ShouldEqual, "")
convey.So(ids, convey.ShouldResemble, make([]common.LogicId, 0))
})
tool := mockAscendTools()
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910A3})
defer mockOption.Reset()
convey.Convey("02-manually fault cache is empty, should return empty string", func() {
info, ids := tool.handleManuallySeparateNPUFaultInfo()
convey.So(info, convey.ShouldEqual, "")
convey.So(ids, convey.ShouldResemble, make([]common.LogicId, 0))
})
mockStatus := gomonkey.ApplyFuncReturn(common.QueryManuallyFaultNPULogicIDsByHandleStatus, []int32{3})
defer mockStatus.Reset()
mockMethod := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"GetManuallySeparateNPUFromDeviceInfo",
func(_ *kubeclient.ClientK8s, _ *v1.ConfigMap) []common.PhyId {
return []common.PhyId{common.PhyId(1)}
}).ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"GetConfigMap", func(_ *kubeclient.ClientK8s, _, _ string) (*v1.ConfigMap, error) {
return nil, nil
})
defer mockMethod.Reset()
convey.Convey("03-handle fault info success, manually separate npu 1, should return Ascend910-1", func() {
info, _ := tool.handleManuallySeparateNPUFaultInfo()
convey.So(info, convey.ShouldEqual, api.Ascend910MinuxPrefix+"1")
})
})
}
func TestIsDeviceStatusChange(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test IsDeviceStatusChange true", t, func() {
devices := map[string][]*common.NpuDevice{api.Ascend910: {{Health: v1beta1.Healthy}}}
aiCoreDevice := []*common.NpuDevice{{Health: v1beta1.Healthy}}
oldDevice := deepCopyGroupDevice(devices)
tool.UpdateHealth(devices, aiCoreDevice, api.Ascend910)
res := tool.GetChange(devices, oldDevice)
convey.So(res, convey.ShouldNotBeNil)
})
tool = AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMockErr{}}
convey.Convey("test IsDeviceStatusChange which chip is unhealthy ", t, func() {
devices := map[string][]*common.NpuDevice{api.Ascend310P: {{Health: v1beta1.Unhealthy}}}
aiCoreDevice := []*common.NpuDevice{{Health: v1beta1.Unhealthy}}
oldDevice := deepCopyGroupDevice(devices)
tool.UpdateHealth(devices, aiCoreDevice, api.Ascend310P)
res := tool.GetChange(devices, oldDevice)
convey.So(res, convey.ShouldNotBeNil)
})
}
func TestSetAICoreHealthyIfVNpu(t *testing.T) {
convey.Convey("test setAICoreHealthyIfVNpu", t, func() {
groupDevices := map[string][]*common.NpuDevice{api.Ascend910: {{LogicID: 1, Health: v1beta1.Healthy}}}
aiCoreDevs := []*common.NpuDevice{{LogicID: 1, Health: v1beta1.Unhealthy}}
convey.Convey("01-presetVDevice is false, not update aiCoreDevs", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{PresetVDevice: false})
defer mockOption.Reset()
setAICoreHealthyIfVNpu(groupDevices, aiCoreDevs)
convey.So(aiCoreDevs[0].Health == v1beta1.Healthy, convey.ShouldBeTrue)
})
convey.Convey("presetVDevice is true, update aiCoreDevs, aiCoreDevs[0] should be Unhealthy", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{PresetVDevice: true})
defer mockOption.Reset()
setAICoreHealthyIfVNpu(groupDevices, aiCoreDevs)
convey.So(aiCoreDevs[0].Health == v1beta1.Healthy, convey.ShouldBeFalse)
})
})
}
func TestSetHealthyIfDuoCard(t *testing.T) {
convey.Convey("test setHealthyIfDuoCard", t, func() {
convey.Convey("01-not contain atlas 300IDuo, should not update groupDevice", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{ProductTypes: []string{}})
defer mockOption.Reset()
setHealthyIfDuoCard(map[string][]*common.NpuDevice{})
})
convey.Convey("02-HotReset is false, should not update groupDevice", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption,
common.Option{ProductTypes: []string{common.Atlas300IDuo}, HotReset: common.HotResetTrainOffLine})
defer mockOption.Reset()
setHealthyIfDuoCard(map[string][]*common.NpuDevice{})
})
convey.Convey("03-not found devices, should not update groupDevice", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption,
common.Option{ProductTypes: []string{common.Atlas300IDuo}, HotReset: common.HotResetInfer})
defer mockOption.Reset()
setHealthyIfDuoCard(map[string][]*common.NpuDevice{})
})
convey.Convey("04-update unhealthy card status, should update groupDevice", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption,
common.Option{ProductTypes: []string{common.Atlas300IDuo}, HotReset: common.HotResetInfer})
defer mockOption.Reset()
groupDevices := map[string][]*common.NpuDevice{
api.Ascend310P: {{CardID: 0, Health: v1beta1.Healthy}, {CardID: 0, Health: v1beta1.Unhealthy}},
}
setHealthyIfDuoCard(groupDevices)
convey.So(groupDevices[api.Ascend310P][0].Health == v1beta1.Healthy, convey.ShouldBeFalse)
})
})
}
func TestIsHealthy(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test isHealthy", t, func() {
device := &common.NpuDevice{
FaultCodes: []int64{int64(0x80C98008), int64(0x80CB8008)},
LogicID: 0,
DeviceName: "Ascend910-0",
}
convey.Convey("01-normal npu is healthy, device should be healthy", func() {
mockFaultType := gomonkey.ApplyFuncReturn(common.GetFaultType, common.NormalNPU)
defer mockFaultType.Reset()
convey.So(tool.isHealthy(device) == v1beta1.Healthy, convey.ShouldBeTrue)
})
convey.Convey("02-PreSeparate npu is healthy and npu is used now, device should be healthy", func() {
mockFaultType := gomonkey.ApplyFuncReturn(common.GetFaultType, common.PreSeparateNPU)
defer mockFaultType.Reset()
mockNpuIsUseNow := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)), "npuIsUsedNow",
func(_ *AscendTools, deviceName string) bool { return true })
defer mockNpuIsUseNow.Reset()
convey.So(tool.isHealthy(device) == v1beta1.Healthy, convey.ShouldBeTrue)
})
convey.Convey("03-PreSeparate npu is healthy and npu is not used now, device should be unhealthy", func() {
mockFaultType := gomonkey.ApplyFuncReturn(common.GetFaultType, common.PreSeparateNPU)
defer mockFaultType.Reset()
mockNpuIsUseNow := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)), "npuIsUsedNow",
func(_ *AscendTools, deviceName string) bool { return false })
defer mockNpuIsUseNow.Reset()
convey.So(tool.isHealthy(device) == v1beta1.Unhealthy, convey.ShouldBeTrue)
})
})
}
func TestIsNetworkHealthy(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test isNetworkHealthy", t, func() {
device := &common.NpuDevice{}
convey.Convey("01-network faultType is NormalNpu, network should be healthy", func() {
mockFaultType := gomonkey.ApplyFuncReturn(common.GetNetworkFaultType, common.NormalNPU)
defer mockFaultType.Reset()
convey.So(tool.isNetworkHealthy(device) == v1beta1.Healthy, convey.ShouldBeTrue)
})
convey.Convey("02-network faultType is PreSeparateNpu, network should be unhealthy", func() {
mockFaultType := gomonkey.ApplyFuncReturn(common.GetNetworkFaultType, common.PreSeparateNPU)
defer mockFaultType.Reset()
convey.So(tool.isNetworkHealthy(device) == v1beta1.Unhealthy, convey.ShouldBeTrue)
})
})
}
func TestMoreThanFiveMin(t *testing.T) {
convey.Convey("test moreThanFiveMin", t, func() {
device := &common.NpuDevice{AlarmRaisedTime: 0}
convey.So(moreThanFiveMin(device), convey.ShouldBeFalse)
device.AlarmRaisedTime = 3000
convey.So(moreThanFiveMin(device), convey.ShouldBeTrue)
})
}
func TestNetworkMoreThanFiveMin(t *testing.T) {
convey.Convey("test networkMoreThanFiveMin", t, func() {
device := &common.NpuDevice{NetworkAlarmRaisedTime: 0}
convey.So(networkMoreThanFiveMin(device), convey.ShouldBeFalse)
device.NetworkAlarmRaisedTime = 3000
convey.So(networkMoreThanFiveMin(device), convey.ShouldBeTrue)
})
}
func TestLogFaultModeChange(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test LogFaultModeChange", t, func() {
convey.Convey("01-faultMode is empty, should update fault mode", func() {
device := &common.NpuDevice{LogicID: 1}
mockFaultMode := gomonkey.ApplyGlobalVar(&faultMode, map[int32]string{})
defer mockFaultMode.Reset()
tool.LogFaultModeChange(device, []int32{0, 1}, common.Polling)
convey.So(faultMode[1] == common.Polling, convey.ShouldBeTrue)
})
convey.Convey("02-old mode is equal to new mode, should not update fault mode", func() {
device := &common.NpuDevice{LogicID: 1}
mockFaultMode := gomonkey.ApplyGlobalVar(&faultMode, map[int32]string{1: common.Polling})
defer mockFaultMode.Reset()
tool.LogFaultModeChange(device, []int32{0, 1}, common.Polling)
convey.So(faultMode[1] == common.Polling, convey.ShouldBeTrue)
})
convey.Convey("03-old mode is different from new mode, should update fault mode", func() {
device := &common.NpuDevice{LogicID: 1, Health: v1beta1.Unhealthy, AlarmRaisedTime: 3000}
mockFaultMode := gomonkey.ApplyGlobalVar(&faultMode, map[int32]string{1: common.Subscribe})
defer mockFaultMode.Reset()
tool.LogFaultModeChange(device, []int32{0, 1}, common.Polling)
convey.So(faultMode[1] == common.Polling, convey.ShouldBeTrue)
})
})
}
func TestGetNPUsByShareMode(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test getNPUsByShareMode", t, func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{ShareCount: 2})
defer mockOption.Reset()
davinCiDev := common.DavinCiDev{LogicID: 1, PhyID: 1, CardID: 1, IP: "127.0.0.1"}
numNum := 2
devs := tool.getNPUsByShareMode(davinCiDev)
convey.So(len(devs) == numNum, convey.ShouldBeTrue)
convey.So(devs[0].IP == "127.0.0.1", convey.ShouldBeTrue)
convey.So(devs[0].DeviceName == api.Ascend910MinuxPrefix+"1-0", convey.ShouldBeTrue)
})
}
func TestAssembleShareModeDevices(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test assembleShareModeDevices", t, func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{ShareCount: 1})
defer mockOption.Reset()
davinCiDev := common.DavinCiDev{LogicID: 1, PhyID: 1, CardID: 1, IP: "127.0.0.1"}
npuDevs := &[]common.NpuDevice{}
devTypes := &[]string{}
tool.assembleShareModeDevices(davinCiDev, npuDevs, devTypes)
convey.So(len(*npuDevs) == 1, convey.ShouldBeTrue)
convey.So(len(*devTypes) == 1, convey.ShouldBeTrue)
convey.So((*npuDevs)[0].IP == "127.0.0.1", convey.ShouldBeTrue)
})
}
func TestGetServerBoardId(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test GetServerBoardId", t, func() {
devLogicID := int32(2)
convey.Convey("01-board id is not empty, should return nil", func() {
tool.boardId = common.A800IA2NoneHccsBoardId
_, err := tool.GetServerBoardId(devLogicID)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("02-board id is empty, should return nil", func() {
tool.boardId = common.EmptyBoardId
boardId, err := tool.GetServerBoardId(devLogicID)
convey.So(err, convey.ShouldBeNil)
convey.So(boardId == 0, convey.ShouldBeTrue)
})
})
}
func TestWriteFaultToEvent(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test writeFaultToEvent", t, func() {
mockDoWriteFaultToEventMethod := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"doWriteFaultToEvent", func(_ *AscendTools, faultInfo npuCommon.DevFaultInfo) error {
return errors.New("write fault to event fail")
})
defer mockDoWriteFaultToEventMethod.Reset()
allFaultInfo <- npuCommon.DevFaultInfo{LogicID: 0, Assertion: FaultOnce, EventID: common.CardDropFaultCode}
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(time.Second)
cancel()
}()
tool.WriteFaultToEvent(ctx)
})
}
func TestDoWriteFaultToEvent(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test doWriteFaultToEvent", t, func() {
mockGetNodeNameFromEnv := gomonkey.ApplyFuncReturn(kubeclient.GetNodeNameFromEnv, "mock node", nil)
defer mockGetNodeNameFromEnv.Reset()
mockGetPodNameFromEnv := gomonkey.ApplyFuncReturn(common.GetPodNameFromEnv, "mock pod", nil)
defer mockGetPodNameFromEnv.Reset()
mockCreateEvent := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"CreateEvent", func(_ *kubeclient.ClientK8s, evt *v1.Event) (*v1.Event, error) { return nil, nil })
defer mockCreateEvent.Reset()
convey.Convey("01-assertion is invalid, should return error", func() {
faultInfo := npuCommon.DevFaultInfo{LogicID: 0, Assertion: 3, EventID: common.CardDropFaultCode}
convey.So(tool.doWriteFaultToEvent(faultInfo), convey.ShouldNotBeNil)
})
convey.Convey("02-write fault to event success, should return nil", func() {
faultInfo := npuCommon.DevFaultInfo{LogicID: 0, Assertion: npuCommon.FaultRecover, EventID: common.LinkDownFaultCode}
convey.So(tool.doWriteFaultToEvent(faultInfo), convey.ShouldBeNil)
})
})
}
func TestHandleDropCardFaultEvents(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test HandleDropCardFaultEvents", t, func() {
mockSaveDevFaultInfo := gomonkey.ApplyFunc(common.DoSaveDevFaultInfo,
func(devFaultInfo npuCommon.DevFaultInfo, enableDelay bool) {})
defer mockSaveDevFaultInfo.Reset()
convey.Convey("01-occur fault event, CardDrop should be true", func() {
mockCheckCardDropFault := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"checkCardDropFault", func(_ *AscendTools, logicID int32) bool { return true })
defer mockCheckCardDropFault.Reset()
npuDevice := &common.NpuDevice{LogicID: 1, CardDrop: false}
tool.generateCardDropFaultEvents(npuDevice)
convey.So(npuDevice.CardDrop, convey.ShouldBeTrue)
})
convey.Convey("recover fault event, CardDrop should be false", func() {
mockCheckCardDropFault := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"checkCardDropFault", func(_ *AscendTools, logicID int32) bool { return false })
defer mockCheckCardDropFault.Reset()
npuDevice := &common.NpuDevice{LogicID: 1, CardDrop: true}
tool.generateCardDropFaultEvents(npuDevice)
convey.So(npuDevice.CardDrop, convey.ShouldBeFalse)
})
})
}
func TestCheckCardDropFault(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test checkCardDropFault", t, func() {
convey.Convey("01-check card drop fault fail, should return false", func() {
convey.So(tool.checkCardDropFault(1), convey.ShouldBeFalse)
})
convey.Convey("02-check card drop fault success, should return true", func() {
mockGetDeviceIPAddress := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceHealth", func(_ *devmanager.DeviceManagerMock, logicID int32) (uint32, error) {
return 0, errors.New("error is " + npuCommon.DeviceNotReadyErrCodeStr)
})
defer mockGetDeviceIPAddress.Reset()
convey.So(tool.checkCardDropFault(1), convey.ShouldBeTrue)
})
})
}
func TestHandleLostChipFaultEvents(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test HandleLostChipFaultEvents", t, func() {
device := &common.NpuDevice{
FaultCodes: []int64{},
LogicID: 1,
}
mockGlobalVar := gomonkey.ApplyGlobalVar(&isFirstFlushFault, true)
defer mockGlobalVar.Reset()
convey.Convey("01-get device all error code fail, devFaultInfoMap should not be updated", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, nil, errors.New("mock failure message")
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostChipFaultEvents(device, nil)
faultInfo := common.GetAndCleanFaultInfo()
convey.So(len(faultInfo) == 0, convey.ShouldBeTrue)
})
convey.Convey("02-handle lost chip fault event, devFaultInfoMap should be updated", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, []int64{common.LinkDownFaultCode, common.CardDropFaultCode}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostChipFaultEvents(device, nil)
faultInfoMap := common.GetAndCleanFaultInfo()
convey.So(len(faultInfoMap) == 1, convey.ShouldBeTrue)
faultInfo, ok := faultInfoMap[1]
convey.So(ok, convey.ShouldBeTrue)
convey.So(len(faultInfo) == 1, convey.ShouldBeTrue)
convey.So(faultInfo[0].EventID == common.CardDropFaultCode, convey.ShouldBeTrue)
})
})
}
func TestHandleLostNetworkFaultEvents(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test HandleLostNetworkFaultEvents", t, func() {
device := &common.NpuDevice{
FaultCodes: []int64{},
LogicID: 1,
}
mockGlobalVar := gomonkey.ApplyGlobalVar(&isFirstFlushFault, true)
defer mockGlobalVar.Reset()
convey.Convey("01-get device all error code fail, devFaultInfoMap should not be updated", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, nil, errors.New("mock failure message")
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostNetworkFaultEvents(device, nil)
faultInfo := common.GetAndCleanFaultInfo()
convey.So(len(faultInfo) == 0, convey.ShouldBeTrue)
})
convey.Convey("02-handle lost network fault event, devFaultInfoMap should be updated", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, []int64{common.LinkDownFaultCode, common.CardDropFaultCode}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostNetworkFaultEvents(device, nil)
faultInfoMap := common.GetAndCleanFaultInfo()
convey.So(len(faultInfoMap) == 1, convey.ShouldBeTrue)
faultInfo, ok := faultInfoMap[1]
convey.So(ok, convey.ShouldBeTrue)
convey.So(len(faultInfo) == 1, convey.ShouldBeTrue)
convey.So(faultInfo[0].EventID == common.LinkDownFaultCode, convey.ShouldBeTrue)
})
})
}
func TestHandleLostHyperPlaneFaultEvents(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test HandleLostHyperPlaneFaultEvents", t, func() {
device := &common.NpuDevice{
FaultCodes: []int64{},
LogicID: 1,
PhyID: 0,
}
mockGlobalVar := gomonkey.ApplyGlobalVar(&isFirstFlushFault, true)
defer mockGlobalVar.Reset()
convey.Convey("01-get device all error code fail, devFaultInfoMap should not be updated", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, nil, errors.New("mock failure message")
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostHyperPlaneFaultEvents(device, nil)
faultInfo := common.GetAndCleanFaultInfo()
convey.So(len(faultInfo) == 0, convey.ShouldBeTrue)
})
convey.Convey("02-handle lost hyper plane fault event, devFaultInfoMap should be updated", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 1, []int64{common.UBPortDownCode, common.CardDropFaultCode}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostHyperPlaneFaultEvents(device, nil)
faultInfoMap := common.GetAndCleanFaultInfo()
convey.So(len(faultInfoMap) == 1, convey.ShouldBeTrue)
faultInfo, ok := faultInfoMap[1]
convey.So(ok, convey.ShouldBeTrue)
convey.So(len(faultInfo) == 1, convey.ShouldBeTrue)
convey.So(faultInfo[0].EventID == common.UBPortDownCode, convey.ShouldBeTrue)
})
convey.Convey("03-no hyper plane fault code and device is not A5, should not query ub port status",
func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, logicID int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
tool.HandleLostHyperPlaneFaultEvents(device, nil)
faultInfo := common.GetAndCleanFaultInfo()
convey.So(len(faultInfo) == 0, convey.ShouldBeTrue)
})
})
}
func TestGenerateHyperPlaneFaultEventsBasedOnFaultCacheChange(t *testing.T) {
convey.Convey("test generateHyperPlaneFaultEventsBasedOnFaultCacheChange", t, func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
device := &common.NpuDevice{LogicID: 0, DeviceID: 0, PhyID: 0, FaultCodes: []int64{}}
convey.Convey("01-get device fault failed, should return directly", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, _ int32) (int32, []int64, error) {
return -1, nil, errors.New("get device fault failed")
})
defer mockGetDeviceAllErrorCode.Reset()
tool.generateHyperPlaneFaultEventsBasedOnFaultCacheChange(device)
faultInfo := common.GetAndCleanFaultInfo()
convey.So(len(faultInfo) == 0, convey.ShouldBeTrue)
})
convey.Convey("02-hyper plane fault codes is not empty, should save fault event", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, _ int32) (int32, []int64, error) {
return 1, []int64{common.UBPortDownCode}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
saveCalled := false
mockSave := gomonkey.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
defer mockSave.Reset()
tool.generateHyperPlaneFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeTrue)
})
convey.Convey("03-no hyper plane fault code, ub link down, should generate fault code and save",
func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
mockGetHyperPlaneStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getHyperPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkDownStatus,
downPortsNum: 8}
})
defer mockGetHyperPlaneStatus.Reset()
saveCalled := false
mockSave := gomonkey.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
defer mockSave.Reset()
tool.generateHyperPlaneFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeTrue)
})
convey.Convey("04-no hyper plane fault code, ub link up, should not save fault event", func() {
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
mockGetHyperPlaneStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getHyperPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
defer mockGetHyperPlaneStatus.Reset()
saveCalled := false
mockSave := gomonkey.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
defer mockSave.Reset()
tool.generateHyperPlaneFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeFalse)
})
convey.Convey("05-device is not A5, should skip ub port status query", func() {
notA5Tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910B}}
mockGetDeviceAllErrorCode := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceAllErrorCode", func(_ *devmanager.DeviceManagerMock, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
defer mockGetDeviceAllErrorCode.Reset()
queryCalled := false
mockGetHyperPlaneStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getHyperPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
queryCalled = true
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
defer mockGetHyperPlaneStatus.Reset()
notA5Tool.generateHyperPlaneFaultEventsBasedOnFaultCacheChange(device)
convey.So(queryCalled, convey.ShouldBeFalse)
})
})
}
func TestAssembleVirtualDevices(t *testing.T) {
convey.Convey("test assembleVirtualDevices", t, func() {
tool := AscendTools{name: api.Ascend910, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
var device []common.NpuDevice
var deivceType []string
davinCiDev := common.DavinCiDev{
PhyID: phyIDNum,
LogicID: logicIDNum,
}
QueryInfo := npuCommon.CgoVDevQueryInfo{
Computing: npuCommon.CgoComputingResource{Aic: aiCoreNum},
Name: "vir16",
}
vDevInfos := npuCommon.VirtualDevInfo{
VDevInfo: []npuCommon.CgoVDevQueryStru{{QueryInfo: QueryInfo, VDevID: vDevIDNum}},
}
tool.assembleVirtualDevices(davinCiDev, vDevInfos, &device, &deivceType)
testRes := common.NpuDevice{
DevType: common.Ascend910vir16,
DeviceName: fmt.Sprintf("%s-%d-%d", common.Ascend910vir16, vDevIDNum, phyIDNum),
Health: v1beta1.Healthy,
NetworkHealth: v1beta1.Healthy,
LogicID: logicIDNum,
PhyID: phyIDNum,
}
convey.So(device, convey.ShouldContain, testRes)
})
}
func TestAddPodAnnotation1(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test AddPodAnnotation 1", t, func() {
convey.Convey("GetDeviceListID failed", func() {
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend910},
}, common.Ascend910vir2, "", nil)
convey.So(err, convey.ShouldBeNil)
})
mockTryUpdatePodAnnotation := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"TryUpdatePodAnnotation", func(_ *kubeclient.ClientK8s, pod *v1.Pod,
annotation map[string]string) error {
return nil
})
defer mockTryUpdatePodAnnotation.Reset()
convey.Convey("physical device 310P", func() {
tool.name = api.Ascend310P
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend310P + "-0"},
}, api.Ascend310P, "", nil)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("virtual device", func() {
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{common.Ascend310Pc2 + "-100-0"},
}, common.Ascend310Pc2, "", nil)
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestAddPodAnnotation2(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test AddPodAnnotation 2", t, func() {
mockTryUpdatePodAnnotation := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"TryUpdatePodAnnotation", func(_ *kubeclient.ClientK8s, pod *v1.Pod,
annotation map[string]string) error {
return nil
})
defer mockTryUpdatePodAnnotation.Reset()
mockGetLogicIDFromPhysicID := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, physicID int32) (int32, error) {
return 0, nil
})
defer mockGetLogicIDFromPhysicID.Reset()
convey.Convey("GetDeviceIPAddress failed", func() {
mockGetDeviceIPAddress := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceIPAddress", func(_ *devmanager.DeviceManagerMock, logicID, ipType int32) (
string, error) {
return "", fmt.Errorf("error")
})
defer mockGetDeviceIPAddress.Reset()
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend910 + "-0"},
}, api.Ascend910, "", nil)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("GetDeviceIPAddress ok", func() {
mockGetDeviceIPAddress := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceIPAddress", func(_ *devmanager.DeviceManagerMock, logicID, ipType int32) (
string, error) {
return "", nil
})
defer mockGetDeviceIPAddress.Reset()
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend910 + "-0"},
}, api.Ascend910, "", nil)
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestAddPodAnnotation3(t *testing.T) {
tool := mockAscendTools()
convey.Convey("test AddPodAnnotation 3", t, func() {
mockTryUpdatePodAnnotation := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)),
"TryUpdatePodAnnotation", func(_ *kubeclient.ClientK8s, pod *v1.Pod,
annotation map[string]string) error {
return nil
})
defer mockTryUpdatePodAnnotation.Reset()
convey.Convey("GetLogicIDFromPhysicID failed", func() {
mockGetLogicIDFromPhysicID := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, physicID int32) (int32, error) {
return 0, fmt.Errorf("error")
})
defer mockGetLogicIDFromPhysicID.Reset()
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend910 + "-0"},
}, api.Ascend910, "", nil)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("ParseInt failed", func() {
tool.name = api.Ascend910
err := tool.AddPodAnnotation(&common.PodDeviceInfo{
Pod: v1.Pod{},
KltDevice: nil,
RealDevice: []string{api.Ascend910 + "-a"},
}, api.Ascend910, "", nil)
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestCreateVirtualDevice(t *testing.T) {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
convey.Convey("test CreateVirtualDevice", t, func() {
convey.Convey("CreateVirtualDevice success", func() {
mockGetLogicIDFromPhysicID := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, physicID int32) (int32, error) {
return 0, nil
})
mockCreate := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"CreateVirtualDevice", func(_ *devmanager.DeviceManagerMock, logicID int32,
vDevInfo npuCommon.CgoCreateVDevRes) (npuCommon.CgoCreateVDevOut, error) {
return npuCommon.CgoCreateVDevOut{}, nil
})
defer mockCreate.Reset()
defer mockGetLogicIDFromPhysicID.Reset()
_, err := tool.CreateVirtualDevice(0, "vir01")
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestDestroyVirtualDevice(t *testing.T) {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
convey.Convey("test DestroyVirtualDevice", t, func() {
convey.Convey("DestroyVirtualDevice success", func() {
mockGetLogicIDFromPhysicID := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, physicID int32) (int32, error) {
return 0, nil
})
mockDestroy := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"DestroyVirtualDevice", func(_ *devmanager.DeviceManagerMock, _ int32, _ uint32) error {
return nil
})
defer mockDestroy.Reset()
defer mockGetLogicIDFromPhysicID.Reset()
err := tool.DestroyVirtualDevice("Ascend310P-1c-100-0")
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestGetChipAiCoreCount(t *testing.T) {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
res := getVirtualDevInfo(aiCoreNum)
mockLogicIDs := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetDeviceList", func(_ *devmanager.DeviceManagerMock) (int32, []int32, error) {
return 1, []int32{0}, nil
})
mockVirtual := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetVirtualDeviceInfo", func(_ *devmanager.DeviceManagerMock, _ int32) (
npuCommon.VirtualDevInfo, error) {
return res, nil
})
defer mockVirtual.Reset()
defer mockLogicIDs.Reset()
convey.Convey("test GetChipAiCoreCount 1", t, func() {
convey.Convey("GetChipAiCoreCount failed", func() {
_, err := tool.GetChipAiCoreCount()
convey.So(err, convey.ShouldNotBeNil)
})
})
res = getVirtualDevInfo(aiCoreCount)
convey.Convey("test GetChipAiCoreCount 2", t, func() {
convey.Convey("GetChipAiCoreCount success", func() {
_, err := tool.GetChipAiCoreCount()
convey.So(err, convey.ShouldBeNil)
})
})
}
func getVirtualDevInfo(aic float32) npuCommon.VirtualDevInfo {
return npuCommon.VirtualDevInfo{
TotalResource: npuCommon.CgoSocTotalResource{
Computing: npuCommon.CgoComputingResource{
Aic: aic,
},
},
VDevInfo: []npuCommon.CgoVDevQueryStru{
{
VDevID: vDevChipID,
},
},
}
}
func TestAppendVGroupInfo(t *testing.T) {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
res := getVirtualDevInfo(aiCoreCount)
convey.Convey("test AppendVGroupInfo", t, func() {
convey.Convey("AppendVGroupInfo success", func() {
mockGetLogicIDFromPhysicID := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, physicID int32) (int32, error) {
return 0, nil
})
mockVirtual := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetVirtualDeviceInfo", func(_ *devmanager.DeviceManagerMock, _ int32) (
npuCommon.VirtualDevInfo, error) {
return res, nil
})
defer mockVirtual.Reset()
defer mockGetLogicIDFromPhysicID.Reset()
allocateDevice := []string{
"Ascend310P-1c-100-0",
}
tool.AppendVGroupInfo(allocateDevice)
convey.So(len(allocateDevice), convey.ShouldEqual, 1)
})
})
}
func TestCheckDeviceTypeLabel(t *testing.T) {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
node := getMockNode()
convey.Convey("test CheckDeviceTypeLabel", t, func() {
convey.Convey("CheckDeviceTypeLabel get node failed", func() {
mockNode := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetNode",
func(_ *kubeclient.ClientK8s) (*v1.Node, error) {
return nil, fmt.Errorf("failed to get node")
})
defer mockNode.Reset()
err := tool.CheckDeviceTypeLabel()
convey.So(err, convey.ShouldNotBeNil)
})
convey.Convey("CheckDeviceTypeLabel success", func() {
mockNode := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetNode",
func(_ *kubeclient.ClientK8s) (*v1.Node, error) {
return node, nil
})
defer mockNode.Reset()
delete(node.Labels, common.ServerTypeLabelKey)
err := tool.CheckDeviceTypeLabel()
convey.So(err, convey.ShouldNotBeNil)
common.ParamOption.AiCoreCount = aiCoreCount
node.Labels[common.ServerTypeLabelKey] = api.Ascend310PMinuxPrefix + "8"
err = tool.CheckDeviceTypeLabel()
convey.So(err, convey.ShouldBeNil)
})
})
}
func getMockNode() *v1.Node {
labels := make(map[string]string, 1)
labels[common.ServerTypeLabelKey] = "Ascend310P-8"
return &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
}
}
func TestAssemble310PMixedPhyDevices(t *testing.T) {
convey.Convey("test assembleVirtualDevices", t, func() {
tool := AscendTools{name: api.Ascend310P, client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{}}
var device []common.NpuDevice
var deivceType []string
davinCiDev := common.DavinCiDev{
PhyID: phyIDNum,
LogicID: logicIDNum,
}
mockProductType := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetProductType",
func(_ *devmanager.DeviceManagerMock, logicID int32) (string, error) {
return atlas300VPro, nil
})
defer mockProductType.Reset()
productTypeMap := common.Get310PProductType()
tool.assemble310PMixedPhyDevices(davinCiDev, &device, &deivceType)
testRes := common.NpuDevice{
DevType: productTypeMap[atlas300VPro],
DeviceName: fmt.Sprintf("%s-%d", productTypeMap[atlas300VPro], phyIDNum),
Health: v1beta1.Healthy,
NetworkHealth: v1beta1.Healthy,
LogicID: logicIDNum,
PhyID: phyIDNum,
}
convey.So(device, convey.ShouldContain, testRes)
})
}
func TestIfCardsInResetting(t *testing.T) {
convey.Convey("test if card in reset func", t, func() {
tool := NewHwAscend910Manager()
tool.SetCardsInResetting(common.FirstDevice, true)
convey.So(tool.GetIfCardsInResetting(common.FirstDevice), convey.ShouldEqual, true)
convey.So(tool.GetIfCardsInResetting(logicIDNum), convey.ShouldBeFalse)
tool.SetCardsInResetting(common.FirstDevice, false)
convey.So(tool.GetIfCardsInResetting(common.FirstDevice), convey.ShouldBeFalse)
})
}
func TestGetResetFailedTimes(t *testing.T) {
convey.Convey("test set reset failed times", t, func() {
tool := NewHwAscend910Manager()
tool.SetResetFailedTimes(common.FirstDevice, FaultOnce)
convey.So(tool.GetResetFailedTimes(common.FirstDevice), convey.ShouldEqual, FaultOnce)
convey.So(tool.GetResetFailedTimes(logicIDNum), convey.ShouldEqual, NoneFault)
})
}
func TestRemoveDuplicateErr(t *testing.T) {
convey.Convey("test remove duplicate errors", t, func() {
code98008 := int64(0x80C98008)
codeB8008 := int64(0x80CB8008)
code98002 := int64(0x80C98002)
code98003 := int64(0x80C98003)
code98009 := int64(0x80C98009)
codeB8002 := int64(0x80CB8002)
codeB8009 := int64(0x80CB8009)
oldErrors := []int64{code98008, code98002, code98003, code98009, codeB8002, codeB8008, codeB8009}
tool := NewHwAscend910Manager()
newErrors := tool.removeDuplicateErr(oldErrors)
convey.So(len(oldErrors), convey.ShouldEqual, len(newErrors))
baseErrors := []int64{code98008, codeB8008}
oldErrors = []int64{code98008, code98008, code98008, code98008, code98008, code98008, codeB8008, codeB8008}
newErrors = tool.removeDuplicateErr(oldErrors)
convey.So(len(baseErrors), convey.ShouldEqual, len(newErrors))
})
}
func TestGetResetInfoData(t *testing.T) {
convey.Convey("test getResetInfoData", t, func() {
resetInfo := &v1.ConfigMap{}
convey.Convey("01-reset.json not exist, should return error", func() {
_, err := getResetInfoData(resetInfo)
convey.So(err, convey.ShouldNotBeNil)
})
convey.Convey("02-unmarshal fail, should return error", func() {
mockUnmarshal := gomonkey.ApplyFuncReturn(json.Unmarshal, errors.New("fail"))
defer mockUnmarshal.Reset()
resetInfo.Data = map[string]string{common.ResetInfoCMDataKey: "yyy"}
_, err := getResetInfoData(resetInfo)
convey.So(err, convey.ShouldNotBeNil)
})
convey.Convey("03-unmarshal success, should return nil", func() {
taskResetInfo := common.TaskResetInfo{RankList: []*common.TaskDevInfo{{RankId: 0}, {RankId: 1}}, UpdateTime: 20}
devNumsFromRestJson := 2
resetByte, err := json.Marshal(taskResetInfo)
convey.So(err, convey.ShouldBeNil)
resetInfo.Data = map[string]string{common.ResetInfoCMDataKey: string(resetByte)}
devInfo, err := getResetInfoData(resetInfo)
convey.So(err, convey.ShouldBeNil)
convey.So(len(devInfo) == devNumsFromRestJson, convey.ShouldBeTrue)
})
})
}
func TestNpuIsUsedNow(t *testing.T) {
convey.Convey("test npuIsUsedNow", t, func() {
tool := mockAscendTools()
annotationTag := fmt.Sprintf("%s%s", api.ResourceNamePrefix, api.Ascend910)
pods := []v1.Pod{
{ObjectMeta: metav1.ObjectMeta{Name: "mock pod1", Annotations: map[string]string{annotationTag: ""}}},
{ObjectMeta: metav1.ObjectMeta{Name: "mock pod2", Annotations: map[string]string{annotationTag: "device1,device2"}}},
}
mockMethod := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetActivePodListCache",
func(_ *kubeclient.ClientK8s) []v1.Pod { return pods })
defer mockMethod.Reset()
convey.Convey("01-device exist, should return true", func() {
convey.So(tool.npuIsUsedNow("device1"), convey.ShouldBeTrue)
})
convey.Convey("02-device not exist, should return false", func() {
convey.So(tool.npuIsUsedNow("device3"), convey.ShouldBeFalse)
})
})
}
func TestGetRealUsedDevices(t *testing.T) {
convey.Convey("test getRealUsedDevices", t, func() {
tool := mockAscendTools()
annotationTag := api.PodAnnotationAscendReal
pods := []v1.Pod{
{ObjectMeta: metav1.ObjectMeta{Name: "mock pod1", Annotations: map[string]string{"test tag": "device3"}}},
{ObjectMeta: metav1.ObjectMeta{Name: "mock pod2", Annotations: map[string]string{annotationTag: "device1,device2"}}},
}
mockMethod := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetActivePodListCache",
func(_ *kubeclient.ClientK8s) []v1.Pod { return pods })
defer mockMethod.Reset()
podUsedDev := tool.getRealUsedDevices()
convey.So(podUsedDev.Has("device1"), convey.ShouldBeTrue)
convey.So(podUsedDev.Has("device3"), convey.ShouldBeFalse)
})
}
func TestGetDevStatesDevSet(t *testing.T) {
convey.Convey("test getDevStatesDevSet", t, func() {
tool := mockAscendTools()
mockGetRealUsedDevices := gomonkey.ApplyPrivateMethod(reflect.TypeOf(&tool), "getRealUsedDevices",
func(_ *AscendTools) sets.String { return sets.String{} })
mockGroupDevsByStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(&tool), "groupDevsByStatus",
func(_ *AscendTools, subClassDevices []*common.NpuDevice, runMode string) common.DevStatusSet {
return common.DevStatusSet{HealthDevices: sets.String{"Ascend910-0": sets.Empty{}}}
})
mockGetPodsUsedNpu := mockGetPodsUsedNpuByCommon()
defer func() {
mockGetRealUsedDevices.Reset()
mockGroupDevsByStatus.Reset()
mockGetPodsUsedNpu.Reset()
}()
mockClassifyDevs := map[string][]*common.NpuDevice{api.Ascend910: {{Health: v1beta1.Healthy}}}
common.ParamOption.PresetVDevice = true
res := tool.getDevStatesDevSet(mockClassifyDevs, 0)
convey.So(len(res.FreeHealthyDevice), convey.ShouldEqual, 1)
convey.So(len(res.UnHealthyDevice), convey.ShouldEqual, 0)
convey.So(len(res.NetUnHealthyDevice), convey.ShouldEqual, 0)
convey.So(len(res.RecoveringDevices), convey.ShouldEqual, 0)
convey.So(len(res.DeviceFault), convey.ShouldEqual, 0)
})
}
func TestGetServerIndex(t *testing.T) {
convey.Convey("test GetServerIndex", t, func() {
tool := mockAscendTools()
convey.Convey("default serverIndex should be 0", func() {
convey.So(tool.GetServerIndex(), convey.ShouldEqual, int32(0))
})
convey.Convey("should return the value set by SetServerIndex", func() {
tool.SetServerIndex(3)
convey.So(tool.GetServerIndex(), convey.ShouldEqual, int32(3))
})
})
}
func mockAscendTools() AscendTools {
return AscendTools{name: api.Ascend910, client: &kubeclient.ClientK8s{}, dmgr: &devmanager.DeviceManagerMock{}}
}
func TestAscendToolsGetDeviceFaults(t *testing.T) {
t.Run("getDeviceFaults", func(t *testing.T) {
tool := &AscendTools{}
base16 := 16
var faultTime1 int64 = 100000
var faultTime2 int64 = 110000
var faultTime3 int64 = 120000
device := &common.NpuDevice{
FaultCodes: []int64{int64(0x80C98008), int64(0x80CB8008)},
AlarmRaisedTime: 100000,
NetworkFaultCodes: []int64{int64(common.LinkDownFaultCode)},
NetworkAlarmRaisedTime: 110000,
FaultTimeMap: map[int64]int64{
int64(0x80C98008): faultTime1,
int64(0x80CB8008): faultTime2,
int64(common.LinkDownFaultCode): faultTime3,
},
DeviceName: "Ascend910-0",
}
got := tool.getDeviceFaults(device)
want := []common.DeviceFault{{
FaultType: common.CardNetworkUnhealthy,
NPUName: "Ascend910-0",
LargeModelFaultLevel: common.GetNetworkFaultType(device.NetworkFaultCodes, device.LogicID),
FaultLevel: common.GetNetworkFaultType(device.NetworkFaultCodes, device.LogicID),
FaultHandling: common.GetNetworkFaultType(device.NetworkFaultCodes, device.LogicID),
FaultCode: strings.ToUpper(common.Int64Tool.ToHexString(device.NetworkFaultCodes)),
FaultTimeAndLevelMap: map[string]common.FaultTimeAndLevel{
strings.ToUpper(strconv.FormatInt(common.LinkDownFaultCode, base16)): {faultTime3,
common.GetNetworkFaultType(device.NetworkFaultCodes, device.LogicID)},
},
}, {
FaultType: common.CardUnhealthy,
NPUName: "Ascend910-0",
LargeModelFaultLevel: common.GetFaultType(device.FaultCodes, device.LogicID),
FaultLevel: common.GetFaultType(device.FaultCodes, device.LogicID),
FaultHandling: common.GetFaultType(device.FaultCodes, device.LogicID),
FaultCode: strings.ToUpper(common.Int64Tool.ToHexString(device.FaultCodes)),
FaultTimeAndLevelMap: map[string]common.FaultTimeAndLevel{
strings.ToUpper(strconv.FormatInt(int64(0x80C98008), base16)): {faultTime1,
common.GetFaultType([]int64{0x80C98008}, device.LogicID),
},
strings.ToUpper(strconv.FormatInt(int64(0x80CB8008), base16)): {faultTime2,
common.GetFaultType([]int64{0x80CB8008}, device.LogicID)},
},
},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("getDeviceFaults() = %v, want %v", got, want)
}
})
}
func initTestObjects() (*AscendTools, *common.NpuDevice) {
tool := &AscendTools{
dmgr: &devmanager.DeviceManager{},
}
device := &common.NpuDevice{
LogicID: 0,
DeviceID: 0,
NetworkHealth: v1beta1.Healthy,
NetworkFaultCodes: []int64{},
}
return tool, device
}
func resetNetWorkLimiter() {
parameterPlaneLimiterMap = make(map[int32]*rate.Limiter, common.GeneralMapSize)
parameterPlaneStatusCache = make(map[int32]networkPlaneStatus, common.GeneralMapSize)
}
func testGetDeviceFaultFailed(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return -1, nil, errors.New("get device fault failed")
})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
}
}
func testNetworkFaultCodesNotEmpty(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
mockFaultCode := int64(0x81078603)
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 1, []int64{mockFaultCode}, nil
})
oldNetworkFaultCodes := common.NetworkFaultCodes
common.NetworkFaultCodes = sets.NewInt64()
common.NetworkFaultCodes.Insert(mockFaultCode)
defer func() { common.NetworkFaultCodes = oldNetworkFaultCodes }()
saveCalled := false
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeTrue)
}
}
func testNetworkFaultCodesEmptyLinkStatusFailed(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkDownStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
}
}
func testNetworkFaultCodesEmptyHealthyLinkDown(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkDownStatus,
downPortsNum: npuCommon.RoceParameterPlanePortAllDownCount}
})
saveCalled := false
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeTrue)
}
}
func testNetworkFaultCodesEmptyHealthMatchLink(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
saveCalled := false
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {
saveCalled = true
})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
convey.So(saveCalled, convey.ShouldBeFalse)
}
}
func TestGenerateNetworkFaultEventsBasedOnFaultCacheChange(t *testing.T) {
convey.Convey("test generateNetworkFaultEventsBasedOnFaultCacheChange", t, func() {
tool, device := initTestObjects()
convey.Convey("get device fault failed",
testGetDeviceFaultFailed(tool, device))
convey.Convey("network fault codes is not empty",
testNetworkFaultCodesNotEmpty(tool, device))
convey.Convey("network fault codes is empty, get link status failed",
testNetworkFaultCodesEmptyLinkStatusFailed(tool, device))
convey.Convey("network fault codes is empty, device healthy but link down",
testNetworkFaultCodesEmptyHealthyLinkDown(tool, device))
convey.Convey("network fault codes is empty, device health status matches link status",
testNetworkFaultCodesEmptyHealthMatchLink(tool, device))
convey.Convey("network limiter initialization",
testNetworkLimiterInitialization(tool, device))
convey.Convey("network limiter rate limiting",
testNetworkLimiterRateLimiting(tool, device))
convey.Convey("network limiter multiple devices",
testNetworkLimiterMultipleDevices(tool,
&common.NpuDevice{LogicID: 0, DeviceID: 3,
NetworkHealth: v1beta1.Healthy, NetworkFaultCodes: []int64{}},
&common.NpuDevice{LogicID: 1, DeviceID: 4,
NetworkHealth: v1beta1.Healthy, NetworkFaultCodes: []int64{}}))
})
}
func testNetworkLimiterInitialization(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
}
}
func testNetworkLimiterRateLimiting(tool *AscendTools, device *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device)
}
}
func testNetworkLimiterMultipleDevices(tool *AscendTools, device1, device2 *common.NpuDevice) func() {
return func() {
defer resetNetWorkLimiter()
patches := gomonkey.NewPatches()
defer patches.Reset()
patches.ApplyMethod(&devmanager.DeviceManager{}, "GetDeviceAllErrorCode",
func(_ *devmanager.DeviceManager, _ int32) (int32, []int64, error) {
return 0, []int64{}, nil
})
patches.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
patches.ApplyFunc(common.DoSaveDevFaultInfo, func(_ npuCommon.DevFaultInfo, _ bool) {})
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device1)
tool.generateNetworkFaultEventsBasedOnFaultCacheChange(device2)
}
}
func TestCompareDeviceList(t *testing.T) {
convey.Convey("test compareDeviceList", t, func() {
convey.Convey("01-deviceList and newDeviceList are both nil, should return true", func() {
res := compareDeviceList(nil, nil)
convey.So(res, convey.ShouldBeTrue)
})
convey.Convey("02-deviceList is nil and newDeviceList is not nil, should return false", func() {
res := compareDeviceList(nil, map[string]string{})
convey.So(res, convey.ShouldBeFalse)
})
convey.Convey("03-deviceList and newDeviceList length are different, should return false", func() {
deviceList := map[string]string{"key1": "value1"}
newDeviceList := map[string]string{"key1": "value1", "key2": "value2"}
res := compareDeviceList(deviceList, newDeviceList)
convey.So(res, convey.ShouldBeFalse)
})
convey.Convey("04-deviceList and newDeviceList only key are different, should return false", func() {
deviceList := map[string]string{"key1": "value"}
newDeviceList := map[string]string{"key2": "value"}
res := compareDeviceList(deviceList, newDeviceList)
convey.So(res, convey.ShouldBeFalse)
})
convey.Convey("05-deviceList and newDeviceList only value are different, should return false", func() {
deviceList := map[string]string{"key": "value1"}
newDeviceList := map[string]string{"key": "value2"}
res := compareDeviceList(deviceList, newDeviceList)
convey.So(res, convey.ShouldBeFalse)
})
convey.Convey("06-deviceList and newDeviceList only fault_time are different, should return true", func() {
deviceList := map[string]string{`key`: `key:value,"fault_time":11111,key:value`}
newDeviceList := map[string]string{`key`: `key:value,"fault_time":22222,key:value`}
res := compareDeviceList(deviceList, newDeviceList)
convey.So(res, convey.ShouldBeTrue)
})
convey.Convey("07-deviceList and newDeviceList are same, should return true", func() {
deviceList := map[string]string{"key": "value"}
newDeviceList := map[string]string{"key": "value"}
res := compareDeviceList(deviceList, newDeviceList)
convey.So(res, convey.ShouldBeTrue)
})
})
}
func TestGetUsedDevices(t *testing.T) {
tool := &AscendTools{}
tests := []struct {
name string
input []*common.NpuDevice
expectedOutput sets.String
}{
{
name: "01-nil input, should return empty sets.String",
input: nil,
expectedOutput: sets.NewString(),
},
{
name: "02-no devices used, should return empty sets.String",
input: []*common.NpuDevice{
{DeviceName: "device0", PodUsed: false},
{DeviceName: "device1", PodUsed: false},
},
expectedOutput: sets.NewString(),
},
{
name: "03-some devices used, , should return used devices",
input: []*common.NpuDevice{
{DeviceName: "device0", PodUsed: true},
{DeviceName: "device1", PodUsed: false},
{DeviceName: "device2", PodUsed: true},
},
expectedOutput: sets.NewString("device0", "device2"),
},
}
convey.Convey("getUsedDevices tests", t, func() {
for _, tt := range tests {
convey.Convey(tt.name, func() {
result := tool.getUsedDevices(tt.input)
convey.So(result, convey.ShouldEqual, tt.expectedOutput)
})
}
})
}
func TestGetCurDeviceFaultCode(t *testing.T) {
convey.Convey("test getCurDeviceFaultCode", t, func() {
tool := mockAscendTools()
convey.Convey("when devFaultInfo is empty, should return empty result", func() {
res := tool.getCurDeviceFaultCode(0, []npuCommon.DevFaultInfo{})
convey.So(res.Len(), convey.ShouldEqual, 0)
})
devFaultInfo := []npuCommon.DevFaultInfo{
{LogicID: 0, Assertion: npuCommon.FaultRecover, EventID: common.LinkDownFaultCode},
}
convey.Convey("when dcmi get device fault code failed, should return empty result", func() {
mockMethod := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetDeviceAllErrorCode", int32(0), []int64{}, errors.New("failed"))
defer mockMethod.Reset()
res := tool.getCurDeviceFaultCode(0, devFaultInfo)
convey.So(res.Len(), convey.ShouldEqual, 0)
})
convey.Convey("when dcmi get device fault code success, should return fault code sets", func() {
mockMethod := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetDeviceAllErrorCode", int32(1), []int64{common.LinkDownFaultCode}, nil)
defer mockMethod.Reset()
res := tool.getCurDeviceFaultCode(0, devFaultInfo)
convey.So(res.Len(), convey.ShouldEqual, 1)
})
})
}
func TestGetNodeDeviceInfoCache(t *testing.T) {
convey.Convey("test getNodeDeviceInfoCache", t, func() {
tool := mockAscendTools()
var superPodID int32 = 1
var serverIndex int32 = 1
var rackID int32 = 1
tool.SetSuperPodID(superPodID)
tool.SetServerIndex(serverIndex)
tool.SetRackID(rackID)
convey.Convey("case 1: not A5 get node device info cache not have rack id", func() {
mockRealCardType := common.ParamOption.RealCardType
common.ParamOption.RealCardType = api.Ascend910
defer func() {
common.ParamOption.RealCardType = mockRealCardType
}()
cache := tool.getNodeDeviceInfoCache(make(map[string]string))
convey.So(cache.SuperPodID, convey.ShouldEqual, superPodID)
convey.So(cache.ServerIndex, convey.ShouldEqual, serverIndex)
convey.So(cache.RackID, convey.ShouldBeNil)
})
convey.Convey("case 2: A5 Pod type get node device info cache have rack id", func() {
mockRealCardType := common.ParamOption.RealCardType
common.ParamOption.RealCardType = api.Ascend910A5
tool.SetSuperPodType(common.ProductType2D)
defer func() {
common.ParamOption.RealCardType = mockRealCardType
}()
cache := tool.getNodeDeviceInfoCache(make(map[string]string))
convey.So(cache.SuperPodID, convey.ShouldEqual, superPodID)
convey.So(cache.ServerIndex, convey.ShouldEqual, serverIndex)
convey.So(*cache.RackID, convey.ShouldEqual, rackID)
})
convey.Convey("case 3: A5 Server type get node device info cache not have rack id", func() {
mockRealCardType := common.ParamOption.RealCardType
common.ParamOption.RealCardType = api.Ascend910A5
tool.SetSuperPodType(common.ProductTypeServer)
defer func() {
common.ParamOption.RealCardType = mockRealCardType
}()
cache := tool.getNodeDeviceInfoCache(make(map[string]string))
convey.So(cache.SuperPodID, convey.ShouldEqual, superPodID)
convey.So(cache.ServerIndex, convey.ShouldEqual, serverIndex)
convey.So(cache.RackID, convey.ShouldBeNil)
})
convey.Convey("case 4: A5 Card type get node device info cache not have rack id", func() {
mockRealCardType := common.ParamOption.RealCardType
common.ParamOption.RealCardType = api.Ascend910A5
tool.SetSuperPodType(common.ProductType1PCard)
defer func() {
common.ParamOption.RealCardType = mockRealCardType
}()
cache := tool.getNodeDeviceInfoCache(make(map[string]string))
convey.So(cache.SuperPodID, convey.ShouldEqual, superPodID)
convey.So(cache.ServerIndex, convey.ShouldEqual, serverIndex)
convey.So(cache.RackID, convey.ShouldBeNil)
})
})
}
type filterSoftShareDevicesTestCase struct {
name string
mockSupportSoft bool
npuChipMemory int
kltUsedDevices sets.String
classifyDev []*common.NpuDevice
expected sets.String
}
func TestFilterSoftShareDevices(t *testing.T) {
tests := []filterSoftShareDevicesTestCase{
{
name: "Not supporting soft shared devices, directly return the original set",
mockSupportSoft: false,
npuChipMemory: common.MBPerGB,
kltUsedDevices: sets.NewString("npu0", "npu1"),
classifyDev: []*common.NpuDevice{},
expected: sets.NewString("npu0", "npu1"),
},
{
name: "Supporting soft shared devices but memory is 0, directly return the original set",
mockSupportSoft: true,
npuChipMemory: 0,
kltUsedDevices: sets.NewString("npu2"),
classifyDev: []*common.NpuDevice{},
expected: sets.NewString("npu2"),
},
{
name: "sufficient device resources filter available devices",
mockSupportSoft: true,
npuChipMemory: numTwo * common.MBPerGB,
kltUsedDevices: sets.NewString("npu0", "npu1", "npu2"),
classifyDev: []*common.NpuDevice{
{
DeviceName: "npu0",
UsedAicoreQuota: numFifty,
UsedHbmQuota: common.MBPerGB,
},
{
DeviceName: "npu1",
UsedAicoreQuota: numOneHundred,
UsedHbmQuota: numTwo * common.MBPerGB,
},
},
expected: sets.NewString("npu1", "npu2"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
patch := gomonkey.ApplyFuncReturn(common.IsSupportSoftShareDevice, tt.mockSupportSoft)
defer patch.Reset()
tool := &AscendTools{}
result := tool.filterSoftShareDevices(tt.kltUsedDevices, tt.classifyDev, tt.npuChipMemory)
assert.True(t, tt.expected.Equal(result), "expected %v, got %v",
tt.expected.List(), result.List())
})
}
}
func TestQueryNetworkStatusWithoutRoCEDev(t *testing.T) {
convey.Convey("Test queryParameterPlaneStatusWithoutFaultCode function", t, func() {
convey.Convey("When faultCodes is not empty, should return directly", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
faultCodes := []int64{123, 456}
device := &common.NpuDevice{DeviceID: 0}
result := tool.queryParameterPlaneStatusWithoutFaultCode(faultCodes, device)
convey.So(result, convey.ShouldResemble, faultCodes)
})
convey.Convey("When WithoutParameterPlane returns true, should return directly", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
faultCodes := []int64{}
device := &common.NpuDevice{DeviceID: 0}
patch := gomonkey.ApplyMethod(reflect.TypeOf(new(AscendTools)), "WithoutParameterPlane",
func(_ *AscendTools) bool { return true })
defer patch.Reset()
result := tool.queryParameterPlaneStatusWithoutFaultCode(faultCodes, device)
convey.So(result, convey.ShouldBeEmpty)
})
convey.Convey("When WithoutParameterPlane false and link status normal", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
faultCodes := []int64{}
device := &common.NpuDevice{DeviceID: 0, PhyID: 0}
patchWithoutRoCE := gomonkey.ApplyMethod(reflect.TypeOf(new(AscendTools)), "WithoutParameterPlane",
func(_ *AscendTools) bool { return false })
defer patchWithoutRoCE.Reset()
patchGetNetworkStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.RoceParameterPlanePortAllDownCount}
})
defer patchGetNetworkStatus.Reset()
result := tool.queryParameterPlaneStatusWithoutFaultCode(faultCodes, device)
convey.So(result, convey.ShouldBeEmpty)
})
convey.Convey("When WithoutParameterPlane false and link status abnormal", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
faultCodes := []int64{}
device := &common.NpuDevice{DeviceID: 0, PhyID: 0}
patchWithoutRoCE := gomonkey.ApplyMethod(reflect.TypeOf(new(AscendTools)), "WithoutParameterPlane",
func(_ *AscendTools) bool { return false })
defer patchWithoutRoCE.Reset()
patchGetNetworkStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkDownStatus,
downPortsNum: npuCommon.PortNoDownCount}
})
defer patchGetNetworkStatus.Reset()
result := tool.queryParameterPlaneStatusWithoutFaultCode(faultCodes, device)
convey.So(result, convey.ShouldNotContain, common.LinkDownFaultCode)
})
convey.Convey("When Ascend910A5 device and all ports up", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
faultCodes := []int64{}
device := &common.NpuDevice{DeviceID: 0, PhyID: 0}
patchWithoutRoCE := gomonkey.ApplyMethod(reflect.TypeOf(new(AscendTools)), "WithoutParameterPlane",
func(_ *AscendTools) bool { return false })
defer patchWithoutRoCE.Reset()
patchGetNetworkStatus := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(AscendTools)),
"getParameterPlaneStatusCache", func(_ *AscendTools, _ int32) networkPlaneStatus {
return networkPlaneStatus{status: npuCommon.NPUNetworkLinkUpStatus,
downPortsNum: npuCommon.UBOEParameterPlanePortAllDownCount}
})
defer patchGetNetworkStatus.Reset()
result := tool.queryParameterPlaneStatusWithoutFaultCode(faultCodes, device)
convey.So(result, convey.ShouldBeEmpty)
})
})
}
func TestWithoutParameterPlane(t *testing.T) {
convey.Convey("test WithoutParameterPlane", t, func() {
convey.Convey("01-Ascend310 without parameter plane, should return true", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend310}}
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeTrue)
})
convey.Convey("02-Ascend310P without parameter plane, should return true", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend310P}}
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeTrue)
})
convey.Convey("03-Ascend310B without parameter plane, should return true", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend310B}}
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeTrue)
})
convey.Convey("04-Ascend910A5 without UBOE device, should return true", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeTrue)
})
convey.Convey("05-Ascend910A5 with UBOE device, should return false", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
patch := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)), "GetMainBoardId",
func(_ *devmanager.DeviceManagerMock) uint32 { return api.Atlas850MainBoardID })
defer patch.Reset()
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeFalse)
})
convey.Convey("06-Ascend910 with parameter plane, should return false", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910}}
convey.So(tool.WithoutParameterPlane(), convey.ShouldBeFalse)
})
})
}
func TestUpdateNodeDeviceInfoRateLimit(t *testing.T) {
convey.Convey("test UpdateNodeDeviceInfo cm report rate limit", t, func() {
tool := &AscendTools{
name: api.Ascend910,
client: &kubeclient.ClientK8s{},
dmgr: &devmanager.DeviceManagerMock{},
}
var writeCmCalled bool
var updateFuncCalled bool
var condSucceeded bool
patches := gomonkey.NewPatches().
ApplyPrivateMethod(tool, "checkAndInitNodeDeviceInfo", func(_ *AscendTools) {}).
ApplyMethod(reflect.TypeOf(&kubeclient.ClientK8s{}), "GetDeviceInfoCMCache",
func(_ *kubeclient.ClientK8s) *common.NodeDeviceInfoCache {
return &common.NodeDeviceInfoCache{
DeviceInfo: common.NodeDeviceInfo{
DeviceList: map[string]string{"Ascend910-0": `{"health":"healthy"}`},
UpdateTime: time.Now().Unix(),
},
}
}).
ApplyPrivateMethod(tool, "handleManuallySeparateNPUFaultInfo",
func(_ *AscendTools) (string, []common.LogicId) { return "", nil }).
ApplyPrivateMethod(tool, "delVirDevInfo", func(_ *AscendTools, _ map[string]string) {}).
ApplyFuncReturn(common.GetSwitchFaultInfo, common.SwitchFaultInfo{}).
ApplyFuncReturn(common.GetSyncMapLen, 0).
ApplyFuncReturn(common.CopyUpgradeFaultCache, common.UpgradeFaultReasonMap[common.LogicId]{}).
ApplyPrivateMethod(tool, "writeDeviceInfoCm",
func(_ *AscendTools, _ string, _ map[string]string,
_ common.UpgradeFaultReasonMap[common.LogicId], _ common.SwitchFaultInfo) (bool, error) {
writeCmCalled = true
return true, nil
}).
ApplyFunc(wait.PollImmediate,
func(_, _ time.Duration, condition wait.ConditionFunc) error {
ok, _ := condition()
condSucceeded = ok
return nil
})
defer patches.Reset()
noopUpdateBase := func(deviceList, newDeviceList map[string]string, _ common.DevStatusSet) error {
return nil
}
changeUpdateBase := func(deviceList, newDeviceList map[string]string, _ common.DevStatusSet) error {
newDeviceList["Ascend910-1"] = `{"health":"healthy"}`
return nil
}
noopUpdate := func(a, b map[string]string, c common.DevStatusSet) error {
updateFuncCalled = true
return noopUpdateBase(a, b, c)
}
changeUpdate := func(a, b map[string]string, c common.DevStatusSet) error {
updateFuncCalled = true
return changeUpdateBase(a, b, c)
}
tests := []struct {
name string
timestamp time.Time
updateFunc func(map[string]string, map[string]string, common.DevStatusSet) error
wantWriteCm bool
wantCondSucceed bool
wantUpdateFuncCalled bool
}{
{
name: "should return false to retry when rate limited within 1s",
timestamp: time.Now(),
updateFunc: changeUpdate,
wantWriteCm: false,
wantCondSucceed: false,
wantUpdateFuncCalled: false,
},
{
name: "should proceed when lastUpdateTimeStamp is zero",
timestamp: time.Time{},
updateFunc: noopUpdate,
wantWriteCm: true,
wantCondSucceed: true,
wantUpdateFuncCalled: true,
},
{
name: "should proceed when min interval has passed",
timestamp: time.Now().Add(-2 * time.Second),
updateFunc: changeUpdate,
wantWriteCm: true,
wantCondSucceed: true,
wantUpdateFuncCalled: true,
},
}
for _, tt := range tests {
convey.Convey(tt.name, func() {
tool.lastUpdateTimeStamp = tt.timestamp
writeCmCalled = false
updateFuncCalled = false
condSucceeded = false
err := tool.UpdateNodeDeviceInfo(common.DevStatusSet{}, tt.updateFunc)
convey.So(err, convey.ShouldBeNil)
convey.So(writeCmCalled, convey.ShouldEqual, tt.wantWriteCm)
convey.So(condSucceeded, convey.ShouldEqual, tt.wantCondSucceed)
convey.So(updateFuncCalled, convey.ShouldEqual, tt.wantUpdateFuncCalled)
})
}
})
}
func TestBuildLogicToPhyMap(t *testing.T) {
convey.Convey("test buildLogicToPhyMap", t, func() {
all := []common.NpuDevice{
{LogicID: 0, PhyID: 10},
{LogicID: 1, PhyID: 11},
{LogicID: 2, PhyID: 12},
}
m := buildLogicToPhyMap(all)
convey.So(m[0], convey.ShouldEqual, 10)
convey.So(m[1], convey.ShouldEqual, 11)
convey.So(m[2], convey.ShouldEqual, 12)
})
}
func TestConvertLogicToPhyIDWithLog(t *testing.T) {
convey.Convey("test convertLogicToPhyIDWithLog", t, func() {
logic2phy := map[int]int{
0: 10,
1: 11,
}
convey.Convey("not A5, return logicID", func() {
common.ParamOption.RealCardType = api.Ascend910B
id, err := convertLogicToPhyIDWithLog(1, logic2phy)
convey.So(err, convey.ShouldBeNil)
convey.So(id, convey.ShouldEqual, 1)
})
convey.Convey("A5 and mapping exists", func() {
common.ParamOption.RealCardType = api.Ascend910A5
id, err := convertLogicToPhyIDWithLog(1, logic2phy)
convey.So(err, convey.ShouldBeNil)
convey.So(id, convey.ShouldEqual, 11)
})
convey.Convey("A5 but mapping missing", func() {
common.ParamOption.RealCardType = api.Ascend910A5
id, err := convertLogicToPhyIDWithLog(99, logic2phy)
convey.So(err, convey.ShouldNotBeNil)
convey.So(id, convey.ShouldEqual, 99)
})
})
}
func TestGetDeviceListIP(t *testing.T) {
tool := &AscendTools{name: api.Ascend910}
convey.Convey("test getDeviceListIP", t, func() {
common.ParamOption.RealCardType = api.Ascend910A5
mockGetDeviceListID := gomonkey.ApplyFunc(
common.GetDeviceListID,
func(devices []string, opt string) (map[int]int, []int, error) {
return nil, []int{0, 1}, nil
})
defer mockGetDeviceListID.Reset()
mockGetDeviceIP := gomonkey.ApplyMethod(
reflect.TypeOf(new(AscendTools)),
"GetDeviceIP",
func(_ *AscendTools, deviceType string, phyID int) (string, error) {
if phyID == 10 {
return "1.1.1.1", nil
}
if phyID == 11 {
return "2.2.2.2", nil
}
return "", fmt.Errorf("unexpected phyID %d", phyID)
})
defer mockGetDeviceIP.Reset()
allDevices := []common.NpuDevice{
{LogicID: 0, PhyID: 10},
{LogicID: 1, PhyID: 11},
}
devices := []string{"0", "1"}
res, err := tool.getDeviceListIP(devices, api.Ascend910, allDevices)
convey.So(err, convey.ShouldBeNil)
convey.So(res[10], convey.ShouldEqual, "1.1.1.1")
convey.So(res[11], convey.ShouldEqual, "2.2.2.2")
})
}
func TestWithUBOEDevice(t *testing.T) {
convey.Convey("test withUBOEDevice", t, func() {
convey.Convey("return false when main board id is Atlas950MainBoardID", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(api.Atlas950MainBoardID))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeFalse)
})
convey.Convey("return false when main board id is Atlas9501DMainBoardID", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(api.Atlas9501DMainBoardID))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeFalse)
})
convey.Convey("return true when main board id is Atlas850MainBoardID", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(api.Atlas850MainBoardID))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeTrue)
})
convey.Convey("return true when main board id is Atlas850MainBoardID2", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(api.Atlas850MainBoardID2))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeTrue)
})
convey.Convey("return true when main board id is Atlas850MainBoardID3", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(api.Atlas850MainBoardID3))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeTrue)
})
convey.Convey("return false when main board id is not in UBOE set", func() {
patches := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{},
"GetMainBoardId", uint32(0xFF))
defer patches.Reset()
tools := mockAscendTools()
convey.So(tools.withUBOEDevice(), convey.ShouldBeFalse)
})
})
}
func TestHandleHangCardFaultEvents(t *testing.T) {
convey.Convey("test HandleHangCardFaultEvents", t, func() {
tool := mockAscendTools()
convey.Convey("when devices is empty, should return immediately", func() {
tool.HandleHangCardFaultEvents(nil)
})
convey.Convey("when devices is not empty, should register logicIDs and process fault cache", func() {
hangdetection.RegisterLogicIDForProducer(int32(1))
devFault1 := &npuCommon.DevFaultInfo{LogicID: 1, EventID: npuCommon.HangFaultCode, Assertion: npuCommon.FaultOccur}
called := false
patches := gomonkey.ApplyFunc(common.DoSaveDevFaultInfo, func(faultInfo npuCommon.DevFaultInfo, _ bool) {
if faultInfo.LogicID == devFault1.LogicID {
convey.So(faultInfo.Assertion, convey.ShouldEqual, devFault1.Assertion)
}
called = true
}).ApplyFunc(hangdetection.GetAndCleanAllHangFaultCache, func() []*npuCommon.DevFaultInfo {
return []*npuCommon.DevFaultInfo{devFault1}
})
defer patches.Reset()
devices := []*common.NpuDevice{{LogicID: 1}}
tool.HandleHangCardFaultEvents(devices)
convey.So(called, convey.ShouldBeTrue)
})
})
}
func TestAscendToolsModifyFaultCodesForMultiDevices(t *testing.T) {
convey.Convey("test modifyFaultCodesForMultiDevices", t, func() {
convey.Convey("base card type, should not modify device fault codes", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910})
defer mockOption.Reset()
tool := mockAscendTools()
devices := []*common.NpuDevice{
{LogicID: 0, FaultCodes: []int64{common.UBPortDownCode, common.UBSeparateFaultCode}},
}
originFaultCodes := make([]int64, len(devices[0].FaultCodes))
copy(originFaultCodes, devices[0].FaultCodes)
tool.modifyFaultCodesForMultiDevices(devices)
convey.So(devices[0].FaultCodes, convey.ShouldResemble, originFaultCodes)
})
convey.Convey("A950 card type, not all devices have UBPortDownCode, should not modify", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910A5})
defer mockOption.Reset()
tool := mockAscendTools()
devices := []*common.NpuDevice{
{LogicID: 0, FaultCodes: []int64{common.UBPortDownCode, common.UBSeparateFaultCode}},
{LogicID: 1, FaultCodes: []int64{common.UBSeparateFaultCode}},
}
tool.modifyFaultCodesForMultiDevices(devices)
convey.So(devices[0].FaultCodes, convey.ShouldResemble,
[]int64{common.UBPortDownCode, common.UBSeparateFaultCode})
convey.So(devices[1].FaultCodes, convey.ShouldResemble,
[]int64{common.UBSeparateFaultCode})
})
convey.Convey("A950 card type, all devices have UBPortDownCode, should convert separate to sub heal", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910A5})
defer mockOption.Reset()
tool := mockAscendTools()
devices := []*common.NpuDevice{
{LogicID: 0, FaultCodes: []int64{common.UBPortDownCode, common.UBSeparateFaultCode}},
{LogicID: 1,
FaultCodes: []int64{common.UBPortDownCode, common.UBSeparateFaultCode, common.UBSubHealFaultCode}},
}
tool.modifyFaultCodesForMultiDevices(devices)
convey.So(devices[0].FaultCodes, convey.ShouldNotContain, common.UBSeparateFaultCode)
convey.So(devices[0].FaultCodes, convey.ShouldContain, common.UBSubHealFaultCode)
convey.So(devices[0].FaultCodes, convey.ShouldContain, common.UBPortDownCode)
convey.So(devices[0].FaultTimeMap[common.UBSeparateFaultCode], convey.ShouldEqual, 0)
convey.So(devices[0].FaultTimeMap[common.UBSubHealFaultCode], convey.ShouldBeGreaterThan, 0)
convey.So(devices[1].FaultCodes, convey.ShouldNotContain, common.UBSeparateFaultCode)
convey.So(devices[1].FaultCodes, convey.ShouldContain, common.UBSubHealFaultCode)
convey.So(countFaultCode(devices[1].FaultCodes, common.UBSubHealFaultCode), convey.ShouldEqual, 1)
})
convey.Convey("A950 card type, device is nil, should return directly", func() {
mockOption := gomonkey.ApplyGlobalVar(&common.ParamOption, common.Option{RealCardType: api.Ascend910A5})
defer mockOption.Reset()
tool := mockAscendTools()
devices := []*common.NpuDevice{
{LogicID: 0, FaultCodes: []int64{common.UBPortDownCode}},
nil,
}
tool.modifyFaultCodesForMultiDevices(devices)
convey.So(devices[0].FaultCodes, convey.ShouldResemble,
[]int64{common.UBPortDownCode})
})
})
}
func countFaultCode(faultCodes []int64, code int64) int {
cnt := 0
for _, c := range faultCodes {
if c == code {
cnt++
}
}
return cnt
}
func TestGetDeviceIP(t *testing.T) {
convey.Convey("test GetDeviceIP", t, func() {
convey.Convey("A5 returns empty ip", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{DevType: api.Ascend910A5}}
ip, err := tool.GetDeviceIP(api.Ascend910, 0)
convey.So(err, convey.ShouldBeNil)
convey.So(ip, convey.ShouldEqual, "")
})
convey.Convey("virtual device returns default ip", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
ip, err := tool.GetDeviceIP("Ascend910-2c", 0)
convey.So(err, convey.ShouldBeNil)
convey.So(ip, convey.ShouldEqual, api.DeviceIPDefaultCodeStr)
})
convey.Convey("GetLogicIDFromPhysicID failed", func() {
tool := AscendTools{dmgr: &devmanager.DeviceManagerMock{}}
patches := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetLogicIDFromPhysicID", func(_ *devmanager.DeviceManagerMock, phyID int32) (int32, error) {
return 0, fmt.Errorf("mock error")
})
defer patches.Reset()
ip, err := tool.GetDeviceIP(api.Ascend910, 0)
convey.So(err, convey.ShouldNotBeNil)
convey.So(ip, convey.ShouldEqual, "")
})
})
}
