+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/agiledragon/gomonkey/v2"
"github.com/smartystreets/goconvey/convey"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
"Ascend-device-plugin/pkg/common"
"Ascend-device-plugin/pkg/device"
"Ascend-device-plugin/pkg/device/deviceswitch"
"Ascend-device-plugin/pkg/kubeclient"
"Ascend-device-plugin/pkg/next/devicefactory/customname"
"ascend-common/api"
"ascend-common/common-utils/utils"
"ascend-common/devmanager"
npuCommon "ascend-common/devmanager/common"
)
const (
serverNum = 2
rqtTaskNum = 4
numTwo = 2
numThree = 3
numTen = 10
numThirty = 30
expectedMemory = 32768
ascend910LogicID0 = api.Ascend910MinuxPrefix + "0"
ascend910LogicID1 = api.Ascend910MinuxPrefix + "1"
testSuperPodId = int32(2)
testRackId = int32(4)
testServerIndex = int32(6)
testNegativeSuperPodId = int32(-1)
testNegativeRackId = int32(-1)
testNegativeServerIdx = int32(-1)
)
var testErr = errors.New("test")
func setPatch() *gomonkey.Patches {
patch := gomonkey.ApplyFuncReturn(kubeclient.NewClientK8s, &kubeclient.ClientK8s{
Clientset: &kubernetes.Clientset{},
NodeName: "node",
DeviceInfoName: common.DeviceInfoCMNamePrefix + "node",
IsApiErr: false,
}, nil).
ApplyMethodReturn((&kubernetes.Clientset{}).CoreV1().Nodes(), "Get", &v1.Node{
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{}},
}, nil)
return patch
}
func createFile(filePath string) error {
f, err := os.Create(filePath)
if err != nil {
return err
}
defer f.Close()
return f.Chmod(common.SocketChmod)
}
func TestNewHwDevManager(t *testing.T) {
patch := setPatch()
defer patch.Reset()
convey.Convey("test NewHwDevManager", t, func() {
if _, err := os.Stat(common.HiAIManagerDevice); err != nil {
if err = createFile(common.HiAIManagerDevice); err != nil {
t.Fatal("TestGetDefaultDevices Run Failed")
}
}
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
mockUpdateNodeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(HwDevManager)), "UpdateNode",
func(_ *HwDevManager) error {
return nil
})
defer mockUpdateNodeLabel.Reset()
convey.Convey("init HwDevManager", func() {
common.ParamOption.UseVolcanoType = true
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
res := NewHwDevManager(&devmanager.DeviceManagerMock{DevType: api.Ascend910})
convey.So(res, convey.ShouldNotBeNil)
})
convey.Convey("init HwDevManager get device type failed", func() {
mockGetDevType := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)), "GetDevType",
func(_ *devmanager.DeviceManagerMock) string {
return "errorType"
})
defer mockGetDevType.Reset()
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
res := NewHwDevManager(&devmanager.DeviceManagerMock{})
convey.So(res, convey.ShouldBeNil)
})
convey.Convey("test NewHwDevManager, product type is not supported", func() {
common.ParamOption.ProductTypes = []string{common.Atlas300IDuo}
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
res := NewHwDevManager(&devmanager.DeviceManagerMock{})
convey.So(res, convey.ShouldBeNil)
})
})
}
func TestSetAscendManager(t *testing.T) {
var hdm HwDevManager
devM := &devmanager.DeviceManagerMock{DevType: api.Ascend910}
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
convey.Convey("test devType is Ascend310", t, func() {
mockGetDevType := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)), "GetDevType",
func(_ *devmanager.DeviceManagerMock) string {
return api.Ascend310
})
defer mockGetDevType.Reset()
err := hdm.setAscendManager(devM)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("test devType is Ascend310P", t, func() {
mockGetDevType := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)), "GetDevType",
func(_ *devmanager.DeviceManagerMock) string {
return api.Ascend310P
})
defer mockGetDevType.Reset()
err := hdm.setAscendManager(devM)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("test devType is Ascend910A3", t, func() {
mockGetDevType := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)), "GetDevType",
func(_ *devmanager.DeviceManagerMock) string {
return api.Ascend910A3
})
defer mockGetDevType.Reset()
err := hdm.setAscendManager(devM)
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("test GetChipAiCoreCount return error", t, func() {
mockGetChipAiCoreCount = gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return 0, fmt.Errorf("getChipAiCoreCount error")
})
defer mockGetChipAiCoreCount.Reset()
err := hdm.setAscendManager(devM)
convey.So(err.Error(), convey.ShouldEqual, "getChipAiCoreCount error")
})
}
func TestUpdateNodeEdgeScene(t *testing.T) {
var hdm HwDevManager
hdm.manager = device.NewHwAscend310Manager()
convey.Convey("test update node when scene is edge", t, func() {
tmpBuildScene := common.ParamOption.BuildScene
common.ParamOption.BuildScene = common.EdgeScene
err := hdm.UpdateNode()
convey.So(err, convey.ShouldBeNil)
common.ParamOption.BuildScene = tmpBuildScene
})
}
func TestUpdateNodeGetNodeError(t *testing.T) {
var hdm HwDevManager
hdm.manager = device.NewHwAscend310Manager()
mockInitPodInformer := gomonkey.ApplyMethod(&kubeclient.ClientK8s{}, "InitPodInformer",
func(_ *kubeclient.ClientK8s) {})
defer mockInitPodInformer.Reset()
convey.Convey("test update node when get node error", t, func() {
mockGetNode := gomonkey.ApplyMethod(&kubeclient.ClientK8s{}, "GetNode",
func(_ *kubeclient.ClientK8s) (*v1.Node, error) { return &v1.Node{}, fmt.Errorf("getNode error") })
defer mockGetNode.Reset()
err := hdm.UpdateNode()
convey.So(err.Error(), convey.ShouldEqual, "getNode error")
})
}
func TestUpdateNodeUpdateLabelSuccess(t *testing.T) {
var hdm HwDevManager
hdm.manager = device.NewHwAscend310Manager()
mockInitPodInformer := gomonkey.ApplyMethod(&kubeclient.ClientK8s{}, "InitPodInformer",
func(_ *kubeclient.ClientK8s) {})
defer mockInitPodInformer.Reset()
mockMarshal := gomonkey.ApplyFuncReturn(json.Marshal, []byte{0}, nil)
defer mockMarshal.Reset()
convey.Convey("test update node when update node label success", t, func() {
testLabel := map[string]string{"testKey": "testValue"}
mockGetNewNodeLabel := gomonkey.ApplyPrivateMethod(
reflect.TypeOf(new(HwDevManager)), "getNewNodeLabel",
func(_ *HwDevManager, _ *v1.Node) (map[string]string, error) { return testLabel, nil },
)
defer mockGetNewNodeLabel.Reset()
mockGetCardType := gomonkey.ApplyPrivateMethod(
reflect.TypeOf(new(HwDevManager)), "getCardType",
func(_ *HwDevManager) (string, error) { return "", nil },
)
defer mockGetCardType.Reset()
mockGetNewNodeAnnotation := gomonkey.ApplyPrivateMethod(
reflect.TypeOf(new(HwDevManager)), "getNewNodeAnnotation",
func(_ *HwDevManager, _ *v1.Node) (map[string]string, error) {
return map[string]string{}, nil
},
)
defer mockGetNewNodeAnnotation.Reset()
mockGetNode := gomonkey.ApplyMethod(&kubeclient.ClientK8s{}, "GetNode",
func(_ *kubeclient.ClientK8s) (*v1.Node, error) {
return &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: make(map[string]string),
Labels: make(map[string]string),
Name: "node",
},
Status: v1.NodeStatus{Addresses: getAddresses()},
}, nil
})
defer mockGetNode.Reset()
mockPatchNodeState := gomonkey.ApplyMethod(&kubeclient.ClientK8s{}, "PatchNodeState",
func(_ *kubeclient.ClientK8s, _, _ *v1.Node) (*v1.Node, []byte, error) {
return &v1.Node{}, []byte{}, nil
})
defer mockPatchNodeState.Reset()
err := hdm.UpdateNode()
convey.So(err, convey.ShouldBeNil)
})
}
func TestGetNewNodeLabel(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend310Manager(),
allInfo: common.NpuAllInfo{
AllDevs: []common.NpuDevice{{LogicID: 0}},
},
}
testNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{common.ServerTypeLabelKey: "test server type"},
Name: "node",
}}
mockGetDmgr := gomonkey.ApplyMethod(reflect.TypeOf(new(device.HwAscend310Manager)), "GetDmgr",
func(_ *device.HwAscend310Manager) devmanager.DeviceInterface { return &devmanager.DeviceManagerMock{} })
defer mockGetDmgr.Reset()
convey.Convey("test getNewNodeLabel when chip info error", t, func() {
mockGetValidChipInfo := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetValidChipInfo", func(_ *devmanager.DeviceManagerMock) (npuCommon.ChipInfo, error) {
return npuCommon.ChipInfo{}, fmt.Errorf("chip info error")
})
defer mockGetValidChipInfo.Reset()
labelMap, err := hdm.getNewNodeLabel(testNode)
convey.So(labelMap, convey.ShouldBeNil)
convey.So(err.Error(), convey.ShouldEqual, "chip info error")
})
convey.Convey("test getNewNodeLabel success", t, func() {
common.ParamOption.RealCardType = api.Ascend310
mockGetBoardInfo := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetBoardInfo", func(_ *devmanager.DeviceManagerMock, _ int32) (npuCommon.BoardInfo, error) {
return npuCommon.BoardInfo{BoardId: common.A300IA2BoardId}, nil
})
defer mockGetBoardInfo.Reset()
mockGetValidChipInfo := gomonkey.ApplyMethod(reflect.TypeOf(new(devmanager.DeviceManagerMock)),
"GetValidChipInfo", func(_ *devmanager.DeviceManagerMock) (npuCommon.ChipInfo, error) {
return npuCommon.ChipInfo{Name: "testName"}, nil
})
defer mockGetValidChipInfo.Reset()
mockIsContainAll300IDuo := gomonkey.ApplyFuncReturn(common.IsContainAll300IDuo, true)
defer mockIsContainAll300IDuo.Reset()
labelMap, err := hdm.getNewNodeLabel(testNode)
convey.So(labelMap, convey.ShouldResemble, map[string]string{
api.AcceleratorLabelKey: api.Accelerator310Label,
common.InferCardKey: api.A300IDuoLabel,
common.ChipNameLabel: "testName", common.DcmiDriverVersion: "v1"})
convey.So(err, convey.ShouldBeNil)
})
}
func TestAddTopologyLabel_NilMap_A3(t *testing.T) {
convey.Convey("Test addTopologyLabel: newLabelMap is nil and RealCardType is A3", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A3
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId)
defer mockGetSuperPodInfo.Reset()
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
})
}
func TestAddTopologyLabel_ExistingMap_A3(t *testing.T) {
convey.Convey("Test addTopologyLabel: newLabelMap is not nil and RealCardType is A3", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A3
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId)
defer mockGetSuperPodInfo.Reset()
newLabelMap := map[string]string{"existing-label": "value"}
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
convey.So(newLabelMap["existing-label"], convey.ShouldEqual, "value")
})
}
func TestAddTopologyLabel_ExistingMap_A5(t *testing.T) {
convey.Convey("Test addTopologyLabel: newLabelMap is not nil and RealCardType is A5", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A5
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetRackID", testRackId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetServerIndex", testServerIndex).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetSuperPodType", int32(common.ProductType2D))
defer mockGetSuperPodInfo.Reset()
newLabelMap := map[string]string{"existing-label": "value"}
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
convey.So(newLabelMap[npuCommon.TopoLabelRackId], convey.ShouldEqual, strconv.Itoa(int(testRackId)))
convey.So(newLabelMap[npuCommon.TopoLabelServerId], convey.ShouldEqual, strconv.Itoa(int(testServerIndex)))
convey.So(newLabelMap["existing-label"], convey.ShouldEqual, "value")
})
}
func TestAddTopologyLabel_NonA3A5(t *testing.T) {
convey.Convey("Test addTopologyLabel: non-A3/A5 card type should not write any topology label", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
convey.Convey("01-Ascend910A should not write topology labels", func() {
common.ParamOption.RealCardType = api.Ascend910A
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(len(newLabelMap), convey.ShouldEqual, 0)
})
convey.Convey("02-Ascend910B should not write topology labels", func() {
common.ParamOption.RealCardType = api.Ascend910B
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(len(newLabelMap), convey.ShouldEqual, 0)
})
convey.Convey("03-Ascend310 should not write topology labels", func() {
common.ParamOption.RealCardType = api.Ascend310
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(len(newLabelMap), convey.ShouldEqual, 0)
})
})
}
func TestAddTopologyLabel_A5_NegativeServerIndex(t *testing.T) {
convey.Convey("Test addTopologyLabel: A5 with negative serverIndex", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A5
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetRackID", testRackId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetServerIndex", testNegativeServerIdx).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetSuperPodType", int32(common.ProductType2D))
defer mockGetSuperPodInfo.Reset()
convey.Convey("01-should skip serverIndex but write superPodId and rackId labels", func() {
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
convey.So(newLabelMap[npuCommon.TopoLabelRackId], convey.ShouldEqual, strconv.Itoa(int(testRackId)))
_, serverExists := newLabelMap[npuCommon.TopoLabelServerId]
convey.So(serverExists, convey.ShouldBeFalse)
})
})
}
func TestAddTopologyLabel_A5_AllNegative(t *testing.T) {
convey.Convey("Test addTopologyLabel: A5 with all negative values", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A5
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testNegativeSuperPodId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetRackID", testNegativeRackId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetServerIndex", testNegativeServerIdx).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetSuperPodType", int32(common.ProductType2D))
defer mockGetSuperPodInfo.Reset()
convey.Convey("01-should not write any topology label", func() {
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(len(newLabelMap), convey.ShouldEqual, 0)
})
})
}
func TestAddTopologyLabel_A5ServerNoRackId(t *testing.T) {
convey.Convey("Test addTopologyLabel: A5 Server type should not add rackid label", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A5
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetRackID", testRackId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetServerIndex", testServerIndex).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetSuperPodType", int32(common.ProductTypeServer))
defer mockGetSuperPodInfo.Reset()
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
_, hasRackId := newLabelMap[npuCommon.TopoLabelRackId]
convey.So(hasRackId, convey.ShouldBeFalse)
convey.So(newLabelMap[npuCommon.TopoLabelServerId], convey.ShouldEqual, strconv.Itoa(int(testServerIndex)))
})
}
func TestAddTopologyLabel_A5CardNoRackId(t *testing.T) {
convey.Convey("Test addTopologyLabel: A5 Card type should not add rackid label", t, func() {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
common.ParamOption.RealCardType = api.Ascend910A5
mockGetSuperPodInfo := gomonkey.ApplyMethodReturn(device.NewHwAscend910Manager(),
"GetSuperPodID", testSuperPodId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetRackID", testRackId).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetServerIndex", testServerIndex).
ApplyMethodReturn(device.NewHwAscend910Manager(), "GetSuperPodType", int32(common.ProductType1PCard))
defer mockGetSuperPodInfo.Reset()
newLabelMap := make(map[string]string)
hdm.addTopologyLabel(newLabelMap)
convey.So(newLabelMap[npuCommon.TopoLabelSuperPodId], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
_, hasRackId := newLabelMap[npuCommon.TopoLabelRackId]
convey.So(hasRackId, convey.ShouldBeFalse)
convey.So(newLabelMap[npuCommon.TopoLabelServerId], convey.ShouldEqual, strconv.Itoa(int(testServerIndex)))
})
}
func TestSetAcceleratorLabel_NilMap(t *testing.T) {
convey.Convey("Test setAcceleratorLabel: newLabelMap is nil", t, func() {
hdm := &HwDevManager{}
hdm.setAcceleratorLabel(nil)
})
}
func TestSetAcceleratorLabel_AllCardTypes(t *testing.T) {
convey.Convey("Test setAcceleratorLabel: all card types", t, func() {
hdm := &HwDevManager{}
convey.Convey("01-Ascend910 should set huawei-Ascend910", func() {
common.ParamOption.RealCardType = api.Ascend910
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.Accelerator910Label)
})
convey.Convey("02-Ascend910B should set huawei-Ascend910", func() {
common.ParamOption.RealCardType = api.Ascend910B
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.Accelerator910Label)
})
convey.Convey("03-Ascend910A3 should set huawei-Ascend910", func() {
common.ParamOption.RealCardType = api.Ascend910A3
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.Accelerator910Label)
})
convey.Convey("04-Ascend910A5 should set huawei-npu", func() {
common.ParamOption.RealCardType = api.Ascend910A5
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.AcceleratorNPULabel)
})
convey.Convey("05-Ascend310 should set huawei-Ascend310", func() {
common.ParamOption.RealCardType = api.Ascend310
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.Accelerator310Label)
})
convey.Convey("06-Ascend310P should set huawei-Ascend310P", func() {
common.ParamOption.RealCardType = api.Ascend310P
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(newLabelMap[api.AcceleratorLabelKey], convey.ShouldEqual, api.Accelerator310PLabel)
})
convey.Convey("07-unknown card type should not set accelerator label", func() {
common.ParamOption.RealCardType = "UnknownType"
newLabelMap := make(map[string]string)
hdm.setAcceleratorLabel(newLabelMap)
convey.So(len(newLabelMap), convey.ShouldEqual, 0)
})
})
}
func TestStartAllServer(t *testing.T) {
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
mockUpdateNodeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(HwDevManager)), "UpdateNode",
func(_ *HwDevManager) error {
return nil
})
defer mockUpdateNodeLabel.Reset()
convey.Convey("test startAllServer", t, func() {
patch := setPatch()
defer patch.Reset()
mockStart := gomonkey.ApplyMethod(reflect.TypeOf(new(PluginServer)), "Start",
func(_ *PluginServer, socketWatcher *common.FileWatch) error {
return fmt.Errorf("error")
})
defer mockStart.Reset()
common.ParamOption.PresetVDevice = true
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
hdm := NewHwDevManager(&devmanager.DeviceManagerMock{DevType: api.Ascend910})
res := hdm.startAllServer(&common.FileWatch{})
convey.So(res, convey.ShouldBeFalse)
})
}
func TestUpdatePodAnnotation(t *testing.T) {
node := getMockNode(api.Ascend310P)
podDeviceInfo := getMockDeviceInfo()
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
mockUpdateNodeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(HwDevManager)), "UpdateNode",
func(_ *HwDevManager) error {
return nil
})
defer mockUpdateNodeLabel.Reset()
convey.Convey("test updatePodAnnotation", t, func() {
convey.Convey("updatePodAnnotation success", func() {
mockNode := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetNode",
func(_ *kubeclient.ClientK8s) (*v1.Node, error) {
return node, nil
})
mockPodDeviceInfo := gomonkey.ApplyMethod(reflect.TypeOf(new(PluginServer)), "GetKltAndRealAllocateDev",
func(_ *PluginServer, _ []v1.Pod) ([]*common.PodDeviceInfo, error) {
return podDeviceInfo, nil
})
mockManager := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "AddPodAnnotation",
func(_ *device.AscendTools, _ *common.PodDeviceInfo, _ string, _ string, _ []common.NpuDevice) error {
return nil
})
mockPodList := gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetActivePodListCache",
func(_ *kubeclient.ClientK8s) []v1.Pod {
return []v1.Pod{}
})
patch := setPatch()
defer patch.Reset()
defer mockPodList.Reset()
defer mockManager.Reset()
defer mockNode.Reset()
defer mockPodDeviceInfo.Reset()
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
hdm := NewHwDevManager(&devmanager.DeviceManagerMock{DevType: api.Ascend910})
err := hdm.updatePodAnnotation()
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestUpdateDevice(t *testing.T) {
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
mockUpdateNodeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(HwDevManager)), "UpdateNode",
func(_ *HwDevManager) error {
return nil
})
defer mockUpdateNodeLabel.Reset()
convey.Convey("test UpdateDevice", t, func() {
convey.Convey("UpdateDevice success", func() {
mockCheckLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)),
"CheckDeviceTypeLabel",
func(_ *device.AscendTools) error {
return nil
})
mockDestroy := gomonkey.ApplyMethod(reflect.TypeOf(new(PluginServer)), "DestroyNotUsedVNPU",
func(_ *PluginServer) error {
return nil
})
patch := setPatch()
defer patch.Reset()
defer mockDestroy.Reset()
defer mockCheckLabel.Reset()
common.ParamOption.PresetVDevice = true
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
hdm := NewHwDevManager(&devmanager.DeviceManagerMock{DevType: api.Ascend910})
hdm.ServerMap[common.AiCoreResourceName] = NewPluginServer(api.Ascend310P, nil, nil, nil)
err := hdm.updateAllInfo()
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestNotifyToK8s(t *testing.T) {
mockGetChipAiCoreCount := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChipAiCoreCount",
func(_ *device.AscendTools) (int32, error) {
return common.DeviceNotSupport, nil
})
defer mockGetChipAiCoreCount.Reset()
mockUpdateNodeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(HwDevManager)), "UpdateNode",
func(_ *HwDevManager) error {
return nil
})
defer mockUpdateNodeLabel.Reset()
convey.Convey("test NotifyToK8s", t, func() {
convey.Convey("NotifyToK8s success", func() {
mockUpdateHealth := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "UpdateHealth",
func(_ *device.AscendTools, _ map[string][]*common.NpuDevice, _ []*common.NpuDevice, _ string) {
return
})
mockGrace := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "graceTolerance",
func(_ *HwDevManager, _ map[string][]*common.NpuDevice) {
return
})
mockChange := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "GetChange",
func(_ *device.AscendTools, _ map[string][]*common.NpuDevice,
_ map[string][]*common.NpuDevice) map[string]bool {
return map[string]bool{api.Ascend310P: true, api.Ascend310: false}
})
patch := setPatch()
defer patch.Reset()
defer mockUpdateHealth.Reset()
defer mockGrace.Reset()
defer mockChange.Reset()
common.ParamOption.PresetVDevice = true
devMgrPatch := gomonkey.ApplyMethodReturn(&devmanager.DeviceManagerMock{}, "GetSuperPodInfo",
npuCommon.CgoSuperPodInfo{RackId: 1, SuperPodType: 1}, nil)
defer devMgrPatch.Reset()
hdm := NewHwDevManager(&devmanager.DeviceManagerMock{DevType: api.Ascend910})
hdm.ServerMap[common.AiCoreResourceName] = NewPluginServer(api.Ascend310P, nil, nil, nil)
initTime := time.Now()
hdm.notifyToK8s(context.TODO(), &initTime)
convey.So(len(hdm.ServerMap), convey.ShouldEqual, serverNum)
})
})
}
func getMockPod() v1.Pod {
limitValue := v1.ResourceList{
api.HuaweiAscend910: *resource.NewQuantity(rqtTaskNum, resource.BinarySI),
}
annotation := map[string]string{
api.HuaweiAscend910: "0-vir01",
}
return v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "mindx-dls-npu-1p-default-2p-0",
Namespace: "btg-test",
Annotations: annotation,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{Resources: v1.ResourceRequirements{
Limits: limitValue,
}},
},
},
Status: v1.PodStatus{
Reason: "UnexpectedAdmissionError1",
ContainerStatuses: []v1.ContainerStatus{
{State: v1.ContainerState{
Waiting: &v1.ContainerStateWaiting{},
}},
},
},
}
}
func getMockNode(ascendType string) *v1.Node {
return &v1.Node{
Status: v1.NodeStatus{
Allocatable: v1.ResourceList{
v1.ResourceName(ascendType): resource.Quantity{},
},
Addresses: getAddresses(),
},
}
}
func getAddresses() []v1.NodeAddress {
return []v1.NodeAddress{
{
Type: v1.NodeHostName,
Address: common.DefaultDeviceIP,
},
}
}
func getMockDeviceInfo() []*common.PodDeviceInfo {
return []*common.PodDeviceInfo{
{
Pod: getMockPod(),
KltDevice: []string{},
RealDevice: []string{},
},
{
Pod: getMockPod(),
KltDevice: []string{""},
RealDevice: []string{""},
},
}
}
type mockDevManager struct {
device.DevManager
}
func (mdm mockDevManager) GetKubeClient() *kubeclient.ClientK8s {
return nil
}
func (mdm mockDevManager) GetNPUs() (common.NpuAllInfo, error) {
return common.NpuAllInfo{}, testErr
}
func TestCheckNodeResetInfo(t *testing.T) {
hdm := HwDevManager{}
flag := false
convey.Convey("test checkNodeResetInfo", t, func() {
patch := gomonkey.ApplyFunc(device.WriteResetInfo,
func(resetInfo device.ResetInfo, writeMode device.WriteMode, update bool) {
flag = true
})
patch.ApplyFuncReturn(checkOverRetryDev, device.ResetInfo{})
defer patch.Reset()
const id0 = 0
hdm.manager = mockDevManager{}
convey.Convey("01-client nil, flag should be false", func() {
hdm.checkNodeResetInfo()
convey.So(flag, convey.ShouldBeFalse)
})
patch.ApplyMethodReturn(&mockDevManager{}, "GetKubeClient", &kubeclient.ClientK8s{})
patch.ApplyFuncReturn(device.GetResetInfoMgr, &device.ResetInfoMgr{})
patch.ApplyFuncReturn(device.ReadResetInfo,
device.ResetInfo{ThirdPartyResetDevs: []device.ResetDevice{
{PhyID: id0},
}})
convey.Convey("02-get npus error, flag should be false", func() {
hdm.checkNodeResetInfo()
convey.So(flag, convey.ShouldBeFalse)
})
patch.ApplyMethodReturn(&mockDevManager{}, "GetNPUs", common.NpuAllInfo{}, nil)
convey.Convey("03-success, flag should be true", func() {
patch1 := gomonkey.ApplyFuncReturn(checkDeviceStatus, []device.ResetDevice{}, true)
defer patch1.Reset()
hdm.checkNodeResetInfo()
convey.So(flag, convey.ShouldBeTrue)
})
})
}
func TestCheckDeviceStatus(t *testing.T) {
convey.Convey("test checkDeviceStatus", t, func() {
const id1, id2, id3 = 1, 2, 3
convey.Convey("01-status change, should return true", func() {
patch := gomonkey.ApplyFunc(device.FreeBusyDev, func(logicID int32) {
return
})
defer patch.Reset()
allInfo := map[string][]*common.NpuDevice{
api.Ascend910: {
&common.NpuDevice{
PhyID: int32(id1),
Health: v1beta1.Healthy,
},
},
}
failDevs := []device.ResetDevice{
{
PhyID: id1,
},
{
PhyID: id2,
},
{
PhyID: id3,
},
}
_, isChange := checkDeviceStatus(failDevs, allInfo)
convey.So(isChange, convey.ShouldBeTrue)
})
})
}
func TestCheckOverRetryDev(t *testing.T) {
const id = 0
const numOne, numZero = 1, 0
convey.Convey("test checkOverRetryDev", t, func() {
input := device.ResetInfo{
ThirdPartyResetDevs: []device.ResetDevice{{
CardId: id,
}},
ManualResetDevs: make([]device.ResetDevice, 0),
}
convey.Convey("01-over retry time, dev should be add to manualDev", func() {
patch1 := gomonkey.ApplyFuncReturn(device.GetResetCnt, common.MaxResetTimes+numOne)
defer patch1.Reset()
ret := checkOverRetryDev(input)
convey.So(len(ret.ManualResetDevs), convey.ShouldEqual, numOne)
})
convey.Convey("02-not over retry times, dev be add to third party", func() {
patch1 := gomonkey.ApplyFuncReturn(device.GetResetCnt, common.MaxResetTimes-numOne)
defer patch1.Reset()
ret := checkOverRetryDev(input)
convey.So(len(ret.ManualResetDevs), convey.ShouldEqual, numZero)
})
})
}
func TestFlattenMap(t *testing.T) {
const id = 0
const targetLen = 2
m := map[string][]*common.NpuDevice{
api.Ascend910: {
&common.NpuDevice{
PhyID: int32(id),
Health: v1beta1.Healthy,
},
},
api.Ascend310P: {
&common.NpuDevice{
PhyID: int32(id),
Health: v1beta1.Healthy,
},
},
}
ret := flattenMap(m)
if len(ret) != targetLen {
t.Errorf("expect len %v, got %v", targetLen, len(ret))
}
}
func TestIsSupportGraceTolerance(t *testing.T) {
var hdm HwDevManager
tmpGraceToleranceOn := common.ParamOption.GraceToleranceOn
tmpHotReset := common.ParamOption.HotReset
common.ParamOption.GraceToleranceOn = false
convey.Convey("test isSupportGraceTolerance when hot reset mode error", t, func() {
common.ParamOption.HotReset = common.HotResetInfer
hdm.isSupportGraceTolerance()
convey.So(common.ParamOption.GraceToleranceOn, convey.ShouldNotEqual, true)
})
common.ParamOption.HotReset = common.HotResetTrainOffLine
convey.Convey("test isSupportGraceTolerance when run mode is not Ascend910", t, func() {
hdm.RunMode = api.Ascend310P
hdm.isSupportGraceTolerance()
convey.So(common.ParamOption.GraceToleranceOn, convey.ShouldNotEqual, true)
})
hdm.RunMode = api.Ascend910
tmpRealCardType := common.ParamOption.RealCardType
common.ParamOption.RealCardType = api.Ascend910A
convey.Convey("test isSupportGraceTolerance when SMP chip mode is not for Ascend910", t, func() {
hdm.WorkMode = common.AMPMode
hdm.isSupportGraceTolerance()
convey.So(common.ParamOption.GraceToleranceOn, convey.ShouldNotEqual, true)
})
hdm.WorkMode = common.SMPMode
convey.Convey("test isSupportGraceTolerance when GraceToleranceOn is true", t, func() {
hdm.isSupportGraceTolerance()
convey.So(common.ParamOption.GraceToleranceOn, convey.ShouldEqual, true)
})
common.ParamOption.HotReset = tmpHotReset
common.ParamOption.RealCardType = tmpRealCardType
common.ParamOption.GraceToleranceOn = tmpGraceToleranceOn
}
func TestUpdateAllInfo(t *testing.T) {
hdm := &HwDevManager{}
tmpPresetVDevice := common.ParamOption.PresetVDevice
convey.Convey("test updateAllInfo when PresetVDevice is true return nil", t, func() {
common.ParamOption.PresetVDevice = true
err := hdm.updateAllInfo()
convey.So(err, convey.ShouldBeNil)
})
common.ParamOption.PresetVDevice = false
convey.Convey("test updateAllInfo when not found npu-core in server map return error", t, func() {
err := hdm.updateAllInfo()
convey.So(err.Error(), convey.ShouldEqual, "not found npu-core plugin server")
})
hdm = &HwDevManager{ServerMap: map[string]InterfaceServer{common.AiCoreResourceName: &PluginServer{}},
manager: &device.HwAscend310Manager{}}
convey.Convey("test updateAllInfo when DestroyNotUsedVNPU is error return error", t, func() {
patch := gomonkey.ApplyMethod(reflect.TypeOf(new(PluginServer)), "DestroyNotUsedVNPU",
func(_ *PluginServer) error { return fmt.Errorf("error") })
defer patch.Reset()
err := hdm.updateAllInfo()
convey.So(err.Error(), convey.ShouldEqual, "error")
})
patch := gomonkey.ApplyMethod(reflect.TypeOf(new(PluginServer)), "DestroyNotUsedVNPU",
func(_ *PluginServer) error { return nil })
defer patch.Reset()
mockCheckDeviceTypeLabel := gomonkey.ApplyMethod(reflect.TypeOf(new(device.AscendTools)), "CheckDeviceTypeLabel",
func(_ *device.AscendTools) error { return fmt.Errorf("error") })
defer mockCheckDeviceTypeLabel.Reset()
convey.Convey("test updateAllInfo when GetNPUs is error return error", t, func() {
patch := gomonkey.ApplyMethod(reflect.TypeOf(new(device.HwAscend310Manager)), "GetNPUs", func(
_ *device.HwAscend310Manager) (common.NpuAllInfo, error) {
return common.NpuAllInfo{}, fmt.Errorf("error")
})
defer patch.Reset()
err := hdm.updateAllInfo()
convey.So(err.Error(), convey.ShouldEqual, "error")
})
mockGetNPUs := gomonkey.ApplyMethod(reflect.TypeOf(new(device.HwAscend310Manager)), "GetNPUs",
func(_ *device.HwAscend310Manager) (common.NpuAllInfo, error) {
return common.NpuAllInfo{AllDevs: []common.NpuDevice{}, AllDevTypes: []string{}}, nil
})
defer mockGetNPUs.Reset()
convey.Convey("test updateAllInfo success return nil", t, func() {
err := hdm.updateAllInfo()
convey.So(err, convey.ShouldBeNil)
})
common.ParamOption.PresetVDevice = tmpPresetVDevice
}
func mockGetConfigMap(configmap *v1.ConfigMap, err error) *gomonkey.Patches {
return gomonkey.ApplyMethod(reflect.TypeOf(new(kubeclient.ClientK8s)), "GetConfigMap",
func(_ *kubeclient.ClientK8s, _, _ string) (*v1.ConfigMap, error) {
return configmap, err
})
}
func TestResetHccsServer(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}}
convey.Convey("Test ResetHccsServer", t, func() {
convey.Convey("When all devices are healthy, do nothing", func() {
devices := []*common.NpuDevice{{Health: v1beta1.Healthy}, {Health: v1beta1.Healthy}}
hdm.ResetHccsServer("devType", devices, &PodResource{})
})
devices := []*common.NpuDevice{{Health: v1beta1.Unhealthy}}
convey.Convey("When reset failed times exceed max limit, log warning and return", func() {
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetResetFailedTimes", common.MaxResetTimes+1)
defer patch.Reset()
hdm.ResetHccsServer("devType", devices, &PodResource{})
})
convey.Convey("When cards are in resetting, do nothing", func() {
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetIfCardsInResetting", true)
defer patch.Reset()
hdm.ResetHccsServer("devType", devices, &PodResource{})
})
patch1 := gomonkey.ApplyMethodReturn(hdm.manager, "GetResetFailedTimes", 0).ApplyMethodReturn(hdm.manager,
"GetIfCardsInResetting", false)
defer patch1.Reset()
convey.Convey("When index out of range, log error and return", func() {
devices := make([]*common.NpuDevice, 0)
hdm.ResetHccsServer("devType", devices, &PodResource{})
})
})
}
func TestSubscribeNpuFaultEvent(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}, RunMode: api.Ascend910}
convey.Convey("Test subscribeNpuFaultEvent", t, func() {
convey.Convey("When LoadFaultCodeFromFile fails, set SubscribeFailed and log error", func() {
patch := gomonkey.ApplyFunc(common.LoadFaultCodeFromFile,
func() error { return errors.New("load faultCode.json failed") })
defer patch.Reset()
hdm.subscribeNpuFaultEvent()
convey.So(common.SubscribeFailed, convey.ShouldBeTrue)
})
patch1 := gomonkey.ApplyFunc(common.LoadFaultCodeFromFile, func() error { return nil })
defer patch1.Reset()
convey.Convey("When RunMode is not Ascend910, set SubscribeFailed and log debug", func() {
hdm.RunMode = "otherMode"
hdm.subscribeNpuFaultEvent()
convey.So(common.SubscribeFailed, convey.ShouldBeTrue)
hdm.RunMode = api.Ascend910
})
mockDmgr := gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", &devmanager.DeviceManager{})
defer mockDmgr.Reset()
convey.Convey("When SetFaultEventCallFunc fails, set SubscribeFailed and log error", func() {
patch2 := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "SetFaultEventCallFunc",
errors.New("set callback failed"))
defer patch2.Reset()
hdm.subscribeNpuFaultEvent()
convey.So(common.SubscribeFailed, convey.ShouldBeTrue)
})
patch2 := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "SetFaultEventCallFunc", nil)
defer patch2.Reset()
tmpSubscribeFailed := common.SubscribeFailed
common.SubscribeFailed = false
convey.Convey("When SubscribeDeviceFaultEvent succeeds, return directly", func() {
patch3 := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "SubscribeDeviceFaultEvent", nil)
defer patch3.Reset()
hdm.subscribeNpuFaultEvent()
convey.So(common.SubscribeFailed, convey.ShouldBeFalse)
})
common.SubscribeFailed = tmpSubscribeFailed
convey.Convey("When SubscribeDeviceFaultEvent fails after retries, set SubscribeFailed and log error", func() {
patch3 := gomonkey.ApplyMethod(hdm.manager.GetDmgr(), "SubscribeDeviceFaultEvent",
func(_ *devmanager.DeviceManager, _ int32) error { return errors.New("subscribe failed") })
patch4 := gomonkey.ApplyFunc(time.Sleep, func(_ time.Duration) {})
defer patch3.Reset()
defer patch4.Reset()
hdm.subscribeNpuFaultEvent()
convey.So(common.SubscribeFailed, convey.ShouldBeTrue)
})
})
}
func TestHotReset(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}, RunMode: api.Ascend910}
npuDevice := &common.NpuDevice{DeviceName: "name", LogicID: 0}
convey.Convey("Test hotReset", t, func() {
patch := gomonkey.ApplyMethod(hdm.manager, "SetCardsInResetting",
func(_ *device.HwAscend310Manager, _ int32, _ bool) {}).
ApplyMethod(hdm.manager, "SetResetFailedTimes",
func(_ *device.HwAscend310Manager, _ int32, _ int) {})
defer patch.Reset()
convey.Convey("When PollImmediate error log warn and return", func() {
mockPollImmediate := gomonkey.ApplyFuncReturn(wait.PollImmediate, errors.New("error"))
defer mockPollImmediate.Reset()
hdm.hotReset(npuDevice, []*common.NpuDevice{npuDevice})
})
mockPollImmediate := gomonkey.ApplyFuncReturn(wait.PollImmediate, nil)
defer mockPollImmediate.Reset()
convey.Convey("When PollImmediate return nil hot rest success", func() {
hdm.hotReset(npuDevice, []*common.NpuDevice{npuDevice})
})
})
}
func TestResetCommonInferCard(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}, allInfo: common.NpuAllInfo{AllDevs: []common.NpuDevice{
{DeviceName: "device1", Health: v1beta1.Healthy}, {DeviceName: "device2", Health: v1beta1.Unhealthy}}}}
devices := []*common.NpuDevice{{DeviceName: "device1", Health: v1beta1.Healthy},
{DeviceName: "device2", Health: v1beta1.Unhealthy}}
convey.Convey("Test resetCommonInferCard", t, func() {
tmpRealCardType := common.ParamOption.RealCardType
defer func() { common.ParamOption.RealCardType = tmpRealCardType }()
convey.Convey("card type is A3", func() {
common.ParamOption.RealCardType = api.Ascend910A3
npuDeviceList := []*common.NpuDevice{{LogicID: 0, Health: v1beta1.Healthy}}
hdm.resetCommonInferCard("devType", npuDeviceList, nil)
convey.ShouldBeFalse(hdm.manager.GetIfCardsInResetting(0))
})
convey.Convey("When hdm is nil, log error and return", func() {
var nilHdm *HwDevManager
nilHdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When allInfo.AllDevs is empty, log error and return", func() {
tmpAllDevs := hdm.allInfo.AllDevs
hdm.allInfo.AllDevs = []common.NpuDevice{}
hdm.resetCommonInferCard("devType", devices, &PodResource{})
hdm.allInfo.AllDevs = tmpAllDevs
})
convey.Convey("When getServerUsageAndBoardId fails, log error and return", func() {
common.ParamOption.RealCardType = api.Ascend910B
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId", uint32(0), errors.New("error"))
defer patch.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
common.ParamOption.RealCardType = api.Ascend910B
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId", uint32(common.A800IA2NoneHccsBoardId),
nil).ApplyMethodReturn(&mockDevManager{}, "GetKubeClient", &kubeclient.ClientK8s{})
defer patch.Reset()
convey.Convey("When boardId is A800IA2NoneHccsBoardId, call ResetWithoutHccsServer", func() {
patch1 := gomonkey.ApplyMethod(hdm, "ResetWithoutHccsServer",
func(_ *HwDevManager, _ string, _ []*common.NpuDevice, _ *PodResource) {})
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When boardId is A800IA2NoneHccsBoardIdOld, call ResetWithoutHccsServer", func() {
patch1 := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId",
uint32(common.A800IA2NoneHccsBoardIdOld), nil).
ApplyMethod(hdm, "ResetWithoutHccsServer",
func(_ *HwDevManager, _ string, _ []*common.NpuDevice, _ *PodResource) {})
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When boardId is A300IA2BoardId, call ResetWithoutHccsServer", func() {
patch1 := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId",
uint32(common.A300IA2BoardId), nil).
ApplyMethod(hdm, "ResetWithoutHccsServer",
func(_ *HwDevManager, _ string, _ []*common.NpuDevice, _ *PodResource) {})
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When boardId is A300IA2GB64BoardId, call ResetWithoutHccsServer", func() {
patch1 := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId",
uint32(common.A300IA2GB64BoardId), nil).
ApplyMethod(hdm, "ResetWithoutHccsServer",
func(_ *HwDevManager, _ string, _ []*common.NpuDevice, _ *PodResource) {})
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When boardId is not NoneHccsBoardId, call ResetHccsServer", func() {
patch1 := gomonkey.ApplyMethodReturn(hdm.manager, "GetServerBoardId", uint32(0), nil).ApplyMethod(hdm,
"ResetHccsServer", func(_ *HwDevManager, _ string, _ []*common.NpuDevice, _ *PodResource) {})
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
common.ParamOption.RealCardType = api.Ascend310P
convey.Convey("When card type is not A3 or 910B and device is healthy, should skip reset", func() {
healthyDevices := []*common.NpuDevice{{LogicID: 0, Health: v1beta1.Healthy}}
hdm.resetCommonInferCard("devType", healthyDevices, &PodResource{})
})
convey.Convey("When card type is not A3 or 910B and isPodRemove returns false, should skip reset", func() {
patch1 := gomonkey.ApplyPrivateMethod(hdm, "isPodRemove",
func(_ *HwDevManager, _ string, _ *common.NpuDevice, _ *PodResource) bool { return false })
defer patch1.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When card type is not A3 or 910B and checkNoProc returns false, should skip reset", func() {
patch1 := gomonkey.ApplyPrivateMethod(hdm, "isPodRemove",
func(_ *HwDevManager, _ string, _ *common.NpuDevice, _ *PodResource) bool { return true })
defer patch1.Reset()
patch2 := gomonkey.ApplyPrivateMethod(hdm, "checkNoProc",
func(_ *HwDevManager, _ int) bool { return false })
defer patch2.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
convey.Convey("When card type is not A3 or 910B and all checks pass, should call hotReset", func() {
patch1 := gomonkey.ApplyPrivateMethod(hdm, "isPodRemove",
func(_ *HwDevManager, _ string, _ *common.NpuDevice, _ *PodResource) bool { return true })
defer patch1.Reset()
patch2 := gomonkey.ApplyPrivateMethod(hdm, "checkNoProc",
func(_ *HwDevManager, _ int) bool { return true })
defer patch2.Reset()
patch3 := gomonkey.ApplyPrivateMethod(hdm, "hotReset",
func(_ *HwDevManager, _ *common.NpuDevice, _ []*common.NpuDevice) {})
defer patch3.Reset()
hdm.resetCommonInferCard("devType", devices, &PodResource{})
})
})
}
func TestResetServerForA3(t *testing.T) {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager()}
convey.Convey("Test ResetServerForA3", t, func() {
logicList := []int32{0, 1, 8, 9}
npuDevices := []*common.NpuDevice{
{LogicID: 9}, {LogicID: 8, Health: v1beta1.Healthy}, {LogicID: 1}, {LogicID: 0}}
sleepTime := 300
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", &devmanager.DeviceManager{}).
ApplyMethod(hdm.manager, "GetAssociatedLogicIDs",
func(_ *device.HwAscend910Manager, _, _, _ int32) ([]int32, error) {
return logicList, nil
}).
ApplyFuncReturn(wait.PollImmediate, nil)
defer patch.Reset()
convey.Convey("get card id device id failed, should not reset device", func() {
patch.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetCardIDDeviceID",
int32(0), int32(0), errors.New("getCardIDDeviceID error"))
hdm.ResetServerForA3("devType", npuDevices, nil)
convey.ShouldBeFalse(hdm.manager.GetIfCardsInResetting(0))
})
convey.Convey("get logic id list success and pod already remove, should reset device", func() {
hdm.manager.SetCardsInResetting(0, false)
patch.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetCardIDDeviceID", int32(0), int32(0), nil).
ApplyPrivateMethod(hdm, "isPodRemove",
func(*HwDevManager, string, *common.NpuDevice, *PodResource) bool { return true }).
ApplyPrivateMethod(hdm, "checkNoProc", func(*HwDevManager, int) bool { return true })
hdm.ResetServerForA3("devType", npuDevices, nil)
convey.ShouldBeTrue(hdm.manager.GetIfCardsInResetting(0))
})
convey.Convey("pod not remove, should not reset device", func() {
hdm.manager.SetCardsInResetting(0, false)
patch.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetCardIDDeviceID", int32(0), int32(0), nil).
ApplyPrivateMethod(hdm, "isPodRemove",
func(*HwDevManager, string, *common.NpuDevice, *PodResource) bool { return false }).
ApplyPrivateMethod(hdm, "checkNoProc", func(*HwDevManager, int) bool { return true })
hdm.ResetServerForA3("devType", npuDevices, nil)
convey.ShouldBeFalse(hdm.manager.GetIfCardsInResetting(0))
})
convey.Convey("process not remove, should not reset device", func() {
hdm.manager.SetCardsInResetting(0, false)
patch.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetCardIDDeviceID", int32(0), int32(0), nil).
ApplyPrivateMethod(hdm, "isPodRemove",
func(*HwDevManager, string, *common.NpuDevice, *PodResource) bool { return true }).
ApplyPrivateMethod(hdm, "checkNoProc", func(*HwDevManager, int) bool { return false })
hdm.ResetServerForA3("devType", npuDevices, nil)
convey.ShouldBeFalse(hdm.manager.GetIfCardsInResetting(0))
})
time.Sleep(time.Duration(sleepTime) * time.Millisecond)
})
}
func TestExecResetChip(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}}
isResetExec := false
convey.Convey("Test execResetChip", t, func() {
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", &devmanager.DeviceManager{})
defer patch.Reset()
convey.Convey("When isResetExec is true, return nil", func() {
isResetExec := true
err := hdm.execResetChip(int32(0), &isResetExec)
convey.So(err, convey.ShouldBeNil)
})
patch1 := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "SetDeviceReset", nil)
defer patch1.Reset()
convey.Convey("When SetDeviceReset fails, log error and return", func() {
patch2 := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "SetDeviceReset",
errors.New("setDeviceReset error"))
defer patch2.Reset()
err := hdm.execResetChip(int32(0), &isResetExec)
convey.So(err.Error(), convey.ShouldEqual, "setDeviceReset error")
})
convey.Convey("When exec set device reset function success, return nil", func() {
err := hdm.execResetChip(int32(0), &isResetExec)
convey.So(err, convey.ShouldBeNil)
})
})
}
func TestPollFaultCodeCM(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}}
convey.Convey("Test pollFaultCodeCM", t, func() {
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetKubeClient", &kubeclient.ClientK8s{})
defer patch.Reset()
convey.Convey("When context is canceled, stop polling", func() {
ctx, cancel := context.WithCancel(context.Background())
cancel()
hdm.pollFaultCodeCM(ctx, common.PollFaultCodeCMInterval)
})
})
}
func TestInitFaultInfoFromFile(t *testing.T) {
originalCardType := common.ParamOption.RealCardType
originalEnableSwitch := common.ParamOption.EnableSwitchFault
convey.Convey("Test initFaultInfoFromFile", t, func() {
convey.Convey("When LoadFaultCodeFromFile fails", func() {
initFaultInfoFromFile()
convey.So(len(common.NotHandleFaultCodes) == 0, convey.ShouldBeTrue)
})
generalFaultCode := "[0x00f103b0,155649,na,NoneExist]"
switchFileInfo := common.SwitchFaultFileInfo{NotHandleFaultCodes: []string{generalFaultCode}}
bytes, err := json.Marshal(switchFileInfo)
convey.So(err, convey.ShouldBeNil)
tmpNotHandleFaultCodes := common.NotHandleFaultCodes
patch := gomonkey.ApplyFuncReturn(json.Unmarshal, nil).ApplyFunc(deviceswitch.UpdateSwitchFaultLevel,
func() {}).ApplyFuncReturn(utils.LoadFile, bytes, nil).ApplyFuncReturn(common.LoadFaultCodeFromFile, nil).
ApplyFuncReturn(common.LoadFaultCustomizationFromFile, nil)
defer patch.Reset()
common.ParamOption.RealCardType = api.Ascend910A3
common.ParamOption.EnableSwitchFault = true
convey.Convey("When load switch fault code from file error", func() {
patch1 := gomonkey.ApplyFuncReturn(utils.LoadFile, nil, errors.New("load error"))
defer patch1.Reset()
initFaultInfoFromFile()
convey.So(len(common.NotHandleFaultCodes) == 0, convey.ShouldBeTrue)
common.NotHandleFaultCodes = tmpNotHandleFaultCodes
})
convey.Convey("When all loads succeed", func() {
initFaultInfoFromFile()
convey.So(len(common.NotHandleFaultCodes) > 0, convey.ShouldBeTrue)
common.NotHandleFaultCodes = tmpNotHandleFaultCodes
})
})
common.ParamOption.RealCardType = originalCardType
common.ParamOption.EnableSwitchFault = originalEnableSwitch
}
func TestGetFaultCodeCMPollInterval(t *testing.T) {
type testCase struct {
name string
configMapData map[string]string
expected int
}
testCases := []testCase{{name: "No PollInterval key", configMapData: map[string]string{},
expected: common.PollFaultCodeCMInterval},
{name: "Invalid PollInterval format", configMapData: map[string]string{common.PollIntervalKey: "invalid"},
expected: common.PollFaultCodeCMInterval},
{name: "PollInterval too small", configMapData: map[string]string{
common.PollIntervalKey: strconv.Itoa(common.PollFaultCodeCMMinInterval - 1)},
expected: common.PollFaultCodeCMInterval},
{name: "PollInterval too large", configMapData: map[string]string{
common.PollIntervalKey: strconv.Itoa(common.PollFaultCodeCMMaxInterval + 1)},
expected: common.PollFaultCodeCMInterval},
{name: "Valid PollInterval at lower bound", expected: common.PollFaultCodeCMMinInterval,
configMapData: map[string]string{common.PollIntervalKey: strconv.Itoa(common.PollFaultCodeCMMinInterval)}},
{name: "Valid PollInterval at upper bound", expected: common.PollFaultCodeCMMaxInterval,
configMapData: map[string]string{common.PollIntervalKey: strconv.Itoa(common.PollFaultCodeCMMaxInterval)}},
{name: "Valid PollInterval in middle", configMapData: map[string]string{common.PollIntervalKey: "30"},
expected: common.PollFaultCodeCMMinInterval},
}
convey.Convey("Test getFaultCodeCMPollInterval", t, func() {
for _, tc := range testCases {
convey.Convey(tc.name, func() {
configMap := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
Data: tc.configMapData}
result := getFaultCodeCMPollInterval(configMap)
convey.So(result, convey.ShouldEqual, tc.expected)
})
}
})
}
func mockGetNewNodeAnnotationDeps(superPodInfo common.SuperPodInfo, cardType string) []*gomonkey.Patches {
patches := make([]*gomonkey.Patches, 0, 4)
p1 := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "getCardType",
func(_ *HwDevManager) (string, error) { return cardType, nil })
patches = append(patches, p1)
p2 := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "getNpuBaseInfo",
func(_ *HwDevManager) map[string]*common.NpuBaseInfo { return map[string]*common.NpuBaseInfo{} })
patches = append(patches, p2)
p3 := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "getSuperPodInfo",
func(_ *HwDevManager) common.SuperPodInfo { return superPodInfo })
patches = append(patches, p3)
p4 := gomonkey.ApplyFuncReturn(customname.ReplaceDevicePublicName, "")
patches = append(patches, p4)
return patches
}
func resetPatches(patches []*gomonkey.Patches) {
for _, p := range patches {
p.Reset()
}
}
func TestGetNewNodeAnnotation_NonA5NoRackID(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910B
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testNegativeSuperPodId, ServerId: testNegativeServerIdx, RackId: int32(0),
SuperPodType: common.ProductTypeServer,
}, "")
defer resetPatches(patches)
convey.Convey("should not contain rackID when RealCardType is Ascend910B", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
_, hasRackID := annotationMap[api.RackIDKey]
convey.So(hasRackID, convey.ShouldBeFalse)
})
}
func TestGetNewNodeAnnotation_A5PodHasRackID(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910A5
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testSuperPodId, ServerId: testServerIndex, RackId: testRackId,
SuperPodType: common.ProductType2D,
}, "")
defer resetPatches(patches)
convey.Convey("should contain rackID when RealCardType is Ascend910A5 and superPodType is Pod", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
convey.So(annotationMap[api.RackIDKey], convey.ShouldEqual, strconv.Itoa(int(testRackId)))
convey.So(annotationMap[common.SuperPodIDKey], convey.ShouldEqual, strconv.Itoa(int(testSuperPodId)))
convey.So(annotationMap[serverIndexKey], convey.ShouldEqual, strconv.Itoa(int(testServerIndex)))
})
}
func TestGetNewNodeAnnotation_A5ServerNoRackID(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910A5
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testSuperPodId, ServerId: testServerIndex, RackId: testRackId,
SuperPodType: common.ProductTypeServer,
}, "")
defer resetPatches(patches)
convey.Convey("should not contain rackID when RealCardType is Ascend910A5 and superPodType is Server", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
_, hasRackID := annotationMap[api.RackIDKey]
convey.So(hasRackID, convey.ShouldBeFalse)
})
}
func TestGetNewNodeAnnotation_A5CardNoRackID(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910A5
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testSuperPodId, ServerId: testServerIndex, RackId: testRackId,
SuperPodType: common.ProductType1PCard,
}, "")
defer resetPatches(patches)
convey.Convey("should not contain rackID when RealCardType is Ascend910A5 and superPodType is Card", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
_, hasRackID := annotationMap[api.RackIDKey]
convey.So(hasRackID, convey.ShouldBeFalse)
})
}
func TestGetNewNodeAnnotation_CardTypeNotEmpty(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910A5
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testSuperPodId, ServerId: testServerIndex, RackId: testRackId,
SuperPodType: common.ProductType2D,
}, "testCardType")
defer resetPatches(patches)
convey.Convey("should contain cardType when getCardType returns non-empty", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
convey.So(annotationMap[cardTypeKey], convey.ShouldEqual, "testCardType")
})
}
func TestGetNewNodeAnnotation_MarshalError(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910B
p1 := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "getCardType",
func(_ *HwDevManager) (string, error) { return "", nil })
defer p1.Reset()
p2 := gomonkey.ApplyPrivateMethod(reflect.TypeOf(new(HwDevManager)), "getNpuBaseInfo",
func(_ *HwDevManager) map[string]*common.NpuBaseInfo { return map[string]*common.NpuBaseInfo{} })
defer p2.Reset()
p3 := gomonkey.ApplyFuncReturn(json.Marshal, []byte{}, fmt.Errorf("marshal error"))
defer p3.Reset()
convey.Convey("should return error when json.Marshal fails", t, func() {
hdm := &HwDevManager{}
_, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldNotBeNil)
})
}
func TestGetNewNodeAnnotation_A3NoRackID(t *testing.T) {
common.ParamOption.RealCardType = api.Ascend910A3
patches := mockGetNewNodeAnnotationDeps(common.SuperPodInfo{
SuperPodId: testSuperPodId, ServerId: testServerIndex, RackId: int32(0),
SuperPodType: common.ProductTypeServer,
}, "")
defer resetPatches(patches)
convey.Convey("should not contain rackID when RealCardType is Ascend910A3", t, func() {
hdm := &HwDevManager{}
annotationMap, err := hdm.getNewNodeAnnotation(&v1.Node{})
convey.So(err, convey.ShouldBeNil)
_, hasRackID := annotationMap[api.RackIDKey]
convey.So(hasRackID, convey.ShouldBeFalse)
})
}
func CapturePanic(f func()) error {
var err error
defer func() {
err = nil
if recovered := recover(); recovered != nil {
err = errors.New("panic error")
}
}()
f()
return err
}
func TestChipHotReset(t *testing.T) {
hdm := &HwDevManager{groupDevice: map[string][]*common.NpuDevice{"type1": {{DeviceName: "device1"}},
"virtual": {{DeviceName: "virtual1"}}}, manager: &device.HwAscend310Manager{},
allInfo: common.NpuAllInfo{AllDevs: []common.NpuDevice{}}}
originalHotReset := common.ParamOption.HotReset
convey.Convey("Test chipHotReset", t, func() {
convey.Convey("When HotReset mode is not Infer, log debug and return", func() {
common.ParamOption.HotReset = common.HotResetTrainOffLine
convey.So(CapturePanic(func() { hdm.chipHotReset() }), convey.ShouldBeNil)
})
common.ParamOption.HotReset = common.HotResetInfer
patch := gomonkey.ApplyFuncReturn(common.IsVirtualDev, false).ApplyFuncReturn(common.IsContainAtlas300IDuo,
false).ApplyMethodReturn(hdm.manager, "GetServerBoardId", uint32(0), errors.New("error"))
defer patch.Reset()
convey.Convey("When device is virtual, skip it", func() {
patch := gomonkey.ApplyFuncReturn(common.IsVirtualDev, true)
defer patch.Reset()
convey.So(CapturePanic(func() { hdm.chipHotReset() }), convey.ShouldBeNil)
})
convey.Convey("When device is Atlas300IDuo, call resetDuoCard", func() {
mockAtlas := gomonkey.ApplyFuncReturn(common.IsContainAtlas300IDuo, true)
defer mockAtlas.Reset()
convey.So(CapturePanic(func() { hdm.chipHotReset() }), convey.ShouldBeNil)
})
convey.Convey("When normal infer device, call resetCommonInferCard", func() { hdm.chipHotReset() })
})
common.ParamOption.HotReset = originalHotReset
}
func TestResetDuoCard(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{}, ServerMap: map[string]InterfaceServer{}}
devices := []*common.NpuDevice{{CardID: 1, DeviceName: "device1", Health: v1beta1.Unhealthy},
{CardID: 1, DeviceName: "device2"}}
prClient := &PodResource{}
convey.Convey("Test resetDuoCard", t, func() {
patch := gomonkey.ApplyMethodReturn(&mockDevManager{}, "GetKubeClient", &kubeclient.ClientK8s{})
defer patch.Reset()
convey.Convey("When duo card not removable, skip reset", func() {
mockRemove := gomonkey.ApplyMethodReturn(hdm.manager.GetKubeClient(), "GetAllPodListCache", []v1.Pod{})
defer mockRemove.Reset()
convey.So(CapturePanic(func() { hdm.resetDuoCard("type1", devices, prClient) }), convey.ShouldBeNil)
})
})
}
func TestIsPodRemove(t *testing.T) {
hdm := &HwDevManager{manager: &device.HwAscend310Manager{},
ServerMap: map[string]InterfaceServer{"type1": &PluginServer{}}}
convey.Convey("Test isPodRemove", t, func() {
patch := gomonkey.ApplyMethodReturn(hdm.manager, "GetKubeClient", &kubeclient.ClientK8s{}).ApplyMethodReturn(
&kubeclient.ClientK8s{}, "GetAllPodListCache", []v1.Pod{})
defer patch.Reset()
convey.Convey("When devType not found in ServerMap, return false", func() {
result := hdm.isPodRemove("invalidType", &common.NpuDevice{DeviceName: "device1"}, &PodResource{})
convey.So(result, convey.ShouldBeFalse)
})
convey.Convey("When pod not removed, return false", func() {
mockIsPodMove := gomonkey.ApplyMethodReturn(&PodResource{}, "IsPodMoveComplete", false)
defer mockIsPodMove.Reset()
result := hdm.isPodRemove("type1", &common.NpuDevice{DeviceName: "device1"}, &PodResource{})
convey.So(result, convey.ShouldBeFalse)
})
convey.Convey("When pod removed, return true", func() {
mockIsPodMove := gomonkey.ApplyMethodReturn(&PodResource{}, "IsPodMoveComplete", true)
defer mockIsPodMove.Reset()
result := hdm.isPodRemove("type1", &common.NpuDevice{DeviceName: "device1"}, &PodResource{})
convey.So(result, convey.ShouldBeTrue)
})
})
}
func TestUpdateFaultConfigFromCm(t *testing.T) {
originalVersion := resourceVersion
originalCardType := common.ParamOption.RealCardType
originalEnableSwitchFault := common.ParamOption.EnableSwitchFault
configMap := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "new-version"}}
convey.Convey("Test updateFaultConfigFromCm", t, func() {
convey.Convey("When version not changed, do nothing", func() {
resourceVersion = "new-version"
convey.So(CapturePanic(func() { updateFaultConfigFromCm(configMap) }), convey.ShouldBeNil)
})
patch := gomonkey.ApplyFunc(loadFaultCode, func(_ *v1.ConfigMap) {}).
ApplyFunc(loadFaultCustomization, func(_ *v1.ConfigMap) {})
defer patch.Reset()
convey.Convey("When version changed, update config", func() {
resourceVersion = "old-version"
updateFaultConfigFromCm(configMap)
convey.So(resourceVersion, convey.ShouldEqual, "new-version")
})
common.ParamOption.RealCardType = api.Ascend910A3
common.ParamOption.EnableSwitchFault = true
convey.Convey("When is Ascend910A3 with switch fault enabled", func() {
mockSwitch := gomonkey.ApplyFunc(loadSwitchFaultCode, func(_ *v1.ConfigMap) {}).
ApplyFunc(deviceswitch.UpdateSwitchFaultLevel, func() {})
defer mockSwitch.Reset()
convey.So(CapturePanic(func() { updateFaultConfigFromCm(configMap) }), convey.ShouldBeNil)
})
})
resourceVersion = originalVersion
common.ParamOption.RealCardType = originalCardType
common.ParamOption.EnableSwitchFault = originalEnableSwitchFault
}
func TestLoadFaultCustomization(t *testing.T) {
configMap := &v1.ConfigMap{Data: map[string]string{common.FaultCustomizationKey: "test-data"}}
emptyConfig := &v1.ConfigMap{Data: map[string]string{}}
convey.Convey("Test loadFaultCustomization", t, func() {
patch := gomonkey.ApplyFunc(common.ResetFaultCustomizationCache, func() {}).
ApplyFunc(common.LoadFaultCustomizationFromFile, func() error { return nil })
defer patch.Reset()
convey.Convey("When key not found, load from file", func() {
convey.So(CapturePanic(func() { loadFaultCustomization(emptyConfig) }), convey.ShouldBeNil)
})
convey.Convey("When key not found, load from file error", func() {
patch1 := gomonkey.ApplyFunc(common.LoadFaultCustomizationFromFile,
func() error { return errors.New("error") })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadFaultCustomization(emptyConfig) }), convey.ShouldBeNil)
})
patch1 := gomonkey.ApplyFunc(common.LoadFaultCustomization,
func([]byte) error { return errors.New("load error") })
defer patch1.Reset()
convey.Convey("When load from cm failed, fallback to file", func() {
convey.So(CapturePanic(func() { loadFaultCustomization(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When load from cm success", func() {
patch1 := gomonkey.ApplyFunc(common.LoadFaultCustomization, func([]byte) error { return nil })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadFaultCustomization(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When both cm and file load failed", func() {
patch2 := gomonkey.ApplyFunc(common.LoadFaultCustomizationFromFile,
func() error { return errors.New("file load error") })
defer patch2.Reset()
convey.So(CapturePanic(func() { loadFaultCustomization(configMap) }), convey.ShouldBeNil)
})
})
}
func TestLoadSwitchFaultCode(t *testing.T) {
configMap := &v1.ConfigMap{Data: map[string]string{common.SwitchFaultCodeKey: "test-data"}}
emptyConfig := &v1.ConfigMap{Data: map[string]string{}}
convey.Convey("Test loadSwitchFaultCode", t, func() {
patch := gomonkey.ApplyFunc(common.LoadSwitchFaultCodeFromFile, func() error { return nil })
defer patch.Reset()
convey.Convey("When key not found, load from file", func() {
convey.So(CapturePanic(func() { loadSwitchFaultCode(emptyConfig) }), convey.ShouldBeNil)
})
convey.Convey("When key not found, load from file error", func() {
patch1 := gomonkey.ApplyFunc(common.LoadSwitchFaultCodeFromFile,
func() error { return errors.New("error") })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadSwitchFaultCode(emptyConfig) }), convey.ShouldBeNil)
})
patch1 := gomonkey.ApplyFunc(common.LoadSwitchFaultCode,
func([]byte) error { return errors.New("load error") })
defer patch1.Reset()
convey.Convey("When load from cm failed, fallback to file", func() {
convey.So(CapturePanic(func() { loadSwitchFaultCode(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When load from cm success", func() {
patch1 := gomonkey.ApplyFunc(common.LoadSwitchFaultCode, func([]byte) error { return nil })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadSwitchFaultCode(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When both cm and file load failed", func() {
patch2 := gomonkey.ApplyFunc(common.LoadSwitchFaultCodeFromFile,
func() error { return errors.New("file load error") })
defer patch2.Reset()
convey.So(CapturePanic(func() { loadSwitchFaultCode(configMap) }), convey.ShouldBeNil)
})
})
}
func TestLoadFaultCode(t *testing.T) {
configMap := &v1.ConfigMap{Data: map[string]string{common.FaultCodeKey: "test-data"}}
emptyConfig := &v1.ConfigMap{Data: map[string]string{}}
convey.Convey("Test loadFaultCode", t, func() {
patch := gomonkey.ApplyFunc(common.LoadFaultCodeFromFile, func() error { return nil })
defer patch.Reset()
convey.Convey("When key not found, load from file", func() {
convey.So(CapturePanic(func() { loadFaultCode(emptyConfig) }), convey.ShouldBeNil)
})
convey.Convey("When key not found, load from file error", func() {
patch1 := gomonkey.ApplyFunc(common.LoadFaultCodeFromFile,
func() error { return errors.New("error") })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadFaultCode(emptyConfig) }), convey.ShouldBeNil)
})
patch1 := gomonkey.ApplyFunc(common.LoadFaultCode,
func([]byte) error { return errors.New("load error") })
defer patch1.Reset()
convey.Convey("When load from cm failed, fallback to file", func() {
convey.So(CapturePanic(func() { loadFaultCode(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When load from cm success", func() {
patch1 := gomonkey.ApplyFunc(common.LoadFaultCode, func([]byte) error { return nil })
defer patch1.Reset()
convey.So(CapturePanic(func() { loadFaultCode(configMap) }), convey.ShouldBeNil)
})
convey.Convey("When both cm and file load failed", func() {
patch2 := gomonkey.ApplyFunc(common.LoadFaultCodeFromFile,
func() error { return errors.New("file load error") })
defer patch2.Reset()
convey.So(CapturePanic(func() { loadFaultCode(configMap) }), convey.ShouldBeNil)
})
})
}
type chipHotResetFor300IDuoTestCase struct {
Name string
mockIsCompleted bool
groupDevice map[string][]*common.NpuDevice
serverMap map[string]InterfaceServer
wantFailedTimes int
}
func buildChipHotResetFor300IDuoTestCases() []chipHotResetFor300IDuoTestCase {
return []chipHotResetFor300IDuoTestCase{
{
Name: "01-pod not move, should not reset chip",
mockIsCompleted: false,
groupDevice: map[string][]*common.NpuDevice{
"Ascend310P-4c": {
{CardID: 0, LogicID: 0, Health: v1beta1.Healthy},
{CardID: 0, LogicID: 1, Health: v1beta1.Unhealthy},
},
},
serverMap: map[string]InterfaceServer{"Ascend310P-4c": &PluginServer{}},
wantFailedTimes: 1,
},
{
Name: "02-reset chip",
mockIsCompleted: true,
groupDevice: map[string][]*common.NpuDevice{
"ascend310-4": {
{CardID: 0, LogicID: 0, Health: v1beta1.Healthy},
{CardID: 0, LogicID: 1, Health: v1beta1.Unhealthy},
},
},
serverMap: map[string]InterfaceServer{"ascend310-4": &PluginServer{}},
wantFailedTimes: 0,
},
}
}
func TestChipHotResetFor300IDuo(t *testing.T) {
testCases := buildChipHotResetFor300IDuoTestCases()
patch := gomonkey.ApplyGlobalVar(&common.ParamOption,
common.Option{ProductTypes: []string{common.Atlas300IDuo}, HotReset: common.HotResetInfer}).
ApplyMethodReturn(&kubeclient.ClientK8s{}, "GetAllPodListCache", nil).
ApplyFuncReturn(wait.PollImmediate, nil)
defer patch.Reset()
for _, tt := range testCases {
t.Run(tt.Name, func(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend310Manager(),
}
hdm.groupDevice = tt.groupDevice
hdm.ServerMap = tt.serverMap
patch1 := gomonkey.ApplyMethodReturn(&PodResource{}, "IsPodMoveComplete", tt.mockIsCompleted)
hdm.manager.SetResetFailedTimes(0, 1)
hdm.chipHotReset()
patch1.Reset()
failedTimes := hdm.manager.GetResetFailedTimes(0)
if failedTimes != tt.wantFailedTimes {
t.Errorf("chipHotReset() ResetFailedTimes = %v, "+
"wantFailedTimes = %v", failedTimes, tt.wantFailedTimes)
}
})
}
}
func TestParseTriggers(t *testing.T) {
deviceInfoHandled := false
patch := gomonkey.ApplyPrivateMethod(&HwDevManager{}, "handleDeviceInfoUpdate",
func(_ *HwDevManager, initTime *time.Time) {
deviceInfoHandled = true
})
defer patch.Reset()
convey.Convey("has signal, should update device info", t, func() {
hdm := &HwDevManager{}
select {
case common.GetUpdateChan() <- struct{}{}:
fmt.Print("send to update channel")
default:
fmt.Println("update channel is full")
}
hdm.parseTriggers(context.TODO(), time.Now())
convey.So(deviceInfoHandled, convey.ShouldBeTrue)
})
convey.Convey("no signal, should not update device info", t, func() {
hdm := &HwDevManager{}
deviceInfoHandled = false
select {
case <-common.GetUpdateChan():
fmt.Print("clear update channel")
default:
fmt.Println("update channel is empty")
}
hdm.parseTriggers(context.TODO(), time.Now())
convey.So(deviceInfoHandled, convey.ShouldBeFalse)
})
}
func TestUpdateNodeAnnotations(t *testing.T) {
convey.Convey("TestUpdateNodeAnnotations", t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
convey.Convey("01-npu IP not changed will do nothing", func() {
patch := gomonkey.ApplyPrivateMethod(hdm, "compareBaseNPUInfo", func(_ *HwDevManager) (bool,
map[string]*common.NpuBaseInfo) {
return false, nil
})
defer patch.Reset()
hdm.doUpdateNodeAnnotations()
convey.So(hdm.baseNPUInfo, convey.ShouldBeNil)
})
patch := gomonkey.ApplyPrivateMethod(hdm, "compareBaseNPUInfo", func(_ *HwDevManager) (bool,
map[string]*common.NpuBaseInfo) {
return true, map[string]*common.NpuBaseInfo{
"Ascend910-0": {IP: "127.0.1.1"}}
})
defer patch.Reset()
preBaseInfo := map[string]*common.NpuBaseInfo{"Ascend910-0": {IP: "127.0.0.1"}}
hdm.baseNPUInfo = preBaseInfo
client := &kubeclient.ClientK8s{}
patch2 := gomonkey.ApplyMethodReturn(hdm.manager, "GetKubeClient", client)
defer patch2.Reset()
convey.Convey("02-update node annotations failed will not refresh cache", func() {
patch3 := gomonkey.ApplyMethodReturn(client, "AddAnnotation", errors.New("patch node failed"))
defer patch3.Reset()
hdm.doUpdateNodeAnnotations()
convey.So(hdm.baseNPUInfo, convey.ShouldResemble, preBaseInfo)
})
convey.Convey("04-update node annotations succeed will refresh cache", func() {
patch3 := gomonkey.ApplyMethodReturn(client, "AddAnnotation", nil)
defer patch3.Reset()
hdm.doUpdateNodeAnnotations()
convey.So(hdm.baseNPUInfo, convey.ShouldResemble, map[string]*common.NpuBaseInfo{
"Ascend910-0": {IP: "127.0.1.1"}})
})
})
}
func TestCompareBaseNPUInfo(t *testing.T) {
convey.Convey("TestCompareBaseNPUInfo", t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
allInfo: common.NpuAllInfo{
AllDevs: []common.NpuDevice{{
DeviceName: "Ascend910-0",
IP: "127.0.0.1",
}},
},
baseNPUInfo: map[string]*common.NpuBaseInfo{
"Ascend910-0": {
IP: "127.0.0.1",
SuperDeviceID: 0,
}},
}
hdm.manager.SetDmgr(&devmanager.DeviceManagerMock{})
convey.Convey("01-get npu IP failed should return false", func() {
patch := gomonkey.ApplyMethod(hdm.manager, "GetDeviceIP", func(_ *device.HwAscend910Manager,
_ string, _ int) (string, error) {
return "", errors.New("get npu IP failed")
})
defer patch.Reset()
res, _ := hdm.compareBaseNPUInfo()
convey.So(res, convey.ShouldBeFalse)
})
convey.Convey("02-npu IP not changed should return false", func() {
patch := gomonkey.ApplyMethod(hdm.manager, "GetDeviceIP", func(_ *device.HwAscend910Manager,
_ string, _ int) (string, error) {
return "127.0.0.1", nil
})
defer patch.Reset()
res, newInfo := hdm.compareBaseNPUInfo()
convey.So(res, convey.ShouldBeFalse)
convey.So(newInfo, convey.ShouldResemble, hdm.baseNPUInfo)
})
convey.Convey("03-npu IP changed should return false return true", func() {
patch := gomonkey.ApplyMethod(hdm.manager, "GetDeviceIP", func(_ *device.HwAscend910Manager,
_ string, _ int) (string, error) {
return "127.0.1.1", nil
})
defer patch.Reset()
res, newInfo := hdm.compareBaseNPUInfo()
convey.So(res, convey.ShouldBeTrue)
convey.So(newInfo, convey.ShouldResemble, map[string]*common.NpuBaseInfo{
"Ascend910-0": {IP: "127.0.1.1"}})
})
})
}
func TestUpdateDeviceUsedInfo(t *testing.T) {
convey.Convey("TestUpdateDeviceUsedInfo when device used by pod, the PodUsed should be true",
t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
client := &kubeclient.ClientK8s{}
mockGetClient := gomonkey.ApplyMethodReturn(hdm.manager, "GetKubeClient", client)
mockGetPodsUsedNPUByKlt := gomonkey.ApplyMethodReturn(client, "GetPodsUsedNPUByKlt",
sets.NewString("Ascend910-1"))
defer mockGetPodsUsedNPUByKlt.Reset()
defer mockGetClient.Reset()
groupDevice := map[string][]*common.NpuDevice{
api.Ascend910: {
&common.NpuDevice{
DeviceName: "Ascend910-0",
},
&common.NpuDevice{
DeviceName: "Ascend910-1",
},
},
}
hdm.updateDeviceUsedInfo(groupDevice)
convey.So(groupDevice[api.Ascend910][1].PodUsed, convey.ShouldBeTrue)
})
}
func TestSetRestartForAll(t *testing.T) {
convey.Convey("TestSetRestartForAll case 1", t, func() {
ps := PluginServer{}
hdm := &HwDevManager{
ServerMap: map[string]InterfaceServer{
"1": &ps,
},
}
hdm.setRestartForAll()
convey.So(ps.GetRestartFlag(), convey.ShouldBeTrue)
})
}
func TestPluginNotify(t *testing.T) {
convey.Convey("TestPluginNotify case 1", t, func() {
dev := &HwDevManager{
ServerMap: map[string]InterfaceServer{
"1": nil,
},
}
dev.pluginNotify(make([]*common.NpuDevice, 0), "2")
})
convey.Convey("TestPluginNotify case 2", t, func() {
dev := &HwDevManager{
ServerMap: map[string]InterfaceServer{
"1": nil,
},
}
dev.pluginNotify(make([]*common.NpuDevice, 0), "1")
})
}
func TestStopAllSever(t *testing.T) {
convey.Convey("test stopAllSever case 1", t, func() {
ps := PluginServer{}
hdm := &HwDevManager{
ServerMap: map[string]InterfaceServer{
"1": &ps,
},
}
hdm.stopAllSever()
})
}
func TestHwDevManagerCheckNoProc(t *testing.T) {
convey.Convey("TestHwDevManagerCheckNoProc when device used by some process, the func should be false",
t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
manager := &devmanager.DeviceManager{}
gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", manager)
info := &npuCommon.DevProcessInfo{
ProcNum: 1,
}
gomonkey.ApplyMethodReturn(manager, "GetDevProcessInfo", info, nil)
checkNoProc := hdm.checkNoProc(0)
convey.So(checkNoProc, convey.ShouldBeFalse)
})
convey.Convey("TestHwDevManagerCheckNoProc when device used by no process, the func should be true",
t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
manager := &devmanager.DeviceManager{}
gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", manager)
info := &npuCommon.DevProcessInfo{
ProcNum: 0,
}
gomonkey.ApplyMethodReturn(manager, "GetDevProcessInfo", info, nil)
checkNoProc := hdm.checkNoProc(0)
convey.So(checkNoProc, convey.ShouldBeTrue)
})
}
type parseAnnotationIntValueTestCase struct {
name string
pod v1.Pod
annoKey string
desc string
expectedVal int
expectError bool
}
func buildParseAnnotationIntValueTestCases() []parseAnnotationIntValueTestCase {
return []parseAnnotationIntValueTestCase{
{
name: "TC1 - Annotations is nil",
pod: v1.Pod{ObjectMeta: metav1.ObjectMeta{}},
annoKey: "test-key",
desc: "test description",
expectedVal: 0,
expectError: true,
},
{
name: "TC2 - Annotation key not found",
pod: v1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"other-key": "value"}}},
annoKey: "test-key",
desc: "test description",
expectedVal: 0,
expectError: true,
},
{
name: "TC3 - Special key with valid value",
pod: v1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare}}},
annoKey: api.SchedulerSoftShareDevPolicyKey,
desc: "scheduler policy",
expectedVal: 1,
expectError: false,
},
{
name: "TC4 - Normal key with invalid value",
pod: v1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
api.SchedulerSoftShareDevAicoreQuotaKey: "aa"}}},
annoKey: "normal-key",
desc: "normal annotation",
expectedVal: 0,
expectError: true,
},
}
}
func TestParseAnnotationIntValue(t *testing.T) {
convey.Convey("Testing parseAnnotationIntValue function", t, func() {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
testCases := buildParseAnnotationIntValueTestCases()
for _, tc := range testCases {
convey.Convey(tc.name, func() {
result, err := hdm.parseAnnotationIntValue(tc.pod, tc.annoKey, tc.desc)
if tc.expectError {
convey.So(err, convey.ShouldNotBeNil)
convey.So(tc.expectedVal, convey.ShouldEqual, result)
} else {
convey.So(err, convey.ShouldBeNil)
convey.So(tc.expectedVal, convey.ShouldEqual, result)
}
})
}
})
}
func TestCalculateCardUsedResourceQuota1(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
convey.Convey("Test calculateCardUsedResourceQuota", t, func() {
convey.Convey("Empty podDeviceInfoList returns empty map", func() {
var emptyList []*common.PodDeviceInfo
result, err := hdm.calculateCardUsedResourceQuota(emptyList)
convey.So(err, convey.ShouldBeNil)
convey.So(len(result), convey.ShouldEqual, 0)
})
convey.Convey("Single pod with valid annotations returns correct quota", func() {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-1",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-0",
api.SchedulerSoftShareDevAicoreQuotaKey: "10",
api.SchedulerSoftShareDevHbmQuotaKey: "1024",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare,
},
},
Status: v1.PodStatus{Phase: v1.PodRunning},
}
podDeviceInfoList := []*common.PodDeviceInfo{{Pod: *pod}}
result, err := hdm.calculateCardUsedResourceQuota(podDeviceInfoList)
convey.So(err, convey.ShouldBeNil)
convey.So(len(result), convey.ShouldEqual, 1)
convey.So(result["card-0"].aicoreQuota, convey.ShouldEqual, numTen)
convey.So(result["card-0"].hbmQuota, convey.ShouldEqual, common.MBPerGB)
convey.So(result["card-0"].schedulingPolicy, convey.ShouldEqual, 1)
})
})
}
func TestCalculateCardUsedResourceQuota2(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
convey.Convey("Test calculateCardUsedResourceQuota", t, func() {
convey.Convey("Multiple pods for same card sum up quota", func() {
pod1 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-1",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-0",
api.SchedulerSoftShareDevAicoreQuotaKey: "10",
api.SchedulerSoftShareDevHbmQuotaKey: "1024",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare,
},
},
}
pod2 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-2",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-0",
api.SchedulerSoftShareDevAicoreQuotaKey: "20",
api.SchedulerSoftShareDevHbmQuotaKey: "2048",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare,
},
},
}
podDeviceInfoList := []*common.PodDeviceInfo{{Pod: *pod1}, {Pod: *pod2}}
result, err := hdm.calculateCardUsedResourceQuota(podDeviceInfoList)
convey.So(err, convey.ShouldBeNil)
convey.So(result["card-0"].aicoreQuota, convey.ShouldEqual, numThirty)
convey.So(result["card-0"].hbmQuota, convey.ShouldEqual, numThree*common.MBPerGB)
convey.So(result["card-0"].schedulingPolicy, convey.ShouldEqual, 1)
})
})
}
func TestCalculateCardUsedResourceQuota3(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
convey.Convey("Test calculateCardUsedResourceQuota", t, func() {
convey.Convey("Pod missing AscendReal annotation is skipped", func() {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-1",
Namespace: "test-ns",
Annotations: map[string]string{
api.SchedulerSoftShareDevAicoreQuotaKey: "10",
},
},
}
podDeviceInfoList := []*common.PodDeviceInfo{{Pod: *pod}}
result, err := hdm.calculateCardUsedResourceQuota(podDeviceInfoList)
convey.So(err, convey.ShouldBeNil)
convey.So(len(result), convey.ShouldEqual, 0)
})
convey.Convey("Pod with invalid annotation value is skipped", func() {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-1",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-0",
api.SchedulerSoftShareDevAicoreQuotaKey: "invalid",
api.SchedulerSoftShareDevHbmQuotaKey: "1024",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare,
},
},
}
podDeviceInfoList := []*common.PodDeviceInfo{{Pod: *pod}}
result, err := hdm.calculateCardUsedResourceQuota(podDeviceInfoList)
convey.So(err, convey.ShouldBeNil)
convey.So(len(result), convey.ShouldEqual, 0)
})
})
}
func TestCalculateCardUsedResourceQuota4(t *testing.T) {
hdm := &HwDevManager{
manager: device.NewHwAscend910Manager(),
}
convey.Convey("Test calculateCardUsedResourceQuota", t, func() {
convey.Convey("Multiple pods for different cards calculate quota independently", func() {
pod1 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-1",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-0",
api.SchedulerSoftShareDevAicoreQuotaKey: "10",
api.SchedulerSoftShareDevHbmQuotaKey: "1024",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyFixedShare,
},
},
}
pod2 := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-2",
Namespace: "test-ns",
Annotations: map[string]string{
api.PodAnnotationAscendReal: "card-1",
api.SchedulerSoftShareDevAicoreQuotaKey: "30",
api.SchedulerSoftShareDevHbmQuotaKey: "3072",
api.SchedulerSoftShareDevPolicyKey: api.SoftShareDeviceSchedulingPolicyElastic,
},
},
}
podDeviceInfoList := []*common.PodDeviceInfo{{Pod: *pod1}, {Pod: *pod2}}
result, err := hdm.calculateCardUsedResourceQuota(podDeviceInfoList)
convey.So(err, convey.ShouldBeNil)
convey.So(len(result), convey.ShouldEqual, numTwo)
convey.So(result["card-0"].aicoreQuota, convey.ShouldEqual, numTen)
convey.So(result["card-0"].hbmQuota, convey.ShouldEqual, common.MBPerGB)
convey.So(result["card-0"].schedulingPolicy, convey.ShouldEqual, 1)
convey.So(result["card-1"].aicoreQuota, convey.ShouldEqual, numThirty)
convey.So(result["card-1"].hbmQuota, convey.ShouldEqual, numThree*common.MBPerGB)
convey.So(result["card-1"].schedulingPolicy, convey.ShouldEqual, numTwo)
})
})
}
func TestGetChipMemory(t *testing.T) {
hdm := &HwDevManager{manager: device.NewHwAscend910Manager(), allInfo: common.NpuAllInfo{
AllDevs: []common.NpuDevice{{LogicID: 0}}}}
convey.Convey("Test getChipMemory", t, func() {
convey.Convey("On-chip memory not enabled, return 0", func() {
patchHasOnChip := gomonkey.ApplyFuncReturn(common.HasOnChipMemory, false)
defer patchHasOnChip.Reset()
result := hdm.getChipMemory()
convey.So(result, convey.ShouldEqual, 0)
})
patchHasOnChip := gomonkey.ApplyFuncReturn(common.HasOnChipMemory, true)
defer patchHasOnChip.Reset()
mockDmgr := gomonkey.ApplyMethodReturn(hdm.manager, "GetDmgr", &devmanager.DeviceManager{})
defer mockDmgr.Reset()
convey.Convey("On-chip memory enabled but get HBM info failed, return 0", func() {
patchGetHbmInfo := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetDeviceHbmInfo",
nil, errors.New("get HBM info failed"))
defer patchGetHbmInfo.Reset()
result := hdm.getChipMemory()
convey.So(result, convey.ShouldEqual, 0)
})
convey.Convey("On-chip memory enabled and get HBM info success, return correct memory size", func() {
patchGetHbmInfo := gomonkey.ApplyMethodReturn(hdm.manager.GetDmgr(), "GetDeviceHbmInfo",
&npuCommon.HbmInfo{MemorySize: uint64(expectedMemory)}, nil)
defer patchGetHbmInfo.Reset()
result := hdm.getChipMemory()
convey.So(result, convey.ShouldEqual, expectedMemory)
})
})
}
type getPluginServerTestCase struct {
name string
hdm *HwDevManager
deviceType string
wantServer *PluginServer
wantErr bool
}
func buildGetPluginServerTestCases() []getPluginServerTestCase {
return []getPluginServerTestCase{
{
name: "device type exists and element is *PluginServer",
hdm: &HwDevManager{
ServerMap: map[string]InterfaceServer{api.Ascend910: &PluginServer{}},
},
deviceType: api.Ascend910,
wantServer: &PluginServer{},
wantErr: false,
},
{
name: "device type not found in ServerMap",
hdm: &HwDevManager{
ServerMap: map[string]InterfaceServer{api.Ascend310: &PluginServer{}},
},
deviceType: api.Ascend910,
wantServer: nil,
wantErr: true,
},
{
name: "element type is not *PluginServer",
hdm: &HwDevManager{
ServerMap: map[string]InterfaceServer{api.Ascend910: nil},
},
deviceType: api.Ascend910,
wantServer: nil,
wantErr: true,
},
{
name: "ServerMap is nil",
hdm: &HwDevManager{
ServerMap: nil,
},
deviceType: api.Ascend910,
wantServer: nil,
wantErr: true,
},
}
}
func TestGetPluginServer(t *testing.T) {
tests := buildGetPluginServerTestCases()
convey.Convey("Test getPluginServer", t, func() {
for _, tt := range tests {
convey.Convey(tt.name, func() {
gotServer, err := tt.hdm.getPluginServer(tt.deviceType)
if tt.wantErr {
convey.So(err, convey.ShouldNotBeNil)
} else {
convey.So(err, convey.ShouldBeNil)
}
convey.So(gotServer, convey.ShouldResemble, tt.wantServer)
})
}
})
}
func TestUpdateQuota(t *testing.T) {
convey.Convey("Test updateQuota", t, func() {
errorMsg := "calculate quota failed"
successQuotaMap := map[string]shareDevResourceQuota{
ascend910LogicID0: {aicoreQuota: api.SoftShareDeviceMaxAICoreQuota, hbmQuota: common.MBPerGB},
}
convey.Convey("calculateCardUsedResourceQuota returns error, no quota update", func() {
hdm := &HwDevManager{
allInfo: common.NpuAllInfo{
AllDevs: []common.NpuDevice{{DeviceName: ascend910LogicID0}},
},
}
patchCalc := gomonkey.ApplyPrivateMethod(reflect.TypeOf(hdm), "calculateCardUsedResourceQuota",
func(_ *HwDevManager, _ []*common.PodDeviceInfo) (map[string]shareDevResourceQuota, error) {
return nil, errors.New(errorMsg)
})
defer patchCalc.Reset()
hdm.updateQuota(nil)
convey.So(hdm.allInfo.AllDevs[0].UsedAicoreQuota, convey.ShouldEqual, 0)
convey.So(hdm.allInfo.AllDevs[0].UsedHbmQuota, convey.ShouldEqual, 0)
})
convey.Convey("calculate success, update quota for matched devices only", func() {
hdm := &HwDevManager{
allInfo: common.NpuAllInfo{
AllDevs: []common.NpuDevice{{DeviceName: ascend910LogicID0}, {DeviceName: ascend910LogicID1}},
},
}
patchCalc := gomonkey.ApplyPrivateMethod(reflect.TypeOf(hdm), "calculateCardUsedResourceQuota",
func(_ *HwDevManager, _ []*common.PodDeviceInfo) (map[string]shareDevResourceQuota, error) {
return successQuotaMap, nil
})
defer patchCalc.Reset()
hdm.updateQuota(nil)
convey.So(hdm.allInfo.AllDevs[0].UsedAicoreQuota, convey.ShouldEqual, api.SoftShareDeviceMaxAICoreQuota)
convey.So(hdm.allInfo.AllDevs[0].UsedHbmQuota, convey.ShouldEqual, common.MBPerGB)
convey.So(hdm.allInfo.AllDevs[1].UsedAicoreQuota, convey.ShouldEqual, 0)
convey.So(hdm.allInfo.AllDevs[1].UsedHbmQuota, convey.ShouldEqual, 0)
})
})
}
func TestLoadDeviceFaultFromUpgradeReason(t *testing.T) {
convey.Convey("Test loadDeviceFaultFromUpgradeReason", t, func() {
manager := device.NewHwAscend910Manager()
hdm := &HwDevManager{
groupDevice: map[string][]*common.NpuDevice{api.Ascend910: {{LogicID: 0}}},
allInfo: common.NpuAllInfo{},
manager: manager,
}
patches := gomonkey.NewPatches()
defer patches.Reset()
addLinkdown := atomic.Bool{}
addAic := atomic.Bool{}
initLogicIDs := make([]int32, 0)
patches.ApplyFunc(common.GetUpgradeFaultLevelAndTime, func(int32, string) map[int64]common.FaultTimeAndLevel {
res := make(map[int64]common.FaultTimeAndLevel)
res[common.LinkDownFaultCode] = common.FaultTimeAndLevel{
FaultTime: 0,
FaultLevel: common.PreSeparateNPU,
}
res[common.AicBusFaultCode] = common.FaultTimeAndLevel{
FaultTime: 0,
FaultLevel: common.RestartRequest,
}
return res
}).ApplyFunc(common.DoSaveDevFaultInfo, func(devFaultInfo npuCommon.DevFaultInfo, enableDelay bool) {
if devFaultInfo.EventID == common.LinkDownFaultCode {
addLinkdown.Store(true)
}
if devFaultInfo.EventID == common.AicBusFaultCode {
addAic.Store(true)
}
}).ApplyFunc(common.SetDeviceInit, func(logicID int32) {
initLogicIDs = append(initLogicIDs, logicID)
}).ApplyMethod(manager, "UpdateHealth",
func(*device.HwAscend910Manager, map[string][]*common.NpuDevice, []*common.NpuDevice, string) {})
hdm.loadDeviceFaultFromUpgradeReason()
convey.So(addLinkdown.Load(), convey.ShouldBeTrue)
convey.So(addAic.Load(), convey.ShouldBeTrue)
convey.So(len(initLogicIDs), convey.ShouldNotBeZeroValue)
convey.So(initLogicIDs[0], convey.ShouldEqual, 0)
})
}
type registerSoftSharePodDeleteHandlerTestCase struct {
name string
supportSoftShare bool
kubeClientNil bool
podInformerNil bool
serverMapEmpty bool
expectRegistered bool
}
func buildRegisterSoftSharePodDeleteHandlerTestCases() []registerSoftSharePodDeleteHandlerTestCase {
return []registerSoftSharePodDeleteHandlerTestCase{
{
name: "soft share not supported, should not register handler",
supportSoftShare: false,
kubeClientNil: false,
podInformerNil: false,
serverMapEmpty: false,
expectRegistered: false,
},
{
name: "kubeClient is nil, should not register handler",
supportSoftShare: true,
kubeClientNil: true,
podInformerNil: false,
serverMapEmpty: false,
expectRegistered: false,
},
{
name: "podInformer is nil, should not register handler",
supportSoftShare: true,
kubeClientNil: false,
podInformerNil: true,
serverMapEmpty: false,
expectRegistered: false,
},
{
name: "serverMap is empty, should not register handler",
supportSoftShare: true,
kubeClientNil: false,
podInformerNil: false,
serverMapEmpty: true,
expectRegistered: false,
},
{
name: "all conditions met, should register handler",
supportSoftShare: true,
kubeClientNil: false,
podInformerNil: false,
serverMapEmpty: false,
expectRegistered: true,
},
}
}
type mockSharedIndexInformer struct {
cache.SharedIndexInformer
handlerAdded bool
}
func (m *mockSharedIndexInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) {
m.handlerAdded = true
return nil, nil
}
func TestRegisterSoftSharePodDeleteHandler(t *testing.T) {
convey.Convey("Test registerSoftSharePodDeleteHandler", t, func() {
tests := buildRegisterSoftSharePodDeleteHandlerTestCases()
for _, tt := range tests {
convey.Convey(tt.name, func() {
patchSoftShare := gomonkey.ApplyFuncReturn(common.IsSupportSoftShareDevice, tt.supportSoftShare)
defer patchSoftShare.Reset()
mockInformer := &mockSharedIndexInformer{}
var kubeClient *kubeclient.ClientK8s
if !tt.kubeClientNil {
kubeClient = &kubeclient.ClientK8s{}
if !tt.podInformerNil {
kubeClient.PodInformer = mockInformer
}
}
manager := device.NewHwAscend910Manager()
patchGetKubeClient := gomonkey.ApplyMethod(manager, "GetKubeClient",
func(_ *device.HwAscend910Manager) *kubeclient.ClientK8s {
return kubeClient
})
defer patchGetKubeClient.Reset()
hdm := &HwDevManager{
manager: manager,
}
if !tt.serverMapEmpty {
ps := &PluginServer{}
hdm.ServerMap = map[string]InterfaceServer{
api.Ascend910: ps,
}
} else {
hdm.ServerMap = map[string]InterfaceServer{}
}
hdm.registerSoftSharePodDeleteHandler()
if tt.expectRegistered {
convey.So(mockInformer.handlerAdded, convey.ShouldBeTrue)
} else {
convey.So(mockInformer.handlerAdded, convey.ShouldBeFalse)
}
})
}
})
}
