package colocation
import (
"fmt"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
. "gitcode.com/openFuyao/e2e-auto-test/e2e/colocation/utils"
)
var _ = SIGDescribe("混部部署测试", Label("colocation-deployment"), func() {
It("【在离线混部-005/015/018/021/030】验证HLS业务部署", func() {
Expect(CheckVolcanoExists(k8sClient, ctx)).To(BeTrue(), "Volcano scheduler not found in cluster")
Expect(WaitVolcanoWebhookReady(k8sClient, ctx, 120*time.Second)).NotTo(HaveOccurred(), "Volcano admission webhook not ready")
enoughCPU, err := CheckNodeAllocatableCPU(k8sClient, ctx, 1100)
if err != nil || !enoughCPU {
Skip("节点可分配 CPU 不足 1 核,跳过 HLS 用例")
}
hlsPod, err := InstallHLSPodNamed(k8sClient, ctx, fmt.Sprintf("hls-pod-%d", time.Now().UnixNano()), "", "")
Expect(err).NotTo(HaveOccurred())
defer k8sClient.Clientset.CoreV1().Pods(hlsPod.Namespace).Delete(ctx, hlsPod.Name, metav1.DeleteOptions{})
Expect(WaitPodCondition(k8sClient, ctx, hlsPod.Name, hlsPod.Namespace, "Running", 360*time.Second)).To(BeTrue(), "HLS Pod failed to reach Running state")
freshPod, err := k8sClient.Clientset.CoreV1().Pods(hlsPod.Namespace).Get(ctx, hlsPod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(freshPod.Status.QOSClass).To(Equal(corev1.PodQOSGuaranteed))
Expect(freshPod.Annotations["openfuyao.com/qos-level"]).To(Equal("HLS"))
Expect(freshPod.Spec.SchedulerName).To(Equal("volcano"))
Expect(*freshPod.Spec.Priority).To(Equal(int32(10000)))
Expect(freshPod.Spec.PriorityClassName).To(Equal("priority-hls"))
Expect(freshPod.Spec.Affinity).NotTo(BeNil())
Expect(freshPod.Spec.Affinity.NodeAffinity).NotTo(BeNil())
container := freshPod.Spec.Containers[0]
Expect(container.Resources.Limits.Cpu()).To(Equal(container.Resources.Requests.Cpu()))
})
It("【在离线混部-009/016/019/031】验证LS业务部署", func() {
Expect(CheckVolcanoExists(k8sClient, ctx)).To(BeTrue(), "Volcano scheduler not found in cluster")
Expect(WaitVolcanoWebhookReady(k8sClient, ctx, 120*time.Second)).NotTo(HaveOccurred(), "Volcano admission webhook not ready")
lsPod, err := InstallLSPodNamed(k8sClient, ctx, fmt.Sprintf("ls-pod-%d", time.Now().UnixNano()), "", "")
Expect(err).NotTo(HaveOccurred())
defer k8sClient.Clientset.CoreV1().Pods(lsPod.Namespace).Delete(ctx, lsPod.Name, metav1.DeleteOptions{})
Expect(WaitPodCondition(k8sClient, ctx, lsPod.Name, lsPod.Namespace, "Running", 240*time.Second)).To(BeTrue(), "LS Pod failed to reach Running state")
freshPod, err := k8sClient.Clientset.CoreV1().Pods(lsPod.Namespace).Get(ctx, lsPod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(freshPod.Status.QOSClass).To(Equal(corev1.PodQOSBurstable))
Expect(freshPod.Annotations["openfuyao.com/qos-level"]).To(Equal("LS"))
Expect(freshPod.Spec.SchedulerName).To(Equal("volcano"))
Expect(*freshPod.Spec.Priority).To(Equal(int32(1000)))
Expect(freshPod.Spec.PriorityClassName).To(Equal("priority-ls"))
Expect(freshPod.Spec.Affinity).NotTo(BeNil())
Expect(freshPod.Spec.Affinity.NodeAffinity).NotTo(BeNil())
})
It("【在离线混部-010/017/020/032/033】验证BE业务部署", func() {
Expect(CheckVolcanoExists(k8sClient, ctx)).To(BeTrue(), "Volcano scheduler not found in cluster")
Expect(WaitVolcanoWebhookReady(k8sClient, ctx, 120*time.Second)).NotTo(HaveOccurred(), "Volcano admission webhook not ready")
bePod, err := InstallBEPodNamed(k8sClient, ctx, fmt.Sprintf("be-pod-%d", time.Now().UnixNano()), "", "")
Expect(err).NotTo(HaveOccurred())
defer k8sClient.Clientset.CoreV1().Pods(bePod.Namespace).Delete(ctx, bePod.Name, metav1.DeleteOptions{})
Expect(WaitPodCondition(k8sClient, ctx, bePod.Name, bePod.Namespace, "Running", 240*time.Second)).To(BeTrue(), "BE Pod failed to reach Running state")
freshPod, err := k8sClient.Clientset.CoreV1().Pods(bePod.Namespace).Get(ctx, bePod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(freshPod.Status.QOSClass).To(Equal(corev1.PodQOSBestEffort))
Expect(freshPod.Annotations["openfuyao.com/qos-level"]).To(Equal("BE"))
Expect(freshPod.Spec.SchedulerName).To(Equal("volcano"))
Expect(*freshPod.Spec.Priority).To(Equal(int32(-1000)))
Expect(freshPod.Spec.PriorityClassName).To(Equal("priority-be"))
Expect(freshPod.Spec.Affinity).NotTo(BeNil())
Expect(freshPod.Spec.Affinity.NodeAffinity).NotTo(BeNil())
Expect(freshPod.Annotations["volcano.sh/preemptable"]).To(Equal("true"))
Expect(freshPod.Annotations).To(HaveKey("openfuyao.com/extender-resource-cfg"))
})
It("【在离线混部-011/012/013/014】验证默认 PriorityClass 资源存在性", func() {
pcList, err := ListPriorityClasses(k8sClient, ctx)
Expect(err).NotTo(HaveOccurred())
expectedPriorities := map[string]int32{
"priority-hls": 10000,
"priority-ls": 1000,
"priority-be": -1000,
}
foundPriorities := make(map[string]int32)
for _, item := range pcList.Items {
name := item.GetName()
value, _, _ := unstructured.NestedInt64(item.Object, "value")
foundPriorities[name] = int32(value)
}
for name, value := range expectedPriorities {
Expect(foundPriorities).To(HaveKeyWithValue(name, value))
}
})
It("【在离线混部-029】colocation-configmap优化", func() {
cm, err := GetColocationConfigMap(k8sClient, ctx)
Expect(err).NotTo(HaveOccurred())
Expect(cm.Data).To(HaveKey("colocation-options"))
Expect(cm.Data).To(HaveKey("rubik-options"))
Expect(cm.Data).To(HaveKey("volcano-scheduler-options"))
Expect(cm.Data).NotTo(HaveKey("overread_namespaced_config_mapsubscription-options"))
})
})