* Copyright (c) 2024 Huawei Technologies Co., Ltd.
* openFuyao is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
package service
import (
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/emicklei/go-restful"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
"k8s.io/klog/v2"
colocationv1 "openfuyao.com/colocation-service/pkg/apis/colocation/v1"
"openfuyao.com/colocation-service/pkg/utils"
)
func (server *Server) initColocationNodeRouteNew() {
ws := server.ws
ws.Route(ws.GET("/colocation-configs/").To(server.getAllColocationNodesNew).
Doc("get all colocation nodes"))
ws.Route(
ws.PATCH("/colocation-nodes/{name}").
To(server.toggleColocationNodeByNameWithRetry).
Doc("toggle single node colocation status").
Param(ws.QueryParameter("colocation-status", "true / false").DataType("boolean").Required(true)),
)
ws.Route(
ws.POST("/colocation-configs/").
To(server.handleColocationConfigWithRetry).
Doc("receive post colocation configuration").
Reads(ColocationConfigRequest{}),
)
}
const (
NodeMasterLabelKey string = "node-role.kubernetes.io/master"
NodeControlPlaneLabelKey string = "node-role.kubernetes.io/control-plane"
)
type NodeAttr struct {
Uid string `json:"uid"`
Name string `json:"name"`
Addresses []string `json:"addresses"`
OSImage string `json:"osImage"`
Status string `json:"status"`
IsColocated bool `json:"isColocated"`
FeatureSupport *NodeFeatureSupportInfo `json:"featureSupport,omitempty"`
}
type NodeFeatureSupportInfo struct {
CPUPreemption string `json:"cpuPreemption"`
MemoryPreemption string `json:"memoryPreemption"`
QuotaTurbo string `json:"quotaTurbo"`
DynMemory string `json:"dynMemory"`
DynCache string `json:"dynCache"`
PSIDetection string `json:"psiDetection"`
}
type RubikConfig struct {
Eviction RubikEviction `json:"eviction"`
QuotaTurbo *QuotaTurboConfig `json:"quotaTurbo,omitempty"`
DynMemory *DynMemoryConfig `json:"dynMemory,omitempty"`
DynCache *DynCacheConfig `json:"dynCache,omitempty"`
PSI *PSIConfig `json:"psi,omitempty"`
}
type RubikEviction struct {
Enabled bool `json:"enable"`
CPUEvict CPUEvictConfig `json:"cpuevict"`
MemoryEvict MemoryEvictConfig `json:"memoryevict"`
}
type QuotaTurboConfig struct {
Enable bool `json:"enable"`
HighWaterMark int `json:"highWaterMark"`
AlarmWaterMark int `json:"alarmWaterMark"`
}
type DynMemoryConfig struct {
Enable bool `json:"enable"`
}
type DynCacheConfig struct {
Enable bool `json:"enable"`
L3Percent *CachePercentConfig `json:"l3Percent,omitempty"`
MemBandPercent *CachePercentConfig `json:"memBandPercent,omitempty"`
}
type CachePercentConfig struct {
Low int `json:"low"`
Mid int `json:"mid"`
High int `json:"high"`
}
type PSIConfig struct {
Enable bool `json:"enable"`
Resource []string `json:"resource"`
Avg10Threshold float64 `json:"avg10Threshold"`
}
type UsageThresholdConfig struct {
Enabled bool `json:"enable"`
CPUThreshold float64 `json:"cpu,omitempty"`
MemoryThreshold float64 `json:"memory,omitempty"`
}
type UsagePluginConfig struct {
Enabled bool `json:"enable"`
UsageThreshold UsageThresholdConfig `json:"usageThreshold,omitempty"`
}
type VolcanoSchedulerConfig struct {
UsagePlugin UsagePluginConfig `json:"usagePlugin"`
}
type NodeInfoWrapper struct {
NodeInfo NodeAttr `json:"nodeInfo"`
RubikConfig RubikConfig `json:"rubikConfig"`
VolcanoSchedulerConfig VolcanoSchedulerConfig `json:"volcanoSchedulerConfig"`
}
type ColocationConfigRequest struct {
NodeInfo []NodeAttr `json:"nodeInfo"`
RubikConfig RubikConfig `json:"rubikConfig"`
VolcanoSchedulerConfig VolcanoSchedulerConfig `json:"volcanoSchedulerConfig"`
}
func (server *Server) buildNodeAttr(node *corev1.Node) NodeAttr {
addr := getNodeAddresses(*node)
var addresses []string
if addr != "" {
addresses = append(addresses, addr)
}
isColocated := isColocationNode(node)
status := getNodeStatus(*node)
featureSupport := server.getNodeFeatureSupport(node)
return NodeAttr{
Uid: string(node.ObjectMeta.UID),
Name: node.ObjectMeta.Name,
Addresses: addresses,
OSImage: node.Status.NodeInfo.OSImage,
Status: status,
IsColocated: isColocated,
FeatureSupport: featureSupport,
}
}
func (server *Server) getAllColocationNodesNew(request *restful.Request, response *restful.Response) {
klog.Info("getAllColocationNodes")
nodes, err := server.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
klog.Error(err)
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
return
}
rubikConfig, err := server.getRubikConfig()
if err != nil {
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
return
}
volcanoSchedulerConfig, err := server.getVolcanoSchedulerConfig()
if err != nil {
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
return
}
var result ColocationConfigRequest
var nodeList []NodeAttr
for _, node := range nodes.Items {
n := node
nodeList = append(nodeList, server.buildNodeAttr(&n))
}
result = ColocationConfigRequest{
NodeInfo: nodeList,
RubikConfig: rubikConfig,
VolcanoSchedulerConfig: volcanoSchedulerConfig,
}
response.WriteHeaderAndEntity(http.StatusOK, utils.NewResponseResultOk(result))
}
func (server *Server) getNodeFeatureSupport(node *corev1.Node) *NodeFeatureSupportInfo {
featureSupport := &NodeFeatureSupportInfo{
CPUPreemption: "检测中",
MemoryPreemption: "检测中",
QuotaTurbo: "检测中",
DynMemory: "检测中",
DynCache: "检测中",
PSIDetection: "检测中",
}
if labels := node.Labels; labels != nil {
if val, exists := labels["openfuyao.com/colocation.feature.cpupreemption"]; exists && val == "true" {
featureSupport.CPUPreemption = "OK"
} else {
featureSupport.CPUPreemption = "内核缺少cpu.qos_level接口"
}
if val, exists := labels["openfuyao.com/colocation.feature.memorypreemption"]; exists && val == "true" {
featureSupport.MemoryPreemption = "OK"
} else {
featureSupport.MemoryPreemption = "内核缺少memory.qos_level接口"
}
if val, exists := labels["openfuyao.com/colocation.feature.quotaturbo"]; exists && val == "true" {
featureSupport.QuotaTurbo = "OK"
} else {
featureSupport.QuotaTurbo = "内核缺少CPU.CFS控制接口"
}
if val, exists := labels["openfuyao.com/colocation.feature.dynmemory"]; exists && val == "true" {
featureSupport.DynMemory = "OK"
} else {
featureSupport.DynMemory = "内核缺少memory.high_async_ratio接口"
}
if val, exists := labels["openfuyao.com/colocation.feature.dyncache"]; exists && val == "true" {
featureSupport.DynCache = "OK"
} else {
featureSupport.DynCache = "硬件不支持RDT/MPAM特性"
}
if val, exists := labels["openfuyao.com/colocation.feature.psi"]; exists && val == "true" {
featureSupport.PSIDetection = "OK"
} else {
featureSupport.PSIDetection = "内核不支持PSI特性"
}
}
return featureSupport
}
func (server *Server) getRubikConfig() (RubikConfig, error) {
colocationConfig, err := server.client.CoreV1().ConfigMaps(colocationConfigMapNamespace).
Get(context.Background(), "colocation-config", metav1.GetOptions{})
if err != nil {
klog.Errorf("Failed to get colocation-config ConfigMap: %v", err)
return RubikConfig{}, err
}
rubikConfig := RubikConfig{}
if rubikOpts, exists := colocationConfig.Data["rubik-options"]; exists {
if err := json.Unmarshal([]byte(rubikOpts), &rubikConfig); err != nil {
klog.Errorf("Failed to parse rubik-options: %v", err)
return rubikConfig, fmt.Errorf("rubik-options parse error: %w", err)
}
}
return rubikConfig, nil
}
func (server *Server) getVolcanoSchedulerConfig() (VolcanoSchedulerConfig, error) {
colocationConfig, err := server.client.CoreV1().ConfigMaps(colocationConfigMapNamespace).
Get(context.Background(), "colocation-config", metav1.GetOptions{})
if err != nil {
klog.Errorf("Failed to get colocation-config ConfigMap: %v", err)
return VolcanoSchedulerConfig{}, err
}
volcanoConfig := VolcanoSchedulerConfig{}
if volcanoOpts, exists := colocationConfig.Data["volcano-scheduler-options"]; exists {
if err := json.Unmarshal([]byte(volcanoOpts), &volcanoConfig); err != nil {
klog.Errorf("Failed to parse volcano-scheduler-options: %v", err)
return volcanoConfig, fmt.Errorf("volcano-scheduler-options parse error: %w", err)
}
}
return volcanoConfig, nil
}
func (server *Server) toggleColocationNodeByNameWithRetry(request *restful.Request, response *restful.Response) {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
return server.toggleColocationNodeByName(request, response)
})
if err != nil {
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
}
}
func (server *Server) handleColocationConfigWithRetry(request *restful.Request, response *restful.Response) {
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
return server.handleColocationConfig(request, response)
})
if err != nil {
klog.Error(err)
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
}
}
func (server *Server) handleColocationConfig(request *restful.Request, response *restful.Response) error {
newConfig := ColocationConfigRequest{}
err := request.ReadEntity(&newConfig)
if err != nil {
response.WriteError(http.StatusBadRequest, err)
return err
}
err = server.coverColocationConfig(newConfig)
if err != nil {
response.WriteError(http.StatusBadRequest, err)
return err
}
updatedConfig, err := server.getUpdatedColocationConfig()
if err != nil {
klog.Errorf("Failed to get updated config after POST: %v", err)
response.WriteHeaderAndEntity(http.StatusOK, utils.NewResponseResultOk(newConfig))
} else {
response.WriteHeaderAndEntity(http.StatusOK, utils.NewResponseResultOk(updatedConfig))
}
return nil
}
func (server *Server) getUpdatedColocationConfig() (*ColocationConfigRequest, error) {
nodes, err := server.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
rubikConfig, err := server.getRubikConfig()
if err != nil {
return nil, err
}
volcanoSchedulerConfig, err := server.getVolcanoSchedulerConfig()
if err != nil {
return nil, err
}
var nodeList []NodeAttr
for _, node := range nodes.Items {
n := node
nodeList = append(nodeList, server.buildNodeAttr(&n))
}
return &ColocationConfigRequest{
NodeInfo: nodeList,
RubikConfig: rubikConfig,
VolcanoSchedulerConfig: volcanoSchedulerConfig,
}, nil
}
func (server *Server) toggleColocationNodeByName(request *restful.Request, response *restful.Response) error {
targetStatus := request.QueryParameter("colocation-status")
if targetStatus != "true" && targetStatus != "false" {
return response.WriteErrorString(http.StatusBadRequest, "colocation-status must be true or false")
}
if targetStatus == "true" {
return server.addColocationNodeByName(request, response)
}
return server.deleteColocationNodeByName(request, response)
}
func (server *Server) addColocationNodeByName(request *restful.Request, response *restful.Response) error {
klog.InfoS(request.Request.URL.Path)
klog.InfoS(request.Request.URL.RawPath)
klog.InfoS(request.Request.URL.RawQuery)
nodeName := request.PathParameter("name")
klog.Info("Adding colocation node:", nodeName)
findNode, err := server.client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
if err != nil {
klog.InfoS("failed to get node:%#v", nodeName)
klog.Error(err)
return err
}
coloNode := []string{findNode.Name}
if err := server.modifyColocationConfig(coloNode, "add"); err != nil {
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
return err
}
response.WriteHeaderAndEntity(
http.StatusOK,
utils.NewResponseResultOk(map[string]interface{}{
"name": nodeName,
"colocation-status": "true",
}),
)
return nil
}
func (server *Server) deleteColocationNodeByName(request *restful.Request, response *restful.Response) error {
nodeName := request.PathParameter("name")
klog.Info("Deleting colocation node:", nodeName)
findNode, err := server.client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
if err != nil {
klog.InfoS("failed to get node:%#v", nodeName)
klog.Error(err)
return err
}
coloNode := []string{findNode.Name}
if err := server.modifyColocationConfig(coloNode, "delete"); err != nil {
response.WriteHeaderAndEntity(http.StatusInternalServerError,
utils.NewResponseResultWithError(AppCode, FeatureCode, err))
return err
}
response.WriteHeaderAndEntity(
http.StatusOK,
utils.NewResponseResultOk(map[string]interface{}{
"name": nodeName,
"colocation-status": "false",
}),
)
return nil
}
func isColocationNode(node *corev1.Node) bool {
if value, ok := node.Labels[colocationv1.ColocationNodeLabel]; ok {
return value == "true"
}
return false
}