*
* * Copyright (c) 2024 Huawei Technologies Co., Ltd.
* * openFuyao is licensed under Mulan PSL v2.
* * You can use this software according to the terms and conditions of the Mulan PSL v2.
* * You may obtain a copy of Mulan PSL v2 at:
* * http://license.coscl.org.cn/MulanPSL2
* * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* * See the Mulan PSL v2 for more details.
*
*/
package controller
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"time"
"gopkg.in/yaml.v2"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"volcano-config-service/pkg/zlog"
)
type NodeConfigController struct {
client.Client
Scheme *runtime.Scheme
}
type ConfigRequest struct {
EnableStaticCPUManager bool `json:"enableStaticCPUManager"`
EnableNUMADistance bool `json:"enableNUMADistance"`
KubeletConfigPath string `json:"kubeletConfigPath,omitempty"`
}
type KubeletConfig struct {
Kind string `yaml:"kind"`
ApiVersion string `yaml:"apiVersion"`
CpuManagerPolicy string `yaml:"cpuManagerPolicy"`
FeatureGates map[string]bool `yaml:"featureGates"`
TopologyManagerPolicyOptions map[string]string `yaml:"topologyManagerPolicyOptions"`
KubeReserved map[string]string `yaml:"kubeReserved"`
SystemReserved map[string]string `yaml:"systemReserved"`
}
const (
RequeueTimeMinutes = 5
featurePrefix = "TopologyManagerPolicy"
FeatureTopologyManagerPolicyOptions = featurePrefix + "Options"
FeatureTopologyManagerPolicyBetaOptions = featurePrefix + "BetaOptions"
FeatureTopologyManagerPolicyAlphaOptions = featurePrefix + "AlphaOptions"
)
func (r *NodeConfigController) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&NodeConfig{}).
WithOptions(controller.Options{
MaxConcurrentReconciles: 1,
}).
Complete(r)
}
func (r *NodeConfigController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
nodeName := os.Getenv("NODE_NAME")
if nodeName == "" {
return ctrl.Result{}, fmt.Errorf("NODE_NAME environment variable not set")
}
nodeConfig := &NodeConfig{}
if err := r.Get(ctx, req.NamespacedName, nodeConfig); err != nil {
zlog.FormatInfo("get nodeConfig")
if !errors.IsNotFound(err) {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}
nodeConf := r.findNodeConfiguration(nodeConfig, nodeName)
if nodeConf == nil {
return r.handleMissingNodeConfig(ctx, nodeConfig, nodeName)
}
return r.handleNodeConfigUpdate(ctx, nodeConfig, nodeName, nodeConf)
}
func (r *NodeConfigController) findNodeConfiguration(nodeConfig *NodeConfig, nodeName string) *NodeConfiguration {
for _, nc := range nodeConfig.Spec.Nodes {
if nc.NodeName == nodeName {
return &nc
}
}
return nil
}
func resolveDesiredNodeFlags(existingNodes []NodeConfiguration,
fallbackStaticCPUEnabled, fallbackNUMAEnabled bool) (bool, bool) {
desiredStaticCPUEnabled := fallbackStaticCPUEnabled
desiredNUMAEnabled := fallbackNUMAEnabled
if len(existingNodes) == 0 {
return desiredStaticCPUEnabled, desiredNUMAEnabled
}
uniformStatic := true
staticValue := existingNodes[0].EnableStaticCPUManager
for i := 1; i < len(existingNodes); i++ {
if existingNodes[i].EnableStaticCPUManager != staticValue {
uniformStatic = false
break
}
}
if uniformStatic {
desiredStaticCPUEnabled = staticValue
}
uniformNUMA := true
numaValue := existingNodes[0].EnableNUMADistance
for i := 1; i < len(existingNodes); i++ {
if existingNodes[i].EnableNUMADistance != numaValue {
uniformNUMA = false
break
}
}
if uniformNUMA {
desiredNUMAEnabled = numaValue
}
return desiredStaticCPUEnabled, desiredNUMAEnabled
}
func (r *NodeConfigController) handleMissingNodeConfig(ctx context.Context,
nodeConfig *NodeConfig, nodeName string) (ctrl.Result, error) {
currentConfig, err := r.readKubeletConfig(KubeletConfigPath)
if err != nil {
zlog.Error(err, "Failed to read kubelet config")
return ctrl.Result{}, err
}
currentStaticCPUEnabled, currentNUMAEnabled := r.parseCurrentConfiguration(currentConfig)
desiredStaticCPUEnabled, desiredNUMAEnabled := resolveDesiredNodeFlags(
nodeConfig.Spec.Nodes, currentStaticCPUEnabled, currentNUMAEnabled)
nodeConfig.Spec.Nodes = append(nodeConfig.Spec.Nodes, NodeConfiguration{
NodeName: nodeName,
EnableStaticCPUManager: desiredStaticCPUEnabled,
EnableNUMADistance: desiredNUMAEnabled,
KubeletConfigPath: KubeletConfigPath,
})
if err := r.Update(ctx, nodeConfig); err != nil {
zlog.Error(err, "Failed to update NodeConfig")
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}
func (r *NodeConfigController) handleNodeConfigUpdate(ctx context.Context,
nodeConfig *NodeConfig, nodeName string, nodeConf *NodeConfiguration) (ctrl.Result, error) {
currentConfig, err := r.readKubeletConfig(KubeletConfigPath)
if err != nil {
zlog.Error(err, "Failed to read kubelet config")
return ctrl.Result{}, err
}
currentStaticCPUEnabled, currentNUMAEnabled := r.parseCurrentConfiguration(currentConfig)
if nodeConf.EnableStaticCPUManager != currentStaticCPUEnabled ||
nodeConf.EnableNUMADistance != currentNUMAEnabled {
return r.updateConfiguration(ctx, nodeConfig, nodeName, nodeConf, currentConfig)
}
return ctrl.Result{RequeueAfter: time.Minute * RequeueTimeMinutes}, nil
}
func (r *NodeConfigController) parseCurrentConfiguration(
currentConfig map[string]interface{}) (bool, bool) {
currentStaticCPUEnabled := false
if cpuManagerPolicy, ok := currentConfig["cpuManagerPolicy"]; ok {
if policyStr, isString := cpuManagerPolicy.(string); isString {
currentStaticCPUEnabled = policyStr == "static"
}
}
currentNUMAEnabled := false
if topologyOptsRaw, exists := currentConfig["topologyManagerPolicyOptions"]; exists {
currentNUMAEnabled = r.parseNUMADistanceOption(topologyOptsRaw)
}
return currentStaticCPUEnabled, currentNUMAEnabled
}
func (r *NodeConfigController) parseNUMADistanceOption(topologyOptsRaw interface{}) bool {
switch opts := topologyOptsRaw.(type) {
case map[interface{}]interface{}:
if numaValue, ok := opts["prefer-closest-numa-nodes"]; ok {
return r.parseNUMABoolValue(numaValue)
}
case map[string]interface{}:
if numaValue, ok := opts["prefer-closest-numa-nodes"]; ok {
return r.parseNUMABoolValue(numaValue)
}
default:
return false
}
return false
}
func (r *NodeConfigController) parseNUMABoolValue(value interface{}) bool {
switch v := value.(type) {
case string:
return strings.EqualFold(v, "true")
case bool:
return v
default:
return false
}
}
func (r *NodeConfigController) updateConfiguration(ctx context.Context,
nodeConfig *NodeConfig, nodeName string, nodeConf *NodeConfiguration,
currentConfig map[string]interface{}) (ctrl.Result, error) {
if err := r.updateNodeStatusWithRetry(ctx, nodeConfig, nodeName,
NodeConfigStateConfiguring, ""); err != nil {
zlog.Error(err, "Failed to update node status")
return ctrl.Result{}, err
}
if err := r.updateKubeletConfig(currentConfig, nodeConf); err != nil {
zlog.Error(err, "Failed to update kubelet config")
if updateErr := r.updateNodeStatusWithRetry(ctx, nodeConfig, nodeName,
NodeConfigStateFailed, err.Error()); updateErr != nil {
zlog.Error(updateErr, "Failed to update failure status")
}
return ctrl.Result{}, err
}
if err := r.updateNodeStatusWithRetry(ctx, nodeConfig, nodeName,
NodeConfigStateConfigured, ""); err != nil {
zlog.Error(err, "Failed to update success status")
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: time.Minute * RequeueTimeMinutes}, nil
}
func (r *NodeConfigController) readKubeletConfig(path string) (map[string]interface{}, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read kubelet config file: %v", err)
}
config := make(map[string]interface{})
if err := yaml.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("failed to unmarshal kubelet config: %v", err)
}
return config, nil
}
func (r *NodeConfigController) updateKubeletConfig(config map[string]interface{}, nodeConf *NodeConfiguration) error {
modified := false
currentPolicy, _ := config["cpuManagerPolicy"].(string)
modifiedCPU := r.updateCPUManagerPolicy(config, nodeConf, currentPolicy)
modifiedNUMA := r.updateNUMADistanceSettings(config, nodeConf, currentPolicy)
modified = modifiedCPU || modifiedNUMA
r.ensureBasicConfigFields(config)
if modified {
return r.applyConfigChanges(nodeConf, config)
}
return nil
}
func (r *NodeConfigController) updateCPUManagerPolicy(
config map[string]interface{}, nodeConf *NodeConfiguration, currentPolicy string) bool {
if config == nil {
return false
}
modified := false
if nodeConf.EnableStaticCPUManager && currentPolicy != "static" {
config["cpuManagerPolicy"] = "static"
r.ensureResourceReservations(config)
modified = true
} else if !nodeConf.EnableStaticCPUManager && currentPolicy == "static" {
config["cpuManagerPolicy"] = "none"
modified = true
} else {
return modified
}
return modified
}
func (r *NodeConfigController) ensureResourceReservations(config map[string]interface{}) bool {
if config == nil {
return false
}
modified := false
if _, ok := config["systemReserved"].(map[string]interface{}); !ok {
config["systemReserved"] = map[string]interface{}{
"cpu": "0.5",
}
modified = true
}
if _, ok := config["kubeReserved"].(map[string]interface{}); !ok {
config["kubeReserved"] = map[string]interface{}{
"cpu": "0.5",
}
modified = true
}
return modified
}
func (r *NodeConfigController) updateNUMADistanceSettings(
config map[string]interface{}, nodeConf *NodeConfiguration, currentPolicy string) bool {
modified := false
if nodeConf.EnableNUMADistance {
modified = r.enableNUMADistance(config, currentPolicy)
} else {
modified = r.disableNUMADistance(config)
}
return modified
}
func (r *NodeConfigController) enableNUMADistance(config map[string]interface{}, currentPolicy string) bool {
if config == nil {
return false
}
modified := false
if currentPolicy != "static" {
config["cpuManagerPolicy"] = "static"
r.ensureResourceReservations(config)
modified = true
}
modifiedFeatures := r.updateFeatureGates(config)
modifiedTopology := r.updateTopologyOptions(config, true)
return modified || modifiedFeatures || modifiedTopology
}
func (r *NodeConfigController) disableNUMADistance(config map[string]interface{}) bool {
modified := false
if topologyOptsRaw, exists := config["topologyManagerPolicyOptions"]; exists {
switch topologyOpts := topologyOptsRaw.(type) {
case map[interface{}]interface{}:
if _, exists := topologyOpts["prefer-closest-numa-nodes"]; exists {
topologyOpts["prefer-closest-numa-nodes"] = "false"
modified = true
}
case map[string]interface{}:
if _, exists := topologyOpts["prefer-closest-numa-nodes"]; exists {
topologyOpts["prefer-closest-numa-nodes"] = "false"
modified = true
}
default:
return modified
}
}
return modified
}
func (r *NodeConfigController) updateFeatureGates(config map[string]interface{}) bool {
if config == nil {
return false
}
featureGates, ok := config["featureGates"].(map[string]interface{})
if !ok {
featureGates = make(map[string]interface{})
config["featureGates"] = featureGates
}
featureGates[FeatureTopologyManagerPolicyOptions] = true
featureGates[FeatureTopologyManagerPolicyBetaOptions] = true
featureGates[FeatureTopologyManagerPolicyAlphaOptions] = true
return true
}
func (r *NodeConfigController) updateTopologyOptions(config map[string]interface{}, enableNUMA bool) bool {
if config == nil {
return false
}
topologyOpts, ok := config["topologyManagerPolicyOptions"].(map[string]interface{})
if !ok {
topologyOpts = make(map[string]interface{})
config["topologyManagerPolicyOptions"] = topologyOpts
}
currentNUMAValue, _ := topologyOpts["prefer-closest-numa-nodes"].(string)
numaValue := "true"
if !enableNUMA {
numaValue = "false"
}
if currentNUMAValue != numaValue {
topologyOpts["prefer-closest-numa-nodes"] = numaValue
return true
}
return false
}
func (r *NodeConfigController) ensureBasicConfigFields(config map[string]interface{}) {
if config == nil {
return
}
if _, ok := config["kind"].(string); !ok {
config["kind"] = "KubeletConfiguration"
}
if _, ok := config["apiVersion"].(string); !ok {
config["apiVersion"] = "kubelet.config.k8s.io/v1beta1"
}
}
func (r *NodeConfigController) applyConfigChanges(nodeConf *NodeConfiguration, config map[string]interface{}) error {
if err := r.deleteCPUManagerStateFile(); err != nil {
return fmt.Errorf("failed to delete cpu_manager_state file: %v", err)
}
if err := r.saveKubeletConfig(nodeConf.KubeletConfigPath, config); err != nil {
return fmt.Errorf("failed to save kubelet config: %v", err)
}
if KubeletConfigPath == "/var/lib/kubelet/config.yaml" {
if err := r.restartKubelet(); err != nil {
return fmt.Errorf("failed to restart kubelet: %v", err)
}
} else {
if err := r.restartBkeKubelet(); err != nil {
return fmt.Errorf("failed to restart kubelet: %v", err)
}
}
return nil
}
func (r *NodeConfigController) deleteCPUManagerStateFile() error {
statePath := "/var/lib/kubelet/cpu_manager_state"
if err := os.Remove(statePath); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
func (r *NodeConfigController) saveKubeletConfig(path string, config map[string]interface{}) error {
data, err := yaml.Marshal(config)
if err != nil {
return fmt.Errorf("failed to marshal kubelet config: %v", err)
}
const FilePermission = 0644
tmpfile := path + ".tmp"
if err := os.WriteFile(tmpfile, data, FilePermission); err != nil {
return fmt.Errorf("failed to write temporary config file: %v", err)
}
if err := os.Rename(tmpfile, path); err != nil {
if err := os.Remove(tmpfile); err != nil {
return fmt.Errorf("failed to remove temporary file: %v", err)
}
return fmt.Errorf("failed to replace config file: %v", err)
}
return nil
}
func (r *NodeConfigController) restartKubelet() error {
cmd := exec.Command("nsenter",
"-i/mnt/proc/1/ns/ipc",
"-m/mnt/proc/1/ns/mnt",
"-n/mnt/proc/1/ns/net",
"systemctl", "restart", "kubelet")
return cmd.Run()
}
func (r *NodeConfigController) restartBkeKubelet() error {
cmd := exec.Command("nsenter",
"-i/mnt/proc/1/ns/ipc",
"-m/mnt/proc/1/ns/mnt",
"-n/mnt/proc/1/ns/net",
"nerdctl", "-n", "k8s.io", "restart", "kubelet")
return cmd.Run()
}
func (r *NodeConfigController) updateNodeStatusWithRetry(ctx context.Context,
nodeConfig *NodeConfig, nodeName, state, errMsg string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
latest := &NodeConfig{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: nodeConfig.Namespace,
Name: nodeConfig.Name,
}, latest); err != nil {
return err
}
updated := false
for i, status := range latest.Status.NodeStatuses {
if status.NodeName == nodeName {
latest.Status.NodeStatuses[i].State = state
latest.Status.NodeStatuses[i].Error = errMsg
latest.Status.NodeStatuses[i].LastUpdateTime = metav1.Now()
if state == NodeConfigStateConfigured {
latest.Status.NodeStatuses[i].StaticCPUManagerEnabled =
nodeConfig.Spec.Nodes[i].EnableStaticCPUManager
latest.Status.NodeStatuses[i].NUMADistanceEnabled =
nodeConfig.Spec.Nodes[i].EnableNUMADistance
}
updated = true
break
}
}
if !updated {
newStatus := NodeStatus{
NodeName: nodeName,
State: state,
Error: errMsg,
LastUpdateTime: metav1.Now(),
}
latest.Status.NodeStatuses = append(latest.Status.NodeStatuses, newStatus)
}
latest.Status.LastUpdateTime = metav1.Now()
return r.Status().Update(ctx, latest)
})
}
func InitializeNodeConfig(client client.Client) error {
ctx := context.Background()
nodeName, err := getNodeName()
if err != nil {
return err
}
nodeConfig := &NodeConfig{}
zlog.FormatInfo("正在检查 nodeConfig 是否存在")
err = client.Get(ctx, types.NamespacedName{Name: "kubelet-config", Namespace: "default"}, nodeConfig)
if err == nil {
zlog.FormatInfo("nodeConfig 已存在,检查是否需要添加当前节点")
nodeFound := false
for _, nc := range nodeConfig.Spec.Nodes {
if nc.NodeName == nodeName {
nodeFound = true
zlog.FormatInfo("节点 %s 已存在于 NodeConfig 中", nodeName)
break
}
}
if !nodeFound {
zlog.FormatInfo("正在将节点 %s 添加到现有 NodeConfig 中", nodeName)
return addNodeToExistingConfig(ctx, client, nodeConfig, nodeName)
}
return nil
}
if !errors.IsNotFound(err) {
return err
}
return createNewNodeConfig(ctx, client, nodeName)
}
func addNodeToExistingConfig(ctx context.Context, client client.Client,
nodeConfig *NodeConfig, nodeName string) error {
currentConfig, err := readCurrentKubeletConfig()
if err != nil {
return err
}
currentStaticCPUEnabled, currentNUMAEnabled := analyzeCurrentConfig(currentConfig)
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
latest := &NodeConfig{}
err := client.Get(ctx, types.NamespacedName{
Name: nodeConfig.Name,
Namespace: nodeConfig.Namespace,
}, latest)
if err != nil {
return err
}
desiredStaticCPUEnabled, desiredNUMAEnabled := resolveDesiredNodeFlags(
latest.Spec.Nodes, currentStaticCPUEnabled, currentNUMAEnabled)
newNodeConf := NodeConfiguration{
NodeName: nodeName,
EnableStaticCPUManager: desiredStaticCPUEnabled,
EnableNUMADistance: desiredNUMAEnabled,
KubeletConfigPath: KubeletConfigPath,
}
latest.Spec.Nodes = append(latest.Spec.Nodes, newNodeConf)
if err := client.Update(ctx, latest); err != nil {
zlog.FormatError("更新 NodeConfig 失败: %v", err)
return err
}
zlog.FormatInfo("成功将节点 %s 添加到 NodeConfig", nodeName)
return nil
})
}
func getNodeName() (string, error) {
nodeName := os.Getenv("NODE_NAME")
if nodeName == "" {
return "", fmt.Errorf("NODE_NAME environment variable not set")
}
return nodeName, nil
}
func createNewNodeConfig(ctx context.Context, client client.Client, nodeName string) error {
currentConfig, err := readCurrentKubeletConfig()
if err != nil {
return err
}
currentStaticCPUEnabled, currentNUMAEnabled := analyzeCurrentConfig(currentConfig)
return createNodeConfigCR(ctx, client, nodeName, currentStaticCPUEnabled, currentNUMAEnabled)
}
func readCurrentKubeletConfig() (map[string]interface{}, error) {
zlog.FormatInfo("reading current kubelet config")
currentConfig := make(map[string]interface{})
data, err := os.ReadFile(KubeletConfigPath)
if err != nil {
zlog.FormatError("Failed to read kubelet config: %v", err)
return nil, fmt.Errorf("failed to read kubelet config: %v", err)
}
if err := yaml.Unmarshal(data, ¤tConfig); err != nil {
zlog.FormatError("Failed to unmarshal kubelet config: %v", err)
return nil, fmt.Errorf("failed to unmarshal kubelet config: %v", err)
}
return currentConfig, nil
}
func analyzeCurrentConfig(currentConfig map[string]interface{}) (bool, bool) {
cpuManagerPolicy, ok := currentConfig["cpuManagerPolicy"].(string)
currentStaticCPUEnabled := ok && cpuManagerPolicy == "static"
var currentNUMAEnabled bool
if topologyOpts, ok := currentConfig["topologyManagerPolicyOptions"].(map[string]interface{}); ok {
numaValue, exists := topologyOpts["prefer-closest-numa-nodes"]
currentNUMAEnabled = exists && numaValue == "true"
}
return currentStaticCPUEnabled, currentNUMAEnabled
}
func createNodeConfigCR(ctx context.Context, client client.Client,
nodeName string, staticCPUEnabled, numaEnabled bool) error {
zlog.FormatInfo("creating new nodeConfig CR")
nodeConfig := &NodeConfig{
ObjectMeta: metav1.ObjectMeta{
Name: "kubelet-config",
Namespace: "default",
},
Spec: NodeConfigSpec{
Nodes: []NodeConfiguration{
{
NodeName: nodeName,
EnableStaticCPUManager: staticCPUEnabled,
EnableNUMADistance: numaEnabled,
KubeletConfigPath: KubeletConfigPath,
},
},
},
}
if err := client.Create(ctx, nodeConfig); err != nil {
zlog.FormatError("Failed to create NodeConfig: %v", err)
return fmt.Errorf("failed to create NodeConfig: %v", err)
}
zlog.FormatInfo("successfully created nodeConfig CR")
return nil
}