* Copyright (c) 2025 Bocloud Technologies Co., Ltd.
* installer is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain n copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
******************************************************************/
package kube
import (
"context"
"fmt"
"time"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/controller-runtime/pkg/client"
bkev1beta1 "gopkg.openfuyao.cn/cluster-api-provider-bke/api/capbke/v1beta1"
bkeaddon "gopkg.openfuyao.cn/cluster-api-provider-bke/common/cluster/addon"
bkenode "gopkg.openfuyao.cn/cluster-api-provider-bke/common/cluster/node"
"gopkg.openfuyao.cn/cluster-api-provider-bke/pkg/phaseframe/phaseutil"
"gopkg.openfuyao.cn/cluster-api-provider-bke/utils"
"gopkg.openfuyao.cn/cluster-api-provider-bke/utils/log"
)
const (
ExpectedErrorCount = 2
DefaultRestConfigTimeout = 10
)
func init() {
if err := apiextv1.AddToScheme(scheme.Scheme); err != nil {
panic(err)
}
if err := apiextv1beta1.AddToScheme(scheme.Scheme); err != nil {
panic(err)
}
}
type RemoteKubeClient interface {
InstallAddon(bkeCluster *bkev1beta1.BKECluster, addonT *bkeaddon.AddonTransfer, addonRecorder *AddonRecorder, localClient client.Client, bkeNodes bkenode.Nodes) error
ApplyYaml(task *Task) error
NewK8sToken() (string, error)
KubeClient() (*kubernetes.Clientset, dynamic.Interface)
Collect() (*CollectResult, []error, []error)
CheckClusterHealth(cluster *bkev1beta1.BKECluster, version string, bkeNodes bkev1beta1.BKENodes) error
NodeHealthCheck(node *corev1.Node, expectVersion string, log *log.Logger) error
CheckComponentHealth(node *corev1.Node) error
ListNodes(option *metav1.ListOptions) (*corev1.NodeList, error)
GetPod(namespace, name string) (*corev1.Pod, error)
SetLogger(logger *log.Logger)
SetBKELogger(bkeLog *bkev1beta1.BKELogger)
}
type Client struct {
ClientSet *kubernetes.Clientset
DynamicClient dynamic.Interface
RestConfig *rest.Config
Log *log.Logger
BKELog *bkev1beta1.BKELogger
Ctx context.Context
}
func NewRemoteClusterClient(ctx context.Context, c client.Client, bkeCluster *bkev1beta1.BKECluster) (client.Client, error) {
var errs []error
restConfig, err := remote.RESTConfig(ctx, "cluster-cache-tracker", c, util.ObjectKey(bkeCluster))
if err != nil {
errs = append(errs, err)
}
if len(errs) != 0 && bkeCluster.Spec.ControlPlaneEndpoint.IsValid() {
restConfig, err = getRestConfigByToken(ctx, c, bkeCluster)
if err != nil {
errs = append(errs, err)
}
} else {
errs = append(errs, errors.Errorf("failed use k8s token create remote cluster client, BKECluster %q controlPlaneEndpoint is invalid", utils.ClientObjNS(bkeCluster)))
}
ret, err := client.New(restConfig, client.Options{Scheme: c.Scheme()})
if err != nil {
return nil, errors.Wrapf(err, "failed to create client for Cluster %s/%s", bkeCluster.Namespace, bkeCluster.Name)
}
return ret, nil
}
func NewClientFromRestConfig(ctx context.Context, config *rest.Config) (RemoteKubeClient, error) {
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, errors.Wrap(err, "failed to create cluster clientset")
}
dynamicClient, err := dynamic.NewForConfig(config)
if err != nil {
return nil, errors.Wrap(err, "failed to create remote cluster dynamicClient")
}
return &Client{
ClientSet: clientSet,
DynamicClient: dynamicClient,
RestConfig: config,
Log: log.With("module", "kube"),
Ctx: ctx,
}, nil
}
func NewRemoteClientByCluster(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) (RemoteKubeClient, error) {
config, err := remote.RESTConfig(ctx, "cluster-cache-tracker", c, util.ObjectKey(cluster))
if err != nil {
return nil, errors.Wrapf(err, "failed to get remote cluster %q config", cluster.Name)
}
return NewClientFromRestConfig(ctx, config)
}
func NewRemoteClientByBKECluster(ctx context.Context, c client.Client, bkeCluster *bkev1beta1.BKECluster) (RemoteKubeClient, error) {
var errs []error
config, err := remote.RESTConfig(ctx, "cluster-cache-tracker", c, util.ObjectKey(bkeCluster))
if err != nil {
errs = append(errs, err)
}
if len(errs) != 0 && bkeCluster.Spec.ControlPlaneEndpoint.IsValid() {
config, err = getRestConfigByToken(ctx, c, bkeCluster)
if err != nil {
errs = append(errs, err)
}
} else {
errs = append(errs, errors.Errorf("failed use k8s token create remote cluster client, BKECluster %q controlPlaneEndpoint is invalid", utils.ClientObjNS(bkeCluster)))
}
if len(errs) == ExpectedErrorCount {
return nil, kerrors.NewAggregate(errs)
}
return NewClientFromRestConfig(ctx, config)
}
func NewClientFromConfig(kubeConfigPath string) (RemoteKubeClient, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
if err != nil {
return nil, err
}
return NewClientFromRestConfig(context.Background(), config)
}
func NewClientFromKubeConfig(kubeConfig []byte) (RemoteKubeClient, error) {
config, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
if err != nil {
return nil, err
}
return NewClientFromRestConfig(context.Background(), config)
}
func NewClientFromK8sToken(host, port, token string) (RemoteKubeClient, error) {
config := &rest.Config{
BearerToken: token,
Host: fmt.Sprintf("https://%s:%s", host, port),
TLSClientConfig: rest.TLSClientConfig{
Insecure: true,
},
}
config.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-cache-tracker")
config.Timeout = DefaultRestConfigTimeout * time.Second
return NewClientFromRestConfig(context.Background(), config)
}
func (c *Client) KubeClient() (*kubernetes.Clientset, dynamic.Interface) {
return c.ClientSet, c.DynamicClient
}
func (c *Client) SetLogger(logger *log.Logger) {
c.Log = logger
}
func (c *Client) SetBKELogger(bkeLog *bkev1beta1.BKELogger) {
c.BKELog = bkeLog
}
func GetTargetClusterClient(ctx context.Context, c client.Client, bkeCluster *bkev1beta1.BKECluster) (*kubernetes.Clientset, dynamic.Interface, error) {
cluster, err := util.GetOwnerCluster(ctx, c, bkeCluster.ObjectMeta)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to get owner cluster for bkeCluster %q", bkeCluster.Name)
}
remoteClient, err := NewRemoteClientByCluster(ctx, c, cluster)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to create remote cluster %q client", cluster.Name)
}
cs, dc := remoteClient.KubeClient()
return cs, dc, nil
}
func getRestConfigByToken(ctx context.Context, c client.Client, bkeCluster *bkev1beta1.BKECluster) (*rest.Config, error) {
secret, err := phaseutil.GetK8sTokenSecret(ctx, c, bkeCluster)
if err != nil {
return nil, err
}
token, ok := secret.Data["token"]
if !ok || string(token) == "" {
return nil, errors.Errorf("token data in secret %q not found", utils.ClientObjNS(secret))
}
config := &rest.Config{
Host: fmt.Sprintf("https://%s", bkeCluster.Spec.ControlPlaneEndpoint.String()),
BearerToken: string(token),
BearerTokenFile: "",
TLSClientConfig: rest.TLSClientConfig{
Insecure: true,
},
}
return config, nil
}