package installation
import (
"fmt"
"path/filepath"
"strings"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gitcode.com/openFuyao/e2e-auto-test/e2e/framework/executor"
config "gitcode.com/openFuyao/e2e-auto-test/e2e/installation/bke-config"
"gitcode.com/openFuyao/e2e-auto-test/e2e/installation/utils"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
)
var _ = Describe("Addon tests", func() {
var (
sshExecutor *executor.SSHExecutor
localExecutor *executor.LocalExecutor
dynamicClient dynamic.Interface
clusterManager *utils.ClusterManager
configPath string
nodeConfigPath string
)
clusterName := fmt.Sprintf("test-addon-order-%d", time.Now().Unix())
BeforeEach(func() {
guideConfig := config.LoadGuideNodeFromEnv()
Expect(guideConfig.Host).NotTo(BeEmpty())
Expect(guideConfig.Password).NotTo(BeEmpty())
var err error
sshExecutor, err = executor.NewSSHExecutor(
guideConfig.Host,
guideConfig.Port,
guideConfig.Username,
guideConfig.Password,
)
Expect(err).NotTo(HaveOccurred())
kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config")
restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
Expect(err).NotTo(HaveOccurred())
dynamicClient, err = dynamic.NewForConfig(restConfig)
Expect(err).NotTo(HaveOccurred())
clusterManager = utils.NewClusterManager(sshExecutor, dynamicClient)
localExecutor = executor.NewLocalExecutor(30 * time.Second)
})
AfterEach(func() {
By("清理测试集群")
if clusterManager.ClusterExistsWithKubeconfig(clusterName, "") {
if err := clusterManager.DeleteClusterWithKubeconfig(clusterName, ""); err != nil {
GinkgoWriter.Printf("Failed to delete cluster: %v\n", err)
}
Eventually(func() bool {
return !clusterManager.ClusterExistsWithKubeconfig(clusterName, "")
}, uninstallTimeout, 10*time.Second).Should(BeTrue())
}
if configPath != "" {
if err := clusterManager.CleanupConfig(configPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup config file: %v\n", err)
}
}
if nodeConfigPath != "" {
if err := clusterManager.CleanupConfig(nodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup node config file: %v\n", err)
}
}
})
It("addon组件部署能力增强-addon顺序安装", Label("installation", "addon", "25.12", "post-init"), SpecTimeout(InstallationItTimeout), func(ctx SpecContext) {
nodes := config.LoadTestNodesFromEnv()
Expect(len(nodes)).To(BeNumerically(">=", 1), "至少需要一个节点用于创建集群")
masterNode := nodes[0]
masterNode.Role = []string{"master/node", "etcd"}
clusterConfig := config.NewDefaultBKEClusterConfig(clusterName, []config.NodeInfo{masterNode})
clusterConfig.ExcludeAddons = append(clusterConfig.ExcludeAddons, "bkeagent-deployer")
clusterConfig.ExtraAddons = append(clusterConfig.ExtraAddons,
config.AddonConfig{
Name: "bkeagent-deployer",
Version: "latest",
Block: true,
Param: map[string]string{
"tagVersion": "latest",
},
},
config.AddonConfig{Name: "redis", Version: "6.2.12", Block: true},
)
By("生成集群配置文件")
var err error
configPath, nodeConfigPath, err = clusterManager.GetConfigGenerator().GenerateAndUpload(clusterConfig)
Expect(err).NotTo(HaveOccurred(), "应该成功生成配置文件")
By("执行集群创建命令")
err = clusterManager.CreateClusterInBackgroundWithKubeconfig(configPath, nodeConfigPath, "")
Expect(err).NotTo(HaveOccurred(), "创建集群命令应该成功执行")
By("等待集群状态变为 Healthy")
Eventually(func() string {
phase, state, clusterStatus, _ := clusterManager.GetClusterFullStatusWithKubeconfig(clusterName, "")
GinkgoWriter.Printf("当前集群状态: phase=%s, state=%s, clusterStatus=%s\n", phase, state, clusterStatus)
failOnClusterFailure(state, clusterStatus)
return state
}, installTimeout, pollInterval).Should(Equal("Healthy"))
checker := utils.NewClusterCheckerWithParentKubeconfig(sshExecutor, clusterName, "")
Eventually(func() bool {
n1, _ := checker.GetPodNodeName("", "app.kubernetes.io/name=redis")
n2, _ := checker.GetPodNodeName("", "app=bkeagent-deployer")
return n1 != "" && n2 != ""
}, 10*time.Minute, 15*time.Second).Should(BeTrue(), "bkeagent-deployer 和 redis 的 Pod 应该被创建并运行")
kubeconfigPath, err := checker.SaveKubeconfigToFile()
Expect(err).NotTo(HaveOccurred())
By("验证addon部署顺序")
getTime := func(namespace, selector, jsonpath string) time.Time {
nsArg := "-A"
if namespace != "" {
nsArg = fmt.Sprintf("-n %s", namespace)
}
sel := ""
if selector != "" {
sel = fmt.Sprintf("-l %s", selector)
}
cmd := fmt.Sprintf("KUBECONFIG=%s kubectl get pods %s %s -o jsonpath=\"%s\"", kubeconfigPath, nsArg, sel, jsonpath)
result, err := localExecutor.Exec(cmd)
Expect(err).NotTo(HaveOccurred())
Expect(result.ExitCode).To(Equal(0))
ts := strings.TrimSpace(strings.Trim(result.Stdout, "'\""))
t, perr := time.Parse(time.RFC3339, ts)
Expect(perr).NotTo(HaveOccurred(), "解析时间戳失败: %s", ts)
return t
}
redisCreate := getTime("", "app.kubernetes.io/name=redis", "{.items[0].metadata.creationTimestamp}")
bkeagentCreate := getTime("", "app=bkeagent-deployer", "{.items[0].metadata.creationTimestamp}")
Expect(bkeagentCreate.Before(redisCreate) || bkeagentCreate.Equal(redisCreate)).To(BeTrue(), "bkeagent-deployer 应先于 redis 组件创建")
By("通过更新 BKECluster 配置删除 redis addon")
newCfg := clusterConfig
filtered := []config.AddonConfig{}
for _, a := range newCfg.ExtraAddons {
if a.Name == "redis" {
continue
}
filtered = append(filtered, a)
}
newCfg.ExtraAddons = filtered
By("生成新的集群配置文件")
newConfigPath, newNodeConfigPath, err := clusterManager.GetConfigGenerator().GenerateAndUpload(newCfg)
Expect(err).NotTo(HaveOccurred(), "应该成功生成用于更新的配置文件")
configPath = newConfigPath
nodeConfigPath = newNodeConfigPath
GinkgoWriter.Printf("newConfigPath: %s\n", newConfigPath)
applyCmd := fmt.Sprintf("kubectl apply -f %s", newConfigPath)
result, err := sshExecutor.Exec(applyCmd)
Expect(err).NotTo(HaveOccurred())
Expect(result.ExitCode).To(Equal(0), "应该成功应用更新的 BKECluster 配置")
By("等待 redis Pod 被删除")
Eventually(func() bool {
nodeName, _ := checker.GetPodNodeName("", "app.kubernetes.io/name=redis")
return nodeName == ""
}, 5*time.Minute, 10*time.Second).Should(BeTrue(), "redis Pod 应该在移除 addon 后被删除")
})
})