package installation
import (
"fmt"
"path/filepath"
"strings"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gitcode.com/openFuyao/e2e-auto-test/e2e/framework/executor"
config "gitcode.com/openFuyao/e2e-auto-test/e2e/installation/bke-config"
"gitcode.com/openFuyao/e2e-auto-test/e2e/installation/utils"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
)
var _ = SIGDescribe("BKE Cluster Installation", func() {
var (
sshExecutor *executor.SSHExecutor
localExecutor *executor.LocalExecutor
guideConfig *config.GuideNodeConfig
dynamicClient dynamic.Interface
clusterManager *utils.ClusterManager
)
BeforeEach(func() {
guideConfig = config.LoadGuideNodeFromEnv()
Expect(guideConfig.Host).NotTo(BeEmpty(), "GUIDE_NODE_HOST 环境变量必须设置")
Expect(guideConfig.Password).NotTo(BeEmpty(), "GUIDE_NODE_PASSWORD 环境变量必须设置")
var err error
sshExecutor, err = executor.NewSSHExecutor(
guideConfig.Host,
guideConfig.Port,
guideConfig.Username,
guideConfig.Password,
)
Expect(err).NotTo(HaveOccurred(), "应该成功连接到引导节点")
kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config")
restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
Expect(err).NotTo(HaveOccurred(), "应该成功加载 kubeconfig")
dynamicClient, err = dynamic.NewForConfig(restConfig)
Expect(err).NotTo(HaveOccurred(), "应该成功创建动态客户端")
clusterManager = utils.NewClusterManager(sshExecutor, dynamicClient)
localExecutor = executor.NewLocalExecutor(30 * time.Second)
})
Describe("创建 1Master1Worker 管理集群和业务集群并使用自定义证书签名", Label("1m1w-custom", "certificate", "post-init"), Ordered, func() {
var (
mgmtCluster *Cluster1M1WConfig
workloadCluster *Cluster1M1WConfig
certGenerator *utils.CertificateConfigGenerator
)
BeforeAll(func() {
By("在引导节点上生成证书链文件")
err := utils.GenerateCertificateChainOnRemote(sshExecutor, "/etc/openFuyao/certs")
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书链文件")
GinkgoWriter.Printf("证书链文件在引导节点上生成于: %s\n", "/etc/openFuyao/certs")
By("在引导节点上生成证书配置")
certConfigDir := "/etc/openFuyao/certs/cert_config"
certGenerator = utils.NewCertificateConfigGenerator(certConfigDir)
err = certGenerator.GenerateAllCertificateConfigs()
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书配置")
GinkgoWriter.Printf("证书配置在引导节点上生成于: %s\n", certConfigDir)
var err2 error
mgmtCluster, err2 = Create1M1WManagementCluster(
clusterManager,
localExecutor,
"test-mgmt-1m1w-cert",
true,
)
Expect(err2).NotTo(HaveOccurred(), "应该成功创建管理集群")
workloadCluster, err2 = Create1M1WWorkloadCluster(
clusterManager,
localExecutor,
mgmtCluster.KubeconfigPath,
"test-workload-1m1w-cert",
)
Expect(err2).NotTo(HaveOccurred(), "应该成功创建业务集群")
})
用例名称：验证1master-1worker集群自定义证书导入
前置条件：
1.在引导节点上生成证书链文件
2.在引导节点上生成证书配置
3.创建1master1worker的管理集群
4.创建1master1worker的业务集群
用例步骤：
1.检查管理集群的master节点的证书配置
2.检查业务集群的master节点的证书配置
预期结果：
*/
It("应该验证所有 master 节点的证书签名", SpecTimeout(InstallationItTimeout), func(ctx SpecContext) {
nodes := config.LoadTestNodesFromEnv()
Expect(len(nodes)).To(BeNumerically(">=", 4), "需要至少 4 个节点（管理集群 2 个 + 业务集群 2 个）")
By("验证管理集群 master 节点的证书签名")
managerMasterNode := nodes[0]
By(fmt.Sprintf("验证管理集群 master 节点 1 (%s) 的证书签名", managerMasterNode.IP))
verifier := utils.NewCertificateVerifierWithSSH("/etc/kubernetes/pki", localExecutor, managerMasterNode.IP, managerMasterNode.Username, managerMasterNode.Password)
results, err := verifier.VerifyCertificateChain()
Expect(err).NotTo(HaveOccurred(), "应该成功验证管理集群 master 节点 1 的证书链")
GinkgoWriter.Printf("=== 管理集群 Master 节点 %d (%s) 证书验证结果 ===\n", 1, managerMasterNode.IP)
verifier.PrintResults(results)
for _, item := range results.Items {
if !item.Success {
GinkgoWriter.Printf("管理集群 master 节点 1 证书验证失败: %s - %v\n", item.Name, item.Error)
}
Expect(item.Success).To(BeTrue(), fmt.Sprintf("管理集群 master 节点 1 证书验证应该成功: %s", item.Name))
}
By("验证业务集群 master 节点的证书签名")
workloadMasterNode := nodes[2]
By(fmt.Sprintf("验证业务集群 master 节点 3 (%s) 的证书签名", workloadMasterNode.IP))
verifier2 := utils.NewCertificateVerifierWithSSH("/etc/kubernetes/pki", localExecutor, workloadMasterNode.IP, workloadMasterNode.Username, workloadMasterNode.Password)
results2, err2 := verifier2.VerifyCertificateChain()
Expect(err2).NotTo(HaveOccurred(), "应该成功验证业务集群 master 节点 3 的证书链")
GinkgoWriter.Printf("=== 业务集群 Master 节点 3 (%s) 证书验证结果 ===\n", workloadMasterNode.IP)
verifier.PrintResults(results2)
for _, item := range results2.Items {
if !item.Success {
GinkgoWriter.Printf("业务集群 master 节点 3 证书验证失败: %s - %v\n", item.Name, item.Error)
}
Expect(item.Success).To(BeTrue(), fmt.Sprintf("业务集群 master 节点 3 证书验证应该成功: %s", item.Name))
}
})
AfterAll(func() {
if workloadCluster != nil && mgmtCluster != nil {
Delete1M1WWorkloadCluster(
clusterManager,
localExecutor,
workloadCluster.ClusterName,
mgmtCluster.KubeconfigPath,
)
if workloadCluster.ConfigPath != "" {
if err := clusterManager.CleanupConfig(workloadCluster.ConfigPath); err != nil {
GinkgoWriter.Printf("清理业务集群配置文件失败: %v\n", err)
}
}
if workloadCluster.NodeConfigPath != "" {
if err := clusterManager.CleanupConfig(workloadCluster.NodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup workload node config file: %v\n", err)
}
}
}
if mgmtCluster != nil {
Delete1M1WManagementCluster(
clusterManager,
localExecutor,
mgmtCluster.ClusterName,
mgmtCluster.KubeconfigPath,
)
if mgmtCluster.ConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.ConfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群配置文件失败: %v\n", err)
}
}
if mgmtCluster.NodeConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.NodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup mgmt node config file: %v\n", err)
}
}
if mgmtCluster.KubeconfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.KubeconfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群 kubeconfig 文件失败: %v\n", err)
}
}
}
By("从引导节点清理证书目录")
cmd := "rm -rf /etc/openFuyao/certs/*"
localExecutor.Exec(cmd)
if _, err := localExecutor.Exec(cmd); err != nil {
GinkgoWriter.Printf("清理证书目录失败: %v\n", err)
}
})
})
Describe("创建 3Master1Worker 管理集群并使用自定义证书签名", Label("3m1w-mgmt-custom", "certificate", "post-init"), Ordered, func() {
var (
mgmtCluster *Cluster3M1WConfig
certGenerator *utils.CertificateConfigGenerator
)
BeforeAll(func() {
By("在引导节点上生成证书链文件")
err := utils.GenerateCertificateChainOnRemote(sshExecutor, "/etc/openFuyao/certs")
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书链文件")
GinkgoWriter.Printf("证书链文件在引导节点上生成于: %s\n", "/etc/openFuyao/certs")
By("在引导节点上生成证书配置")
certConfigDir := "/etc/openFuyao/certs/cert_config"
certGenerator = utils.NewCertificateConfigGenerator(certConfigDir)
err = certGenerator.GenerateAllCertificateConfigs()
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书配置")
GinkgoWriter.Printf("证书配置在引导节点上生成于: %s\n", certConfigDir)
var err2 error
mgmtCluster, err2 = Create3M1WManagementCluster(
clusterManager,
localExecutor,
"test-mgmt-3m1w-cert",
true,
)
Expect(err2).NotTo(HaveOccurred(), "应该成功创建 3Master1Worker 管理集群")
})
用例名称：验证 3master1worker 管理集群自定义证书导入
前置条件：
1.在引导节点上生成证书链文件
2.在引导节点上生成证书配置
3.创建 3master1worker 的管理集群
用例步骤：
1.检查管理集群 3 个 master 节点的证书配置
预期结果：管理集群所有 master 节点证书验证项全部通过
*/
It("应该验证 3Master1Worker 管理集群所有 master 节点的证书签名", SpecTimeout(InstallationItTimeout), func(ctx SpecContext) {
nodes := config.LoadTestNodesFromEnv()
Expect(len(nodes)).To(BeNumerically(">=", 4), "需要至少 4 个节点（管理集群 3 个 master + 1 个 worker）")
By("验证管理集群所有 master 节点的证书签名")
for i := 0; i < 3; i++ {
masterNode := nodes[i]
By(fmt.Sprintf("验证管理集群 master 节点 %d (%s) 的证书签名", i+1, masterNode.IP))
verifier := utils.NewCertificateVerifierWithSSH("/etc/kubernetes/pki", localExecutor, masterNode.IP, masterNode.Username, masterNode.Password)
results, err := verifier.VerifyCertificateChain()
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("应该成功验证管理集群 master 节点 %d 的证书链", i+1))
GinkgoWriter.Printf("=== 管理集群 Master 节点 %d (%s) 证书验证结果 ===\n", i+1, masterNode.IP)
verifier.PrintResults(results)
for _, item := range results.Items {
if !item.Success {
GinkgoWriter.Printf("管理集群 master 节点 %d 证书验证失败: %s - %v\n", i+1, item.Name, item.Error)
}
Expect(item.Success).To(BeTrue(), fmt.Sprintf("管理集群 master 节点 %d 证书验证应该成功: %s", i+1, item.Name))
}
}
})
AfterAll(func() {
if mgmtCluster != nil {
Delete3M1WManagementCluster(
clusterManager,
localExecutor,
mgmtCluster.ClusterName,
mgmtCluster.KubeconfigPath,
)
if mgmtCluster.ConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.ConfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群配置文件失败: %v\n", err)
}
}
if mgmtCluster.NodeConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.NodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup mgmt node config file: %v\n", err)
}
}
if mgmtCluster.KubeconfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtCluster.KubeconfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群 kubeconfig 文件失败: %v\n", err)
}
}
}
By("从引导节点清理证书目录")
cmd := "rm -rf /etc/openFuyao/certs/*"
localExecutor.Exec(cmd)
if _, err := localExecutor.Exec(cmd); err != nil {
GinkgoWriter.Printf("清理证书目录失败: %v\n", err)
}
})
})
Describe("创建 1Master 管理集群和 3Master1Worker 业务集群并使用自定义证书签名", Label("1m-mgmt-3m1w-workload-custom", "certificate", "post-init", "skip-temporarily"), Ordered, func() {
var (
mgmtClusterName string
mgmtConfigPath string
mgmtNodeConfigPath string
mgmtKubeconfigPath string
workloadClusterName string
workloadConfigPath string
workloadNodeConfigPath string
certGenerator *utils.CertificateConfigGenerator
)
BeforeAll(func() {
By("在引导节点上生成证书链文件")
err := utils.GenerateCertificateChainOnRemote(sshExecutor, "/etc/openFuyao/certs")
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书链文件")
By("在引导节点上生成证书配置")
certConfigDir := "/etc/openFuyao/certs/cert_config"
certGenerator = utils.NewCertificateConfigGenerator(certConfigDir)
err = certGenerator.GenerateAllCertificateConfigs()
Expect(err).NotTo(HaveOccurred(), "应该成功生成证书配置")
nodes := config.LoadTestNodesFromEnv()
if len(nodes) < 5 {
Skip(fmt.Sprintf("跳过: 管理集群+业务集群场景需要至少 5 个节点，当前只有 %d 个", len(nodes)))
}
By("创建 1Master 管理集群")
mgmtClusterName = fmt.Sprintf("test-mgmt-1m-cert-%d", time.Now().Unix())
mgmtMaster := nodes[0]
mgmtMaster.Role = []string{"master/node", "etcd"}
mgmtClusterConfig := config.NewBKEClusterConfigForMgmt(mgmtClusterName, []config.NodeInfo{mgmtMaster})
mgmtConfigPath, mgmtNodeConfigPath, err = clusterManager.GetConfigGenerator().GenerateAndUpload(mgmtClusterConfig)
Expect(err).NotTo(HaveOccurred(), "应该成功生成管理集群配置文件")
err = clusterManager.CreateClusterInBackgroundWithKubeconfig(mgmtConfigPath, mgmtNodeConfigPath, "")
Expect(err).NotTo(HaveOccurred(), "应该成功触发管理集群创建")
Eventually(func() bool {
phase, state, clusterStatus, _ := clusterManager.GetClusterFullStatusWithKubeconfig(mgmtClusterName, "")
GinkgoWriter.Printf("当前管理集群状态: phase=%s, state=%s, clusterStatus=%s\n", phase, state, clusterStatus)
failOnClusterFailure(state, clusterStatus)
return state == "Healthy" && clusterStatus == "Ready"
}, mgmtInstallTimeout, pollInterval).Should(BeTrue(), "管理集群应该变为 Healthy 和 Ready")
mgmtChecker := utils.NewClusterCheckerWithParentKubeconfig(localExecutor, mgmtClusterName, "")
mgmtKubeconfigPath, err = mgmtChecker.SaveKubeconfigToFile()
Expect(err).NotTo(HaveOccurred(), "应该成功获取管理集群 kubeconfig")
By("在管理集群上创建 3Master1Worker 业务集群")
workloadClusterName = fmt.Sprintf("test-workload-3m1w-cert-%d", time.Now().Unix())
workloadNodes := make([]config.NodeInfo, 4)
copy(workloadNodes, nodes[1:5])
for i := 0; i < 3; i++ {
workloadNodes[i].Role = []string{"master/node", "etcd"}
}
workloadNodes[3].Role = []string{"node"}
workloadClusterConfig := config.NewDefaultBKEClusterConfig(workloadClusterName, workloadNodes)
workloadConfigPath, workloadNodeConfigPath, err = clusterManager.GetConfigGenerator().GenerateAndUpload(workloadClusterConfig)
Expect(err).NotTo(HaveOccurred(), "应该成功生成业务集群配置文件")
err = clusterManager.CreateClusterWithConfig(workloadConfigPath, workloadNodeConfigPath, mgmtKubeconfigPath)
Expect(err).NotTo(HaveOccurred(), "应该成功创建业务集群")
Eventually(func() bool {
phase, state, clusterStatus, _ := clusterManager.GetClusterFullStatusWithKubeconfig(workloadClusterName, mgmtKubeconfigPath)
GinkgoWriter.Printf("当前业务集群状态: phase=%s, state=%s, clusterStatus=%s\n", phase, state, clusterStatus)
failOnClusterFailure(state, clusterStatus)
return state == "Healthy" && clusterStatus == "Ready"
}, installTimeout, pollInterval).Should(BeTrue(), "业务集群应该变为 Healthy 和 Ready")
})
It("应该验证管理和业务集群所有节点的证书签名", SpecTimeout(InstallationItTimeout), func(ctx SpecContext) {
nodes := config.LoadTestNodesFromEnv()
Expect(len(nodes)).To(BeNumerically(">=", 5), "需要至少 5 个节点（管理集群 1 个 + 业务集群 4 个）")
By(fmt.Sprintf("验证管理集群 master 节点 (%s) 的证书签名", nodes[0].IP))
verifier := utils.NewCertificateVerifierWithSSH("/etc/kubernetes/pki", localExecutor, nodes[0].IP, nodes[0].Username, nodes[0].Password)
results, err := verifier.VerifyCertificateChain()
Expect(err).NotTo(HaveOccurred(), "应该成功验证管理集群 master 节点证书链")
verifier.PrintResults(results)
for _, item := range results.Items {
Expect(item.Success).To(BeTrue(), fmt.Sprintf("管理集群 master 节点证书验证应该成功: %s", item.Name))
}
for i := 1; i < 5; i++ {
node := nodes[i]
isWorker := i == 4
roleDesc := "master"
if isWorker {
roleDesc = "worker"
}
By(fmt.Sprintf("验证业务集群 %s 节点 %d (%s) 的证书签名", roleDesc, i+1, node.IP))
verifier := utils.NewCertificateVerifierWithSSH("/etc/kubernetes/pki", localExecutor, node.IP, node.Username, node.Password)
var results *utils.VerificationResult
var err error
if isWorker {
results, err = verifier.VerifyWorkerCertificateChain()
} else {
results, err = verifier.VerifyCertificateChain()
}
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("应该成功执行业务集群 %s 节点 %d 证书校验", roleDesc, i+1))
verifier.PrintResults(results)
for _, item := range results.Items {
Expect(item.Success).To(BeTrue(), fmt.Sprintf("业务集群 %s 节点 %d 证书验证应该成功: %s", roleDesc, i+1, item.Name))
}
}
})
AfterAll(func() {
if workloadClusterName != "" && mgmtKubeconfigPath != "" {
Delete3M1WWorkloadCluster(clusterManager, localExecutor, workloadClusterName, mgmtKubeconfigPath)
if workloadConfigPath != "" {
if err := clusterManager.CleanupConfig(workloadConfigPath); err != nil {
GinkgoWriter.Printf("清理业务集群配置文件失败: %v\n", err)
}
}
if workloadNodeConfigPath != "" {
if err := clusterManager.CleanupConfig(workloadNodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup workload node config file: %v\n", err)
}
}
}
if mgmtClusterName != "" {
if clusterManager.ClusterExistsWithKubeconfig(mgmtClusterName, "") {
if err := clusterManager.DeleteClusterWithKubeconfig(mgmtClusterName, ""); err != nil {
GinkgoWriter.Printf("触发管理集群删除失败: %v\n", err)
}
}
if mgmtKubeconfigPath != "" {
if err := clusterManager.DeleteManagementClusterSelfBC(mgmtKubeconfigPath); err != nil {
GinkgoWriter.Printf("触发管理集群自身 BC 删除失败: %v\n", err)
}
}
Eventually(func() bool {
existsOnBootstrap := clusterManager.ClusterExistsWithKubeconfig(mgmtClusterName, "")
existsOnSelf := false
if mgmtKubeconfigPath != "" {
checkSelfCmd := fmt.Sprintf("KUBECONFIG=%s kubectl get bc bke-cluster -n bke-cluster --no-headers 2>/dev/null", mgmtKubeconfigPath)
result, _ := localExecutor.Exec(checkSelfCmd)
existsOnSelf = result.ExitCode == 0 && strings.TrimSpace(result.Stdout) != ""
}
return !existsOnBootstrap && !existsOnSelf
}, uninstallTimeout, 60*time.Second).Should(BeTrue(), "管理集群应该被完全删除")
}
if mgmtConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtConfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群配置文件失败: %v\n", err)
}
}
if mgmtNodeConfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtNodeConfigPath); err != nil {
GinkgoWriter.Printf("Failed to cleanup mgmt node config file: %v\n", err)
}
}
if mgmtKubeconfigPath != "" {
if err := clusterManager.CleanupConfig(mgmtKubeconfigPath); err != nil {
GinkgoWriter.Printf("清理管理集群 kubeconfig 文件失败: %v\n", err)
}
}
By("从引导节点清理证书目录")
cmd := "rm -rf /etc/openFuyao/certs/*"
localExecutor.Exec(cmd)
if _, err := localExecutor.Exec(cmd); err != nil {
GinkgoWriter.Printf("清理证书目录失败: %v\n", err)
}
})
})
})