Copyright 2025.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/HuaweiCloudDeveloper/cluster-api-provider-huawei/test/utils"
)
const namespace = "caphw-system"
const serviceAccountName = "cluster-api-provider-huawei-controller-manager"
const metricsServiceName = "cluster-api-provider-huawei-controller-manager-metrics-service"
const metricsRoleBindingName = "cluster-api-provider-huawei-metrics-binding"
var _ = Describe("Manager", Ordered, func() {
var controllerPodName string
BeforeAll(func() {
By("creating manager namespace")
cmd := exec.Command("kubectl", "create", "ns", namespace)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
By("installing CRDs")
cmd = exec.Command("make", "install")
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
By("deploying the controller-manager")
_ = os.Setenv("CLOUD_SDK_AK", base64.StdEncoding.EncodeToString([]byte("ak_test")))
_ = os.Setenv("CLOUD_SDK_SK", base64.StdEncoding.EncodeToString([]byte("sk_test")))
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
})
AfterAll(func() {
By("cleaning up the curl pod for metrics")
cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace)
_, _ = utils.Run(cmd)
By("undeploying the controller-manager")
cmd = exec.Command("make", "undeploy")
_, _ = utils.Run(cmd)
By("uninstalling CRDs")
cmd = exec.Command("make", "uninstall")
_, _ = utils.Run(cmd)
By("removing manager namespace")
cmd = exec.Command("kubectl", "delete", "ns", namespace)
_, _ = utils.Run(cmd)
})
AfterEach(func() {
specReport := CurrentSpecReport()
if specReport.Failed() {
By("Fetching controller manager pod logs")
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
controllerLogs, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
}
By("Fetching Kubernetes events")
cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp")
eventsOutput, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err)
}
By("Fetching curl-metrics logs")
cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err)
}
By("Fetching controller manager pod description")
cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace)
podDescription, err := utils.Run(cmd)
if err == nil {
fmt.Println("Pod description:\n", podDescription)
} else {
fmt.Println("Failed to describe controller pod")
}
}
})
SetDefaultEventuallyTimeout(2 * time.Minute)
SetDefaultEventuallyPollingInterval(time.Second)
Context("Manager", func() {
It("should run successfully", func() {
By("validating that the controller-manager pod is running as expected")
verifyControllerUp := func(g Gomega) {
cmd := exec.Command("kubectl", "get",
"pods", "-l", "control-plane=controller-manager",
"-o", "go-template={{ range .items }}"+
"{{ if not .metadata.deletionTimestamp }}"+
"{{ .metadata.name }}"+
"{{ \"\\n\" }}{{ end }}{{ end }}",
"-n", namespace,
)
podOutput, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information")
podNames := utils.GetNonEmptyLines(podOutput)
g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running")
controllerPodName = podNames[0]
g.Expect(controllerPodName).To(ContainSubstring("controller-manager"))
cmd = exec.Command("kubectl", "get",
"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
"-n", namespace,
)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status")
}
Eventually(verifyControllerUp).Should(Succeed())
})
It("should ensure the metrics endpoint is serving metrics", func() {
Skip("Metrics service is not implemented yet")
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
"--clusterrole=cluster-api-provider-huawei-metrics-reader",
fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
By("validating that the metrics service is available")
cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Metrics service should exist")
By("validating that the ServiceMonitor for Prometheus is applied in the namespace")
cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace)
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist")
By("getting the service account token")
token, err := serviceAccountToken()
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())
By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
"--image=curlimages/curl:7.78.0",
"--", "/bin/sh", "-c", fmt.Sprintf(
"curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics",
token, metricsServiceName, namespace))
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
By("waiting for the curl-metrics pod to complete.")
verifyCurlUp := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
"-o", "jsonpath={.status.phase}",
"-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status")
}
Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed())
By("getting the metrics by checking curl-metrics logs")
metricsOutput := getMetricsOutput()
Expect(metricsOutput).To(ContainSubstring(
"controller_runtime_reconcile_total",
))
})
})
})
func serviceAccountToken() (string, error) {
const tokenRequestRawString = `{
"apiVersion": "authentication.k8s.io/v1",
"kind": "TokenRequest"
}`
secretName := fmt.Sprintf("%s-token-request", serviceAccountName)
tokenRequestFile := filepath.Join("/tmp", secretName)
err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644))
if err != nil {
return "", err
}
var out string
verifyTokenCreation := func(g Gomega) {
cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
"/api/v1/namespaces/%s/serviceaccounts/%s/token",
namespace,
serviceAccountName,
), "-f", tokenRequestFile)
output, err := cmd.CombinedOutput()
g.Expect(err).NotTo(HaveOccurred())
var token tokenRequest
err = json.Unmarshal(output, &token)
g.Expect(err).NotTo(HaveOccurred())
out = token.Status.Token
}
Eventually(verifyTokenCreation).Should(Succeed())
return out, err
}
func getMetricsOutput() string {
By("getting the curl-metrics logs")
cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK"))
return metricsOutput
}
type tokenRequest struct {
Status struct {
Token string `json:"token"`
} `json:"status"`
}