package apigateway
import (
"context"
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"time"
apisixclientv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/clientset/versioned/typed/config/v2"
v13 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
v12 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
kbutil "github.com/goodrain/rainbond/util/kubeblocks"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
v2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2"
"github.com/go-chi/chi"
"github.com/goodrain/rainbond/api/handler"
apimodel "github.com/goodrain/rainbond/api/model"
"github.com/goodrain/rainbond/api/util"
"github.com/goodrain/rainbond/api/util/bcode"
ctxutil "github.com/goodrain/rainbond/api/util/ctx"
"github.com/goodrain/rainbond/db"
dbmodel "github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/pkg/component/k8s"
httputil "github.com/goodrain/rainbond/util/http"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"sigs.k8s.io/yaml"
)
func (g Struct) OpenOrCloseDomains(w http.ResponseWriter, r *http.Request) {
c := k8s.Default().ApiSixClient.ApisixV2()
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
serviceAlias := r.URL.Query().Get("service_alias")
if idx := strings.Index(serviceAlias, ","); idx != -1 {
serviceAlias = serviceAlias[:idx]
}
list, _ := c.ApisixRoutes(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: serviceAlias + "=service_alias" + ",port=" + r.URL.Query().Get("port"),
})
for _, itemL := range list.Items {
item := itemL
var plugins = item.Spec.HTTP[0].Plugins
var newPlugins = make([]v2.ApisixRoutePlugin, 0)
for _, plugin := range plugins {
if plugin.Name != util.ResponseRewrite {
newPlugins = append(newPlugins, plugin)
}
}
if r.URL.Query().Get("act") == "close" {
newPlugins = append(newPlugins, v2.ApisixRoutePlugin{
Name: util.ResponseRewrite,
Enable: true,
Config: map[string]interface{}{
"status_code": 404,
"body": "请打开对外访问",
},
})
}
item.Spec.HTTP[0].Plugins = newPlugins
item.Status = v2.ApisixStatus{}
_, err := c.ApisixRoutes(tenant.Namespace).Update(r.Context(), &item, v1.UpdateOptions{})
if err != nil {
if errors.IsConflict(err) {
logrus.Warnf("update route %v conflict", item.Name)
continue
}
logrus.Errorf("update route %v failure: %v", item.Name, err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteUpdate)
}
}
httputil.ReturnSuccess(r, w, nil)
}
func (g Struct) GetHTTPBindDomains(w http.ResponseWriter, r *http.Request) {
c := k8s.Default().ApiSixClient.ApisixV2()
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
serviceAlias := r.URL.Query().Get("service_alias")
if idx := strings.Index(serviceAlias, ","); idx != -1 {
serviceAlias = serviceAlias[:idx]
}
list, err := c.ApisixRoutes(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: serviceAlias + "=service_alias" + ",port=" + r.URL.Query().Get("port"),
})
if err != nil {
logrus.Errorf("get route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
var hosts = make([]string, 0)
for _, item := range list.Items {
var has bool
for _, plugin := range item.Spec.HTTP[0].Plugins {
if plugin.Name == util.ResponseRewrite {
has = true
break
}
}
if !has {
hosts = append(hosts, item.Spec.HTTP[0].Match.Hosts[0])
}
}
httputil.ReturnSuccess(r, w, hosts)
}
func (g Struct) GetTCPBindDomains(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
k := k8s.Default().Clientset.CoreV1()
serviceAlias := r.URL.Query().Get("service_alias")
port := r.URL.Query().Get("port")
if idx := strings.Index(serviceAlias, ","); idx != -1 {
serviceAlias = serviceAlias[:idx]
}
labelSelector := fmt.Sprintf("tcp=true,service_alias=%v,outer=true", serviceAlias)
if port != "" {
labelSelector = fmt.Sprintf("tcp=true,service_alias=%v,outer=true,port=%v", serviceAlias, port)
}
list, err := k.Services(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: labelSelector,
})
if err != nil {
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
var resp []int32
for _, v := range list.Items {
resp = append(resp, v.Spec.Ports[0].NodePort)
}
httputil.ReturnSuccess(r, w, resp)
}
func (g Struct) GetHTTPAPIRoute(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
type routeResponse struct {
*v2.ApisixRouteHTTP
Enabled bool `json:"enabled"`
RegionAppID string `json:"region_app_id"`
}
var resp = make([]*routeResponse, 0)
c := k8s.Default().ApiSixClient.ApisixV2()
appID := r.URL.Query().Get("appID")
labelSelector := ""
if appID != "" {
labelSelector = "app_id=" + appID
}
list, err := c.ApisixRoutes(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: labelSelector,
})
if err != nil {
logrus.Errorf("get route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
for _, v := range list.Items {
httpRoute := v.Spec.HTTP[0].DeepCopy()
labels := v.Labels
service_alias := ""
regionAppID := ""
enabled := false
for labelK, labelV := range labels {
if labelV == "service_alias" {
service_alias = service_alias + "-" + labelK
}
if labelK == "app_id" {
regionAppID = labelV
}
if labelK == "cert-manager-enabled" {
enabled = labelV == "true"
}
}
httpRoute.Name = regionAppID + "|" + v.Name + "|" + service_alias
resp = append(resp, &routeResponse{
ApisixRouteHTTP: httpRoute,
Enabled: enabled,
RegionAppID: regionAppID,
})
}
httputil.ReturnSuccess(r, w, resp)
}
func (g Struct) UpdateHTTPAPIRoute(w http.ResponseWriter, r *http.Request) {
panic("implement me")
}
func addResponseRewritePlugin(apisixRouteHTTP v2.ApisixRouteHTTP) v2.ApisixRouteHTTP {
for _, v := range apisixRouteHTTP.Plugins {
if v.Name == util.ResponseRewrite {
return apisixRouteHTTP
}
}
apisixRouteHTTP.Plugins = append(apisixRouteHTTP.Plugins, v2.ApisixRoutePlugin{
Name: util.ResponseRewrite,
Enable: false,
Config: map[string]interface{}{
"status_code": 404,
},
})
return apisixRouteHTTP
}
func (g Struct) CreateHTTPAPIRoute(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
var apisixRouteHTTP v2.ApisixRouteHTTP
if !httputil.ValidatorRequestStructAndErrorResponse(r, w, &apisixRouteHTTP, nil) {
return
}
sa := r.URL.Query().Get("service_alias")
if idx := strings.Index(sa, ","); idx != -1 {
sa = sa[:idx]
}
sLabel := strings.Split(sa, ",")
labels := make(map[string]string)
labels["creator"] = "Rainbond"
labels["port"] = r.URL.Query().Get("port")
labels["component_sort"] = sa
if r.URL.Query().Get("appID") != "" {
labels["app_id"] = r.URL.Query().Get("appID")
}
defaultDomain := r.URL.Query().Get("default") == "true"
for _, sl := range sLabel {
if sl != "" {
labels[sl] = "service_alias"
}
}
c := k8s.Default().ApiSixClient.ApisixV2()
routeName := strings.ToLower(strings.ReplaceAll(apisixRouteHTTP.Match.Hosts[0], "*", "wildcard") + apisixRouteHTTP.Match.Paths[0])
routeName = strings.ReplaceAll(routeName, "/", "p-p")
routeName = strings.ReplaceAll(routeName, "*", "s-s")
routeName = strings.ReplaceAll(routeName, "_", "")
for _, host := range apisixRouteHTTP.Match.Hosts {
safeHost := sanitizeLabelKey(host)
labels[safeHost] = "host"
}
apisixRouteHTTP.Name = uuid.New().String()[0:8]
route, err := c.ApisixRoutes(tenant.Namespace).Create(r.Context(), &v2.ApisixRoute{
TypeMeta: v1.TypeMeta{
Kind: util.ApisixRoute,
APIVersion: util.APIVersion,
},
ObjectMeta: v1.ObjectMeta{
Labels: labels,
Name: routeName,
GenerateName: "rbd",
},
Spec: v2.ApisixRouteSpec{
IngressClassName: "apisix",
HTTP: []v2.ApisixRouteHTTP{
apisixRouteHTTP,
},
},
}, v1.CreateOptions{})
if err == nil {
name := r.URL.Query().Get("name")
if name != "" {
name = removeLeadingDigits(name)
err = c.ApisixRoutes(tenant.Namespace).Delete(r.Context(), name, v1.DeleteOptions{})
if err != nil {
logrus.Errorf("delete route %v failure: %v", name, err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
}
httputil.ReturnSuccess(r, w, marshalApisixRoute(route))
return
}
logrus.Warnf("create route error %s, will update route", err.Error())
get, err := c.ApisixRoutes(tenant.Namespace).Get(r.Context(), routeName, v1.GetOptions{})
if err != nil {
logrus.Errorf("get route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
if defaultDomain {
httputil.ReturnSuccess(r, w, marshalApisixRoute(get))
return
}
get.Spec.HTTP[0] = apisixRouteHTTP
if get.ObjectMeta.Labels["cert-manager-enabled"] == "true" {
labels["cert-manager-enabled"] = "true"
}
get.ObjectMeta.Labels = labels
update, err := c.ApisixRoutes(tenant.Namespace).Update(r.Context(), get, v1.UpdateOptions{})
if err != nil {
logrus.Errorf("update route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteUpdate)
return
}
httputil.ReturnSuccess(r, w, marshalApisixRoute(update))
}
func marshalApisixRoute(r *v2.ApisixRoute) map[string]interface{} {
r.TypeMeta.Kind = util.ApisixRoute
r.TypeMeta.APIVersion = util.APIVersion
r.ObjectMeta.ManagedFields = nil
resp := make(map[string]interface{})
contentBytes, _ := yaml.Marshal(r)
resp["name"] = r.Name
resp["kind"] = r.TypeMeta.Kind
resp["content"] = string(contentBytes)
return resp
}
func (g Struct) DeleteHTTPAPIRoute(w http.ResponseWriter, r *http.Request) {
var deleteName = make([]string, 0)
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
name := chi.URLParam(r, "name")
name = removeLeadingDigits(name)
c := k8s.Default().ApiSixClient.ApisixV2()
err := c.ApisixRoutes(tenant.Namespace).Delete(r.Context(), name, v1.DeleteOptions{})
if err == nil {
deleteName = append(deleteName, name)
httputil.ReturnSuccess(r, w, deleteName)
return
}
logrus.Errorf("delete route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteDelete)
}
func (g Struct) GetTCPRoute(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
k := k8s.Default().Clientset.CoreV1()
appID := r.URL.Query().Get("appID")
labelSelector := "tcp=true"
if appID != "" {
labelSelector += ",app_id=" + appID
}
list, err := k.Services(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: labelSelector,
})
if err != nil {
httputil.ReturnBcodeError(r, w, bcode.ErrRouteNotFound)
return
}
var resp []apimodel.TCPRouteServicePort
for _, v := range list.Items {
if len(v.Spec.Ports) == 0 {
continue
}
servicePort := v.Spec.Ports[0]
item := apimodel.TCPRouteServicePort{
ServicePort: servicePort,
ServiceName: v.Name,
ServiceAlias: v.Labels["service_alias"],
ServiceID: v.Labels["service_id"],
AppID: v.Labels["app_id"],
ContainerPort: servicePort.Port,
}
if portLabel := v.Labels["port"]; portLabel != "" {
if containerPort, err := strconv.Atoi(portLabel); err == nil {
item.ContainerPort = int32(containerPort)
}
}
resp = append(resp, item)
}
httputil.ReturnSuccess(r, w, resp)
}
func (g Struct) CreateTCPRoute(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
serviceID := r.URL.Query().Get("service_id")
k := k8s.Default().Clientset.CoreV1()
var apisixRouteStream v2.ApisixRouteStream
if !httputil.ValidatorRequestStructAndErrorResponse(r, w, &apisixRouteStream, nil) {
return
}
serviceName := apisixRouteStream.Backend.ServiceName
serviceAlias := serviceName
resolvedServiceID := serviceID
logrus.Infof("apisixRouteStream.Match.IngressPort is %v", apisixRouteStream.Match.IngressPort)
if apisixRouteStream.Match.IngressPort == 0 {
logrus.Infof("change ingressPort")
h := handler.GetGatewayHandler()
res, err := h.GetAvailablePort("0.0.0.0", true)
if err != nil {
logrus.Errorf("GetAvailablePort error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrPortExists)
return
}
apisixRouteStream.Match.IngressPort = int32(res)
}
name := fmt.Sprintf("%v-%v", serviceName, apisixRouteStream.Match.IngressPort)
spec := corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Protocol: corev1.Protocol(strings.ToUpper(apisixRouteStream.Protocol)),
Name: name,
Port: apisixRouteStream.Backend.ServicePort.IntVal,
TargetPort: apisixRouteStream.Backend.ServicePort,
NodePort: apisixRouteStream.Match.IngressPort,
},
},
Type: corev1.ServiceTypeNodePort,
}
isThirdParty := r.URL.Query().Get("service_type") == "third_party"
if isThirdParty {
defer func() {
list, err := k8s.Default().RainbondClient.RainbondV1alpha1().ThirdComponents(tenant.Namespace).List(r.Context(), v1.ListOptions{
LabelSelector: "service_id=" + serviceID,
})
if err != nil {
logrus.Errorf("get route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteUpdate)
return
}
for _, v := range list.Items {
for i := range v.Spec.Ports {
v.Spec.Ports[i].OpenOuter = !v.Spec.Ports[i].OpenOuter
_, err = k8s.Default().RainbondClient.RainbondV1alpha1().ThirdComponents(tenant.Namespace).Update(r.Context(), &v, v1.UpdateOptions{})
if err != nil {
logrus.Errorf("update third component failure: %v", err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteUpdate)
return
}
}
}
}()
}
var rbdService *dbmodel.TenantServices
var err error
if serviceID != "" {
rbdService, err = db.GetManager().TenantServiceDao().GetServiceByID(serviceID)
if err != nil {
logrus.Errorf("get service by id %s error: %v", serviceID, err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteUpdate)
return
}
} else {
rbdService, err = db.GetManager().TenantServiceDao().GetServiceByTenantIDAndServiceAlias(tenant.UUID, serviceName)
if err != nil {
logrus.Warnf("get service by tenant_id %s and service_alias %s error: %v, will use default selector", tenant.UUID, serviceName, err)
}
}
if rbdService != nil {
serviceAlias = rbdService.ServiceAlias
if resolvedServiceID == "" {
resolvedServiceID = rbdService.ServiceID
}
}
if !isThirdParty {
if backendService, err := k.Services(tenant.Namespace).Get(r.Context(), serviceName, v1.GetOptions{}); err == nil {
if alias := backendService.Labels["service_alias"]; alias != "" && serviceAlias == serviceName {
serviceAlias = alias
}
if resolvedServiceID == "" {
resolvedServiceID = backendService.Labels["service_id"]
}
} else if !errors.IsNotFound(err) {
logrus.Warnf("get backend service %s in namespace %s error: %v", serviceName, tenant.Namespace, err)
}
if rbdService == nil && resolvedServiceID != "" {
rbdService, err = db.GetManager().TenantServiceDao().GetServiceByID(resolvedServiceID)
if err != nil {
logrus.Warnf("get service by resolved service_id %s error: %v", resolvedServiceID, err)
} else if rbdService != nil {
serviceAlias = rbdService.ServiceAlias
}
}
spec.Selector = map[string]string{
"service_alias": serviceAlias,
}
}
if rbdService != nil && rbdService.ExtendMethod == "kubeblocks_component" {
spec.Selector = kbutil.GenerateKubeBlocksSelector(rbdService.K8sComponentName)
logrus.Infof("Using KubeBlocks selector for service %s (k8s_component_name: %s)", serviceName, rbdService.K8sComponentName)
}
service, err := k.Services(tenant.Namespace).Get(r.Context(), name, v1.GetOptions{})
if err != nil {
if !errors.IsNotFound(err) {
logrus.Errorf("get route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrPortExists)
return
}
logrus.Infof("Service %s does not exist, creating a new service", name)
labels := make(map[string]string)
labels["creator"] = "Rainbond"
labels["tcp"] = "true"
labels["app_id"] = r.URL.Query().Get("appID")
labels["service_id"] = resolvedServiceID
labels["service_alias"] = serviceAlias
labels["outer"] = "true"
labels["port"] = apisixRouteStream.Backend.ServicePort.String()
service = &corev1.Service{
ObjectMeta: v1.ObjectMeta{
Labels: labels,
Name: name,
},
Spec: spec,
}
for {
nodePort := service.Spec.Ports[0].NodePort
_, err = k.Services(tenant.Namespace).Create(r.Context(), service, v1.CreateOptions{})
if err != nil {
if strings.Contains(err.Error(), "provided port is already allocated") {
logrus.Infof("NodePort %d is already allocated, trying next port...", nodePort)
nodePort++
service.Spec.Ports[0].NodePort = nodePort
service.ObjectMeta.Name = fmt.Sprintf("%v-%v", serviceName, nodePort)
service.Spec.Ports[0].Name = service.ObjectMeta.Name
continue
} else {
logrus.Errorf("create tcp rule func, create svc failure: %s", err.Error())
httputil.ReturnBcodeError(r, w, fmt.Errorf("create tcp rule func, create svc failure: %s", err.Error()))
return
}
}
apisixRouteStream.Match.IngressPort = nodePort
logrus.Infof("Service created successfully with NodePort %d", nodePort)
break
}
} else {
logrus.Infof("Service %s already exists, updating it", name)
service.Spec = spec
_, err = k.Services(tenant.Namespace).Update(r.Context(), service, v1.UpdateOptions{})
if err != nil {
logrus.Errorf("update route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrPortExists)
return
}
}
tcpRule := &dbmodel.TCPRule{
UUID: resolvedServiceID,
ServiceID: resolvedServiceID,
ContainerPort: int(apisixRouteStream.Backend.ServicePort.IntVal),
IP: "0.0.0.0",
Port: int(apisixRouteStream.Match.IngressPort),
}
if err := db.GetManager().TCPRuleDao().AddModel(tcpRule); err != nil {
logrus.Errorf("add tcp %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrPortExists)
return
}
httputil.ReturnSuccess(r, w, service.Spec.Ports[0].NodePort)
return
}
func (g Struct) UpdateTCPRoute(w http.ResponseWriter, r *http.Request) {
panic("implement me")
}
func (g Struct) DeleteTCPRoute(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
name := chi.URLParam(r, "name")
k := k8s.Default().Clientset.CoreV1()
service, err := k.Services(tenant.Namespace).Get(r.Context(), name, v1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
service, err = findTCPRouteServiceByNodePort(r.Context(), k.Services(tenant.Namespace), name)
if err != nil {
logrus.Errorf("failed to find tcp route service by node port for %s: %v", name, err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteDelete)
return
}
if service == nil {
logrus.Infof("Service %s not found, treating as already deleted", name)
httputil.ReturnSuccess(r, w, name)
return
}
logrus.Infof("Service %s not found, fallback to TCP route Service %s by NodePort", name, service.Name)
name = service.Name
} else {
logrus.Errorf("failed to get service %s: %v", name, err)
httputil.ReturnBcodeError(r, w, bcode.ErrRouteDelete)
return
}
}
logrus.Infof("Deleting TCP route Service: %s, labels: %v, port: %v",
name, service.Labels, service.Spec.Ports[0].Port)
err = k.Services(tenant.Namespace).Delete(r.Context(), name, v1.DeleteOptions{})
if err != nil {
logrus.Errorf("delete route error %s", err.Error())
httputil.ReturnBcodeError(r, w, bcode.ErrRouteDelete)
return
}
logrus.Infof("Successfully deleted TCP route Service: %s", name)
httputil.ReturnSuccess(r, w, name)
}
func findTCPRouteServiceByNodePort(ctx context.Context, services typedcorev1.ServiceInterface, routeName string) (*corev1.Service, error) {
nodePort, ok := nodePortFromTCPRouteName(routeName)
if !ok {
return nil, nil
}
list, err := services.List(ctx, v1.ListOptions{
LabelSelector: "tcp=true,outer=true",
})
if err != nil {
return nil, err
}
for i := range list.Items {
service := &list.Items[i]
for _, port := range service.Spec.Ports {
if port.NodePort == nodePort {
return service.DeepCopy(), nil
}
}
}
return nil, nil
}
func nodePortFromTCPRouteName(routeName string) (int32, bool) {
idx := strings.LastIndex(routeName, "-")
if idx == -1 || idx == len(routeName)-1 {
return 0, false
}
port, err := strconv.Atoi(routeName[idx+1:])
if err != nil {
return 0, false
}
return int32(port), true
}
func removeLeadingDigits(name string) string {
re := regexp.MustCompile(`^\d+`)
name = re.ReplaceAllString(name, "")
parts := strings.Split(name, "-")
if len(parts) <= 1 {
return ""
}
if parts[len(parts)-1] == "s" {
return name
}
return strings.Join(parts[:len(parts)-1], "-")
}
func (g Struct) CheckCertManager(w http.ResponseWriter, r *http.Request) {
kubeConfig := config.GetConfigOrDie()
apiextensionsClient, err := clientset.NewForConfig(kubeConfig)
if err != nil {
logrus.Errorf("failed to create apiextensions client: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to create apiextensions client: %v", err))
return
}
crdName := "certificates.cert-manager.io"
_, err = apiextensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(r.Context(), crdName, metav1.GetOptions{})
if err != nil {
if k8sErrors.IsNotFound(err) {
httputil.ReturnSuccess(r, w, map[string]interface{}{
"exists": false,
"message": "Certificate CRD not found. cert-manager may not be installed.",
})
return
}
logrus.Errorf("error checking Certificate CRD: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("error checking Certificate CRD: %v", err))
return
}
httputil.ReturnSuccess(r, w, map[string]interface{}{
"exists": true,
"message": "Certificate CRD exists. cert-manager is installed.",
})
}
func (g Struct) CreateCertManager(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
var req struct {
RouteName string `json:"route_name"`
Domains []string `json:"domains"`
RegionAppID string `json:"region_app_id"`
}
if err := httputil.ReadEntity(r, &req); err != nil {
httputil.ReturnError(r, w, 400, err.Error())
return
}
resourceLabel := make(map[string]string)
resourceLabel["app_id"] = req.RegionAppID
if len(req.Domains) == 0 {
httputil.ReturnError(r, w, 400, "domains cannot be empty")
return
}
req.RouteName = removeLeadingDigits(req.RouteName)
cert := &cmapi.Certificate{
ObjectMeta: v1.ObjectMeta{
Name: req.RouteName,
Namespace: tenant.Namespace,
Labels: resourceLabel,
},
Spec: cmapi.CertificateSpec{
DNSNames: req.Domains,
SecretName: req.RouteName,
IssuerRef: v12.ObjectReference{
Kind: "ClusterIssuer",
Name: "letsencrypt-http",
},
},
}
scheme := runtime.NewScheme()
_ = cmapi.AddToScheme(scheme)
kubeConfig := config.GetConfigOrDie()
k8sClient, err := client.New(kubeConfig, client.Options{Scheme: scheme})
if err != nil {
logrus.Errorf("failed to create k8s client: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to create k8s client: %v", err))
return
}
err = k8sClient.Create(r.Context(), cert)
if err != nil && !errors.IsAlreadyExists(err) {
logrus.Errorf("create certificate error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("create certificate error: %v", err))
return
}
hosts := make([]v2.HostType, len(req.Domains))
for i, domain := range req.Domains {
hosts[i] = v2.HostType(domain)
}
if err := util.CheckDomainConflict(r.Context(), hosts, tenant.Namespace, req.RouteName); err != nil {
logrus.Errorf("domain conflict detected: %s", err.Error())
httputil.ReturnError(r, w, http.StatusConflict, fmt.Sprintf("domain conflict: %v", err))
return
}
apisixTls := &v2.ApisixTls{
TypeMeta: v1.TypeMeta{
Kind: util.ApisixTLS,
APIVersion: util.APIVersion,
},
ObjectMeta: v1.ObjectMeta{
Name: req.RouteName,
Namespace: tenant.Namespace,
Labels: resourceLabel,
},
Spec: &v2.ApisixTlsSpec{
IngressClassName: "apisix",
Hosts: hosts,
Secret: v2.ApisixSecret{
Name: req.RouteName,
Namespace: tenant.Namespace,
},
},
}
c := k8s.Default().ApiSixClient.ApisixV2()
_, err = c.ApisixTlses(tenant.Namespace).Create(r.Context(), apisixTls, v1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
logrus.Errorf("create certificate error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("create certificate error: %v", err))
return
}
route, err := c.ApisixRoutes(tenant.Namespace).Get(r.Context(), req.RouteName, v1.GetOptions{})
if err != nil {
logrus.Errorf("get apisix route error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("get apisix route error: %v", err))
return
}
if route.Labels == nil {
route.Labels = make(map[string]string)
}
route.Labels["cert-manager-enabled"] = "true"
_, err = c.ApisixRoutes(tenant.Namespace).Update(r.Context(), route, v1.UpdateOptions{})
if err != nil {
logrus.Errorf("update apisix route error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("update apisix route error: %v", err))
return
}
httputil.ReturnSuccess(r, w, nil)
}
func (g Struct) GetCertManager(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
type CertificateInfo struct {
Domains []string `json:"domains"`
Status string `json:"status"`
ExpiryDate time.Time `json:"expiry_date"`
AutoRenew bool `json:"auto_renew"`
IssueDetail string `json:"issue_detail"`
Name string `json:"name"`
}
scheme := runtime.NewScheme()
if err := cmapi.AddToScheme(scheme); err != nil {
logrus.Errorf("failed to add cert-manager scheme: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to add cert-manager scheme: %v", err))
return
}
if err := v13.AddToScheme(scheme); err != nil {
logrus.Errorf("failed to add acme scheme: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to add acme scheme: %v", err))
return
}
kubeConfig := config.GetConfigOrDie()
k8sClient, err := client.New(kubeConfig, client.Options{Scheme: scheme})
if err != nil {
logrus.Errorf("failed to create k8s client: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to create k8s client: %v", err))
return
}
appID := r.URL.Query().Get("region_app_id")
if appID == "" {
httputil.ReturnError(r, w, 400, "region_app_id is required")
return
}
selector, _ := labels.Parse(labels.FormatLabels(map[string]string{
"app_id": appID,
}))
certList := &cmapi.CertificateList{}
err = k8sClient.List(r.Context(), certList, &client.ListOptions{
Namespace: tenant.Namespace,
LabelSelector: selector,
})
if err != nil {
logrus.Errorf("list certificates error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("list certificates error: %v", err))
return
}
challengeList := &v13.ChallengeList{}
err = k8sClient.List(r.Context(), challengeList, &client.ListOptions{
Namespace: tenant.Namespace,
})
if err != nil {
logrus.Errorf("list challenges error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("list challenges error: %v", err))
return
}
challengeMap := make(map[string]*v13.Challenge)
for i := range challengeList.Items {
challenge := &challengeList.Items[i]
baseName := extractBaseName(challenge.Name)
challengeMap[baseName] = challenge
}
var certInfoList []CertificateInfo
for _, cert := range certList.Items {
certInfo := CertificateInfo{
Name: cert.Name,
Domains: cert.Spec.DNSNames,
AutoRenew: true,
}
if len(cert.Status.Conditions) > 0 {
for _, condition := range cert.Status.Conditions {
if condition.Type == cmapi.CertificateConditionReady {
certInfo.Status = string(condition.Status)
certInfo.IssueDetail = condition.Message
break
}
}
}
if cert.Status.NotAfter != nil {
certInfo.ExpiryDate = cert.Status.NotAfter.Time
}
if challenge, exists := challengeMap[cert.Name]; exists {
if certInfo.Status != "True" {
certInfo.IssueDetail = fmt.Sprintf("%s: %s",
challenge.Status.State,
challenge.Status.Reason)
if challenge.Status.Processing {
certInfo.IssueDetail = fmt.Sprintf("%v\nProcessing: %v", certInfo.IssueDetail, challenge.Status.Presented)
}
}
}
certInfoList = append(certInfoList, certInfo)
}
httputil.ReturnSuccess(r, w, certInfoList)
}
func (g Struct) DeleteCertManager(w http.ResponseWriter, r *http.Request) {
tenant := r.Context().Value(ctxutil.ContextKey("tenant")).(*dbmodel.Tenants)
RouteName := r.URL.Query().Get("route_name")
regionAppID := r.URL.Query().Get("region_app_id")
domains := parseCertManagerDomains(r.URL.Query().Get("domains"))
if RouteName == "" && len(domains) == 0 {
httputil.ReturnError(r, w, 400, "route_name or domains is required")
return
}
if len(domains) > 0 && regionAppID == "" {
httputil.ReturnError(r, w, 400, "region_app_id is required")
return
}
if RouteName != "" {
RouteName = removeLeadingDigits(RouteName)
}
scheme := runtime.NewScheme()
_ = cmapi.AddToScheme(scheme)
kubeConfig := config.GetConfigOrDie()
k8sClient, err := client.New(kubeConfig, client.Options{Scheme: scheme})
if err != nil {
logrus.Errorf("failed to create k8s client: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("failed to create k8s client: %v", err))
return
}
c := k8s.Default().ApiSixClient.ApisixV2()
if len(domains) > 0 {
if err = deleteCertificatesByDomains(r.Context(), k8sClient, c, tenant.Namespace, regionAppID, domains); err != nil {
logrus.Errorf("delete certificate by domains error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("delete certificate by domains error: %v", err))
return
}
if err = clearCertManagerLabelsByDomains(r.Context(), c, tenant.Namespace, domains); err != nil {
logrus.Errorf("clear route cert-manager labels error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("clear route cert-manager labels error: %v", err))
return
}
httputil.ReturnSuccess(r, w, nil)
return
}
cert := &cmapi.Certificate{
ObjectMeta: v1.ObjectMeta{
Name: RouteName,
Namespace: tenant.Namespace,
},
}
err = k8sClient.Delete(r.Context(), cert)
if err != nil && !errors.IsNotFound(err) {
logrus.Errorf("delete certificate error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("delete certificate error: %v", err))
return
}
err = c.ApisixTlses(tenant.Namespace).Delete(r.Context(), RouteName, v1.DeleteOptions{})
if err != nil && !errors.IsNotFound(err) {
logrus.Errorf("delete apisix tls error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("delete apisix tls error: %v", err))
return
}
route, err := c.ApisixRoutes(tenant.Namespace).Get(r.Context(), RouteName, v1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
httputil.ReturnSuccess(r, w, nil)
return
}
logrus.Errorf("get apisix route error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("get apisix route error: %v", err))
return
}
if route.Labels != nil {
delete(route.Labels, "cert-manager-enabled")
_, err = c.ApisixRoutes(tenant.Namespace).Update(r.Context(), route, v1.UpdateOptions{})
if err != nil {
logrus.Errorf("update apisix route error: %v", err)
httputil.ReturnError(r, w, 500, fmt.Sprintf("update apisix route error: %v", err))
return
}
}
httputil.ReturnSuccess(r, w, nil)
}
func extractBaseName(challengeName string) string {
parts := strings.Split(challengeName, "-")
if len(parts) < 4 {
return challengeName
}
return strings.Join(parts[:len(parts)-3], "-")
}
func parseCertManagerDomains(raw string) []string {
if raw == "" {
return nil
}
parts := strings.Split(raw, ",")
domains := make([]string, 0, len(parts))
for _, part := range parts {
domain := strings.TrimSpace(part)
if domain == "" {
continue
}
domains = append(domains, domain)
}
return domains
}
func hasMatchingCertManagerDomain(certDomains []string, routeDomains []string) bool {
for _, certDomain := range certDomains {
for _, routeDomain := range routeDomains {
if certManagerDomainsConflict(certDomain, routeDomain) {
return true
}
}
}
return false
}
func certManagerDomainsConflict(domain1, domain2 string) bool {
return domain1 == domain2
}
func routeMatchesCertManagerDomains(route *v2.ApisixRoute, domains []string) bool {
if route == nil || len(domains) == 0 {
return false
}
for _, httpRoute := range route.Spec.HTTP {
if hasMatchingCertManagerDomain(httpRoute.Match.Hosts, domains) {
return true
}
}
return false
}
func deleteCertificatesByDomains(ctx context.Context, k8sClient client.Client, c versionedApisixV2, namespace, regionAppID string, domains []string) error {
selector, _ := labels.Parse(labels.FormatLabels(map[string]string{
"app_id": regionAppID,
}))
certList := &cmapi.CertificateList{}
if err := k8sClient.List(ctx, certList, &client.ListOptions{
Namespace: namespace,
LabelSelector: selector,
}); err != nil {
return err
}
for i := range certList.Items {
cert := &certList.Items[i]
if !hasMatchingCertManagerDomain(cert.Spec.DNSNames, domains) {
continue
}
if err := k8sClient.Delete(ctx, cert); err != nil && !errors.IsNotFound(err) {
return err
}
if err := c.ApisixTlses(namespace).Delete(ctx, cert.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
return err
}
}
return nil
}
func clearCertManagerLabelsByDomains(ctx context.Context, c versionedApisixV2, namespace string, domains []string) error {
routeList, err := c.ApisixRoutes(namespace).List(ctx, v1.ListOptions{})
if err != nil {
return err
}
for i := range routeList.Items {
route := &routeList.Items[i]
if !routeMatchesCertManagerDomains(route, domains) || route.Labels == nil {
continue
}
if _, ok := route.Labels["cert-manager-enabled"]; !ok {
continue
}
delete(route.Labels, "cert-manager-enabled")
if _, err := c.ApisixRoutes(namespace).Update(ctx, route, v1.UpdateOptions{}); err != nil {
return err
}
}
return nil
}
type versionedApisixV2 = apisixclientv2.ApisixV2Interface
func sanitizeLabelKey(key string) string {
key = strings.ReplaceAll(key, "*", "wildcard")
return key
}