package utils
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"os"
"path/filepath"
"time"
"gitcode.com/openFuyao/e2e-auto-test/e2e/framework/executor"
)
type CSRConfig struct {
CN string `json:"CN"`
O string `json:"O"`
C string `json:"C"`
ST string `json:"ST"`
L string `json:"L"`
OU string `json:"OU"`
Key KeyConfig `json:"key"`
Hosts []string `json:"hosts"`
}
type KeyConfig struct {
Algo string `json:"algo"`
Size int `json:"size"`
}
type SignPolicyConfig struct {
Signing SigningConfig `json:"signing"`
}
type SigningConfig struct {
Default *ProfileConfig `json:"default,omitempty"`
Profiles map[string]*ProfileConfig `json:"profiles,omitempty"`
}
type ProfileConfig struct {
Usages []string `json:"usages,omitempty"`
Expiry string `json:"expiry,omitempty"`
CAConstraint *CAConstraint `json:"ca_constraint,omitempty"`
}
type CAConstraint struct {
IsCA bool `json:"is_ca,omitempty"`
MaxPathLen *int `json:"max_path_len,omitempty"`
MaxPathLenZero bool `json:"max_path_len_zero,omitempty"`
}
type CertificateConfigOptions struct {
NodeName string
}
type CertificateConfigGenerator struct {
baseDir string
options *CertificateConfigOptions
}
func NewCertificateConfigGenerator(baseDir string) *CertificateConfigGenerator {
return &CertificateConfigGenerator{
baseDir: baseDir,
options: &CertificateConfigOptions{
NodeName: "test-node",
},
}
}
func NewCertificateConfigGeneratorWithOptions(baseDir string, options *CertificateConfigOptions) *CertificateConfigGenerator {
if options == nil {
options = &CertificateConfigOptions{
NodeName: "test-node",
}
}
return &CertificateConfigGenerator{
baseDir: baseDir,
options: options,
}
}
func (g *CertificateConfigGenerator) GenerateAllCertificateConfigs() error {
if err := os.MkdirAll(g.baseDir, 0755); err != nil {
return fmt.Errorf("创建证书配置目录失败: %w", err)
}
csrConfigs := g.getAllCSRConfigs()
for filename, config := range csrConfigs {
if err := g.generateCSRConfig(filename, config); err != nil {
return fmt.Errorf("生成 %s 失败: %w", filename, err)
}
}
if err := g.generateSignPolicyConfig(); err != nil {
return fmt.Errorf("生成 sign-policy.json 失败: %w", err)
}
if err := g.generateClusterCAPolicyConfig(); err != nil {
return fmt.Errorf("生成 cluster-ca-policy.json 失败: %w", err)
}
return nil
}
func (g *CertificateConfigGenerator) generateCSRConfig(filename string, config *CSRConfig) error {
filePath := filepath.Join(g.baseDir, filename)
return g.writeJSONFile(filePath, config)
}
func (g *CertificateConfigGenerator) generateSignPolicyConfig() error {
config := &SignPolicyConfig{
Signing: SigningConfig{
Default: &ProfileConfig{
Usages: []string{"digital signature", "key encipherment"},
Expiry: "8760h",
},
Profiles: map[string]*ProfileConfig{
"ca": {
Usages: []string{"cert sign", "crl sign"},
Expiry: "87600h",
CAConstraint: &CAConstraint{
IsCA: true,
MaxPathLen: intPtr(0),
},
},
"apiserver": {
Usages: []string{"digital signature", "key encipherment", "server auth"},
Expiry: "8760h",
},
"apiserver-etcd-client": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"apiserver-kubelet-client": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"front-proxy-client": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"front-proxy-ca": {
Usages: []string{"cert sign", "crl sign"},
Expiry: "87600h",
CAConstraint: &CAConstraint{
IsCA: true,
MaxPathLen: intPtr(1),
},
},
"etcd/ca": {
Usages: []string{"cert sign", "crl sign"},
Expiry: "87600h",
CAConstraint: &CAConstraint{
IsCA: true,
MaxPathLen: intPtr(1),
},
},
"etcd/server": {
Usages: []string{"digital signature", "key encipherment", "server auth", "client auth"},
Expiry: "8760h",
},
"etcd/peer": {
Usages: []string{"digital signature", "key encipherment", "server auth", "client auth"},
Expiry: "8760h",
},
"etcd/healthcheck-client": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"controller-manager": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"scheduler": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"kubelet": {
Usages: []string{"digital signature", "key encipherment", "server auth", "client auth"},
Expiry: "8760h",
},
"admin": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
"kube-proxy": {
Usages: []string{"digital signature", "key encipherment", "client auth"},
Expiry: "8760h",
},
},
},
}
filePath := filepath.Join(g.baseDir, "sign-policy.json")
return g.writeJSONFile(filePath, config)
}
func (g *CertificateConfigGenerator) generateClusterCAPolicyConfig() error {
config := &SignPolicyConfig{
Signing: SigningConfig{
Default: &ProfileConfig{
Usages: []string{"cert sign", "crl sign"},
Expiry: "87600h",
CAConstraint: &CAConstraint{
IsCA: true,
MaxPathLen: intPtr(0),
},
},
Profiles: map[string]*ProfileConfig{
"ca": {
Usages: []string{"cert sign", "crl sign"},
Expiry: "87600h",
CAConstraint: &CAConstraint{
IsCA: true,
MaxPathLenZero: true,
},
},
},
},
}
filePath := filepath.Join(g.baseDir, "cluster-ca-policy.json")
return g.writeJSONFile(filePath, config)
}
func (g *CertificateConfigGenerator) getAllCSRConfigs() map[string]*CSRConfig {
return map[string]*CSRConfig{
"cluster-ca-csr.json": {
CN: "kubernetes",
O: "system:masters",
C: "CN",
ST: "Beijing-cluster-ca",
L: "Beijing-cluster-ca",
OU: "Kubernetes-cluster-ca",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{"kubernetes", "kubernetes.default"},
},
"apiserver-csr.json": g.buildAPIServerCSRConfig(),
"apiserver-kubelet-client-csr.json": {
CN: "apiserver-kubelet-client",
O: "system:masters",
C: "CN",
ST: "Beijing-apiserver-kubelet-client",
L: "Beijing-apiserver-kubelet-client",
OU: "Kubernetes-apiserver-kubelet-client",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"apiserver-etcd-client-csr.json": {
CN: "apiserver-etcd-client",
O: "system:masters",
C: "CN",
ST: "Beijing-apiserver-etcd-client",
L: "Beijing-apiserver-etcd-client",
OU: "Kubernetes-apiserver-etcd-client",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"controller-manager-csr.json": {
CN: "system:kube-controller-manager",
O: "system:kube-controller-manager",
C: "CN",
ST: "Beijing-controller-manager",
L: "Beijing-controller-manager",
OU: "Kubernetes-controller-manager",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"scheduler-csr.json": {
CN: "system:kube-scheduler",
O: "system:kube-scheduler",
C: "CN",
ST: "Beijing-scheduler",
L: "Beijing-scheduler",
OU: "Kubernetes-scheduler",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"kube-proxy-csr.json": {
CN: "system:kube-proxy",
O: "system:node-proxier",
C: "CN",
ST: "Beijing-kube-proxy",
L: "Beijing-kube-proxy",
OU: "Kubernetes-kube-proxy",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"admin-kubeconfig-csr.json": {
CN: "kubernetes-admin",
O: "system:masters",
C: "CN",
ST: "Beijing-admin",
L: "Beijing-admin",
OU: "kubernetes-admin",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"kubelet-kubeconfig-csr.json": g.buildKubeletCSRConfig(),
"etcd-ca-csr.json": {
CN: "etcd-ca",
O: "system:masters",
C: "CN",
ST: "Beijing-etcd-ca",
L: "Beijing-etcd-ca",
OU: "Kubernetes-etcd-ca",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"etcd-server-csr.json": {
CN: "etcd-server",
O: "system:masters",
C: "CN",
ST: "Beijing-etcd-server",
L: "Beijing-etcd-server",
OU: "Kubernetes-etcd-server",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{"localhost", "127.0.0.1"},
},
"etcd-peer-csr.json": {
CN: "etcd-peer",
O: "system:masters",
C: "CN",
ST: "Beijing-etcd-peer",
L: "Beijing-etcd-peer",
OU: "Kubernetes-etcd-peer",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"etcd-healthcheck-client-csr.json": {
CN: "etcd-healthcheck-client",
O: "system:masters",
C: "CN",
ST: "Beijing-etcd-healthcheck-client",
L: "Beijing-etcd-healthcheck-client",
OU: "Kubernetes-etcd-healthcheck-client",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"front-proxy-ca-csr.json": {
CN: "front-proxy-ca",
O: "system:masters",
C: "CN",
ST: "Beijing-front-proxy-ca",
L: "Beijing-front-proxy-ca",
OU: "Kubernetes-front-proxy-ca",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
"front-proxy-client-csr.json": {
CN: "front-proxy-client",
O: "system:masters",
C: "CN",
ST: "Beijing-front-proxy-client",
L: "Beijing-front-proxy-client",
OU: "Kubernetes-front-proxy-client",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
},
}
}
func (g *CertificateConfigGenerator) writeJSONFile(filePath string, config interface{}) error {
jsonData, err := json.MarshalIndent(config, "", "\t")
if err != nil {
return fmt.Errorf("序列化 JSON 失败: %w", err)
}
if err := os.WriteFile(filePath, jsonData, 0644); err != nil {
return fmt.Errorf("写入文件失败: %w", err)
}
return nil
}
func (g *CertificateConfigGenerator) CleanupCertificateConfigs() error {
if err := os.RemoveAll(g.baseDir); err != nil {
return fmt.Errorf("清理证书配置目录失败: %w", err)
}
return nil
}
func (g *CertificateConfigGenerator) GetBaseDir() string {
return g.baseDir
}
func (g *CertificateConfigGenerator) buildAPIServerCSRConfig() *CSRConfig {
hosts := []string{
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster.local",
"10.0.0.1",
"{{.ClusterName}}",
"{{.AdvertiseAddress}}",
}
return &CSRConfig{
CN: "kube-apiserver",
O: "system:masters",
C: "CN",
ST: "Beijing-apiserver",
L: "Beijing-apiserver",
OU: "Kubernetes-apiserver",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: hosts,
}
}
func (g *CertificateConfigGenerator) buildKubeletCSRConfig() *CSRConfig {
nodeName := g.options.NodeName
if nodeName == "" {
nodeName = "test-node"
}
return &CSRConfig{
CN: fmt.Sprintf("system:node:%s", nodeName),
O: "system:nodes",
C: "CN",
ST: "Beijing-kubelet",
L: "Beijing-kubelet",
OU: "Kubernetes-kubelet",
Key: KeyConfig{
Algo: "rsa",
Size: 2048,
},
Hosts: []string{},
}
}
func intPtr(i int) *int {
return &i
}
func GenerateCertificateChain(outputDir string) error {
if err := os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("创建输出目录失败: %w", err)
}
rootCAKey, rootCACert, err := generateRootCA()
if err != nil {
return fmt.Errorf("生成根CA失败: %w", err)
}
intermediateCAKey, intermediateCACert, err := generateIntermediateCA(rootCAKey, rootCACert)
if err != nil {
return fmt.Errorf("生成中间CA失败: %w", err)
}
globalCAKey, globalCACert, err := generateGlobalCA(intermediateCAKey, intermediateCACert)
if err != nil {
return fmt.Errorf("生成global-ca失败: %w", err)
}
globalCAKeyPath := filepath.Join(outputDir, "global-ca.key")
if err := savePrivateKey(globalCAKeyPath, globalCAKey); err != nil {
return fmt.Errorf("保存global-ca.key失败: %w", err)
}
globalCACertPath := filepath.Join(outputDir, "global-ca.crt")
if err := saveCertificate(globalCACertPath, globalCACert); err != nil {
return fmt.Errorf("保存global-ca.crt失败: %w", err)
}
trustChainPath := filepath.Join(outputDir, "trust-chain.crt")
if err := saveCertificateChain(trustChainPath, globalCACert, intermediateCACert, rootCACert); err != nil {
return fmt.Errorf("保存trust-chain.crt失败: %w", err)
}
return nil
}
func GenerateCertificateChainOnRemote(exec *executor.SSHExecutor, remoteDir string) error {
rootCAKey, rootCACert, err := generateRootCA()
if err != nil {
return fmt.Errorf("生成根CA失败: %w", err)
}
intermediateCAKey, intermediateCACert, err := generateIntermediateCA(rootCAKey, rootCACert)
if err != nil {
return fmt.Errorf("生成中间CA失败: %w", err)
}
globalCAKey, globalCACert, err := generateGlobalCA(intermediateCAKey, intermediateCACert)
if err != nil {
return fmt.Errorf("生成global-ca失败: %w", err)
}
mkdirCmd := fmt.Sprintf("mkdir -p %s", remoteDir)
result, err := exec.Exec(mkdirCmd)
if err != nil {
return fmt.Errorf("创建远程目录失败: %w", err)
}
if result.ExitCode != 0 {
return fmt.Errorf("创建远程目录失败: %s", result.Stderr)
}
globalCAKeyPath := filepath.Join(remoteDir, "global-ca.key")
if err := savePrivateKeyToRemote(exec, globalCAKeyPath, globalCAKey); err != nil {
return fmt.Errorf("保存global-ca.key到远程节点失败: %w", err)
}
globalCACertPath := filepath.Join(remoteDir, "global-ca.crt")
if err := saveCertificateToRemote(exec, globalCACertPath, globalCACert); err != nil {
return fmt.Errorf("保存global-ca.crt到远程节点失败: %w", err)
}
trustChainPath := filepath.Join(remoteDir, "trust-chain.crt")
if err := saveCertificateChainToRemote(exec, trustChainPath, globalCACert, intermediateCACert, rootCACert); err != nil {
return fmt.Errorf("保存trust-chain.crt到远程节点失败: %w", err)
}
return nil
}
func savePrivateKeyToRemote(exec *executor.SSHExecutor, remotePath string, key *rsa.PrivateKey) error {
keyBytes := x509.MarshalPKCS1PrivateKey(key)
keyBlock := &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: keyBytes,
}
keyPEM := pem.EncodeToMemory(keyBlock)
if keyPEM == nil {
return fmt.Errorf("编码私钥失败")
}
encoded := base64.StdEncoding.EncodeToString(keyPEM)
cmd := fmt.Sprintf("cat <<EOF | base64 -d > %s\n%s\nEOF", remotePath, encoded)
result, err := exec.Exec(cmd)
if err != nil {
return fmt.Errorf("写入私钥到远程节点失败: %w", err)
}
if result.ExitCode != 0 {
return fmt.Errorf("写入私钥到远程节点失败: %s", result.Stderr)
}
return nil
}
func saveCertificateToRemote(exec *executor.SSHExecutor, remotePath string, cert *x509.Certificate) error {
certBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
certPEM := pem.EncodeToMemory(certBlock)
if certPEM == nil {
return fmt.Errorf("编码证书失败")
}
encoded := base64.StdEncoding.EncodeToString(certPEM)
cmd := fmt.Sprintf("cat <<EOF | base64 -d > %s\n%s\nEOF", remotePath, encoded)
result, err := exec.Exec(cmd)
if err != nil {
return fmt.Errorf("写入证书到远程节点失败: %w", err)
}
if result.ExitCode != 0 {
return fmt.Errorf("写入证书到远程节点失败: %s", result.Stderr)
}
return nil
}
func saveCertificateChainToRemote(exec *executor.SSHExecutor, remotePath string, certs ...*x509.Certificate) error {
var chainPEM []byte
for _, cert := range certs {
certBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
certPEM := pem.EncodeToMemory(certBlock)
if certPEM == nil {
return fmt.Errorf("编码证书失败")
}
chainPEM = append(chainPEM, certPEM...)
}
encoded := base64.StdEncoding.EncodeToString(chainPEM)
cmd := fmt.Sprintf("cat <<EOF | base64 -d > %s\n%s\nEOF", remotePath, encoded)
result, err := exec.Exec(cmd)
if err != nil {
return fmt.Errorf("写入证书链到远程节点失败: %w", err)
}
if result.ExitCode != 0 {
return fmt.Errorf("写入证书链到远程节点失败: %s", result.Stderr)
}
return nil
}
func generateRootCA() (*rsa.PrivateKey, *x509.Certificate, error) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, nil, fmt.Errorf("生成私钥失败: %w", err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Country: []string{"CN"},
Province: []string{"Guangdong"},
Locality: []string{"Guangdong"},
Organization: []string{"Test Company"},
OrganizationalUnit: []string{"Root CA"},
CommonName: "Test Root CA",
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
MaxPathLen: 0,
MaxPathLenZero: true,
}
certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
return nil, nil, fmt.Errorf("创建证书失败: %w", err)
}
cert, err := x509.ParseCertificate(certDER)
if err != nil {
return nil, nil, fmt.Errorf("解析证书失败: %w", err)
}
return key, cert, nil
}
func generateIntermediateCA(parentKey *rsa.PrivateKey, parentCert *x509.Certificate) (*rsa.PrivateKey, *x509.Certificate, error) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, nil, fmt.Errorf("生成私钥失败: %w", err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{
Country: []string{"CN"},
Province: []string{"Guangdong"},
Locality: []string{"Guangdong"},
Organization: []string{"Test Company"},
OrganizationalUnit: []string{"Intermediate CA"},
CommonName: "Test Intermediate CA",
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
MaxPathLen: 1,
MaxPathLenZero: false,
}
certDER, err := x509.CreateCertificate(rand.Reader, template, parentCert, &key.PublicKey, parentKey)
if err != nil {
return nil, nil, fmt.Errorf("创建证书失败: %w", err)
}
cert, err := x509.ParseCertificate(certDER)
if err != nil {
return nil, nil, fmt.Errorf("解析证书失败: %w", err)
}
return key, cert, nil
}
func generateGlobalCA(parentKey *rsa.PrivateKey, parentCert *x509.Certificate) (*rsa.PrivateKey, *x509.Certificate, error) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, nil, fmt.Errorf("生成私钥失败: %w", err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(3),
Subject: pkix.Name{
Country: []string{"CN"},
Province: []string{"Guangdong"},
Locality: []string{"Guangdong"},
Organization: []string{"Test Company"},
OrganizationalUnit: []string{"Cluster Issuer"},
CommonName: "Global CA",
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
MaxPathLen: 1,
MaxPathLenZero: false,
}
certDER, err := x509.CreateCertificate(rand.Reader, template, parentCert, &key.PublicKey, parentKey)
if err != nil {
return nil, nil, fmt.Errorf("创建证书失败: %w", err)
}
cert, err := x509.ParseCertificate(certDER)
if err != nil {
return nil, nil, fmt.Errorf("解析证书失败: %w", err)
}
return key, cert, nil
}
func savePrivateKey(filePath string, key *rsa.PrivateKey) error {
keyFile, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("创建文件失败: %w", err)
}
defer keyFile.Close()
keyBytes := x509.MarshalPKCS1PrivateKey(key)
keyBlock := &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: keyBytes,
}
if err := pem.Encode(keyFile, keyBlock); err != nil {
return fmt.Errorf("编码私钥失败: %w", err)
}
return nil
}
func saveCertificate(filePath string, cert *x509.Certificate) error {
certFile, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("创建文件失败: %w", err)
}
defer certFile.Close()
certBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
if err := pem.Encode(certFile, certBlock); err != nil {
return fmt.Errorf("编码证书失败: %w", err)
}
return nil
}
func saveCertificateChain(filePath string, certs ...*x509.Certificate) error {
chainFile, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("创建文件失败: %w", err)
}
defer chainFile.Close()
for _, cert := range certs {
certBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
if err := pem.Encode(chainFile, certBlock); err != nil {
return fmt.Errorf("编码证书失败: %w", err)
}
}
return nil
}