* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright The KubeVirt Authors.
*
*/
package migrationproxy
import (
"crypto/tls"
"fmt"
"io"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"kubevirt.io/client-go/log"
diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"
"kubevirt.io/kubevirt/pkg/util"
"kubevirt.io/kubevirt/pkg/util/net/ip"
virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
)
const (
LibvirtDirectMigrationPort = 49152
LibvirtBlockMigrationPort = 49153
)
var migrationPortsRange = []int{LibvirtDirectMigrationPort, LibvirtBlockMigrationPort}
type ProxyManager interface {
StartTargetListener(key string, targetUnixFiles []string) error
GetTargetListenerPorts(key string) map[string]int
StopTargetListener(key string)
StartSourceListener(key string, targetAddress string, destSrcPortMap map[string]int, baseDir string) error
GetSourceListenerFiles(key string) []string
StopSourceListener(key string)
OpenListenerCount() int
InitiateGracefulShutdown()
}
type migrationProxyManager struct {
sourceProxies map[string][]*migrationProxy
targetProxies map[string][]*migrationProxy
managerLock sync.Mutex
serverTLSConfig *tls.Config
clientTLSConfig *tls.Config
migrationTLSConfig *tls.Config
isShuttingDown bool
config *virtconfig.ClusterConfig
}
type MigrationProxyListener interface {
Start() error
Stop()
}
type migrationProxy struct {
unixSocketPath string
tcpBindAddress string
tcpBindPort int
targetAddress string
targetProtocol string
stopChan chan struct{}
listenErrChan chan error
fdChan chan net.Conn
listener net.Listener
serverTLSConfig *tls.Config
clientTLSConfig *tls.Config
migrationTLSConfig *tls.Config
logger *log.FilteredLogger
}
func (m *migrationProxyManager) InitiateGracefulShutdown() {
m.managerLock.Lock()
defer m.managerLock.Unlock()
m.isShuttingDown = true
}
func (m *migrationProxyManager) OpenListenerCount() int {
m.managerLock.Lock()
defer m.managerLock.Unlock()
return len(m.sourceProxies) + len(m.targetProxies)
}
func GetMigrationPortsList(isBlockMigration bool) (ports []int) {
ports = append(ports, migrationPortsRange[0])
if isBlockMigration {
ports = append(ports, migrationPortsRange[1])
}
return
}
func NewMigrationProxyManager(serverTLSConfig *tls.Config, clientTLSConfig, migrationTLSConfig *tls.Config, config *virtconfig.ClusterConfig) ProxyManager {
return &migrationProxyManager{
sourceProxies: make(map[string][]*migrationProxy),
targetProxies: make(map[string][]*migrationProxy),
serverTLSConfig: serverTLSConfig,
clientTLSConfig: clientTLSConfig,
migrationTLSConfig: migrationTLSConfig,
config: config,
}
}
func SourceUnixFile(baseDir string, key string) string {
return filepath.Join(baseDir, "migrationproxy", key+"-source.sock")
}
func (m *migrationProxyManager) StartTargetListener(key string, targetUnixFiles []string) error {
m.managerLock.Lock()
defer m.managerLock.Unlock()
if m.isShuttingDown {
return fmt.Errorf("unable to process new migration connections during virt-handler shutdown")
}
isExistingProxy := func(curProxies []*migrationProxy, targetUnixFiles []string) bool {
if len(curProxies) != len(targetUnixFiles) {
return false
}
existingSocketFiles := make(map[string]bool)
for _, file := range targetUnixFiles {
existingSocketFiles[file] = true
}
for _, curProxy := range curProxies {
if _, ok := existingSocketFiles[curProxy.targetAddress]; !ok {
return false
}
}
return true
}
curProxies, exists := m.targetProxies[key]
if exists {
if isExistingProxy(curProxies, targetUnixFiles) {
return nil
} else {
for _, curProxy := range curProxies {
curProxy.logger.Infof("Manager stopping proxy on target node due to new unix filepath location")
curProxy.Stop()
}
}
}
zeroAddress := ip.GetIPZeroAddress()
proxiesList := []*migrationProxy{}
serverTLSConfig := m.serverTLSConfig
if m.config.GetMigrationConfiguration().DisableTLS != nil && *m.config.GetMigrationConfiguration().DisableTLS {
serverTLSConfig = nil
}
for _, targetUnixFile := range targetUnixFiles {
proxy := NewTargetProxy(zeroAddress, 0, serverTLSConfig, targetUnixFile, key)
err := proxy.Start()
if err != nil {
proxy.Stop()
for _, curProxy := range proxiesList {
curProxy.Stop()
}
return err
}
proxiesList = append(proxiesList, proxy)
proxy.logger.Infof("Manager created proxy on target")
}
m.targetProxies[key] = proxiesList
return nil
}
func (m *migrationProxyManager) GetSourceListenerFiles(key string) []string {
m.managerLock.Lock()
defer m.managerLock.Unlock()
curProxies, exists := m.sourceProxies[key]
socketsList := []string{}
if exists {
for _, curProxy := range curProxies {
socketsList = append(socketsList, curProxy.unixSocketPath)
}
}
return socketsList
}
func ConstructProxyKey(id string, port int) string {
key := id
if port != 0 {
key += fmt.Sprintf("-%d", port)
}
return key
}
func (m *migrationProxyManager) GetTargetListenerPorts(key string) map[string]int {
m.managerLock.Lock()
defer m.managerLock.Unlock()
getPortFromSocket := func(id string, path string) int {
for _, port := range migrationPortsRange {
key := ConstructProxyKey(id, port)
if strings.Contains(path, key) {
return port
}
}
return 0
}
curProxies, exists := m.targetProxies[key]
targetSrcPortMap := make(map[string]int)
if exists {
for _, curProxy := range curProxies {
port := strconv.Itoa(curProxy.tcpBindPort)
targetSrcPortMap[port] = getPortFromSocket(key, curProxy.targetAddress)
}
}
return targetSrcPortMap
}
func (m *migrationProxyManager) StopTargetListener(key string) {
m.managerLock.Lock()
defer m.managerLock.Unlock()
curProxies, exists := m.targetProxies[key]
if exists {
for _, curProxy := range curProxies {
curProxy.logger.Info("Manager stopping proxy on target node")
curProxy.Stop()
delete(m.targetProxies, key)
}
}
}
func (m *migrationProxyManager) StartSourceListener(key string, targetAddress string, destSrcPortMap map[string]int, baseDir string) error {
m.managerLock.Lock()
defer m.managerLock.Unlock()
if m.isShuttingDown {
return fmt.Errorf("unable to process new migration connections during virt-handler shutdown")
}
isExistingProxy := func(curProxies []*migrationProxy, targetAddress string, destSrcPortMap map[string]int) bool {
if len(curProxies) != len(destSrcPortMap) {
return false
}
destSrcLookup := make(map[string]int)
for dest, src := range destSrcPortMap {
addr := net.JoinHostPort(targetAddress, dest)
destSrcLookup[addr] = src
}
for _, curProxy := range curProxies {
if _, ok := destSrcLookup[curProxy.targetAddress]; !ok {
return false
}
}
return true
}
curProxies, exists := m.sourceProxies[key]
if exists {
if isExistingProxy(curProxies, targetAddress, destSrcPortMap) {
return nil
} else {
for _, curProxy := range curProxies {
curProxy.logger.Infof("Manager is stopping proxy on source node due to new target location")
curProxy.Stop()
}
}
}
clientTLSConfig := m.clientTLSConfig
migrationTLSConfig := m.migrationTLSConfig
if m.config.GetMigrationConfiguration().DisableTLS != nil && *m.config.GetMigrationConfiguration().DisableTLS {
clientTLSConfig = nil
migrationTLSConfig = nil
}
proxiesList := []*migrationProxy{}
for destPort, srcPort := range destSrcPortMap {
proxyKey := ConstructProxyKey(key, srcPort)
targetFullAddr := net.JoinHostPort(targetAddress, destPort)
filePath := SourceUnixFile(baseDir, proxyKey)
os.RemoveAll(filePath)
proxy := NewSourceProxy(filePath, targetFullAddr, clientTLSConfig, migrationTLSConfig, key)
err := proxy.Start()
if err != nil {
proxy.Stop()
for _, curProxy := range proxiesList {
curProxy.Stop()
}
return err
}
proxiesList = append(proxiesList, proxy)
proxy.logger.Infof("Manager created proxy on source node")
}
m.sourceProxies[key] = proxiesList
return nil
}
func (m *migrationProxyManager) StopSourceListener(key string) {
m.managerLock.Lock()
defer m.managerLock.Unlock()
curProxies, exists := m.sourceProxies[key]
if exists {
for _, curProxy := range curProxies {
curProxy.logger.Infof("Manager stopping proxy on source node")
curProxy.Stop()
os.RemoveAll(curProxy.unixSocketPath)
}
delete(m.sourceProxies, key)
}
}
func NewSourceProxy(unixSocketPath string, tcpTargetAddress string, clientTLSConfig, migrationTLSConfig *tls.Config, vmiUID string) *migrationProxy {
return &migrationProxy{
unixSocketPath: unixSocketPath,
targetAddress: tcpTargetAddress,
targetProtocol: "tcp",
stopChan: make(chan struct{}),
fdChan: make(chan net.Conn, 1),
listenErrChan: make(chan error, 1),
clientTLSConfig: clientTLSConfig,
migrationTLSConfig: migrationTLSConfig,
logger: log.Log.With("uid", vmiUID).With("listening", filepath.Base(unixSocketPath)).With("outbound", tcpTargetAddress),
}
}
func NewTargetProxy(tcpBindAddress string, tcpBindPort int, serverTLSConfig *tls.Config, virtqemudSocketPath string, vmiUID string) *migrationProxy {
return &migrationProxy{
tcpBindAddress: tcpBindAddress,
tcpBindPort: tcpBindPort,
targetAddress: virtqemudSocketPath,
targetProtocol: "unix",
stopChan: make(chan struct{}),
fdChan: make(chan net.Conn, 1),
listenErrChan: make(chan error, 1),
serverTLSConfig: serverTLSConfig,
logger: log.Log.With("uid", vmiUID).With("outbound", filepath.Base(virtqemudSocketPath)),
}
}
func (m *migrationProxy) createTcpListener() error {
var listener net.Listener
var err error
laddr := net.JoinHostPort(m.tcpBindAddress, strconv.Itoa(m.tcpBindPort))
if m.serverTLSConfig != nil {
listener, err = tls.Listen("tcp", laddr, m.serverTLSConfig)
} else {
listener, err = net.Listen("tcp", laddr)
}
if err != nil {
m.logger.Reason(err).Error("failed to create unix socket for proxy service")
return err
}
if m.tcpBindPort == 0 {
m.tcpBindPort = listener.Addr().(*net.TCPAddr).Port
m.logger = m.logger.With("listening", fmt.Sprintf("%s:%d", m.tcpBindAddress, m.tcpBindPort))
}
m.listener = listener
return nil
}
func (m *migrationProxy) createUnixListener() error {
os.RemoveAll(m.unixSocketPath)
err := util.MkdirAllWithNosec(filepath.Dir(m.unixSocketPath))
if err != nil {
m.logger.Reason(err).Error("unable to create directory for unix socket")
return err
}
listener, err := net.Listen("unix", m.unixSocketPath)
if err != nil {
m.logger.Reason(err).Error("failed to create unix socket for proxy service")
return err
}
if err := diskutils.DefaultOwnershipManager.UnsafeSetFileOwnership(m.unixSocketPath); err != nil {
log.Log.Reason(err).Error("failed to change ownership on migration unix socket")
return err
}
m.listener = listener
return nil
}
func (m *migrationProxy) Stop() {
close(m.stopChan)
if m.listener != nil {
m.logger.Infof("proxy stopped listening")
m.listener.Close()
}
}
func (m *migrationProxy) handleConnection(fd net.Conn) {
defer fd.Close()
outBoundErr := make(chan error, 1)
inBoundErr := make(chan error, 1)
var conn net.Conn
var err error
if m.targetProtocol == "tcp" && m.clientTLSConfig != nil {
conn, err = tls.Dial(m.targetProtocol, m.targetAddress, m.migrationTLSConfig)
if err != nil {
m.logger.Reason(err).Info("fallback to old tls config")
conn, err = tls.Dial(m.targetProtocol, m.targetAddress, m.clientTLSConfig)
} else if tlsErr := conn.(*tls.Conn).Handshake(); tlsErr != nil {
m.logger.Reason(err).Info("handshake failed, fallback to old tls config")
conn, err = tls.Dial(m.targetProtocol, m.targetAddress, m.clientTLSConfig)
}
} else {
conn, err = net.Dial(m.targetProtocol, m.targetAddress)
}
if err != nil {
m.logger.Reason(err).Error("unable to create outbound leg of proxy to host")
return
}
go func() {
n, err := io.Copy(fd, conn)
m.logger.Infof("%d bytes copied outbound to inbound", n)
inBoundErr <- err
}()
go func() {
n, err := io.Copy(conn, fd)
m.logger.Infof("%d bytes copied from inbound to outbound", n)
outBoundErr <- err
}()
select {
case err = <-outBoundErr:
if err != nil {
m.logger.Reason(err).Errorf("error encountered copying data to outbound connection")
}
case err = <-inBoundErr:
if err != nil {
m.logger.Reason(err).Errorf("error encountered copying data into inbound connection")
}
case <-m.stopChan:
m.logger.Info("stop channel terminated proxy")
}
}
func (m *migrationProxy) Start() error {
if m.unixSocketPath != "" {
err := m.createUnixListener()
if err != nil {
return err
}
} else {
err := m.createTcpListener()
if err != nil {
return err
}
}
go func(ln net.Listener, fdChan chan net.Conn, listenErr chan error, stopChan chan struct{}) {
for {
fd, err := ln.Accept()
if err != nil {
listenErr <- err
select {
case <-stopChan:
m.logger.Reason(err).V(3).Infof("stopChan is closed. Listener exited with expected error.")
default:
m.logger.Reason(err).Error("proxy unix socket listener returned error.")
}
break
} else {
fdChan <- fd
}
}
}(m.listener, m.fdChan, m.listenErrChan, m.stopChan)
go func(m *migrationProxy) {
for {
select {
case fd := <-m.fdChan:
go m.handleConnection(fd)
case <-m.stopChan:
return
case <-m.listenErrChan:
return
}
}
}(m)
m.logger.Infof("proxy started listening")
return nil
}