* Copyright (c) 2024 Huawei Technologies Co., Ltd.
* openFuyao is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
package common
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
)
const (
DefaultKVCacheNotHitRateWeight = 1.0
DefaultXPUCacheUsageWeight = 1.0
DefaultWaitingRequestWeight = 1.0
DefaultPrefillWaitingRequestWeight = 1.0
DefaultDecodeWaitingRequestWeight = 1.0
DefaultPrefillPodScoreWeight = 1.0
DefaultDecodePodScoreWeight = 1.0
DefaultKVCacheManagerTimeout = 5 * time.Second
)
type KVCacheAwareParameters struct {
KVCacheNotHitRateWeight float64 `json:"kvCacheHitNotRateWeight"`
XPUCacheUsageWeight float64 `json:"xpuCacheUsageWeight"`
WaitingRequestWeight float64 `json:"waitingRequestWeight"`
PrefillWaitingRequestWeight float64 `json:"prefillWaitingRequestWeight"`
DecodeWaitingRequestWeight float64 `json:"decodeWaitingRequestWeight"`
PrefillPodScoreWeight float64 `json:"prefillPodScoreWeight"`
DecodePodScoreWeight float64 `json:"decodePodScoreWeight"`
KVCacheManagerIP string `json:"kvCacheManagerIP"`
KVCacheManagerPort int `json:"kvCacheManagerPort"`
KVCacheManagerPath string `json:"kvCacheManagerPath"`
KVCacheManagerTimeout time.Duration `json:"kvCacheManagerTimeout"`
}
type KVCacheAware struct {
managerIP string
managerPort int
managerPath string
timeout time.Duration
}
func NewKVCacheAware(managerIP string, managerPort int, managerPath string, timeout time.Duration) *KVCacheAware {
return &KVCacheAware{
managerIP: managerIP,
managerPort: managerPort,
managerPath: managerPath,
timeout: timeout,
}
}
func (k *KVCacheAware) GetKVCacheHitRate(ctx context.Context, pods []types.Pod, request *types.LLMRequest) (
map[string]float64, error) {
if len(pods) == 0 {
return nil, fmt.Errorf("pods list is empty")
}
requestData, err := k.buildRequestBody(pods, request)
if err != nil {
return nil, err
}
response, err := k.sendRequest(ctx, requestData)
if err != nil {
return nil, err
}
hitRateMap := k.parseResponse(response)
return hitRateMap, nil
}
type KVCacheManagerRequestBody struct {
Model string `json:"model"`
Messages interface{} `json:"messages"`
}
type KVCacheManagerResponse struct {
ServerScoreList []map[string]float64 `json:"server_score_list"`
Message string `json:"message"`
Status int `json:"status"`
}
func (k *KVCacheAware) buildRequestBody(pods []types.Pod, request *types.LLMRequest) (
map[string]interface{}, error) {
if request == nil {
return nil, fmt.Errorf("llm request is nil")
}
if len(pods) == 0 {
return nil, fmt.Errorf("pods list is empty")
}
serverIPs := make([]string, 0, len(pods))
for _, pod := range pods {
k8sPod := pod.GetPod()
if k8sPod == nil {
return nil, fmt.Errorf("underlying kubernetes pod is nil")
}
serverIPs = append(serverIPs, k8sPod.Address)
}
if request.Body == nil {
return nil, fmt.Errorf("request body is nil")
}
var messages interface{}
if request.Body.ChatCompletions != nil {
messages = request.Body.ChatCompletions.Messages
} else if request.Body.Completions != nil {
messages = request.Body.Completions.Prompt
} else {
return nil, fmt.Errorf("unsupported request body type: neither Completions nor ChatCompletions")
}
requestBody := KVCacheManagerRequestBody{
Model: request.TargetModel,
Messages: messages,
}
requestData := map[string]interface{}{
"server_ip": serverIPs,
"body": requestBody,
}
return requestData, nil
}
func (k *KVCacheAware) sendRequest(ctx context.Context, requestData map[string]interface{}) (
KVCacheManagerResponse, error) {
requestBody, err := json.Marshal(requestData)
if err != nil {
return KVCacheManagerResponse{}, fmt.Errorf("failed to marshal request body: %w", err)
}
requestURL := fmt.Sprintf("http://%s:%d%s", k.managerIP, k.managerPort, k.managerPath)
client := &http.Client{Timeout: k.timeout}
response, err := client.Post(requestURL, "application/json", bytes.NewBuffer(requestBody))
if err != nil {
return KVCacheManagerResponse{}, fmt.Errorf("failed to send request, error is: %w", err)
}
if response.StatusCode != http.StatusOK {
return KVCacheManagerResponse{}, fmt.Errorf("failed to send request, status is: %s", response.Status)
}
defer response.Body.Close()
var responseBody KVCacheManagerResponse
if err := json.NewDecoder(response.Body).Decode(&responseBody); err != nil {
return KVCacheManagerResponse{}, fmt.Errorf("failed to decode response body: %w", err)
}
serverScoreListJSON, err := json.Marshal(responseBody.ServerScoreList)
if err != nil {
return KVCacheManagerResponse{}, fmt.Errorf("failed to marshal server score list: %w", err)
}
log.FromContext(ctx).V(logging.DEBUG).Info("Successfully received response from KV cache manager",
"url", requestURL, "status", responseBody.Status,
"message", responseBody.Message, "serverCount", len(responseBody.ServerScoreList),
"serverScoreList", string(serverScoreListJSON))
return responseBody, nil
}
func (k *KVCacheAware) parseResponse(response KVCacheManagerResponse) map[string]float64 {
scoreMap := make(map[string]float64, len(response.ServerScoreList))
for _, serverScore := range response.ServerScoreList {
for serverIP, hitRate := range serverScore {
scoreMap[serverIP] = hitRate
}
}
return scoreMap
}
func CalculateMaxWaitingRequest(scoredPods []types.ScoredPod) int {
maxWaitingRequest := 0
for _, scoredPod := range scoredPods {
pod := scoredPod.Pod
metrics := pod.GetMetrics()
if metrics != nil && metrics.WaitingQueueSize > maxWaitingRequest {
maxWaitingRequest = metrics.WaitingQueueSize
}
}
return maxWaitingRequest
}
func CalculateWaitingRequestWeight(gamma float64, waitingQueueSize int, maxWaitingRequest int) float64 {
if maxWaitingRequest > 0 {
return gamma * float64(waitingQueueSize) / float64(maxWaitingRequest)
}
return 0
}
type KVCacheScoreWeights struct {
KVCacheNotHitRateWeight float64
XPUCacheUsageWeight float64
WaitingRequestWeight float64
}
func CalculatePodKVScore(ctx context.Context, weights KVCacheScoreWeights,
hitRateMap map[string]float64, scoredPods []types.ScoredPod) {
alpha := weights.KVCacheNotHitRateWeight
beta := weights.XPUCacheUsageWeight
gamma := weights.WaitingRequestWeight
maxWaitingRequest := CalculateMaxWaitingRequest(scoredPods)
for i := range scoredPods {
scoredPod := &scoredPods[i]
pod := scoredPod.Pod
backendPod := pod.GetPod()
if backendPod == nil {
continue
}
hitRate := hitRateMap[backendPod.Address]
metrics := pod.GetMetrics()
var kvCacheUsagePercent float64
var waitingQueueSize int
if metrics != nil {
kvCacheUsagePercent = metrics.KVCacheUsagePercent
waitingQueueSize = metrics.WaitingQueueSize
}
waitingRequestWeight := CalculateWaitingRequestWeight(gamma, waitingQueueSize, maxWaitingRequest)
score := alpha*(1.0-hitRate) + beta*kvCacheUsagePercent + waitingRequestWeight
scoredPod.Score = score
log.FromContext(ctx).V(logging.DEBUG).Info("Calculated pod score",
"podAddress", backendPod.Address, "score", score, "kvCacheHitRate", hitRate,
"kvCacheUsagePercent", kvCacheUsagePercent, "waitingQueueSize", waitingQueueSize,
"maxWaitingRequest", maxWaitingRequest)
}
}