* Copyright (c) 2024 Huawei Technologies Co., Ltd.
* openFuyao is licensed under Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PSL v2 for more details.
*/
package picker
import (
"context"
"encoding/json"
"errors"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
"hermes-router/pkg/plugins/common"
)
const (
PickerPDKVCacheAwareType = "picker-pd-kv-cache-aware"
)
type PickerPDKVCacheAware struct {
typedName plugins.TypedName
}
var _ framework.Picker = &PickerPDKVCacheAware{}
func PickerPDKVCacheAwareFactory(name string, rawParameters json.RawMessage, handle plugins.Handle) (plugins.Plugin, error) {
return NewPickerPDKVCacheAware(name, handle.Context()), nil
}
func NewPickerPDKVCacheAware(name string, ctx context.Context) *PickerPDKVCacheAware {
return &PickerPDKVCacheAware{
typedName: plugins.TypedName{Type: PickerPDKVCacheAwareType, Name: name},
}
}
func (p *PickerPDKVCacheAware) TypedName() plugins.TypedName {
return p.typedName
}
func (p *PickerPDKVCacheAware) WithName(name string) *PickerPDKVCacheAware {
p.typedName.Name = name
return p
}
func (p *PickerPDKVCacheAware) Pick(ctx context.Context, cycleState *types.CycleState, scoredPods []*types.ScoredPod) *types.ProfileRunResult {
logger := log.FromContext(ctx)
pdGroupList, err := types.ReadCycleStateKey[*common.PDGroupList](cycleState, common.PDGroupsCycleStateKey)
if err != nil {
logger.V(logging.DEFAULT).Error(err, "failed to read pd group list from state")
return nil
}
if len(pdGroupList.Groups) == 0 {
logger.V(logging.DEFAULT).Error(errors.New("pd group list is empty"), "pd group list", pdGroupList)
return nil
}
bestPDGroup := pdGroupList.Groups[0]
for _, pdGroup := range pdGroupList.Groups {
if pdGroup.LeaderPod.Score < bestPDGroup.LeaderPod.Score {
bestPDGroup = pdGroup
}
}
return &types.ProfileRunResult{
TargetPods: []types.Pod{
bestPDGroup.LeaderPod.Pod,
bestPDGroup.SelectedPrefillPod.Pod,
bestPDGroup.SelectedDecodePod.Pod},
}
}