global:
image:
# Optional override for all chart-managed container imagePullPolicy values.
pullPolicy: ""
images:
hermes-router:
repository: cr.openfuyao.cn/openfuyao/hermes-router
tag: latest
pullPolicy: IfNotPresent
hermes-router-tokenizer:
repository: cr.openfuyao.cn/openfuyao/hermes-router-tokenizer
tag: latest
pullPolicy: IfNotPresent
hermes-router-prediction:
repository: cr.openfuyao.cn/openfuyao/hermes-router-prediction
tag: latest
pullPolicy: IfNotPresent
inferenceExtension:
flags:
v: "1"
replicas: 1
podSecurityContext:
runAsUser: 65532
runAsGroup: 65532
extProcPort: 9002
extraServicePorts:
- name: http
port: 8081
protocol: TCP
targetPort: 8081
env: []
pluginsConfigFile: "default-plugins.yaml"
routing:
deploymentMode: pd
# Supported profiles: "" (disabled), random, kv-cache-aware, bucket (pd only), prediction.
profile: "kv-cache-aware"
pd:
pdLabelName: openfuyao.com/pdRole
pdGroupLabelName: openfuyao.com/pdGroupID
prefillValue: prefill
decodeValue: decode
leaderValue: leader
tokenizer:
model: Qwen/Qwen3-8B
# tokenizerSource: /models/glm-base-tok # set only when the tokenizer
# source differs from model (quantized/renamed served models). Empty
# falls back to model.
requestTracking:
storeName: hermes-inflight
persistence:
enabled: false
flushThreshold: 100
outputPath: /tmp/hermes-inflight/completed.jsonl
kvCacheAware:
prefillKVUsageWeight: 1.0
prefillPrefixWeight: 1.0
prefillQueueWeight: 1.0
prefillInflightWeight: 1.0
decodeKVUsageWeight: 1.0
decodeQueueWeight: 1.0
decodeInflightWeight: 1.0
prefillScoreWeight: 1.0
decodeScoreWeight: 1.0
cacheIndexer:
address: http://cache-indexer-service:8080
endpointsServer:
# set it to false when you want to deploy EPP with inferencepool
createInferencePool: true
# Required when createInferencePool is false
# endpointSelector: app=vllm-qwen3-32b
# unused when createInferencePool is true
targetPorts: 8000
# unused when createInferencePool is true
modelServerType: vllm # vllm, sglang, triton-tensorrt-llm, trtllm-serve
tokenizer:
enabled: true
# routing.tokenizer.socketPath defaults to this socket when the sidecar is enabled.
socketPath: /var/run/tokenizer/tokenizer.sock
provider: huggingface
extraArgs: []
extraEnv: []
# Extra tokenizer-sidecar-only mounts. Use this when you want to mount a
# hostPath or PVC into the tokenizer container without affecting the main
# EPP container. If TOKENIZER_CACHE_DIR is not set here, the sidecar image
# default remains in effect: /workspace/.cache/huggingface/hub.
volumeMounts:
- name: tokenizer-cache
mountPath: /workspace/.cache/huggingface/hub
# Extra tokenizer-sidecar pod volumes. Use this for tokenizer-local cache
# or model artifacts without routing them through the main EPP container.
volumes:
- name: tokenizer-cache
hostPath:
path: /home/llm_cache/huggingface/hub
type: Directory
# Tokenizer sidecar resources. The CPU *limit* is what matters: the sidecar
# reads the cgroup CPU quota (set by limits.cpu, NOT requests.cpu) to
# auto-size its tokenizer thread pool to one worker thread per core. Always
# set a CPU limit so the pool matches the cores actually granted. This is a
# co-located sidecar, so size the node for the EPP container + this together.
resources:
requests:
cpu: "4"
memory: 2Gi
limits:
cpu: "4"
memory: 4Gi
# Tokenizer thread-pool size (parallel-tokenize worker threads, excluding the
# event-loop thread). When unset it is auto-derived from resources.limits.cpu
# (one worker per core), the measured sweet spot. Set an integer to override.
# threadPoolSize: 4
prediction:
enabled: false
socketPath: /var/run/hermes/prediction.sock
# predictionMode: "active" routes on predicted latency and requires the
# prediction sidecar (enabled: true). "shadow" only collects PredictionInput
# training data and routes via the snapshot fallback; pair it with
# enabled: false to bootstrap a dataset with no prediction sidecar at all.
predictionMode: active
timeout: 1s
maxBatchSize: 128
# Scorer weights passed to prediction-scorer (defaults match plugin.go).
ttftWeight: 0.8
tpotWeight: 0.2
kvWeight: 1.0
queueWeight: 1.0
prefixWeight: 1.0
inflightWeight: 1.0
# Set targetModel+modelVersion+modelVolume to load a bundle from
# <artifactRoot>/<targetModel>/<modelVersion>/ (must match manifest.json).
# Example bundle: examples/prediction-model/ (Qwen/Qwen3-32B / aggregate-Qwen-Qwen3-32B).
targetModel: ""
modelVersion: ""
# Bound the latency-critical predictor. Leaving resources empty makes it
# BestEffort: first to be CPU-throttled and OOM-killed under node pressure,
# which pushes Predict past its timeout and forces routing fallback. The
# OpenMP cap matters as much as the limit: an uncapped booster spawns one
# thread per node core and oversubscribes the cgroup. K=2 predict workers x
# OMP_NUM_THREADS=2 = ~4 busy threads, matched to the 4-core limit so the
# model's internal threads do not fight the worker pool.
resources:
requests:
cpu: "2"
memory: 1Gi
limits:
cpu: "4"
memory: 2Gi
env:
- name: OMP_NUM_THREADS
value: "2"
# The bundle is mounted at a fixed in-pod path by the shared chart helper, so
# supply only the volume SOURCE here (PVC / hostPath / CSI) — not a mountPath.
# The bundle must be laid out as <targetModel>/<modelVersion>/ under the volume
# root. Left empty it falls back to an empty no-op volume; pair that with
# predictionMode: shadow. Example:
# modelVolume:
# persistentVolumeClaim:
# claimName: hermes-models
modelVolume: {}
# Main-EPP-container-only pod volumes. This does not apply to tokenizer or
# prediction sidecars; use inferenceExtension.tokenizer.volumes or
# inferenceExtension.prediction.volumes for sidecar-local storage.
volumes: []
# Main-EPP-container-only mounts. This does not apply to the tokenizer
# sidecar; use inferenceExtension.tokenizer.volumeMounts for that.
volumeMounts: []
# Pod scheduling configuration for the standalone EPP Pod.
# To pin the Pod to a specific node, configure nodeAffinity with a node label
# such as kubernetes.io/hostname.
affinity: {}
sidecar:
enabled: true
configMap:
name: envoy
# Because the template just dumps this section, the keys become filenames.
# The values MUST be strings (note the literal block scalar '|')
data:
envoy.yaml: |
admin:
address:
socket_address:
address: 127.0.0.1
port_value: 19000
access_log:
- name: envoy.access_loggers.file
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: /dev/null
static_resources:
listeners:
- name: envoy-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
port_value: 19001
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: envoy-ready-http
route_config:
name: local_route
virtual_hosts:
- name: prometheus_stats
domains: ["*"]
routes:
- match:
prefix: "/stats/prometheus"
route:
cluster: "prometheus_stats"
http_filters:
- name: envoy.filters.http.health_check
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck
pass_through_mode: false
headers:
- name: ":path"
string_match:
exact: "/ready"
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
- name: vllm
address:
socket_address:
address: 0.0.0.0
port_value: 8081
per_connection_buffer_limit_bytes: 32768
access_log:
- name: envoy.access_loggers.file
filter:
response_flag_filter:
flags: ["NR"]
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: /dev/stdout
log_format:
text_format_source:
inline_string: "{\"start_time\":\"%START_TIME%\",\"method\":\"%REQ(:METHOD)%\",...}\n"
filter_chains:
- name: vllm
filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: http-8081
route_config:
name: vllm
virtual_hosts:
- name: vllm-default
domains: ["*"]
routes:
- match:
prefix: "/"
route:
cluster: original_destination_cluster
timeout: 86400s
idle_timeout: 86400s
upgrade_configs:
- upgrade_type: websocket
typed_per_filter_config:
envoy.filters.http.ext_proc:
"@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
config: {}
http_filters:
- name: envoy.filters.http.ext_proc
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
grpc_service:
envoy_grpc:
cluster_name: ext_proc
authority: localhost:9002
timeout: 10s
processing_mode:
request_header_mode: SEND
response_header_mode: SEND
request_body_mode: FULL_DUPLEX_STREAMED
response_body_mode: FULL_DUPLEX_STREAMED
request_trailer_mode: SEND
response_trailer_mode: SEND
message_timeout: 1000s
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
suppress_envoy_headers: true
http2_protocol_options:
max_concurrent_streams: 100
initial_stream_window_size: 65536
initial_connection_window_size: 1048576
use_remote_address: true
normalize_path: true
merge_slashes: true
server_header_transformation: PASS_THROUGH
common_http_protocol_options:
headers_with_underscores_action: REJECT_REQUEST
path_with_escaped_slashes_action: UNESCAPE_AND_REDIRECT
access_log:
- name: envoy.access_loggers.file
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: /dev/stdout
log_format:
text_format_source:
inline_string: "{\"start_time\":\"%START_TIME%\",\"method\":\"%REQ(:METHOD)%\",...}\n"
clusters:
- name: prometheus_stats
type: STATIC
connect_timeout: 0.250s
load_assignment:
cluster_name: prometheus_stats
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: 127.0.0.1
port_value: 19000
- name: original_destination_cluster
type: ORIGINAL_DST
connect_timeout: 1000s
lb_policy: CLUSTER_PROVIDED
circuit_breakers:
thresholds:
- max_connections: 40000
max_pending_requests: 40000
max_requests: 40000
original_dst_lb_config:
use_http_header: true
http_header_name: x-gateway-destination-endpoint
- name: ext_proc
type: STATIC
connect_timeout: 86400s
lb_policy: LEAST_REQUEST
circuit_breakers:
thresholds:
- max_connections: 40000
max_pending_requests: 40000
max_requests: 40000
max_retries: 1024
health_checks:
- timeout: 2s
interval: 10s
unhealthy_threshold: 3
healthy_threshold: 2
reuse_connection: true
grpc_health_check:
service_name: "envoy.service.ext_proc.v3.ExternalProcessor"
tls_options:
alpn_protocols: ["h2"]
transport_socket:
name: "envoy.transport_sockets.tls"
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
common_tls_context:
validation_context:
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options:
initial_stream_window_size: 65536
initial_connection_window_size: 1048576
load_assignment:
cluster_name: ext_proc
endpoints:
- locality:
region: ext_proc/e2e/0
lb_endpoints:
- endpoint:
address:
socket_address:
address: 127.0.0.1
port_value: 9002
load_balancing_weight: 1
name: envoy-sidecar
image: docker.io/envoyproxy/envoy:distroless-v1.33.2
command: "envoy"
args:
- "--service-node"
- "envoy-sidecar"
- "--log-level"
- "trace"
- "--cpuset-threads"
- "--drain-strategy"
- "immediate"
- "--drain-time-s"
- "60"
- "-c"
- "/etc/envoy/envoy.yaml"
env:
- name: NS_NAME
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
ports:
- containerPort: 8081
name: http-8081
- containerPort: 19001
name: metrics-19001
resources:
requests:
cpu: 100m
memory: 512Mi
readinessProbe:
failureThreshold: 1
httpGet:
path: /ready
port: 19001
scheme: HTTP
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
volumeMounts:
- name: config
mountPath: /etc/envoy
readOnly: true
volumes:
- name: config
configMap:
name: envoy
items:
- key: envoy.yaml
path: envoy.yaml
monitoring:
interval: "10s"
# Prometheus ServiceMonitor will be created when enabled for EPP metrics collection
prometheus:
enabled: false
auth:
# To allow unauthenticated /metrics access (e.g., for debugging with curl), set to false
enabled: true
tracing:
enabled: false
latencyPredictor:
# common latencyPredictor setting exists in config/charts/inference-extension/values.yaml
enabled: false
# Metrics DataSource Configuration
# These values configure how the EPP scrapes metrics from model server pods.
metricsDataSource:
# scheme is the HTTP scheme used to scrape metrics (http or https).
scheme: "http"
# path is the URL path on the model server pod that exposes Prometheus metrics.
path: "/metrics"
# insecureSkipVerify disables TLS certificate verification when scheme is https.
insecureSkipVerify: true
# Options: ["gke"]
provider:
name: none
# GKE-specific configuration.
# This block is only used if name is "gke".
gke:
# Set to true if the cluster is an Autopilot cluster.
autopilot: false
# This is not used when you deploy standalone with inferenceExtension.endpointsServer.createInferencePool=false
inferencePool:
targetPorts:
- number: 8000
modelServerType: vllm # vllm, sglang, triton-tensorrt-llm, trtllm-serve
apiVersion: inference.networking.k8s.io/v1
modelServers: # REQUIRED
matchLabels:
openfuyao.com/model: qwen-qwen3-8b