inferenceGateway:
  enabled: true        # 是否部署 inference gateway（含数据面 Gateway 资源 gateway.yaml）
  name: "inference-gateway"
  className: "istio"
  enable_http10: false # 允许 HTTP/1.0（及 HTTP/0.9）客户端访问网关。true 时注入 EnvoyFilter 设置 accept_http_10/default_host_for_http_10
  listeners:
    - name: http
      port: 80
      protocol: HTTP

  # type=NodePort 且配置 nodePorts 时可固定对外端口；多套部署时 nodePort 须唯一。
  service:
    type: NodePort
    nodePorts:
      http: 30088

  istio:
    enabled: true

global:
  image:
    pullPolicy: IfNotPresent
  imagePullSecrets: [] # 私有镜像仓库的 Secret，例如: [{"name": "registry-secret"}]

  tracingEnabled: false
  tracingImage:
    repository: cr.openfuyao.cn/openfuyao/eagle-eye-tracing
    tag: "latest"

  env:
    - name: HF_HUB_OFFLINE # HuggingFace Hub 离线开关（1=离线；0=在线）
      value: "0"
    - name: PYTHONHASHSEED # cache-indexer 要求与所有目标 vLLM Pod 保持一致
      value: "0"
    - name: VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES # 与 cache-indexer 的 BlockHash 归一化规则保持一致
      value: "1"

  modelName: "Qwen/Qwen3-8B" # 推理模型名称（用于 served-model-name 和标签）
  modelPath: "" # 推理模型本地路径（可选）。若配置，则 vLLM 使用该路径启动模型，否则使用 modelName。
  autoDownloadModel: true # 是否自动通过 huggingface (hf download) 下载模型，本地已有模型时建议关闭自动下载
  cachePath: "/home/llm_cache" # 推理侧缓存目录（如 HuggingFace / vLLM 的下载与编译缓存）

hermes-router: 
  enabled: true

  inferenceExtension:
    replicas: 1
    pluginsConfigFile: "default-plugins.yaml"
    routing:
      deploymentMode: pd
      # Supported profiles: "" (disabled), random, kv-cache-aware, bucket (pd only),
      # prediction, decode-first (pd only).
      profile: kv-cache-aware
      pd:
        pdLabelName: openfuyao.com/pdRole
        pdGroupLabelName: openfuyao.com/pdGroupID
        prefillValue: prefill
        decodeValue: decode
        leaderValue: leader
      tokenizer:
        model: Qwen/Qwen3-8B
        # tokenizerSource: /models/glm-base-tok. set only when the tokenizer
        # source differs from model (quantized/renamed served models). Empty
        # falls back to model.
      kvCacheAware:
        prefillKVUsageWeight: 1.0
        prefillPrefixWeight: 1.0
        prefillQueueWeight: 1.0
        prefillInflightWeight: 1.0
        decodeKVUsageWeight: 1.0
        decodeQueueWeight: 1.0
        decodeInflightWeight: 1.0
        prefillScoreWeight: 1.0
        decodeScoreWeight: 1.0
      decodeFirst:
        prefillDecider:
          type: prefix-based
          minHitRatio: 0.8
          maxUncachedTokens: 128
        decodeKVCacheAware:
          prefixWeight: 1.0
          runningWeight: 1.0
          kvUsageWeight: 1.0
          queueWeight: 1.0
          inflightWeight: 1.0
      requestTracking:
        persistence:
          enabled: false
          flushThreshold: 100
          outputPath: /tmp/hermes-inflight/completed.jsonl
      cacheIndexer:
        address: http://cache-indexer-service:8080
        blockSize: 128
        prefill:
          l1Weight: 0.7
          l3Weight: 0.3
        decode:
          l1Weight: 1.0
          l3Weight: 0.0

    tokenizer:
      enabled: true
      # routing.tokenizer.socketPath defaults to this socket when the sidecar is enabled.
      socketPath: /tmp/tokenizer/tokenizer-uds.socket
      provider: huggingface
      extraArgs: []
      extraEnv: []
      # Extra tokenizer-sidecar-only mounts. Use this when you want to mount a
      # hostPath or PVC into the tokenizer container without affecting the main
      # EPP container. If TOKENIZER_CACHE_DIR is not set here, the sidecar image
      # default remains in effect: /.cache/huggingface/hub.
      volumeMounts:
        - name: tokenizer-cache
          mountPath: /.cache/huggingface/hub
      # Extra tokenizer-sidecar pod volumes. Use this for tokenizer-local cache
      # or model artifacts without routing them through the main EPP container.
      volumes:
        - name: tokenizer-cache
          hostPath:
            path: /home/llm_cache/huggingface/hub
            type: Directory
    prediction:
      enabled: false
      socketPath: /var/run/hermes/prediction.sock
      # active: route by prediction (enabled: true); shadow: collect training data only (enabled: false).
      predictionMode: active
      timeout: 1s
      maxBatchSize: 128
      ttftWeight: 0.8
      tpotWeight: 0.2
      kvWeight: 1.0
      queueWeight: 1.0
      prefixWeight: 1.0
      inflightWeight: 1.0
      # Set targetModel+modelVersion+modelVolume to load a bundle from
      # <artifactRoot>/<targetModel>/<modelVersion>/ (must match manifest.json).
      # Example bundle: examples/prediction-model/ (Qwen/Qwen3-32B / aggregate-Qwen-Qwen3-32B).
      targetModel: ""
      modelVersion: ""
      env:
        - name: OMP_NUM_THREADS
          value: "2"
      modelVolume:
        hostPath:
          # Example path only; override for your environment. Bundle layout: <targetModel>/<modelVersion>/.
          path: /home/llm_cache/hermes-models
          type: Directory
    affinity: {}

  inferencePool:
    targetPorts:
      - number: 8000
    modelServerType: vllm

  httpRoute:
    inferenceGatewayName: "inference-gateway"

  provider:
    name: istio
    istio:
      destinationRule:
        trafficPolicy:
          tls:
            mode: SIMPLE         
            insecureSkipVerify: true
      retryConfig:
        enabled: true
        retryOn: "connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes,5xx,reset"
        numRetries: 3

inference-backend:
  images: 
    inferenceEngine:
      repository: "quay.io/ascend/vllm-ascend"
      tag: "v0.23.0"
  inferenceDevice: "huawei.com/Ascend910"

  # 默认 volumeMounts（所有 vLLM deployment 共享）
  volumeMounts:
    ascend: # Ascend 设备相关的 volumeMounts
      enable: true

  # 默认 volumes（所有 vLLM deployment 共享）
  volumes:
    ascend: # Ascend 设备相关的 volumes
      enable: true

  services:
  - name: qwen3-8b-2p2d-01
    enabled: true
    mode: pd
    service:
      port: 8000
    pdGroupID: "qwen3-8b-2p2d-01"
    kvTransferConfig:
      connectorConfig:
        kv_connector: "MultiConnector"
        kv_connector_extra_config:
          connectors:
            - kv_connector: "MooncakeConnectorV1"
              kv_buffer_device: "npu"
              kv_parallel_size: 1
              kv_connector_extra_config:
                use_ascend_direct: true
            - kv_connector: "AscendStoreConnector"
              kv_buffer_device: "npu"
              kv_parallel_size: 1
              kv_connector_extra_config:
                lookup_rpc_port: "0"
                backend: "mooncake"
                consumer_is_to_put: true
      mooncake: # 使用mooncake的配置
        use_store: true # 是否使用 mooncake store
        metadataServer: # 共享 Redis metadata server 配置（多 service 启用时取首个已配置项）
          # 设置为 false 可复用已有的 Redis metadata server；此时请将下方 metadata_server 改为已有服务的地址。
          create: true
        master: # 共享 mooncake master 配置（多 service 启用时取首个已配置项）
          # 设置为 false 可复用已有的 Mooncake Master；此时请将下方 master_server_address 改为已有 Master 的地址。
          create: true
          rpcPort: 50051
          servicePort: 30089
          httpPort: 9003 # cache-indexer L3 依赖该 admin HTTP 端口（/get_all_keys、/query_key、/get_all_segments）
          extraArgs: # mooncake_master 启动参数
            - "--eviction_high_watermark_ratio 0.9"
            - "--eviction_ratio 0.1"
        # mooncake 配置文件内容（YAML 格式，initContainer 会自动替换 $POD_IP 并转换为 JSON）
        config: |
          local_hostname: "$POD_IP"
          metadata_server: "redis://redis-service:6379"
          master_server_address: "mooncake-master-service:30089"
          device_name: ""
          protocol: "ascend"
          global_segment_size: 42949672960
          use_ascend_direct: true

    pd:
      prefill:
        replicas: 2
        # 以下为部署拓扑结构化字段：参与 NPU 资源、LWS Worker 数量、DP Rank 与
        # KV Transfer 等拓扑计算，由 chart 统一管理，禁止在 extraArgs 中重复配置。
        tensorParallelSize: 1
        pipelineParallelSize: 1
        # 多DP部署时，data-parallel-size、data-parallel-size-local、data-parallel-start-rank、
        # data-parallel-address、data-parallel-rpc-port、data-parallel-hybrid-lb 等由 chart 
        # 自动填充，无需在 extraArgs 中手动配置。
        dataParallelSize: 1
        dataParallelSizeLocal: 1
        dataParallelRpcPort: 12890
        env:
          - name: HCCL_IF_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        volumeMounts: []
        volumes: []
        # extraArgs 是推理引擎 vLLM 自身的启动参数（仅由 vLLM 消费的模型配置与
        # 推理引擎调优参数），由 chart 直接透传给 `vllm serve`，不参与部署拓扑
        # 计算。未配置的参数使用对应版本 vLLM / vLLM-Ascend 的上游默认值。
        # 禁止重复配置 Chart 管理的参数（如 --tensor-parallel-size、--data-parallel-*、
        # --served-model-name、--port、--kv-transfer-config）；prefill 还禁止
        # --kv-events-config，冲突时渲染直接报错。
        extraArgs:
          - --max-model-len 10000
          - --max-num-batched-tokens 40960
          - --gpu-memory-utilization 0.8
          - --block-size 128
          - --enable-prefix-caching # 默认开启 prefill 节点前缀缓存；如需关闭，请将本参数替换为 --no-enable-prefix-caching
          - --prefix-caching-hash-algo sha256_cbor # 仅在前缀缓存开启时生效，关闭时可移除
          - --trust-remote-code # 默认开启，允许加载远端仓库中的自定义模型代码
          - --enable-prompt-tokens-details
          - --disable-access-log-for-endpoints=/health,/metrics
          # 默认过滤 vLLM 的 /health、/metrics 访问日志。
          # 该配置也会过滤对应接口的 4xx/5xx 请求日志，如需 Debug 请调整或移除该参数（--disable-access-log-for-endpoints）。

      decode:
        replicas: 2
        # 以下为部署拓扑结构化字段：参与 NPU 资源、LWS Worker 数量、DP Rank 与
        # KV Transfer 等拓扑计算，由 chart 统一管理，禁止在 extraArgs 中重复配置。
        tensorParallelSize: 1
        pipelineParallelSize: 1
        dataParallelSize: 1
        dataParallelSizeLocal: 1
        dataParallelRpcPort: 12777
        env:
          - name: HCCL_IF_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        volumeMounts: []
        volumes: []
        # extraArgs 是推理引擎 vLLM 自身的启动参数，语义与 prefill.extraArgs 相同；
        extraArgs:
          - --max-model-len 10000
          - --max-num-batched-tokens 40960
          - --gpu-memory-utilization 0.8
          - --block-size 128
          - --enable-prefix-caching
          - --prefix-caching-hash-algo sha256_cbor
          - --trust-remote-code # 默认开启，允许加载远端仓库中的自定义模型代码
          - --enable-prompt-tokens-details
          - --disable-access-log-for-endpoints=/health,/metrics
          # 默认过滤 vLLM 的 /health、/metrics 访问日志。
          # 该配置也会过滤对应接口的 4xx/5xx 请求日志，如需 Debug 请调整或移除该参数（--disable-access-log-for-endpoints）。

      proxyServer:
        enabled: true
        discoveryInterval: 10
        accessLog:
          # 默认开启 HTTP 访问日志，并过滤 /health、/metrics 接口的成功请求。
          enabled: true # false 时关闭全部 HTTP 访问日志；true 时，开启访问日志，可通过配置 excludedEndpoints 来过滤对应端点的成功请求，4xx/5xx 仍会记录
          excludedEndpoints:
            - /health
            - /metrics


    resources: # p/d 节点资源配置
      requests:
        cpu: "4"
        memory: "64Gi"
      limits:
        cpu: "8"
        memory: "128Gi"

# cache indexer
cache-indexer:
  enabled: true
  resources: # cache-indexer 运行资源，生产环境可按集群规模调整
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: "1"
      memory: 2Gi
  blockKey: # 需与 vLLM 的 PYTHONHASHSEED / hash algo / int hash 开关保持一致
    pythonHashSeed: "0"
    prefixCachingHashAlgo: sha256_cbor
    useIntBlockHashes: true
  discovery:
    refreshInterval: 10s
    labels:
      pdRoleValue:
        - prefill
        - decode
        - aggregate
    vllm:
      zmqPortName: zmq-pub
    mooncakeMaster:
      httpPortName: http
      httpPort: 9003
  service:
    name: cache-indexer-service
    port: 8080

eagle-eye:
  enabled: true

  network-performance-exporter:
    networkPerformanceExporter:
      metricsPort: 8222
      collectInterval: 15

  tracing:
    enabled: false

    jaeger:
      query:
        service:
          type: NodePort
          nodePort: 30686

pd-orchestrator:
  elastic-scaler:
    enabled: true # elastic-scaler 组件部署开关

    elasticScaler:
      enabled: true # chart包中默认提供的 elastic-scaler CR 实例开关，开启后会默认安装 elastic-scaler CR

    name: elastic-scaler # 通过 HPA 扩缩的最大副本数和最小副本数
    minReplicas: 1
    maxReplicas: 10

    targetRef:
      kind: resourcescalinggroup
      name: rsg # 该 name 需要对应 resourcescalinggroup.instanceConfig.name
      apiVersion: autoscaling.openfuyao.com/v1alpha1

    trigger:
      scalingAlgorithm: apa
      # 设置 apa 扩缩资源指标
      resource:
        metricsName: cpu
        targetType: Utilization
        targetValue: 70

  resourcescalinggroup:
    enabled: true # resourcescalinggroup 组件的部署开关

    instanceConfig:
      enabled: true # chart包中默认提供的 resource scaling group CR 实例开关，开启后会默认安装 CR 实例
      name: rsg
      targetResources:
      - name: prefill
        resourceRef:
          apiVersion: leaderworkerset.x-k8s.io/v1
          kind: LeaderWorkerSet
          name: qwen3-8b-2p2d-01-prefill
      - name: decode
        resourceRef:
          apiVersion: leaderworkerset.x-k8s.io/v1
          kind: LeaderWorkerSet
          name: qwen3-8b-2p2d-01-decode
      scalingStrategy:
        type: GroupReplication
        groupConfig: 
          groupName: pd 
          scaleDown: 
            metric: 
              expr: "vllm:num_requests_running" 
              type: "Gauge"
              window: "2m"
              aggregator: "avg"
            order: ascending

  tidal:
    enabled: true # tidal 组件的部署开关