inferenceGateway:
  enabled: true        # 是否部署 inference gateway（含数据面 Gateway 资源 gateway.yaml）
  name: "inference-gateway"
  className: "istio"
  listeners:
    - name: http
      port: 80
      protocol: HTTP

  # type=NodePort 且配置 nodePorts 时可固定对外端口；多套部署时 nodePort 须唯一。
  service:
    type: NodePort
    nodePorts:
      http: 30088

  istio:
    enabled: true

global:
  image:
    pullPolicy: IfNotPresent
  imagePullSecrets: [] # 私有镜像仓库的 Secret，例如: [{"name": "registry-secret"}]

  env:
    - name: HF_HUB_OFFLINE # HuggingFace Hub 离线开关（1=离线；0=在线）
      value: "0"
    - name: PYTHONHASHSEED # cache-indexer 要求与所有目标 vLLM Pod 保持一致
      value: "0"
    - name: VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES # 与 cache-indexer 的 BlockHash 归一化规则保持一致
      value: "1"
    - name: VLLM_ENGINE_READY_TIMEOUT_S
      value: "1800"

  modelName: "glm-5.1" # 推理模型名称（用于 served-model-name 和标签）
  modelPath: "/root/.cache/GLM-5.1-w4a8" # 推理模型本地路径（可选）。若配置，则 vLLM 使用该路径启动模型，否则使用 modelName。
  autoDownloadModel: false # 是否自动通过 huggingface (hf download) 下载模型，本地已有模型时建议关闭自动下载
  cachePath: "/home/llm_cache" # 推理侧缓存目录（如 HuggingFace / vLLM 的下载与编译缓存）

hermes-router: 
  enabled: true

  inferenceExtension:
    replicas: 1
    pluginsConfigFile: "default-plugins.yaml"
    routing:
      deploymentMode: pd
      # Supported profiles: "" (disabled), random, kv-cache-aware, bucket (pd only), prediction.
      profile: random
      pd:
        pdLabelName: openfuyao.com/pdRole
        pdGroupLabelName: openfuyao.com/pdGroupID
        prefillValue: prefill
        decodeValue: decode
        leaderValue: leader
    tokenizer:
      enabled: false
    prediction:
      enabled: false

  inferencePool:
    targetPorts:
      - number: 8000
    modelServerType: vllm

  httpRoute:
    inferenceGatewayName: "inference-gateway"

  provider:
    name: istio
    istio:
      destinationRule:
        trafficPolicy:
          tls:
            mode: SIMPLE         
            insecureSkipVerify: true
      retryConfig:
        enabled: true
        retryOn: "connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes,5xx,reset"
        numRetries: 3

inference-backend:
  images: 
    inferenceEngine:
      repository: "quay.io/ascend/vllm-ascend"
      tag: "v0.21.0rc1"
  inferenceDevice: "huawei.com/Ascend910"

  env:
    - name: HCCL_OP_EXPANSION_MODE
      value: "AIV"
    - name: GLOO_SOCKET_IFNAME
      value: "eth0"
    - name: TP_SOCKET_IFNAME
      value: "eth0"
    - name: HCCL_SOCKET_IFNAME
      value: "eth0"
    - name: OMP_PROC_BIND
      value: "false"
    - name: OMP_NUM_THREADS
      value: "10"
    - name: ASCEND_TRANSPORT_PRINT
      value: "1"
    - name: ACL_OP_INIT_MODE
      value: "1"
    - name: VLLM_NIXL_ABORT_REQUEST_TIMEOUT
      value: "300000"
    - name: HCCL_INTRA_ROCE_ENABLE
      value: "1"
    - name: PYTORCH_NPU_ALLOC_CONF
      value: "expandable_segmens: True"
    - name: HCCL_BUFFSIZE
      value: "1024"
    - name: ASCEND_BUFFER_POOL
      value: "16:8"
    - name: TASK_QUEUE_ENABLE
      value: "1"
    - name: CPU_AFFINITY_CONF
      value: "1"
    - name: HCCL_CONNECT_TIMEOUT
      value: "600"

  # 默认 volumeMounts（所有 vLLM deployment 共享）
  volumeMounts:
    ascend: # Ascend 设备相关的 volumeMounts
      enable: true

  # 默认 volumes（所有 vLLM deployment 共享）
  volumes:
    ascend: # Ascend 设备相关的 volumes
      enable: true

  services:
  - name: glm51-w4a8-pd-0
    enabled: true
    mode: pd
    service:
      port: 8000
    pdGroupID: "glm51-w4a8-pd"
    kvTransferConfig:
      connectorConfig:
        kv_connector: "MultiConnector"
        kv_connector_extra_config:
          connectors:
            - kv_connector: "MooncakeConnectorV1"
              kv_buffer_device: "npu"
              kv_parallel_size: 1
              kv_connector_extra_config:
                use_ascend_direct: true
            - kv_connector: "AscendStoreConnector"
              kv_buffer_device: "npu"
              kv_parallel_size: 1
              kv_connector_extra_config:
                lookup_rpc_port: "0"
                backend: "mooncake"
                consumer_is_to_put: true
      mooncake: # 使用mooncake的配置
        use_store: true # 是否使用 mooncake store
        master: # 共享 mooncake master 配置（多 service 启用时取首个已配置项）
          rpcPort: 50051
          servicePort: 30089
          httpPort: 9003 # cache-indexer L3 依赖该 admin HTTP 端口（/get_all_keys、/query_key、/get_all_segments）
          extraArgs: # mooncake_master 启动参数
            - --eviction_high_watermark_ratio 0.9
            - --eviction_ratio 0.1
        # mooncake 配置文件内容（YAML 格式，initContainer 会自动替换 $POD_IP 并转换为 JSON）
        config: |
          local_hostname: "$POD_IP"
          metadata_server: "redis://redis-service:6379"
          master_server_address: "mooncake-master-service:30089"
          device_name: ""
          protocol: "ascend"
          global_segment_size: "128GB"
          use_ascend_direct: true

    pd:
      prefill:
        replicas: 1
        tensorParallelSize: 8
        pipelineParallelSize: 1
        dataParallelSize: 2
        dataParallelSizeLocal: 1
        dataParallelRpcPort: 12890
        env:
          - name: HCCL_IF_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        volumeMounts: []
        volumes: []
        # extraArgs 是推理引擎 vLLM 自身的启动参数（模型配置与推理引擎调优参数），
        # 与部署拓扑无关；结构化字段仅保留 tensorParallelSize 等拓扑相关配置。
        extraArgs:
          - --max-model-len 194560
          - --max-num-batched-tokens 8192
          - --gpu-memory-utilization 0.96
          - --block-size 128
          - --trust-remote-code
          - --enable-prefix-caching
          - --prefix-caching-hash-algo sha256_cbor
          - --enable-prompt-tokens-details
          - --enable-auto-tool-choice
          - --tool-call-parser glm47
          - --reasoning-parser glm45
          - --enable-expert-parallel
          - --async-scheduling
          - --enforce-eager
          - --additional-config '{"enable_flashcomm1":true, "enable_dsa_cp":false}'
          - --disable-access-log-for-endpoints=/health,/metrics
          # 默认过滤 vLLM 的 /health、/metrics 访问日志。
          # 该配置也会过滤对应接口的 4xx/5xx 请求日志，如需 Debug 请调整或移除该参数（--disable-access-log-for-endpoints）。

      decode:
        replicas: 1
        tensorParallelSize: 2
        pipelineParallelSize: 1
        dataParallelSize: 8
        dataParallelSizeLocal: 4
        dataParallelRpcPort: 12777
        env:
          - name: HCCL_IF_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
        volumeMounts: []
        volumes: []
        extraArgs:
          - --max-model-len 194560
          - --max-num-batched-tokens 256
          - --gpu-memory-utilization 0.95
          - --block-size 128
          - --trust-remote-code
          - --no-enable-prefix-caching
          - --prefix-caching-hash-algo sha256_cbor
          - --enable-prompt-tokens-details
          - --enable-auto-tool-choice
          - --tool-call-parser glm47
          - --reasoning-parser glm45
          - --enable-expert-parallel
          - --async-scheduling
          - --max-num-seqs 128
          - --additional-config '{"enable_flashcomm1":true, "enable_dsa_cp":false}'
          - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes":[1,4,8,12,16,20,24,28,32,36,40,48,56,64,80,96,128,256]}'
          - --disable-access-log-for-endpoints=/health,/metrics
          # 默认过滤 vLLM 的 /health、/metrics 访问日志。
          # 该配置也会过滤对应接口的 4xx/5xx 请求日志，如需 Debug 请调整或移除该参数（--disable-access-log-for-endpoints）。


      proxyServer:
        enabled: true
        discoveryInterval: 10
        accessLog:
          # 默认开启 HTTP 访问日志，并过滤 /health、/metrics 接口的成功请求。
          enabled: true # false 时关闭全部 HTTP 访问日志；true 时，开启访问日志，可通过配置 excludedEndpoints 来过滤对应端点的成功请求，4xx/5xx 仍会记录
          excludedEndpoints:
            - /health
            - /metrics

    resources: # p/d 节点资源配置
      requests:
        cpu: "16"
        memory: "1200Gi"
      limits:
        cpu: "16"
        memory: "1200Gi"

# cache indexer
cache-indexer:
  enabled: false
  resources: # cache-indexer 运行资源，生产环境可按集群规模调整
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: "1"
      memory: 2Gi
  blockKey: # 需与 vLLM 的 PYTHONHASHSEED / hash algo / int hash 开关保持一致
    pythonHashSeed: "0"
    prefixCachingHashAlgo: sha256_cbor
    useIntBlockHashes: true
  discovery:
    refreshInterval: 10s
    vllm:
      zmqPortName: zmq-pub
    mooncakeMaster:
      httpPortName: http
      httpPort: 9003
  service:
    name: cache-indexer-service
    port: 8080

eagle-eye:
  enabled: false

  network-performance-exporter:
    networkPerformanceExporter:
      metricsPort: 8222
      collectInterval: 15

pd-orchestrator:
  elastic-scaler:
    enabled: false # elastic-scaler 组件部署开关

    elasticScaler:
      enabled: false # chart包中默认提供的 elastic-scaler CR 实例开关，开启后会默认安装 elastic-scaler CR

    name: elastic-scaler # 通过 HPA 扩缩的最大副本数和最小副本数
    minReplicas: 1
    maxReplicas: 10

    targetRef:
      kind: resourcescalinggroup
      name: rsg # 该 name 需要对应 resourcescalinggroup.instanceConfig.name
      apiVersion: autoscaling.openfuyao.com/v1alpha1

    trigger:
      scalingAlgorithm: apa
      # 设置 apa 扩缩资源指标
      resource:
        metricsName: cpu
        targetType: Utilization
        targetValue: 70

  resourcescalinggroup:
    enabled: false # resourcescalinggroup 组件的部署开关

    instanceConfig:
      enabled: false # chart包中默认提供的 resource scaling group CR 实例开关，开启后会默认安装 CR 实例
      name: rsg
      targetResources:
      - name: prefill
        resourceRef:
          apiVersion: leaderworkerset.x-k8s.io/v1
          kind: LeaderWorkerSet
          name: qwen3-8b-2p2d-01-prefill
      - name: decode
        resourceRef:
          apiVersion: leaderworkerset.x-k8s.io/v1
          kind: LeaderWorkerSet
          name: qwen3-8b-2p2d-01-decode
      scalingStrategy:
        type: GroupReplication
        groupConfig: 
          groupName: pd 
          scaleDown: 
            metric: 
              expr: "vllm:num_requests_running" 
              type: "Gauge"
              window: "2m"
              aggregator: "avg"
            order: ascending

  tidal:
    enabled: false # tidal 组件的部署开关