# -- Default values for llmstack helm chart
# -- Declare variables to be passed into your templates.
# -- Serving engine configuration
servingEngineSpec:
# -- Whether to enable the serving engine
enableEngine: true
# @schema skipProperties:true
# -- Customized labels for the serving engine deployment
labels:
environment: "test"
release: "test"
# -- Extra service ports for models
extraPorts: []
# -- API key for securing the vLLM models. Can be either a string that will be stored in a generated secret or an object referencing an existing secret.
vllmApiKey: # @schema type:[string, object]
# -- Name of the existing Kubernetes secret that contains the vLLM API key
secretName: ""
# -- Key within the secret that contains the vLLM API key
secretKey: ""
# -- Array of specifications for configuring multiple serving engine deployments running different models. Each entry in the modelSpec array should contain the following fields
## The model below serves as an example. It is not deployed by default.
modelSpec:
# -- The name of the model.
- name: "mistral"
# -- Whether to deploy the model. it Default to true if not set
enabled: false
# -- The annotations to add to the deployment
annotations: {}
# -- The annotations to add to the pods
podAnnotations: {}
# -- The name of the service account to use for the deployment
serviceAccountName: ""
# -- The name of the priority class name for the deployment
priorityClassName: ""
# -- Runtime class for the pod, e.g., "nvidia". If not specified, falls back to servingEngineSpec.runtimeClassName
runtimeClassName: "nvidia"
# -- The repository of the model
repository: "lmcache/vllm-openai"
# -- The tag of the model
tag: "latest"
# -- Name of secret with credentials to private container repository
imagePullSecret: ""
# -- The URL of the model
modelURL: "mistralai/Mistral-7B-Instruct-v0.2"
# -- Chat template (Jinga2) specifying tokenizer configuration
# chatTemplate: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n' + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}"
# -- The number of replicas for the model
replicaCount: 1
# -- a resources block containing requests and limits. If specified, this takes priority over simplified resource fields
resources: {}
# -- The number of CPUs requested for the model
requestCPU: 6
# -- The amount of memory requested for the model
requestMemory: "16Gi"
# -- The number of GPUs requested for the model
requestGPU: 1 # @schema type:[integer, number, string]
# -- The type of GPU requested. If not specified, defaults to "nvidia.com/gpu"
requestGPUType: "nvidia.com/mig-4g.71gb"
# -- Requires HAMi. The amount of GPU memory requested. Each unit equals to 1M. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-memory-usage
requestGPUMem: "3000"
# -- Requires HAMi. The percentage of GPU memory requested, e.g., "80". It cannot be used together with requestGPUMem.
requestGPUMemPercentage: "80"
# -- Requires HAMi. The percentage of GPU cores requested, e.g., "10". Each unit equals to 1% device cores. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-core-usage
requestGPUCores: "10"
# -- The CPU limit for the model. If limitCPU and limitMemory are not specified, only GPU resources will have limits set equal to their requests.
limitCPU: 8 # @schema type:[integer, number, string]
# -- The memory limit for the model. If limitCPU and limitMemory are not specified, only GPU resources will have limits set equal to their requests.
limitMemory: "32Gi"
# -- Requires HAMi. The limit of GPU memory. Each unit equals to 1M. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-memory-usage
limitGPUMem: "3000"
# -- Requires HAMi. The limit of GPU memory of GPU
limitGPUMemPercentage: "80"
# -- Requires HAMi. The limit of GPU cores. Each unit equals to 1% device cores. https://project-hami.io/docs/userguide/NVIDIA-device/specify-device-core-usage
limitGPUCores: "10"
# -- The amount of storage requested for the model
pvcStorage: "50Gi"
# -- The access mode policy for the mounted volume.
pvcAccessMode:
- ReadWriteOnce
# -- The storage class of the PVC
storageClass: ""
# -- The labels to match the PVC.
pvcMatchLabels: {}
# -- The labels to add to the PVC
pvcLabels: {}
# -- The annotations to add to the PVC
pvcAnnotations: {}
# -- Additional volumes to add to the pod, in Kubernetes volume format
extraVolumes: []
# - name: tmp-volume
# emptyDir:
# medium: ""
# sizeLimit: 5Gi
# -- Additional volume mounts to add to the container, in Kubernetes volumeMount format
extraVolumeMounts: []
# - name: tmp-volume
# mountPath: /tmp
# -- The configuration for the init container to be run before the main container.
initContainer:
# -- The name of the init container
name: "init"
# -- The Docker image for the init container
image: "busybox:latest"
# -- The command to run in the init container
command:
- "sh"
- "-c"
# -- Additional arguments to pass to the command
args:
- "ls"
# -- List of environment variables to set in the container
env: []
# -- The resource requests and limits for the container
resources: {}
# -- Whether to mount the model's volume
mountPvcStorage: true
# -- Additional volume mounts to add to the init container, in Kubernetes volumeMount format. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#volumemount-v1-core
extraVolumeMounts: []
# -- The configuration for the VLLM model
vllmConfig:
# -- Specify to 1 to use vLLM v0, otherwise vLLM v1 is used
v0: "1"
# -- Enable prefix caching
enablePrefixCaching: false
# -- Enable chunked prefill.
enableChunkedPrefill: false
# -- The maximum model length
maxModelLen: 16384
# -- The data type
dtype: "bfloat16"
# -- The degree of tensor parallelism
tensorParallelSize: 2
# -- Maximum number of sequences to be processed in a single iteration.
maxNumSeqs: 32
# -- The maximum number of LoRA models to be loaded in a single batch
maxLoras: 4
# -- The fraction of GPU memory to be used for the model executor, which can range from 0 to 1
gpuMemoryUtilization: 0.95 # @schema type:[integer, number, string]
# -- The runner type for the model, can be auto or pooling
runner: "pooling" # @schema enum:[auto,pooling]
# -- The conversion type for the model, can be "token_embed", "embed", "token_classify", "classify", or "score".
convert: "embed" # @schema enum:[token_embed,embed,token_classify,classify,score]
# -- Extra command line arguments to pass to vLLM
extraArgs:
- "--gpu-memory-utilization"
- "0.4"
# -- The configuration of the LMCache for KV offloading
lmcacheConfig:
# -- Enable LMCache
enabled: true
# -- The CPU offloading buffer size
cpuOffloadingBufferSize: "30" # @schema type:[integer, number, string]
# -- The log level for LMCache, e.g., "DEBUG", "INFO", "WARNING", "ERROR"
logLevel: INFO # @schema enum:[DEBUG,INFO,WARNING,ERROR]
# -- Hugging Face token configuration. Can be either a string that will be stored in a generated secret or an object referencing an existing secret.
hf_token: # @schema type:[string, object]
# -- Name of the existing Kubernetes secret that contains the Hugging Face token
secretName: ""
# -- Key within the secret that contains the Hugging Face token
secretKey: ""
# -- Reference to an existing Kubernetes Secret from which all key/value pairs will be loaded as environment variables into the container
envFromSecret: {}
# name: s3-registry
# The referenced Secret could look like:
# apiVersion: v1
# kind: Secret
# metadata:
# name: s3-registry
# namespace: vllm
# data:
# AWS_ACCESS_KEY_ID: <base64-encoded-value>
# AWS_SECRET_ACCESS_KEY: <base64-encoded-value>
# AWS_ENDPOINT_URL: <base64-encoded-value>
# -- The environment variables to set in the container, e.g., your HF_TOKEN
env: []
# -- Affinity configuration
affinity: {}
# -- The node selector terms to match the nodes. When both affinity and nodeSelectorTerms are defined, nodeSelectorTerms will be ignored.
nodeSelectorTerms: []
# -- Directly assigns a pod to a specific node (e.g., "192.168.56.5"). When both nodeName and nodeSelectorTerms are defined, the preference is given to nodeName.
nodeName: ""
# -- Per-model tolerations. Interacts with servingEngineSpec.tolerations according to tolerationsPolicy.
tolerations: []
# Example:
# tolerations:
# - key: "dedicated"
# operator: "Equal"
# value: "gpu-model-a"
# effect: "NoSchedule"
# -- How per-model tolerations interact with global servingEngineSpec.tolerations.
tolerationsPolicy: append
# - "append" (default): model tolerations are appended to the global tolerations (union of both lists).
# - "override": model tolerations replace the global tolerations entirely for this model.
# -- The size of the shared memory
shmSize: "20Gi"
# -- The model's service annotations
serviceAnnotations: {}
# -- Whether to enable LoRA
enableLoRA: true
# -- KEDA autoscaling configuration for this model deployment. Requires KEDA to be installed in the cluster.
keda:
# -- Whether to enable KEDA autoscaling for this model
enabled: true
# -- Minimum number of replicas (supports 0 for scale-to-zero)
minReplicaCount: 1 # @schema minimum:0
# -- Maximum number of replicas
maxReplicaCount: 5
# -- How often KEDA should check the metrics (in seconds)
pollingInterval: 15
# -- How long to wait before scaling down after scaling up (in seconds).
cooldownPeriod: 360
# -- Number of replicas to scale to when no triggers are active
idleReplicaCount: 0
# -- Initial cooldown period in seconds before scaling down after creation
initialCooldownPeriod: 60
# -- Fallback configuration when scaler fails
fallback:
# -- Number of consecutive failures before fallback
failureThreshold: 3
# -- Number of replicas to scale to in fallback
replicas: 2
# -- List of KEDA trigger configurations
triggers:
# -- Trigger type
type: "prometheus"
# -- Trigger metadata
metadata:
# -- Prometheus server address
serverAddress: "http://prometheus-operated.monitoring.svc:9090"
# -- Name of the metric to monitor
metricName: "vllm:num_requests_waiting"
# -- Prometheus query to fetch the metric
query: "vllm:num_requests_waiting"
# -- Threshold value that triggers scaling
threshold: "5"
# -- Advanced KEDA configuration
advanced:
# -- Restore original replica count when ScaledObject is deleted
restoreToOriginalReplicaCount: false
# -- HPA-specific configuration
horizontalPodAutoscalerConfig:
# -- Custom name for the HPA resource
name: "keda-hpa-{scaled-object-name}"
# -- HPA scaling behavior configuration, see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/
behavior: {}
# -- Scaling modifiers for composite metrics
scalingModifiers:
# -- Target value for the composed metric
target: ""
# -- Activation target for the composed metric
activationTarget: ""
# -- Metric type (AverageValue or Value)
metricType: "AverageValue"
# -- Formula to compose metrics together
formula: ""
# -- RaySec configuration, to deploy the model as a rayCluster instead of a deployment
raySpec:
# -- Whether to deploy the model as a rayCluster
enabled: false
# -- Head node configuration
headNode:
# -- CPU request for the head node
requestCPU: 1
# -- Memory request for the head node
requestMemory: "1Gi"
# -- GPU request for the head node
requestGPU: 1
# -- Container port
containerPort: 8000
# -- Service port
servicePort: 80
# -- Set other environment variables from config map
configs: {}
# -- deployment strategy
strategy: {}
# -- Readiness probe configuration
startupProbe:
# -- Number of seconds after the container has started before startup probe is initiated
initialDelaySeconds: 15
# -- How often (in seconds) to perform the startup probe
periodSeconds: 10
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready
failureThreshold:
60
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: 8000 # @schema type:[integer, string]
# -- Liveness probe configuration
livenessProbe:
# -- Number of seconds after the container has started before liveness probe is initiated
initialDelaySeconds: 15
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
failureThreshold: 3
# -- How often (in seconds) to perform the liveness probe
periodSeconds: 10
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: 8000 # @schema type:[integer, string]
# -- Readiness probe configuration
readinessProbe:
# -- Number of seconds after the container has started before readiness probe is initiated
initialDelaySeconds: 15
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
failureThreshold: 3
# -- How often (in seconds) to perform the readiness probe
periodSeconds: 5
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: 8000 # @schema type:[integer, string]
# -- Tolerations configuration (when there are taints on nodes)
# Example:
# tolerations:
# - key: "node-role.kubernetes.io/control-plane"
# operator: "Exists"
# effect: "NoSchedule"
tolerations: []
# -- RuntimeClassName configuration, set to "nvidia" if the model requires GPU
runtimeClassName: "nvidia"
# -- SchedulerName configuration
schedulerName: ""
# -- image pull policy for the model deployment
imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never]
# -- Pod-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#podsecuritycontext-v1-core
securityContext: {}
# -- Run as a non-root user ID
# runAsUser: 1000
# -- Run with a non-root group ID
# runAsGroup: 1000
# -- Run as non-root
# runAsNonRoot: true
# -- Set the seccomp profile
# seccompProfile:
# type: RuntimeDefault
# -- Drop all capabilities
# capabilities:
# drop:
# - ALL
# -- Set the file system group ID for all containers
# fsGroup: 1000
# -- Container-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#securitycontext-v1-core
containerSecurityContext:
# -- Run as non-root
runAsNonRoot: false
# -- Don't allow privilege escalation
# allowPrivilegeEscalation: false
# -- Drop all capabilities
# capabilities:
# drop:
# - ALL
# -- Read-only root filesystem
# readOnlyRootFilesystem: true
# -- Sidecar configuration
sidecar:
# -- Docker image repository and tag for the sidecar
image: "lmcache/lmstack-sidecar:latest"
# -- image pull policy for the sidecar
imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never]
# -- ServiceMonitor configuration to collect serving engine metrics
serviceMonitor:
# -- Specifies whether to create a ServiceMonitor resource for collecting Prometheus metrics
enabled: false
# -- Additional labels
additionalLabels: {}
# -- Interval to scrape metrics
interval: 30s
# -- Timeout if metrics can't be retrieved in given time interval
scrapeTimeout: 25s
# -- Let prometheus add an exported_ prefix to conflicting labels
honorLabels: false
# -- Metric relabel configs to apply to samples before ingestion. [Metric Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs)
metricRelabelings: []
# - action: replace
# regex: (.*)
# replacement: $1
# sourceLabels:
# - exported_namespace
# targetLabel: namespace
# -- Relabel configs to apply to samples before ingestion. [Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config)
relabelings: []
# - sourceLabels: [__meta_kubernetes_pod_node_name]
# separator: ;
# regex: ^(.*)$
# targetLabel: nodename
# replacement: $1
# action: replace
# -- Router configuration
routerSpec:
# -- Whether to enable the router service
enableRouter: true
# -- The docker image of the router. The following values are defaults:
repository: "lmcache/lmstack-router"
# -- docker tag for the router pod
tag: "latest"
# -- image pull policy for the router pod
imagePullPolicy: "Always" # @schema enum:[Always,IfNotPresent,Never]
# -- Hugging Face token configuration
hf_token: ""
# -- Pod-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#podsecuritycontext-v1-core
securityContext: {}
# -- Container-level security context configuration. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#securitycontext-v1-core
containerSecurityContext: {}
# -- Image pull secrets for private container registries
# Example:
# imagePullSecrets:
# - name: my-registry-secret
imagePullSecrets: []
# -- Number of replicas
replicaCount: 1
# -- List of environment variables to set for the router
env: []
# -- autoscaling configuration
autoscaling:
enabled: false
minReplicas: 1 # @schema minimum:0
maxReplicas: 3 # @schema minimum:0
targetCPUUtilizationPercentage: 80
# -- Router Pod Disruption Budget
# -- Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/
pdb:
# -- Deploy a PodDisruptionBudget for the router deployment
enabled: false
# -- Labels to be added to router pdb
labels: {}
# -- Annotations to be added to router pdb
annotations: {}
# -- Number of pods that are available after eviction as number or percentage (eg.: 50%)
minAvailable: # @schema type:[string, integer, null];minimum:0
# -- Number of pods that are unavailable after eviction as number or percentage (eg.: 50%).
maxUnavailable: # @schema type:[string, integer, null];minimum:0
# -- Priority Class
priorityClassName: ""
# -- Container port
containerPort: 8000
# -- Service type
serviceType: ClusterIP # @schema enum:[ClusterIP,NodePort,LoadBalancer,ExternalName]
# -- Service annotations if you use a LoadBalancer or NodePort service type
serviceAnnotations: {}
# -- Fixed NodePort for the router service when serviceType is NodePort.
# -- If not set, Kubernetes assigns a random port (range 30000-32767).
nodePort: # @schema type:[integer, null];minimum:30000;maximum:32767
# -- Service port
servicePort: 80
# -- Service discovery mode, supports "k8s" or "static". Defaults to "k8s" if not set.
serviceDiscovery: "k8s" # @schema enum:[k8s,static]
# -- Service discovery mode type, supports "pod-ip" or "service-name". Defaults to "pod-ip" if not set.
k8sServiceDiscoveryType: "pod-ip" # @schema enum:[pod-ip,service-name]
# -- If serviceDiscovery is set to "static", the comma-separated values below are required. There needs to be the same number of backends and models
staticBackends: ""
staticModels: ""
# -- routing logic, supports "roundrobin", "session", "prefixaware", "kvaware" or "disaggregated_prefill"
routingLogic: "roundrobin" # @schema enum:[roundrobin,session,prefixaware,kvaware,disaggregated_prefill]
# -- session key if using "session" routing logic
sessionKey: ""
# -- FastAPI root path for hosting under a subpath (e.g. /vllm)
rootPath: ""
# -- extra router commandline arguments
extraArgs: []
# -- extra service ports
extraPorts: []
# -- Interval in seconds to scrape the serving engine metrics
engineScrapeInterval: 15
# -- Window size in seconds to calculate the request statistics
requestStatsWindow: 60
# -- OpenTelemetry tracing configuration
# When otelEndpoint is set, tracing is automatically enabled
otel:
# -- OTLP endpoint for tracing (e.g., "localhost:4317" or "otel-collector:4317")
endpoint: ""
# -- Service name for traces (default: "vllm-router")
serviceName: "vllm-router"
# -- Use secure (TLS) connection for OTLP exporter (default: false, i.e., insecure)
secure: false
# -- deployment strategy
strategy: {}
# -- API key for securing the vLLM models. Must be an object referencing an existing secret. If not set, it defaults to .Values.servingEngineSpec.vllmApiKey.
vllmApiKey:
# -- Name of the existing Kubernetes secret that contains the vLLM API key
secretName: ""
# -- Key within the secret that contains the vLLM API key
secretKey: ""
# -- lmcache configuration for the router deployment
lmcacheConfig:
# -- LMCache log level for the router deployment.
logLevel: "INFO"
# -- lmcache controller port, used when routingLogic is "kvaware"
lmcacheControllerPort: "" # @schema type:[integer, string]
# -- router resource requests and limits
resources:
requests:
cpu: 400m
memory: 1000Mi
limits:
memory: 1000Mi
# @schema skipProperties:true
# -- Customized labels for the router deployment
labels:
environment: "router"
release: "router"
# -- Customized pod annotations for the router pods
podAnnotations: {}
# -- Ingress configuration for the router
ingress:
# -- Enable ingress controller resource
enabled: false
# -- IngressClass that will be used to implement the Ingress
className: ""
# -- Additional annotations for the Ingress resource
annotations: {}
# kubernetes.io/ingress.class: alb
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
# The list of hostnames to be covered with this ingress record.
hosts:
- host: vllm-router.local
paths:
- path: /
pathType: Prefix
# -- The tls configuration for hostnames to be covered with this ingress record.
tls: []
# - secretName: vllm-router-tls
# hosts:
# - vllm-router.local
# @schema mergeProperties: true
# -- Expose the service via gateway-api HTTPRoute
# More routes can be added by adding a dictionary key like the 'main' route.
# Requires Gateway API resources and suitable controller installed within the cluster
# (see: https://gateway-api.sigs.k8s.io/guides/)
route:
main:
# -- Enables or disables the route
enabled: false
# -- Set the route apiVersion, e.g. gateway.networking.k8s.io/v1 or gateway.networking.k8s.io/v1alpha2
apiVersion: gateway.networking.k8s.io/v1
# -- Set the route kind
kind: HTTPRoute # @schema enum:[HTTPRoute,GRPCRoute,TCPRoute,TLSRoute,UDPRoute]
# -- Additional annotations to add to the route
annotations: {}
# -- Additional labels to add to the route
labels: {}
# -- List of hostnames for the Route
hostnames: []
# - my-filter.example.com
# -- Gateway API parentRefs for the Route
## Must reference an existing Gateway
parentRefs: []
# - name: acme-gw
# -- create http route for redirect (https://gateway-api.sigs.k8s.io/guides/http-redirect-rewrite/#http-to-https-redirects)
## Take care that you only enable this on the http listener of the gateway to avoid an infinite redirect.
## matches, filters and additionalRules will be ignored if this is set to true.
httpsRedirect: false
# -- Route match configuration
matches:
- path:
type: PathPrefix
value: /
# -- Filters define the filters that are applied to requests that match this rule.
filters: []
# - type: RequestHeaderModifier
# requestHeaderModifier:
# add:
# - name: X-Custom-Header
# value: custom-value
# -- Additional custom rules that can be added to the route
additionalRules: []
# -- Affinity configuration
affinity: {}
# -- The node selector terms to match the nodes. Will be ignored if affinity is configured.
nodeSelectorTerms: []
# - matchExpressions:
# - key: nvidia.com/gpu.product
# operator: "In"
# values:
# - "NVIDIA-RTX-A6000"
# -- Liveness probe configuration for the router pod.
livenessProbe:
# -- Number of seconds after the container has started before liveness probe is initiated
initialDelaySeconds: 30
# -- How often (in seconds) to perform the liveness probe
periodSeconds: 5
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
failureThreshold: 3
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: router-port # @schema type:[integer, string]
# Startup probe configuration for the router pod.
startupProbe:
# -- Number of seconds after the container has started before liveness probe is initiated
initialDelaySeconds: 5
# -- How often (in seconds) to perform the startup probe
periodSeconds: 5
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready
failureThreshold: 3
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: router-port # @schema type:[integer, string]
# Readiness probe configuration for the router pod.
readinessProbe:
# -- Number of seconds after the container has started before liveness probe is initiated
initialDelaySeconds: 30
# -- How often (in seconds) to perform the readiness probe
periodSeconds: 5
# -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
failureThreshold: 3
# -- Configuration of the Kubelet http request on the server
httpGet:
# -- Path to access on the HTTP server
path: /health
# -- Name or number of the port to access on the container, on which the server is listening
port: router-port # @schema type:[integer, string]
# -- ServiceMonitor configuration to collect router metrics
serviceMonitor:
# -- Specifies whether to create a ServiceMonitor resource for collecting Prometheus metrics
enabled: false
# -- Additional labels
additionalLabels: {}
# -- Interval to scrape metrics
interval: 30s
# -- Timeout if metrics can't be retrieved in given time interval
scrapeTimeout: 25s
# -- Let prometheus add an exported_ prefix to conflicting labels
honorLabels: false
# -- Metric relabel configs to apply to samples before ingestion. [Metric Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs)
metricRelabelings: []
# - action: replace
# regex: (.*)
# replacement: $1
# sourceLabels:
# - exported_namespace
# targetLabel: namespace
# -- Relabel configs to apply to samples before ingestion. [Relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config)
relabelings: []
# - sourceLabels: [__meta_kubernetes_pod_node_name]
# separator: ;
# regex: ^(.*)$
# targetLabel: nodename
# replacement: $1
# action: replace
# -- Configuration for the LMCache cache server deployment
cacheserverSpec:
# -- Whether to enable the cache server
enabled: false
# -- Image configuration for the cache Server
image:
# -- Docker image repository for the cache server
repository: "lmcache/vllm-openai"
# -- Docker image tag for the cache server
tag: "latest"
# -- Image pull policy for the cache server
imagePullPolicy: "IfNotPresent" # @schema enum:[Always,IfNotPresent,Never]
# -- Image pull secrets for private container registries
imagePullSecrets: []
# -- Number of replicas for the cache server pod
replicaCount: 1
# -- Port the cache server container is listening on
containerPort: 8000
# -- Kubernetes service type for the cache server
serviceType: ClusterIP # @schema enum:[ClusterIP,NodePort,LoadBalancer,ExternalName]
# -- Port the cache server service will listen on
servicePort: 80
# -- Service annotations for the cache server service
serviceAnnotations: {}
# -- Resource requests and limits for the cache server container
resources: {}
# -- Customized annotations for the cache server deployment
annotations: {}
# -- Customized labels for the cache server deployment and service
labels: {}
# -- Deployment strategy for the cache server pods
strategy: # @schema additionalProperties: false
# -- Customize updateStrategy of the cache server deployment.
type: RollingUpdate # @schema enum:[RollingUpdate, Recreate]
rollingUpdate:
maxUnavailable: 0 # @schema type:[integer, string, null]
maxSurge: 1 # @schema type:[integer, string, null]
# -- Configuration for the liveness probe
livenessProbe: # @schema additionalProperties: false
# -- The number of seconds to wait before starting the first probe.
initialDelaySeconds: 30
# -- The number of seconds to wait between consecutive probes.
periodSeconds: 10
# -- The number of consecutive failures allowed before considering the probe as failed.
failureThreshold: 3
# -- The number of seconds to wait for a probe response before considering it as failed.
timeoutSeconds: 5
# -- Tolerations configuration for the cache server pods
tolerations: []
# -- RuntimeClassName configuration for the cache server pods
runtimeClassName: ""
# -- SchedulerName configuration for the cache server pods
schedulerName: ""
# -- Pod-level security context configuration
securityContext: {}
# -- Container-level security context configuration
containerSecurityContext: {}
# -- Priority class for the cache server pods
priorityClassName: ""
# -- Node selector for the cache server pods
nodeSelector: {}
# -- Affinity configuration. If specified, takes precedence over nodeSelectorTerms.
affinity: {}
# -- Serialization/deserialization format for the cache server
serde: ""
# -- LoRA Adapters Configuration
loraAdapters:
# -- The name of the LoRA adapter instance
- name: "llama3-nemoguard-adapter"
# -- Whether to create le LoRa adapter
enabled: false
# -- The name of the base model this adapter is for
baseModel: llama3-8b-instr
# -- API key configuration for vLLM authentication. If not Set, it defaults to servingEngineSpec.vllmApiKey
vllmApiKey:
# -- Name of the secret
secretName: "vllm-api-key"
# -- Key in the secret containing the API key
secretKey: "VLLM_API_KEY"
# -- Configuration for the adapter source
adapterSource:
# -- Type of adapter source (local, s3, http, huggingface)
type: "local"
# -- Name of the adapter to apply
adapterName: "llama-3.1-nemoguard-8b-topic-control"
# -- Path to the LoRA adapter weights
adapterPath: "/data/lora-adapters/llama-3.1-nemoguard-8b-topic-control"
# -- Repository to get the LoRA adapter from
repository: ""
# -- Pattern to use for the adapter name
pattern: ""
# -- Maximum number of adapters to load
maxAdapters: 1
# -- Reference to secret with storage credentials
## Can be either a string that will be stored in a generated secret or an object referencing an existing secret.
credentials: # @schema type:[string, object]
# -- Name of the secret
secretName: "secret"
# -- Key in the secret containing the credentials
secretKey: "key"
# -- Configuration for adapter deployment
loraAdapterDeploymentConfig:
# -- Placement algorithm to use (default, ordered, equalized)
algorithm: "default"
# -- Number of replicas that should load this adapter
replicas: 1
# -- Additional labels for the LoRA adapter
labels: {}
# -- lora controller Configuration
loraController:
# -- Whether to enable the Lora Controller
enableLoraController: false
# -- kubernetes cluster domain
kubernetesClusterDomain: "cluster.local"
# -- Number of lora controller replicas
replicaCount: 1
# -- lora controller image configuration
image:
# -- Docker image repository
repository: "lmcache/lmstack-lora-controller"
# -- Docker image tag
tag: "latest"
# -- Image pull policy
pullPolicy: "IfNotPresent" # @schema enum:[Always,IfNotPresent,Never]
# -- Image pull secrets
imagePullSecrets: []
# -- Deployment annotations
annotations: {}
# -- Deployment labels
labels: {}
# -- Pod annotations
podAnnotations: {}
# -- Pod labels
podLabels: {}
# @schema skipProperties:true
# -- Pod security context
podSecurityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
# @schema skipProperties:true
# -- Container security context
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
# -- lora controller resources
resources: {}
# -- Node selector
nodeSelector: {}
# -- Affinity
affinity: {}
# -- Tolerations
tolerations: []
# -- Environment variables
env: []
# -- Extra arguments for the lora controller
extraArgs: []
# -- Lora Controller metrics configuration
metrics:
# -- Expose lora controller metrics
enabled: true
# -- loraController Webhook configuration
webhook:
# -- Deploy a webhook inside lora controller
enabled: false
# -- loraController Pod Disruption Budget
# -- Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/
pdb:
# -- Deploy a PodDisruptionBudget for the loraController deployment
enabled: false
# -- Labels to be added to loraController pdb
labels: {}
# -- Annotations to be added to loraController pdb
annotations: {}
# -- Number of pods that are available after eviction as number or percentage (eg.: 50%)
minAvailable: # @schema type:[string, integer, null];minimum:0
# -- Number of pods that are unavailable after eviction as number or percentage (eg.: 50%).
maxUnavailable: # @schema type:[string, integer, null];minimum:0
# -- Configuration for a shared PVC that can be mounted to multiple model deployments.
sharedPvcStorage:
# -- Whether to enable shared storage for the models
enabled: false
# -- Size of the shared storage volume
size: "100Gi"
# -- Access modes for the shared storage volume
accessModes:
- ReadWriteMany
# -- Storage class name for the shared storage volume
storageClass: ""
# -- Host path for the shared storage volume. Specifying a hostPath or nfs server will create a PersistentVolume. These fields should be omitted to rely on dynamic provisioning of PersistentVolumeClaim.
hostPath: ""
# -- NFS configuration for the shared storage volume
nfs:
# -- NFS server address for the shared storage volume
server: ""
# -- NFS export path for the shared storage volume
path: ""
# -- Docker image used in the lora Controller as an init container to set permissions in the PVC
image:
# -- Docker image repository
repository: "busybox"
# -- Docker image tag
tag: "1.37"
# -- Array of extra Kubernetes objects to deploy. Each object should be a valid Kubernetes manifest in YAML format. This can be used to deploy additional resources such as ConfigMaps, Secrets, or custom resources that are not directly supported by the chart's built-in configuration. Supports use of custom Helm templates.
extraObjects: []
# -- Set to true do deploy dashboards stored in the "dashboards" directory as configmaps. This requires the kube-prometheus-stack to be deployed with Grafana enabled and properly configured to pick up dashboards from configmaps.
grafanaDashboards:
enabled: false
# -- Additional annotations to add to the dashboard configmaps
annotations: {}
# @schema skipProperties:true
# -- Additional labels to add to the dashboard configmaps
labels:
grafana_dashboard: "1"
# @schema skipProperties:true
# -- Kube Prometheus Stack dependency chart configuration. More values can be found [here](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml).
kube-prometheus-stack:
enabled: false
# -- Configure the Prometheus Operator to automatically discover and scrape metrics from the vLLM chart using ServiceMonitors.
prometheus:
prometheusSpec:
serviceMonitorSelector:
matchLabels:
app.kubernetes.io/part-of: vllm-stack
# @schema skipProperties:true
# -- Prometheus Adapter dependency chart configuration. More values can be found [here](https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-adapter/values.yaml).
prometheus-adapter:
enabled: false
prometheus:
url: http://{{ .Release.Name }}-kube-prometheus-stack-prometheus
port: 9090
# rules:
# default: true
# custom:
# # Example metric to export for HPA
# - seriesQuery: '{__name__=~"^vllm:num_requests_waiting$"}'
# resources:
# overrides:
# namespace:
# resource: "namespace"
# name:
# matches: ""
# as: "vllm_num_requests_waiting"
# metricsQuery: sum by(namespace) (vllm:num_requests_waiting)
# # Export num_incoming_requests_total by model name
# - seriesQuery: '{__name__=~"^vllm:num_incoming_requests_total$"}'
# resources:
# overrides:
# namespace:
# resource: "namespace"
# name:
# matches: ""
# as: "vllm_num_incoming_requests_total"
# metricsQuery: sum by(namespace, model) (vllm:num_incoming_requests_total)