Star46
157
代码
代码
Issues192
Pull Requests184
项目讨论
Wiki
分析
Star46
157
已合并
Fix 解决vllm-ascend中扩展指标带空格导致的解析失败，调整grafana dashboard json #339
LinWei100创建于 6月24日
Fix 解决vllm-ascend中扩展指标带空格导致的解析失败，调整grafana dashboard json #339
已合并
LinWei100创建于 6月24日
从LinWei100/MindIE-PyMotorfix/metric-parse合入到Ascend/MindIE-Motormaster
共 2 个文件变更+288-105
Mexamples/features/observability/stack/grafana/dashboards/motor-vllm-profiling.json+286-103
@@ -9,7 +9,7 @@
9 ],9 ],
10 "timezone": "browser",10 "timezone": "browser",
11 "schemaVersion": 39,11 "schemaVersion": 39,
12- "version": 1,12+ "version": 2,
13 "refresh": "10s",13 "refresh": "10s",
14 "time": {14 "time": {
15 "from": "now-30m",15 "from": "now-30m",
@@ -93,13 +93,13 @@
93 },93 },
94 {94 {
95 "name": "role",95 "name": "role",
96- "label": "Role",96+ "label": "Role（profiling 数据仅 mixed）",
97 "type": "query",97 "type": "query",
98 "datasource": {98 "datasource": {
99 "type": "prometheus",99 "type": "prometheus",
100 "uid": "prometheus"100 "uid": "prometheus"
101 },101 },
102- "query": "label_values({source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role!=\"\"}, role)",102+ "query": "label_values(vllm_profiling_batch_size_count{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\"}, role)",
103 "refresh": 2,103 "refresh": 2,
104 "includeAll": true,104 "includeAll": true,
105 "multi": true,105 "multi": true,
@@ -117,7 +117,7 @@
117 "type": "prometheus",117 "type": "prometheus",
118 "uid": "prometheus"118 "uid": "prometheus"
119 },119 },
120- "query": "label_values(vllm_profiling_engine_core:engine_core_step:duration_count{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\"}, phase)",120+ "query": "label_values(vllm_profiling_engine_core:process_engine_step:duration_count{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\"}, phase)",
121 "refresh": 2,121 "refresh": 2,
122 "includeAll": true,122 "includeAll": true,
123 "multi": true,123 "multi": true,
@@ -135,7 +135,7 @@
135 "type": "prometheus",135 "type": "prometheus",
136 "uid": "prometheus"136 "uid": "prometheus"
137 },137 },
138- "query": "label_values({source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", dp_rank!=\"\"}, dp_rank)",138+ "query": "label_values(vllm_profiling_batch_size_count{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\"}, dp)",
139 "refresh": 2,139 "refresh": 2,
140 "includeAll": true,140 "includeAll": true,
141 "multi": true,141 "multi": true,
@@ -171,7 +171,7 @@
171 {171 {
172 "id": 100,172 "id": 100,
173 "type": "row",173 "type": "row",
174- "title": "静态显存 (engine:memory:* — Gauge)",174+ "title": "KV Cache Blocks 利用率（实时）",
175 "gridPos": {175 "gridPos": {
176 "h": 1,176 "h": 1,
177 "w": 24,177 "w": 24,
@@ -183,7 +183,7 @@
183 {183 {
184 "id": 1,184 "id": 1,
185 "type": "stat",185 "type": "stat",
186- "title": "显存利用率 utilization",186+ "title": "KV 使用率 (allocated/total)",
187 "gridPos": {187 "gridPos": {
188 "h": 6,188 "h": 6,
189 "w": 4,189 "w": 4,
@@ -196,7 +196,7 @@
196 },196 },
197 "targets": [197 "targets": [
198 {198 {
199- "expr": "max(vllm_profiling_engine:memory:utilization_ratio{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",199+ "expr": "clamp_max( sum(vllm_profiling_allocated_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_total_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1), 1)",
200 "refId": "A"200 "refId": "A"
201 }201 }
202 ],202 ],
@@ -214,18 +214,18 @@
214 },214 },
215 {215 {
216 "color": "yellow",216 "color": "yellow",
217- "value": 0.85217+ "value": 0.7
218 },218 },
219 {219 {
220 "color": "red",220 "color": "red",
221- "value": 0.95221+ "value": 0.9
222 }222 }
223 ]223 ]
224 }224 }
225 }225 }
226 },226 },
227 "options": {227 "options": {
228- "colorMode": "value",228+ "colorMode": "background",
229 "graphMode": "area",229 "graphMode": "area",
230 "reduceOptions": {230 "reduceOptions": {
231 "calcs": [231 "calcs": [
@@ -237,7 +237,7 @@
237 {237 {
238 "id": 2,238 "id": 2,
239 "type": "stat",239 "type": "stat",
240- "title": "总显存 total (GB)",240+ "title": "KV 总块数 total_kvcache_blocks",
241 "gridPos": {241 "gridPos": {
242 "h": 6,242 "h": 6,
243 "w": 4,243 "w": 4,
@@ -250,14 +250,14 @@
250 },250 },
251 "targets": [251 "targets": [
252 {252 {
253- "expr": "max(vllm_profiling_engine:memory:total_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",253+ "expr": "sum(vllm_profiling_total_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_total_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
254 "refId": "A"254 "refId": "A"
255 }255 }
256 ],256 ],
257 "fieldConfig": {257 "fieldConfig": {
258 "defaults": {258 "defaults": {
259- "unit": "decgbytes",259+ "unit": "short",
260- "decimals": 1260+ "decimals": 0
261 }261 }
262 },262 },
263 "options": {263 "options": {
@@ -273,7 +273,7 @@
273 {273 {
274 "id": 3,274 "id": 3,
275 "type": "piechart",275 "type": "piechart",
276- "title": "显存构成 composition (GB)",276+ "title": "KV 块构成 (allocated / free)",
277 "gridPos": {277 "gridPos": {
278 "h": 6,278 "h": 6,
279 "w": 8,279 "w": 8,
@@ -286,40 +286,22 @@
286 },286 },
287 "targets": [287 "targets": [
288 {288 {
289- "expr": "max(vllm_profiling_engine:memory:weights_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",289+ "expr": "sum(vllm_profiling_allocated_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_allocated_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
290- "legendFormat": "weights",290+ "legendFormat": "allocated",
291 "refId": "A",291 "refId": "A",
292 "instant": true292 "instant": true
293 },293 },
294 {294 {
295- "expr": "max(vllm_profiling_engine:memory:kvcache_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",295+ "expr": "sum(vllm_profiling_free_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_free_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
296- "legendFormat": "kv_cache",296+ "legendFormat": "free",
297 "refId": "B",297 "refId": "B",
298 "instant": true298 "instant": true
299- },
300- {
301- "expr": "max(vllm_profiling_engine:memory:activation_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",
302- "legendFormat": "activation",
303- "refId": "C",
304- "instant": true
305- },
306- {
307- "expr": "max(vllm_profiling_engine:memory:non_torch_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",
308- "legendFormat": "non_torch",
309- "refId": "D",
310- "instant": true
311- },
312- {
313- "expr": "max(vllm_profiling_engine:memory:graph_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",
314- "legendFormat": "npu_graph",
315- "refId": "E",
316- "instant": true
317 }299 }
318 ],300 ],
319 "fieldConfig": {301 "fieldConfig": {
320 "defaults": {302 "defaults": {
321- "unit": "decgbytes",303+ "unit": "short",
322- "decimals": 2304+ "decimals": 0
323 }305 }
324 },306 },
325 "options": {307 "options": {
@@ -341,7 +323,7 @@
341 {323 {
342 "id": 4,324 "id": 4,
343 "type": "timeseries",325 "type": "timeseries",
344- "title": "reserved vs total (GB) by role",326+ "title": "KV Cache blocks 趋势 (total / allocated / free) by dp",
345 "gridPos": {327 "gridPos": {
346 "h": 6,328 "h": 6,
347 "w": 8,329 "w": 8,
@@ -354,19 +336,24 @@
354 },336 },
355 "targets": [337 "targets": [
356 {338 {
357- "expr": "vllm_profiling_engine:memory:reserved_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}",339+ "expr": "sum by (dp) (vllm_profiling_total_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_total_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
358- "legendFormat": "reserved {{role}}",340+ "legendFormat": "total dp={{dp}}",
359 "refId": "A"341 "refId": "A"
360 },342 },
361 {343 {
362- "expr": "vllm_profiling_engine:memory:total_gb{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}",344+ "expr": "sum by (dp) (vllm_profiling_allocated_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_allocated_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
363- "legendFormat": "total {{role}}",345+ "legendFormat": "allocated dp={{dp}}",
364 "refId": "B"346 "refId": "B"
347+ },
348+ {
349+ "expr": "sum by (dp) (vllm_profiling_free_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_free_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
350+ "legendFormat": "free dp={{dp}}",
351+ "refId": "C"
365 }352 }
366 ],353 ],
367 "fieldConfig": {354 "fieldConfig": {
368 "defaults": {355 "defaults": {
369- "unit": "decgbytes",356+ "unit": "short",
370 "custom": {357 "custom": {
371 "drawStyle": "line",358 "drawStyle": "line",
372 "lineWidth": 1,359 "lineWidth": 1,
@@ -399,7 +386,7 @@
399 {386 {
400 "id": 12,387 "id": 12,
401 "type": "timeseries",388 "type": "timeseries",
402- "title": "各阶段平均耗时 avg (sum/count rate)",389+ "title": "各阶段平均耗时 avg（process_engine_step / executor / get_output）",
403 "gridPos": {390 "gridPos": {
404 "h": 8,391 "h": 8,
405 "w": 12,392 "w": 12,
@@ -412,27 +399,27 @@
412 },399 },
413 "targets": [400 "targets": [
414 {401 {
415- "expr": "sum(rate(vllm_profiling_scheduler:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_scheduler:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",402+ "expr": "sum(rate(vllm_profiling_engine_core:process_engine_step:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_engine_core:process_engine_step:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
416- "legendFormat": "scheduler",403+ "legendFormat": "process_engine_step",
417 "refId": "A"404 "refId": "A"
418 },405 },
419 {406 {
420- "expr": "sum(rate(vllm_profiling_executor:prepare_inputs:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:prepare_inputs:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",407+ "expr": "sum(rate(vllm_profiling_executor:prepare_inputs:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:prepare_inputs:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
421 "legendFormat": "prepare_inputs",408 "legendFormat": "prepare_inputs",
422 "refId": "B"409 "refId": "B"
423 },410 },
424 {411 {
425- "expr": "sum(rate(vllm_profiling_executor:execute_model:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:execute_model:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",412+ "expr": "sum(rate(vllm_profiling_executor:execute_model:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:execute_model:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
426 "legendFormat": "execute_model",413 "legendFormat": "execute_model",
427 "refId": "C"414 "refId": "C"
428 },415 },
429 {416 {
430- "expr": "sum(rate(vllm_profiling_executor:sample_tokens:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:sample_tokens:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",417+ "expr": "sum(rate(vllm_profiling_executor:sample_tokens:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:sample_tokens:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
431 "legendFormat": "sample_tokens",418 "legendFormat": "sample_tokens",
432 "refId": "D"419 "refId": "D"
433 },420 },
434 {421 {
435- "expr": "sum(rate(vllm_profiling_worker:model_runner_get_output:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_worker:model_runner_get_output:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",422+ "expr": "sum(rate(vllm_profiling_worker:model_runner_get_output:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_worker:model_runner_get_output:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
436 "legendFormat": "get_output",423 "legendFormat": "get_output",
437 "refId": "E"424 "refId": "E"
438 }425 }
@@ -480,7 +467,7 @@
480 },467 },
481 "targets": [468 "targets": [
482 {469 {
483- "expr": "histogram_quantile(0.95, sum by (le, name) (rate(vllm_profiling_record_function_or_nullcontext_bucket{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",470+ "expr": "histogram_quantile(0.95, sum by (le, name) (rate(vllm_profiling_record_function_or_nullcontext_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
484 "legendFormat": "{{name}}",471 "legendFormat": "{{name}}",
485 "refId": "A"472 "refId": "A"
486 }473 }
@@ -520,7 +507,7 @@
520 {507 {
521 "id": 20,508 "id": 20,
522 "type": "timeseries",509 "type": "timeseries",
523- "title": "NPU 计算时间线 avg (forward / kernel_launch / non_forward)",510+ "title": "NPU 计算时间线 avg (process_input_queue / forward / kernel_launch)",
524 "gridPos": {511 "gridPos": {
525 "h": 8,512 "h": 8,
526 "w": 12,513 "w": 12,
@@ -533,18 +520,18 @@
533 },520 },
534 "targets": [521 "targets": [
535 {522 {
536- "expr": "sum(rate(vllm_profiling_npu:forward_duration_sum{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_npu:forward_duration_count{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",523+ "expr": "sum(rate(vllm_profiling_engine_core:process_input_queue:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_engine_core:process_input_queue:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
537- "legendFormat": "forward",524+ "legendFormat": "process_input_queue",
538 "refId": "A"525 "refId": "A"
539 },526 },
540 {527 {
541- "expr": "sum(rate(vllm_profiling_npu:kernel_launch_sum{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_npu:kernel_launch_count{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",528+ "expr": "sum(rate(vllm_profiling_npu:forward_duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_npu:forward_duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
542- "legendFormat": "kernel_launch",529+ "legendFormat": "npu_forward",
543 "refId": "B"530 "refId": "B"
544 },531 },
545 {532 {
546- "expr": "sum(rate(vllm_profiling_npu:non_forward_duration_sum{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_npu:non_forward_duration_count{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",533+ "expr": "sum(rate(vllm_profiling_npu:kernel_launch_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_npu:kernel_launch_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
547- "legendFormat": "non_forward",534+ "legendFormat": "kernel_launch",
548 "refId": "C"535 "refId": "C"
549 }536 }
550 ],537 ],
@@ -571,7 +558,7 @@
571 {558 {
572 "id": 21,559 "id": 21,
573 "type": "timeseries",560 "type": "timeseries",
574- "title": "请求链路时延 P95 (server / generate / tokenizer / output)",561+ "title": "请求链路时延 P95 (async_add_request / generate / tokenizer / output)",
575 "gridPos": {562 "gridPos": {
576 "h": 8,563 "h": 8,
577 "w": 12,564 "w": 12,
@@ -584,22 +571,22 @@
584 },571 },
585 "targets": [572 "targets": [
586 {573 {
587- "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_server:create_chat_completion:duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",574+ "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_engine:async_add_request:duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
588- "legendFormat": "create_chat_completion",575+ "legendFormat": "async_add_request",
589 "refId": "A"576 "refId": "A"
590 },577 },
591 {578 {
592- "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_engine:generate:duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",579+ "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_engine:generate:duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
593 "legendFormat": "generate",580 "legendFormat": "generate",
594 "refId": "B"581 "refId": "B"
595 },582 },
596 {583 {
597- "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_engine:tokenizer_encode_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",584+ "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_engine:tokenizer_encode_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
598 "legendFormat": "tokenizer_encode",585 "legendFormat": "tokenizer_encode",
599 "refId": "C"586 "refId": "C"
600 },587 },
601 {588 {
602- "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_output_processor:duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",589+ "expr": "histogram_quantile(0.95, sum by (le) (rate(vllm_profiling_output_processor_duration_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
603 "legendFormat": "output_processor",590 "legendFormat": "output_processor",
604 "refId": "D"591 "refId": "D"
605 }592 }
@@ -639,7 +626,7 @@
639 {626 {
640 "id": 30,627 "id": 30,
641 "type": "timeseries",628 "type": "timeseries",
642- "title": "Batch size / Running queue avg",629+ "title": "Batch size / Waiting queue（instant avg per step）",
643 "gridPos": {630 "gridPos": {
644 "h": 8,631 "h": 8,
645 "w": 8,632 "w": 8,
@@ -652,13 +639,13 @@
652 },639 },
653 "targets": [640 "targets": [
654 {641 {
655- "expr": "sum(rate(vllm_profiling_scheduler:batch_size_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_scheduler:batch_size_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",642+ "expr": "sum(vllm_profiling_batch_size_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_batch_size_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
656 "legendFormat": "batch_size",643 "legendFormat": "batch_size",
657 "refId": "A"644 "refId": "A"
658 },645 },
659 {646 {
660- "expr": "sum(rate(vllm_profiling_scheduler:running_queue_size_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_scheduler:running_queue_size_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",647+ "expr": "sum(vllm_profiling_waiting_batch_size_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_waiting_batch_size_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
661- "legendFormat": "running_queue",648+ "legendFormat": "waiting_queue",
662 "refId": "B"649 "refId": "B"
663 }650 }
664 ],651 ],
@@ -685,7 +672,7 @@
685 {672 {
686 "id": 31,673 "id": 31,
687 "type": "timeseries",674 "type": "timeseries",
688- "title": "序列长度 seqlen (avg / sum)",675+ "title": "每步 Token 数量 avg (input / output / total_tokens)",
689 "gridPos": {676 "gridPos": {
690 "h": 8,677 "h": 8,
691 "w": 8,678 "w": 8,
@@ -698,14 +685,19 @@
698 },685 },
699 "targets": [686 "targets": [
700 {687 {
701- "expr": "avg by (phase) (vllm_profiling_scheduler:seqlen:avg{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",688+ "expr": "sum(vllm_profiling_input_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_input_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
702- "legendFormat": "avg {{phase}}",689+ "legendFormat": "input_tokens",
703 "refId": "A"690 "refId": "A"
704 },691 },
705 {692 {
706- "expr": "avg by (phase) (vllm_profiling_scheduler:seqlen:sum{source=~\"$source\", phase=~\"$phase\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",693+ "expr": "sum(vllm_profiling_output_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_output_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
707- "legendFormat": "sum {{phase}}",694+ "legendFormat": "output_tokens",
708 "refId": "B"695 "refId": "B"
696+ },
697+ {
698+ "expr": "sum(vllm_profiling_total_tokens_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_total_tokens_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
699+ "legendFormat": "total_tokens",
700+ "refId": "C"
709 }701 }
710 ],702 ],
711 "fieldConfig": {703 "fieldConfig": {
@@ -731,7 +723,7 @@
731 {723 {
732 "id": 32,724 "id": 32,
733 "type": "timeseries",725 "type": "timeseries",
734- "title": "调度 token 速率 by req_phase",726+ "title": "Token 吞吐速率 (tokens/s: input / output / total)",
735 "gridPos": {727 "gridPos": {
736 "h": 8,728 "h": 8,
737 "w": 8,729 "w": 8,
@@ -744,9 +736,19 @@
744 },736 },
745 "targets": [737 "targets": [
746 {738 {
747- "expr": "sum by (req_phase) (rate(vllm_profiling_scheduler:phase_scheduled_token_counter_total{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"}[5m]))",739+ "expr": "sum(rate(vllm_profiling_input_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_input_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
748- "legendFormat": "{{req_phase}}",740+ "legendFormat": "input tokens/s",
749 "refId": "A"741 "refId": "A"
742+ },
743+ {
744+ "expr": "sum(rate(vllm_profiling_output_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_output_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
745+ "legendFormat": "output tokens/s",
746+ "refId": "B"
747+ },
748+ {
749+ "expr": "sum(rate(vllm_profiling_total_tokens_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_total_tokens_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
750+ "legendFormat": "total tokens/s",
751+ "refId": "C"
750 }752 }
751 ],753 ],
752 "fieldConfig": {754 "fieldConfig": {
@@ -770,26 +772,26 @@
770 }772 }
771 },773 },
772 {774 {
773- "id": 500,775+ "id": 600,
774 "type": "row",776 "type": "row",
775- "title": "EPLB 专家负载 (MoE / EP 场景)",777+ "title": "Engine Core & Executor（metrics_extra 补充）",
776 "gridPos": {778 "gridPos": {
777 "h": 1,779 "h": 1,
778 "w": 24,780 "w": 24,
779 "x": 0,781 "x": 0,
780- "y": 50782+ "y": 42
781 },783 },
782- "collapsed": true784+ "collapsed": false
783 },785 },
784 {786 {
785- "id": 40,787+ "id": 50,
786 "type": "timeseries",788 "type": "timeseries",
787- "title": "Expert hotness (current / update, mean / max)",789+ "title": "Engine Core / Executor 平均耗时 avg",
788 "gridPos": {790 "gridPos": {
789 "h": 8,791 "h": 8,
790 "w": 12,792 "w": 12,
791 "x": 0,793 "x": 0,
792- "y": 51794+ "y": 43
793 },795 },
794 "datasource": {796 "datasource": {
795 "type": "prometheus",797 "type": "prometheus",
@@ -797,23 +799,195 @@
797 },799 },
798 "targets": [800 "targets": [
799 {801 {
800- "expr": "max(vllm_profiling_eplb:expert_hotness:current_mean{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",802+ "expr": "sum(rate(vllm_profiling_engine_core:process_input_queue:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_engine_core:process_input_queue:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
801- "legendFormat": "current_mean",803+ "refId": "A",
804+ "legendFormat": "process_input_queue"
805+ },
806+ {
807+ "expr": "sum(rate(vllm_profiling_engine_core:process_engine_step:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_engine_core:process_engine_step:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
808+ "refId": "B",
809+ "legendFormat": "process_engine_step"
810+ },
811+ {
812+ "expr": "sum(rate(vllm_profiling_engine:async_add_request:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_engine:async_add_request:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
813+ "refId": "C",
814+ "legendFormat": "async_add_request"
815+ },
816+ {
817+ "expr": "sum(rate(vllm_profiling_executor:model_runner_execute_model:duration_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum(rate(vllm_profiling_executor:model_runner_execute_model:duration_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
818+ "refId": "D",
819+ "legendFormat": "model_runner_execute_model"
820+ }
821+ ],
822+ "fieldConfig": {
823+ "defaults": {
824+ "unit": "s",
825+ "custom": {
826+ "drawStyle": "line",
827+ "lineWidth": 1,
828+ "fillOpacity": 5
829+ }
830+ }
831+ },
832+ "options": {
833+ "tooltip": {
834+ "mode": "multi"
835+ },
836+ "legend": {
837+ "displayMode": "table",
838+ "placement": "bottom"
839+ }
840+ }
841+ },
842+ {
843+ "id": 51,
844+ "type": "timeseries",
845+ "title": "Engine Core 输出数量 outputs_len avg by dp",
846+ "gridPos": {
847+ "h": 8,
848+ "w": 12,
849+ "x": 12,
850+ "y": 43
851+ },
852+ "datasource": {
853+ "type": "prometheus",
854+ "uid": "prometheus"
855+ },
856+ "targets": [
857+ {
858+ "expr": "sum by (dp) (rate(vllm_profiling_engine_core_outputs_len_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])) / clamp_min(sum by (dp) (rate(vllm_profiling_engine_core_outputs_len_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])), 1)",
859+ "refId": "A",
860+ "legendFormat": "dp={{dp}}"
861+ }
862+ ],
863+ "fieldConfig": {
864+ "defaults": {
865+ "unit": "short",
866+ "custom": {
867+ "drawStyle": "line",
868+ "lineWidth": 1,
869+ "fillOpacity": 5
870+ }
871+ }
872+ },
873+ "options": {
874+ "tooltip": {
875+ "mode": "multi"
876+ },
877+ "legend": {
878+ "displayMode": "table",
879+ "placement": "bottom"
880+ }
881+ }
882+ },
883+ {
884+ "id": 610,
885+ "type": "row",
886+ "title": "KV Cache Blocks（metrics_extra 补充）",
887+ "gridPos": {
888+ "h": 1,
889+ "w": 24,
890+ "x": 0,
891+ "y": 51
892+ },
893+ "collapsed": false
894+ },
895+ {
896+ "id": 52,
897+ "type": "timeseries",
898+ "title": "KV Cache blocks 明细 instant avg by dp",
899+ "gridPos": {
900+ "h": 8,
901+ "w": 24,
902+ "x": 0,
903+ "y": 52
904+ },
905+ "datasource": {
906+ "type": "prometheus",
907+ "uid": "prometheus"
908+ },
909+ "targets": [
910+ {
911+ "expr": "sum by (dp) (vllm_profiling_total_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_total_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
912+ "legendFormat": "total dp={{dp}}",
802 "refId": "A"913 "refId": "A"
803 },914 },
804 {915 {
805- "expr": "max(vllm_profiling_eplb:expert_hotness:current_max{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",916+ "expr": "sum by (dp) (vllm_profiling_free_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_free_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
806- "legendFormat": "current_max",917+ "legendFormat": "free dp={{dp}}",
807 "refId": "B"918 "refId": "B"
808 },919 },
809 {920 {
810- "expr": "max(vllm_profiling_eplb:expert_hotness:update_mean{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",921+ "expr": "sum by (dp) (vllm_profiling_allocated_kvcache_blocks_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum by (dp) (vllm_profiling_allocated_kvcache_blocks_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
811- "legendFormat": "update_mean",922+ "legendFormat": "allocated dp={{dp}}",
923+ "refId": "C"
924+ }
925+ ],
926+ "fieldConfig": {
927+ "defaults": {
928+ "unit": "short",
929+ "custom": {
930+ "drawStyle": "line",
931+ "lineWidth": 1,
932+ "fillOpacity": 5
933+ }
934+ }
935+ },
936+ "options": {
937+ "tooltip": {
938+ "mode": "multi"
939+ },
940+ "legend": {
941+ "displayMode": "table",
942+ "placement": "bottom"
943+ }
944+ }
945+ },
946+ {
947+ "id": 620,
948+ "type": "row",
949+ "title": "Token & Latency（metrics_extra 补充）",
950+ "gridPos": {
951+ "h": 1,
952+ "w": 24,
953+ "x": 0,
954+ "y": 60
955+ },
956+ "collapsed": false
957+ },
958+ {
959+ "id": 53,
960+ "type": "timeseries",
961+ "title": "Per-step Token 数量 avg (total / input / output / spec)",
962+ "gridPos": {
963+ "h": 8,
964+ "w": 12,
965+ "x": 0,
966+ "y": 61
967+ },
968+ "datasource": {
969+ "type": "prometheus",
970+ "uid": "prometheus"
971+ },
972+ "targets": [
973+ {
974+ "expr": "sum(vllm_profiling_total_tokens_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_total_tokens_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
975+ "legendFormat": "total_tokens/step",
976+ "refId": "A"
977+ },
978+ {
979+ "expr": "sum(vllm_profiling_input_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_input_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
980+ "legendFormat": "input/step",
981+ "refId": "B"
982+ },
983+ {
984+ "expr": "sum(vllm_profiling_output_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_output_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
985+ "legendFormat": "output/step",
812 "refId": "C"986 "refId": "C"
813 },987 },
814 {988 {
815- "expr": "max(vllm_profiling_eplb:expert_hotness:update_max{source=~\"$source\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\", phase=~\"$phase\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",989+ "expr": "sum(vllm_profiling_num_spec_tokens_sum{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}) / clamp_min(sum(vllm_profiling_num_spec_tokens_count{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}), 1)",
816- "legendFormat": "update_max",990+ "legendFormat": "spec_tokens/step",
817 "refId": "D"991 "refId": "D"
818 }992 }
819 ],993 ],
@@ -838,14 +1012,14 @@
838 }1012 }
839 },1013 },
840 {1014 {
841- "id": 41,1015+ "id": 54,
842 "type": "timeseries",1016 "type": "timeseries",
843- "title": "Expert imbalance by layer",1017+ "title": "细粒度时延 P95 (TTFT / TPOT / 2nd token)",
844 "gridPos": {1018 "gridPos": {
845 "h": 8,1019 "h": 8,
846 "w": 12,1020 "w": 12,
847 "x": 12,1021 "x": 12,
848- "y": 511022+ "y": 61
849 },1023 },
850 "datasource": {1024 "datasource": {
851 "type": "prometheus",1025 "type": "prometheus",
@@ -853,20 +1027,29 @@
853 },1027 },
854 "targets": [1028 "targets": [
855 {1029 {
856- "expr": "max by (layer) (vllm_profiling_eplb:expert_hotness:imbalance{source=~\"$source\", phase=\"current\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", role=~\"$role\", dp_rank=~\"$dp\", instance_id=~\"$instance_id\"})",1030+ "expr": "histogram_quantile(0.95, sum by (le, dp) (rate(vllm_profiling_fine_grained_ttft_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
857- "legendFormat": "layer {{layer}}",1031+ "refId": "A",
858- "refId": "A"1032+ "legendFormat": "ttft p95 dp={{dp}}"
1033+ },
1034+ {
1035+ "expr": "histogram_quantile(0.95, sum by (le, dp) (rate(vllm_profiling_fine_grained_tpot_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
1036+ "refId": "B",
1037+ "legendFormat": "tpot p95 dp={{dp}}"
1038+ },
1039+ {
1040+ "expr": "histogram_quantile(0.95, sum by (le, dp) (rate(vllm_profiling_second_token_latency_bucket{source=~\"$source\", role=~\"$role\", cluster=~\"$cluster\", job=~\"$job\", pd_role=~\"$pd_role\", dp=~\"$dp\", instance_id=~\"$instance_id\"}[5m])))",
1041+ "refId": "C",
1042+ "legendFormat": "2nd_token p95 dp={{dp}}"
859 }1043 }
860 ],1044 ],
861 "fieldConfig": {1045 "fieldConfig": {
862 "defaults": {1046 "defaults": {
863- "unit": "short",1047+ "unit": "s",
864 "custom": {1048 "custom": {
865 "drawStyle": "line",1049 "drawStyle": "line",
866 "lineWidth": 1,1050 "lineWidth": 1,
867 "fillOpacity": 51051 "fillOpacity": 5
868- },1052+ }
869- "min": 1
870 }1053 }
871 },1054 },
872 "options": {1055 "options": {
Mmotor/coordinator/metrics/metrics_collector.py+2-2
@@ -585,7 +585,7 @@ class MetricsCollector(ThreadSafeSingleton):
585 return True585 return True
586 586 
587 def _parse_metric_text(self, metrics_str: str) -> list[Metric]:587 def _parse_metric_text(self, metrics_str: str) -> list[Metric]:
588- lines = [ln for ln in metrics_str.strip().split("\n") if ln]588+ lines = [ln for ln in metrics_str.splitlines() if ln.strip()]
atomgit-bot
atomgit-botatomgit-bot6月24日
已过期

🔵 Low Priority

变更行：line 588 lines = [ln for ln in metrics_str.splitlines() if ln.strip()]

与旧代码 [ln for ln in metrics_str.strip().split("\n") if ln] 相比，新代码使用 splitlines() 替代 split("\n")。

splitlines() 除了识别 \n、\r\n、\r 外，还会将以下 Unicode 字符视为行分隔符：\v、\f、\x1c、\x1d、\x1e、\x85、\u2028、\u2029。

Prometheus exposition format 严格使用 \n 作为行分隔符。如果 Prometheus 指标文本的标签值中恰好包含上述任一字符（例如模型名称中含有 \x85），splitlines() 会将其误判为换行，导致该行被错误截断，后续解析失败或产生错误数据。

失败模式：标签值中含特殊 Unicode 字符时，一行被拆成多行，_parse_metric_body_block 收到残缺行导致解析失败，整个 metric family 被丢弃。

这是一种极端边界情况（标签值中出现 Unicode 行分隔符的概率极低），但 splitlines() 的语义确实比实际需要的更宽泛。

建议：如果希望保持与 Prometheus 格式严格一致，可以使用 metrics_str.strip().split("\n") 或显式指定 splitlines(keepends=False) + 对 \r\n 做归一化处理。但考虑到标签值中出现 Unicode 行分隔符的概率极低，且 splitlines() 修复了 \r\n 兼容性问题，当前实现可以接受。此处报告为低优先级可选项。

likedislike
不准确？
LinWei100
LinWei100
6月25日 评论：
589 if not lines:589 if not lines:
590 return []590 return []
591 591 
@@ -641,7 +641,7 @@ class MetricsCollector(ThreadSafeSingleton):
641 metric: Metric,641 metric: Metric,
642 line: str,642 line: str,
643 ) -> bool:643 ) -> bool:
644- parts = line.split()644+ parts = line.rsplit(None, 1)
atomgit-bot
atomgit-botatomgit-bot6月24日
已过期

🟡 Medium Priority

变更行：line 644 parts = line.rsplit(None, 1)（旧代码为 parts = line.split()）

行为变化：

  1. 带时间戳的行：Prometheus 格式允许 metric_name{labels} value timestamp（可选时间戳）。rsplit(None, 1) 会把 timestamp 当作 value，实际 value 混入 label 部分，产生错误数据且无日志报错。
  2. 多余 token 的行：metric_name 1.0 extra_token 同理，extra_token 被误当作 value，真实值被吞入 label。

旧代码会拒绝这些行并记录 [Metrics] Parse metric body failed.，新代码静默产生错误数据。

失败模式：指标值被替换为时间戳或多余 token，标签字符串被污染，导致后续聚合和输出指标数据错误，且没有错误日志可追踪。

建议：建议改用 line.rsplit(None, 2) 获取最多 3 个 token，显式处理可选时间戳：取 parts[-2] 为 value，若 len(parts) == 3 则 parts[-1] 为时间戳（可忽略或记录日志），同时保留 len(parts) < 2 的报错逻辑。这样既能处理标签值中的空格，又能检测并正确处理带时间戳的行。

likedislike
不准确？
LinWei100
LinWei100
6月25日 评论：
645 if len(parts) != 2:645 if len(parts) != 2:
646 logger.error("[Metrics] Parse metric body failed.")646 logger.error("[Metrics] Parse metric body failed.")
647 return False647 return False