{
"annotations": { "list": [] },
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 100,
"title": "Controller Overview",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "steps": [{ "color": "green", "value": null }] },
"unit": "s"
}
},
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"id": 1,
"options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } },
"title": "Controller Uptime",
"targets": [
{ "expr": "tq_controller_uptime_seconds", "legendFormat": "" }
],
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 2147483648 },
{ "color": "red", "value": 4294967296 }
]},
"unit": "bytes"
}
},
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"id": 2,
"options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] } },
"title": "Controller RSS Memory",
"targets": [
{ "expr": "tq_controller_memory_rss_bytes", "legendFormat": "" }
],
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "steps": [{ "color": "blue", "value": null }] }
}
},
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"id": 3,
"options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } },
"title": "Active Partitions",
"targets": [
{ "expr": "tq_partitions_total", "legendFormat": "" }
],
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "steps": [{ "color": "purple", "value": null }] }
}
},
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"id": 4,
"options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } },
"title": "Global Indexes Allocated",
"targets": [
{ "expr": "tq_global_index_allocated_total", "legendFormat": "" }
],
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "steps": [{ "color": "orange", "value": null }] }
}
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"id": 5,
"options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } },
"title": "Reusable Indexes",
"targets": [
{ "expr": "tq_global_index_reusable_total", "legendFormat": "" }
],
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 101,
"title": "Request Throughput & Latency",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 20, "lineWidth": 2, "stacking": { "mode": "none" } },
"unit": "ops"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"id": 10,
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Controller Request Rate (per second)",
"targets": [
{ "expr": "sum by (op_type) (rate(tq_controller_request_total{op_type=~\"$op_type\"}[$__rate_interval]))", "legendFormat": "{{ op_type }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 },
"unit": "s"
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"id": 11,
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Controller Request Latency P50 / P99",
"targets": [
{ "expr": "histogram_quantile(0.50, sum by (op_type, le) (rate(tq_controller_request_duration_seconds_bucket{op_type=~\"$op_type\"}[$__rate_interval])))", "legendFormat": "p50 {{ op_type }}" },
{ "expr": "histogram_quantile(0.99, sum by (op_type, le) (rate(tq_controller_request_duration_seconds_bucket{op_type=~\"$op_type\"}[$__rate_interval])))", "legendFormat": "p99 {{ op_type }}" }
],
"transformations": [{ "id": "filterFieldsByName", "options": { "include": { "pattern": "Time|(${quantile:regex}).*" } } }],
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
"id": 102,
"title": "Partition Status",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 }
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 },
"id": 20,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Samples per Partition",
"targets": [
{ "expr": "tq_partition_samples_total", "legendFormat": "{{ partition_id }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "continuous-GrYlRd" },
"custom": { "fillOpacity": 20, "lineWidth": 2 },
"unit": "percentunit",
"min": 0,
"max": 1
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 },
"id": 22,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Production Progress",
"targets": [
{ "expr": "tq_partition_production_progress{task_name=~\"$task_name\"}", "legendFormat": "{{ partition_id }} / {{ task_name }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "continuous-GrYlRd" },
"custom": { "fillOpacity": 20, "lineWidth": 2 },
"unit": "percentunit",
"min": 0,
"max": 1
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 },
"id": 23,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Consumption Progress",
"targets": [
{ "expr": "tq_partition_consumption_progress{task_name=~\"$task_name\"}", "legendFormat": "{{ partition_id }} / {{ task_name }}" }
],
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 },
"id": 103,
"title": "Storage Units",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "continuous-GrYlRd" },
"thresholds": { "steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "red", "value": 0.9 }
]},
"unit": "percentunit",
"min": 0,
"max": 1
}
},
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 40 },
"id": 30,
"options": { "orientation": "horizontal", "displayMode": "gradient", "reduceOptions": { "calcs": ["lastNotNull"] }, "showUnfilled": true },
"title": "Storage Utilization",
"targets": [
{ "expr": "tq_storage_utilization_ratio", "legendFormat": "{{ storage_unit_id }}" }
],
"type": "bargauge"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 }
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 },
"id": 31,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Active Keys per Storage Unit",
"targets": [
{ "expr": "tq_storage_active_keys_total", "legendFormat": "{{ storage_unit_id }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 }
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 },
"id": 32,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Storage Capacity vs Active Keys",
"targets": [
{ "expr": "tq_storage_capacity_total", "legendFormat": "capacity {{ storage_unit_id }}" },
{ "expr": "tq_storage_active_keys_total", "legendFormat": "active {{ storage_unit_id }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 },
"unit": "bytes"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 },
"id": 33,
"options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Storage Process RSS Memory",
"targets": [
{ "expr": "tq_storage_memory_rss_bytes", "legendFormat": "{{ storage_unit_id }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 20, "lineWidth": 2, "stacking": { "mode": "none" } },
"unit": "ops"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 },
"id": 36,
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Storage Request Rate (per second)",
"targets": [
{ "expr": "sum by (op_type) (rate(tq_storage_request_ops{op_type=~\"$op_type\"}[$__rate_interval]))", "legendFormat": "{{ op_type }}" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 10, "lineWidth": 2 },
"unit": "s"
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 62 },
"id": 37,
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Storage Request Latency P50 / P99",
"targets": [
{ "expr": "tq_storage_request_latency_p50{op_type=~\"$op_type\"}", "legendFormat": "p50 {{ op_type }} ({{ storage_unit_id }})" },
{ "expr": "tq_storage_request_latency_p99{op_type=~\"$op_type\"}", "legendFormat": "p99 {{ op_type }} ({{ storage_unit_id }})" }
],
"transformations": [{ "id": "filterFieldsByName", "options": { "include": { "pattern": "Time|(${quantile:regex}).*" } } }],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": { "fillOpacity": 15, "lineWidth": 2 },
"unit": "short"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 },
"id": 34,
"options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Produced vs Cleared Samples (per second)",
"targets": [
{ "expr": "sum(rate(tq_controller_request_samples_total{op_type=\"NOTIFY_DATA_UPDATE\"}[$__rate_interval]))", "legendFormat": "Produced samples/s" },
{ "expr": "sum(rate(tq_controller_request_samples_total{op_type=\"CLEAR_META\"}[$__rate_interval]))", "legendFormat": "Cleared samples/s" }
],
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"fieldConfig": {
"defaults": {
"color": { "mode": "continuous-GrYlRd" },
"custom": { "fillOpacity": 20, "lineWidth": 2 },
"unit": "short"
}
},
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 78 },
"id": 35,
"options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
"title": "Active Keys Delta (PUT - CLEAR accumulation)",
"targets": [
{ "expr": "sum(tq_storage_active_keys_total)", "legendFormat": "Total Active Keys (all storage units)" }
],
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": ["transferqueue", "prometheus"],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"type": "datasource"
},
{
"current": {},
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"definition": "label_values(tq_partition_consumption_progress, task_name)",
"hide": 0,
"includeAll": true,
"label": "Task",
"multi": true,
"name": "task_name",
"options": [],
"query": "label_values(tq_partition_consumption_progress, task_name)",
"refresh": 2,
"sort": 1,
"type": "query"
},
{
"current": { "selected": true, "text": "All", "value": "$__all" },
"hide": 0,
"includeAll": true,
"label": "Op Type",
"multi": true,
"name": "op_type",
"options": [
{ "selected": true, "text": "All", "value": "$__all" },
{ "selected": false, "text": "PUT_DATA", "value": "PUT_DATA" },
{ "selected": false, "text": "GET_DATA", "value": "GET_DATA" },
{ "selected": false, "text": "CLEAR_DATA", "value": "CLEAR_DATA" },
{ "selected": false, "text": "GET_META", "value": "GET_META" },
{ "selected": false, "text": "CLEAR_META", "value": "CLEAR_META" },
{ "selected": false, "text": "NOTIFY_DATA_UPDATE", "value": "NOTIFY_DATA_UPDATE" }
],
"query": "PUT_DATA,GET_DATA,CLEAR_DATA,GET_META,CLEAR_META,NOTIFY_DATA_UPDATE",
"type": "custom"
},
{
"current": { "selected": true, "text": "All", "value": "$__all" },
"hide": 0,
"includeAll": true,
"label": "Quantile",
"multi": true,
"name": "quantile",
"options": [
{ "selected": true, "text": "All", "value": "$__all" },
{ "selected": false, "text": "p50", "value": "p50" },
{ "selected": false, "text": "p99", "value": "p99" }
],
"query": "p50,p99",
"type": "custom"
}
]
},
"time": { "from": "now-1h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "transfer_queue_dashboard",
"uid": "tq-metrics-dashboard",
"version": 1
}