[observability] 支持Loki在grafana中显示-MindIE-Motor-AtomGit

M.gitignore+2-2

	@@ -115,7 +115,7 @@ venv.bak/
115	# deployer	115	# deployer
116	deployer/output/*	116	deployer/output/*
117		117
118	-# observability stack runtime artifacts	118	+# observability stack local build artifacts
		119	+examples/features/observability/stack/loki/.build/
119	examples/features/observability/stack/generated/	120	examples/features/observability/stack/generated/
120	examples/features/observability/stack/.native-runtime/	121	examples/features/observability/stack/.native-runtime/
121	-examples/features/observability/stack/.env

Mexamples/features/observability/stack/.env.example+6-2

	@@ -12,6 +12,8 @@ GRAFANA_VERSION=11.3.0
12	PROMETHEUS_VERSION=v2.55.1	12	PROMETHEUS_VERSION=v2.55.1
13	TEMPO_VERSION=2.6.1	13	TEMPO_VERSION=2.6.1
14	LOKI_VERSION=3.3.0	14	LOKI_VERSION=3.3.0
		15	+# Loki container image; start.sh sets this after pull/build (override to pin local build).
		16	+# LOKI_IMAGE=grafana/loki:3.3.0
15	OTEL_COLLECTOR_VERSION=0.115.1	17	OTEL_COLLECTOR_VERSION=0.115.1
16	NODE_EXPORTER_VERSION=v1.8.2	18	NODE_EXPORTER_VERSION=v1.8.2
17	CADVISOR_VERSION=v0.49.1	19	CADVISOR_VERSION=v0.49.1
	@@ -32,8 +34,10 @@ MOTOR_USER_CONFIG=
32	MOTOR_ENGINE_MGMT_PORT=10001	34	MOTOR_ENGINE_MGMT_PORT=10001
33	# pyMotor 上报 tracing 使用的观测主机	35	# pyMotor 上报 tracing 使用的观测主机
34	OBS_HOST=	36	OBS_HOST=
35	-# Docker stack mode: minimal or full	37	+# Docker stack mode: minimal (core + Loki) or full (+ infra exporters)
36	-OBS_STACK_MODE=full	38	+OBS_STACK_MODE=minimal
		39	+# Set to 1 to fall back to full native runtime when Docker startup fails (default: stay on Docker).
		40	+# OBS_FORCE_NATIVE_FALLBACK=0
37	# First host port used when Docker needs PodIP bridge forwards	41	# First host port used when Docker needs PodIP bridge forwards
38	MOTOR_PORT_FORWARD_BASE=19000	42	MOTOR_PORT_FORWARD_BASE=19000
39	# -------- Proxy（可选）--------	43	# -------- Proxy（可选）--------

Mexamples/features/observability/stack/GRAFANA_GUIDE.md+1-1

	@@ -30,7 +30,7 @@
30	\|--------\|-----\|------\|------\|	30	\|--------\|-----\|------\|------\|
31	\| Prometheus \| `prometheus` \| `http://prometheus:9090` \| 指标查询（默认数据源） \|	31	\| Prometheus \| `prometheus` \| `http://prometheus:9090` \| 指标查询（默认数据源） \|
32	\| Tempo \| `tempo` \| `http://tempo:3200` \| 分布式追踪（Trace） \|	32	\| Tempo \| `tempo` \| `http://tempo:3200` \| 分布式追踪（Trace） \|
33	-\| Loki \| `loki` \| `http://loki:3100` \| 日志（仅 full 模式拉起） \|	33	+\| Loki \| `loki` \| `http://loki:3100` \| 日志（minimal / full 均包含） \|
34		34
35	并已配置三者间的联动跳转：	35	并已配置三者间的联动跳转：
36		36

Mexamples/features/observability/stack/docker-compose.yml+6-6

	@@ -73,10 +73,10 @@ services:
73	# Core: Loki (logs)	73	# Core: Loki (logs)
74	# ---------------------------------------------------------------	74	# ---------------------------------------------------------------
75	loki:	75	loki:
76	- image: ${REGISTRY_PREFIX:-}grafana/loki:${LOKI_VERSION:-3.3.0}	76	+ image: ${LOKI_IMAGE:-grafana/loki:3.3.0}
		77	+ pull_policy: if_not_present
77	container_name: pymotor-loki	78	container_name: pymotor-loki
78	restart: unless-stopped	79	restart: unless-stopped
79	- profiles: [full]
80	networks: [obs]	80	networks: [obs]
81	ports:	81	ports:
82	- "${LOKI_PORT:-3100}:3100"	82	- "${LOKI_PORT:-3100}:3100"
	@@ -94,7 +94,7 @@ services:
94	container_name: pymotor-otel-collector	94	container_name: pymotor-otel-collector
95	restart: unless-stopped	95	restart: unless-stopped
96	networks: [obs]	96	networks: [obs]
97	- depends_on: [tempo]	97	+ depends_on: [tempo, loki]
98	command: ["--config=/etc/otelcol/otel-collector.yaml"]	98	command: ["--config=/etc/otelcol/otel-collector.yaml"]
99	volumes:	99	volumes:
100	- ${OTEL_CONFIG_FILE:-./otel-collector/otel-collector.yaml}:/etc/otelcol/otel-collector.yaml:ro	100	- ${OTEL_CONFIG_FILE:-./otel-collector/otel-collector.yaml}:/etc/otelcol/otel-collector.yaml:ro
	@@ -112,7 +112,7 @@ services:
112	container_name: pymotor-grafana	112	container_name: pymotor-grafana
113	restart: unless-stopped	113	restart: unless-stopped
114	networks: [obs]	114	networks: [obs]
115	- depends_on: [prometheus, tempo]	115	+ depends_on: [prometheus, tempo, loki]
116	environment:	116	environment:
117	GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER:-motor}	117	GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER:-motor}
118	GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:-motor}	118	GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:-motor}
	@@ -123,8 +123,8 @@ services:
123	HTTPS_PROXY: ""	123	HTTPS_PROXY: ""
124	http_proxy: ""	124	http_proxy: ""
125	https_proxy: ""	125	https_proxy: ""
126	- NO_PROXY: prometheus,tempo,otel-collector,localhost,127.0.0.1,host.docker.internal,.local,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16	126	+ NO_PROXY: prometheus,tempo,loki,otel-collector,localhost,127.0.0.1,host.docker.internal,.local,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
127	- no_proxy: prometheus,tempo,otel-collector,localhost,127.0.0.1,host.docker.internal,.local,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16	127	+ no_proxy: prometheus,tempo,loki,otel-collector,localhost,127.0.0.1,host.docker.internal,.local,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
128	ports:	128	ports:
129	- "${GRAFANA_PORT:-3000}:3000"	129	- "${GRAFANA_PORT:-3000}:3000"
130	volumes:	130	volumes:

Mexamples/features/observability/stack/grafana/provisioning/datasources/datasources-minimal.yml+33-0

	@@ -1,4 +1,5 @@
1	# Grafana datasource provisioning for minimal mode.	1	# Grafana datasource provisioning for minimal mode.
		2	+# Loki + Trace<->Log linking, aligned with full mode.
2	apiVersion: 1	3	apiVersion: 1
3		4
4	datasources:	5	datasources:
	@@ -19,9 +20,41 @@ datasources:
19	url: http://tempo:3200	20	url: http://tempo:3200
20	jsonData:	21	jsonData:
21	httpMethod: GET	22	httpMethod: GET
		23	+ tracesToLogsV2:
		24	+ datasourceUid: loki
		25	+ tags:
		26	+ - { key: service.name, value: service_name }
		27	+ - { key: x_request_id, value: x_request_id }
		28	+ spanStartTimeShift: -5m
		29	+ spanEndTimeShift: 5m
		30	+ filterByTraceID: true
		31	+ filterBySpanID: false
		32	+ tracesToMetrics:
		33	+ datasourceUid: prometheus
		34	+ tags:
		35	+ - { key: service.name, value: service }
22	nodeGraph:	36	nodeGraph:
23	enabled: true	37	enabled: true
24	search:	38	search:
25	hide: false	39	hide: false
26	serviceMap:	40	serviceMap:
27	datasourceUid: prometheus	41	datasourceUid: prometheus
		42	+ lokiSearch:
		43	+ datasourceUid: loki
		44	+
		45	+ - name: Loki
		46	+ type: loki
		47	+ uid: loki
		48	+ access: proxy
		49	+ url: http://loki:3100
		50	+ jsonData:
		51	+ derivedFields:
		52	+ - name: TraceID
		53	+ matcherRegex: '(?:trace_id\|traceID\|traceId)\s[:=]\s"?([A-Fa-f0-9]{32})"?'
		54	+ url: '${__value.raw}'
		55	+ datasourceUid: tempo
		56	+ - name: x_request_id
		57	+ matcherRegex: '(?:x_request_id\|x-request-id)\s[:=]\s"?([A-Za-z0-9_-]+)"?'
		58	+ url: '{ .x_request_id = "${__value.raw}" }'
		59	+ datasourceUid: tempo
		60	+ urlDisplayLabel: 'Find by request_id'

Mexamples/features/observability/stack/grafana/provisioning/datasources/datasources.yml+4-4

	@@ -1,5 +1,5 @@
1	# Grafana datasource provisioning.	1	# Grafana datasource provisioning.
2	-# 预置 Prometheus / Tempo / Loki，并配置 Trace<->Log 双向跳转。	2	+# Provision Prometheus / Tempo / Loki with bidirectional Trace<->Log linking.
3	apiVersion: 1	3	apiVersion: 1
4		4
5	datasources:	5	datasources:
	@@ -50,11 +50,11 @@ datasources:
50	jsonData:	50	jsonData:
51	derivedFields:	51	derivedFields:
52	- name: TraceID	52	- name: TraceID
53	- matcherRegex: '(?:trace_id\|traceID\|traceId)\s[:=]\s"?([A-Fa-f0-9]+)"?'	53	+ matcherRegex: '(?:trace_id\|traceID\|traceId)\s[:=]\s"?([A-Fa-f0-9]{32})"?'
54	- url: '$${__value.raw}'	54	+ url: '${__value.raw}'
55	datasourceUid: tempo	55	datasourceUid: tempo
56	- name: x_request_id	56	- name: x_request_id
57	matcherRegex: '(?:x_request_id\|x-request-id)\s[:=]\s"?([A-Za-z0-9_-]+)"?'	57	matcherRegex: '(?:x_request_id\|x-request-id)\s[:=]\s"?([A-Za-z0-9_-]+)"?'
58	- url: '$${__value.raw}'	58	+ url: '{ .x_request_id = "${__value.raw}" }'
59	datasourceUid: tempo	59	datasourceUid: tempo
60	urlDisplayLabel: 'Find by request_id'	60	urlDisplayLabel: 'Find by request_id'

Mexamples/features/observability/stack/grafana/scripts/build-profiling-dashboard.py+1-3

	@@ -42,9 +42,7 @@ def fetch_metric_names(prometheus_url: str) -> List[str]:
42	def infer_query(metric_name: str) -> str:	42	def infer_query(metric_name: str) -> str:
43	if metric_name.endswith("_bucket"):	43	if metric_name.endswith("_bucket"):
44	metric = metric_name	44	metric = metric_name
45	- return (	45	+ return f"histogram_quantile(0.95, sum(rate({metric}[5m])) by (le, pd_role, role, instance_id))"
46	- f"histogram_quantile(0.95, sum(rate({metric}[5m])) by (le, pd_role, role, instance_id))"
47	- )
48	if metric_name.endswith("_total") or metric_name.endswith("_count"):	46	if metric_name.endswith("_total") or metric_name.endswith("_count"):
49	return f"sum(rate({metric_name}[5m])) by (pd_role, role, instance_id)"	47	return f"sum(rate({metric_name}[5m])) by (pd_role, role, instance_id)"
50	if metric_name.endswith("_sum"):	48	if metric_name.endswith("_sum"):

Mexamples/features/observability/stack/launch.sh+82-6

	@@ -15,7 +15,7 @@ OBS_HOST_INPUT="${OBS_HOST:-}"
15	FORCE_NATIVE=0	15	FORCE_NATIVE=0
16	DISCOVER_ONLY=0	16	DISCOVER_ONLY=0
17	DRY_RUN=0	17	DRY_RUN=0
18	-STACK_MODE="${OBS_STACK_MODE:-full}"	18	+STACK_MODE="${OBS_STACK_MODE:-minimal}"
19		19
20	usage() {	20	usage() {
21	cat <<'EOF'	21	cat <<'EOF'
	@@ -25,8 +25,8 @@ Options:
25	--namespace <namespace> Kubernetes namespace / job_id	25	--namespace <namespace> Kubernetes namespace / job_id
26	--node-ip <node-ip> Node IP used for NodePort access	26	--node-ip <node-ip> Node IP used for NodePort access
27	--user-config <path> pyMotor user_config.json path	27	--user-config <path> pyMotor user_config.json path
28	- --minimal Start minimal Docker stack (Prometheus/Grafana/Tempo/OTel)	28	+ --minimal Start core stack with Loki (default)
29	- --full Start full Docker stack (adds Loki/node-exporter/cAdvisor)	29	+ --full Add node-exporter/cAdvisor infra exporters
30	--discover-only Only run discovery, do not start stack	30	--discover-only Only run discovery, do not start stack
31	--dry-run Run discovery and print generated Prometheus config	31	--dry-run Run discovery and print generated Prometheus config
32	--native Skip Docker Compose and run native runtime	32	--native Skip Docker Compose and run native runtime
	@@ -46,6 +46,7 @@ Proxy (see SERVICE_GUIDE.md §2.4):
46	- Native runtime: set PROXY_SH=/path/to/dotenv in .env (optional; default empty).	46	- Native runtime: set PROXY_SH=/path/to/dotenv in .env (optional; default empty).
47	- Grafana container: HTTP_PROXY cleared for in-stack prometheus/tempo.	47	- Grafana container: HTTP_PROXY cleared for in-stack prometheus/tempo.
48	- OBS_COMPOSE_PULL=never\|missing\|always OBS_COMPOSE_BUILD=0\|1	48	- OBS_COMPOSE_PULL=never\|missing\|always OBS_COMPOSE_BUILD=0\|1
		49	+ - OBS_FORCE_NATIVE_FALLBACK=1 only then fall back to full native runtime on Docker failure
49	EOF	50	EOF
50	}	51	}
51		52
	@@ -144,6 +145,65 @@ run_native() {
144	--prometheus-file "./generated/prometheus.yml"	145	--prometheus-file "./generated/prometheus.yml"
145	}	146	}
146		147
		148	+wait_for_http() {
		149	+ local url=$1
		150	+ local label=$2
		151	+ local max_attempts="${3:-30}"
		152	+ local sleep_sec="${4:-2}"
		153	+
		154	+ local attempt=1
		155	+ while (( attempt <= max_attempts )); do
		156	+ if curl -fsS "${url}" >/dev/null 2>&1; then
		157	+ return 0
		158	+ fi
		159	+ if (( attempt == 1 )); then
		160	+ echo "[launch] waiting for ${label}..."
		161	+ fi
		162	+ sleep "${sleep_sec}"
		163	+ attempt=$((attempt + 1))
		164	+ done
		165	+
		166	+ echo "[launch] ${label} readiness check failed: ${url}" >&2
		167	+ return 1
		168	+}
		169	+
		170	+check_core_stack() {
		171	+ local docker_bin="${DOCKER_BIN:-docker}"
		172	+ local missing=()
		173	+ local core_containers=(
		174	+ pymotor-prometheus
		175	+ pymotor-grafana
		176	+ pymotor-tempo
		177	+ pymotor-loki
		178	+ pymotor-otel-collector
		179	+ )
		180	+
		181	+ for name in "${core_containers[@]}"; do
		182	+ if ! "${docker_bin}" inspect -f '{{.State.Running}}' "${name}" 2>/dev/null \| grep -qx 'true'; then
		183	+ missing+=("${name}")
		184	+ fi
		185	+ done
		186	+
		187	+ if ((${#missing[@]} > 0)); then
		188	+ echo "[launch] core stack unhealthy; not running containers: ${missing[*]}" >&2
		189	+ return 1
		190	+ fi
		191	+
		192	+ if command -v curl >/dev/null 2>&1; then
		193	+ local loki_port="${LOKI_PORT:-3100}"
		194	+ local loki_attempts="${LOKI_READY_MAX_ATTEMPTS:-30}"
		195	+ local loki_sleep="${LOKI_READY_SLEEP_SEC:-2}"
		196	+ wait_for_http \
		197	+ "http://127.0.0.1:${loki_port}/ready" \
		198	+ "Loki :${loki_port}" \
		199	+ "${loki_attempts}" \
		200	+ "${loki_sleep}" \|\| return 1
		201	+ fi
		202	+
		203	+ echo "[launch] core stack healthy (includes pymotor-loki)"
		204	+ return 0
		205	+}
		206	+
147	if [[ "${FORCE_NATIVE}" -eq 1 ]]; then	207	if [[ "${FORCE_NATIVE}" -eq 1 ]]; then
148	run_native	208	run_native
149	exit 0	209	exit 0
	@@ -158,7 +218,23 @@ DOCKER_RC=$?
158	set -e	218	set -e
159		219
160	if [[ "${DOCKER_RC}" -ne 0 ]]; then	220	if [[ "${DOCKER_RC}" -ne 0 ]]; then
161	- echo "[launch] Docker startup failed (exit=${DOCKER_RC}), cleaning partial Docker stack before native fallback."	221	+ if [[ "${OBS_FORCE_NATIVE_FALLBACK:-0}" == "1" ]]; then
162	- ./stop.sh \|\| true	222	+ echo "[launch] Docker startup failed (exit=${DOCKER_RC}); OBS_FORCE_NATIVE_FALLBACK=1, switching to native runtime."
163	- run_native	223	+ ./stop.sh \|\| true
		224	+ run_native
		225	+ exit 0
		226	+ fi
		227	+ echo "[launch] Docker startup failed (exit=${DOCKER_RC}). Set OBS_FORCE_NATIVE_FALLBACK=1 to fall back to native runtime." >&2
		228	+ exit "${DOCKER_RC}"
		229	+fi
		230	+
		231	+if ! check_core_stack; then
		232	+ if [[ "${OBS_FORCE_NATIVE_FALLBACK:-0}" == "1" ]]; then
		233	+ echo "[launch] core stack check failed; OBS_FORCE_NATIVE_FALLBACK=1, switching to native runtime."
		234	+ ./stop.sh \|\| true
		235	+ run_native
		236	+ exit 0
		237	+ fi
		238	+ echo "[launch] core stack check failed. Set OBS_FORCE_NATIVE_FALLBACK=1 to fall back to native runtime." >&2
		239	+ exit 1
164	fi	240	fi

Aexamples/features/observability/stack/loki/Dockerfile+10-0

@@ -0,0 +1,10 @@
	1	+# Minimal Loki image: static binary on scratch (no base image pull required).
	2	+# Built by scripts/build-loki-image.sh when Docker Hub is unavailable.
	3	+
	4	+FROM scratch
	5	+
	6	+COPY loki /loki
	7	+
	8	+EXPOSE 3100 9096
	9	+
	10	+ENTRYPOINT ["/loki"]

Mexamples/features/observability/stack/otel-collector/otel-collector-minimal.yaml+16-0

	@@ -1,4 +1,9 @@
1	# OpenTelemetry Collector configuration for minimal mode.	1	# OpenTelemetry Collector configuration for minimal mode.
		2	+# Receives OTLP traces/logs from pyMotor and routes:
		3	+# traces → Tempo
		4	+# logs → Loki
		5	+#
		6	+# start.sh ensure_loki_image() 会在 Hub 拉取失败时本地 build Loki 镜像。
2		7
3	receivers:	8	receivers:
4	otlp:	9	otlp:
	@@ -17,12 +22,19 @@ processors:
17	- key: deployment.environment	22	- key: deployment.environment
18	value: pymotor-observability-minimal	23	value: pymotor-observability-minimal
19	action: upsert	24	action: upsert
		25	+ attributes/loki:
		26	+ actions:
		27	+ - key: loki.format
		28	+ value: json
		29	+ action: insert
20		30
21	exporters:	31	exporters:
22	otlp/tempo:	32	otlp/tempo:
23	endpoint: tempo:4317	33	endpoint: tempo:4317
24	tls:	34	tls:
25	insecure: true	35	insecure: true
		36	+ loki:
		37	+ endpoint: http://loki:3100/loki/api/v1/push
26	debug:	38	debug:
27	verbosity: basic	39	verbosity: basic
28		40
	@@ -32,6 +44,10 @@ service:
32	receivers: [otlp]	44	receivers: [otlp]
33	processors: [batch, resource]	45	processors: [batch, resource]
34	exporters: [otlp/tempo, debug]	46	exporters: [otlp/tempo, debug]
		47	+ logs:
		48	+ receivers: [otlp]
		49	+ processors: [batch, resource, attributes/loki]
		50	+ exporters: [loki, debug]
35	telemetry:	51	telemetry:
36	logs:	52	logs:
37	level: info	53	level: info

Aexamples/features/observability/stack/scripts/build-loki-image.sh+125-0

@@ -0,0 +1,125 @@

1

+#!/usr/bin/env bash

2

+# Build a local grafana/loki image from a static binary (no Docker Hub / alpine pull).

3

+

4

+set -euo pipefail

5

+

6

+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

7

+STACK_DIR="$(cd -- "${SCRIPT_DIR}/.." &>/dev/null && pwd)"

8

+cd "${STACK_DIR}"

9

+. "${SCRIPT_DIR}/load-dotenv.sh"

10

+

11

+if [[ -f "${STACK_DIR}/.env" ]]; then

12

+ load_dotenv "${STACK_DIR}/.env"

13

+elif [[ -f "${STACK_DIR}/.env.example" ]]; then

14

+ load_dotenv "${STACK_DIR}/.env.example"

15

+fi

16

+

17

+LOKI_VERSION="${LOKI_VERSION:-3.3.0}"

18

+REGISTRY_PREFIX="${REGISTRY_PREFIX:-}"

19

+LOKI_IMAGE="${LOKI_IMAGE:-${REGISTRY_PREFIX}grafana/loki:${LOKI_VERSION}}"

20

+BUILD_DIR="${STACK_DIR}/loki/.build"

21

+BINARY="${BUILD_DIR}/loki"

22

+DOCKER_BIN="${DOCKER_BIN:-docker}"

23

+LOKI_DOWNLOAD_INSECURE="${LOKI_DOWNLOAD_INSECURE:-0}"

24

+

25

+usage() {

26

+ cat <<EOF

27

+Usage: $0

28

+

29

+Build ${LOKI_IMAGE} from a static Loki binary (scratch-based Dockerfile).

30

+

31

+Environment:

32

+ LOKI_VERSION Loki release tag (default: 3.3.0)

33

+ REGISTRY_PREFIX Optional image registry prefix

34

+ LOKI_IMAGE Output image tag (default: \${REGISTRY_PREFIX}grafana/loki:\${LOKI_VERSION})

35

+ LOKI_DOWNLOAD_INSECURE Set to 1 to skip TLS verification (curl -k / wget --no-check-certificate)

36

+ DOCKER_BIN docker binary (default: docker)

37

+

38

+Tip: place a pre-downloaded binary at loki/.build/loki to skip download.

39

+EOF

40

+}

41

+

42

+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then

43

+ usage

44

+ exit 0

45

+fi

46

+

47

+download_file() {

48

+ local url="$1"

49

+ local out_file="$2"

50

+ local curl_args=(-fL --retry 3 --retry-delay 2 -o "${out_file}" "${url}")

51

+ if [[ "${LOKI_DOWNLOAD_INSECURE}" == "1" ]]; then

52

+ curl_args=(-fkL --retry 3 --retry-delay 2 -o "${out_file}" "${url}")

53

+ fi

54

+ if command -v curl >/dev/null 2>&1; then

55

+ curl "${curl_args[@]}"

56

+ return

57

+ fi

58

+ if command -v wget >/dev/null 2>&1; then

atomgit-bot6月9日

🟡 Medium Priority

download_file() 在 curl 路径通过 -k 支持 LOKI_DOWNLOAD_INSECURE=1 跳过 TLS 校验，但 wget 回退路径未传 --no-check-certificate。若环境仅安装了 wget 且 GitHub TLS 握手失败，用户显式设置的 INSECURE 模式不生效，下载将失败。

改动建议

-  if command -v wget >/dev/null 2>&1; then
+if command -v wget >/dev/null 2>&1; then
+    local wget_args=("-O" "${out_file}")
+    if [[ "${LOKI_DOWNLOAD_INSECURE}" == "1" ]]; then
+      wget_args+=("--no-check-certificate")
+    fi
+    wget "${wget_args[@]}" "${url}"
+    return
+  fi

应用建议

不准确？

59

+ local wget_args=("-O" "${out_file}")

60

+ if [[ "${LOKI_DOWNLOAD_INSECURE}" == "1" ]]; then

61

+ wget_args+=("--no-check-certificate")

62

+ fi

63

+ wget "${wget_args[@]}" "${url}"

64

+ return

65

+ fi

66

+ echo "[build-loki] neither curl nor wget is available" >&2

67

+ exit 1

68

+}

69

+

70

+install_binary() {

71

+ if [[ -x "${BINARY}" ]]; then

72

+ echo "[build-loki] using existing binary: ${BINARY}"

73

+ return 0

74

+ fi

75

+

76

+ mkdir -p "${BUILD_DIR}"

77

+ local ver="${LOKI_VERSION#v}"

78

+ local archive="${BUILD_DIR}/loki-${ver}.zip"

79

+ local url="https://github.com/grafana/loki/releases/download/v${ver}/loki-linux-amd64.zip"

80

+

81

+ echo "[build-loki] downloading Loki ${LOKI_VERSION} from GitHub..."

82

+ download_file "${url}" "${archive}"

83

+

84

+ if ! command -v unzip >/dev/null 2>&1; then

85

+ echo "[build-loki] error: unzip is required to extract ${archive}" >&2

86

+ exit 1

87

+ fi

88

+

89

+ local extract_dir="${BUILD_DIR}/extract"

90

+ rm -rf "${extract_dir}"

91

+ mkdir -p "${extract_dir}"

92

+ unzip -o -q "${archive}" -d "${extract_dir}"

atomgit-bot6月9日

🟡 Medium Priority

install_binary() 直接调用 unzip 解压下载的 zip 归档，但未先检查 unzip 是否在 PATH 中。若系统未安装 unzip，脚本会在解压步骤以 "command not found" 失败，无法给出明确错误提示，影响离线/精简环境下的可用性。

不准确？

93

+

94

+ if [[ -f "${extract_dir}/loki-linux-amd64" ]]; then

95

+ cp "${extract_dir}/loki-linux-amd64" "${BINARY}"

96

+ elif [[ -f "${extract_dir}/loki" ]]; then

97

+ cp "${extract_dir}/loki" "${BINARY}"

98

+ else

99

+ echo "[build-loki] unexpected archive layout in ${archive}" >&2

100

+ exit 1

101

+ fi

102

+ chmod +x "${BINARY}"

103

+ echo "[build-loki] binary ready: ${BINARY}"

104

+}

105

+

106

+if ! "${DOCKER_BIN}" info >/dev/null 2>&1; then

107

+ echo "[build-loki] error: docker daemon is not available" >&2

108

+ exit 1

109

+fi

110

+

111

+install_binary

112

+

113

+echo "[build-loki] building image ${LOKI_IMAGE}..."

114

+"${DOCKER_BIN}" build \

115

+ -f "${STACK_DIR}/loki/Dockerfile" \

116

+ -t "${LOKI_IMAGE}" \

117

+ "${BUILD_DIR}"

118

+

119

+if [[ -n "${REGISTRY_PREFIX}" ]] \

120

+ && [[ "${LOKI_IMAGE}" != "grafana/loki:${LOKI_VERSION}" ]] \

121

+ && ! "${DOCKER_BIN}" image inspect "grafana/loki:${LOKI_VERSION}" >/dev/null 2>&1; then

122

+ "${DOCKER_BIN}" tag "${LOKI_IMAGE}" "grafana/loki:${LOKI_VERSION}"

123

+fi

124

+

125

+echo "[build-loki] done: ${LOKI_IMAGE}"

Mexamples/features/observability/stack/scripts/discover-targets.py+15-15

	@@ -51,6 +51,10 @@ def _kubectl_env() -> Dict[str, str]:
51	return env	51	return env
52		52
53		53
		54	+def _kubectl_path() -> Optional[str]:
		55	+ return shutil.which("kubectl")
		56	+
		57	+
54	@dataclass	58	@dataclass
55	class PortForwardSpec:	59	class PortForwardSpec:
56	namespace: str	60	namespace: str
	@@ -85,10 +89,11 @@ class DiscoveryResult:
85		89
86		90
87	def _run_kubectl_json(args: Sequence[str]) -> Dict[str, Any]:	91	def _run_kubectl_json(args: Sequence[str]) -> Dict[str, Any]:
88	- cmd = ["kubectl", *args, "-o", "json"]	92	+ kubectl = _kubectl_path()
89	- output = subprocess.run(	93	+ if kubectl is None:
90	- cmd, check=True, capture_output=True, text=True, env=_kubectl_env()	94	+ raise FileNotFoundError("kubectl not found in PATH")
91	- )	95	+ cmd = [kubectl, *args, "-o", "json"]
		96	+ output = subprocess.run(cmd, check=True, capture_output=True, text=True, env=_kubectl_env())
92	return json.loads(output.stdout)	97	return json.loads(output.stdout)
93		98
94		99
	@@ -110,19 +115,20 @@ def _read_user_config_job_id(path: Optional[str]) -> Optional[str]:
110		115
111		116
112	def _is_kubectl_ready() -> bool:	117	def _is_kubectl_ready() -> bool:
113	- if shutil.which("kubectl") is None:	118	+ kubectl = _kubectl_path()
		119	+ if kubectl is None:
114	return False	120	return False
115	kubectl_env = _kubectl_env()	121	kubectl_env = _kubectl_env()
116	try:	122	try:
117	subprocess.run(	123	subprocess.run(
118	- ["kubectl", "version", "--client"],	124	+ [kubectl, "version", "--client"],
119	check=True,	125	check=True,
120	capture_output=True,	126	capture_output=True,
121	text=True,	127	text=True,
122	env=kubectl_env,	128	env=kubectl_env,
123	)	129	)
124	subprocess.run(	130	subprocess.run(
125	- ["kubectl", "get", "ns"],	131	+ [kubectl, "get", "ns"],
126	check=True,	132	check=True,
127	capture_output=True,	133	capture_output=True,
128	text=True,	134	text=True,
	@@ -283,11 +289,7 @@ def _match_nodeport_for_service(
283	service_port = int(port.get("port") or 0)	289	service_port = int(port.get("port") or 0)
284	target_port = str(port.get("targetPort", ""))	290	target_port = str(port.get("targetPort", ""))
285	name = str(port.get("name", ""))	291	name = str(port.get("name", ""))
286	- if (	292	+ if service_port == expected_port or target_port == str(expected_port) or _has_keyword(name, allow_keywords):
287	- service_port == expected_port
288	- or target_port == str(expected_port)
289	- or _has_keyword(name, allow_keywords)
290	- ):
291	return int(node_port)	293	return int(node_port)
292	return None	294	return None
293		295
	@@ -549,9 +551,7 @@ def _discover(namespace: str, node_ip: str, args: argparse.Namespace) -> Discove
549		551
550	except subprocess.CalledProcessError as exc:	552	except subprocess.CalledProcessError as exc:
551	if _has_explicit_namespace(args):	553	if _has_explicit_namespace(args):
552	- raise RuntimeError(	554	+ raise RuntimeError(f"kubernetes discovery failed for explicit namespace '{namespace}': {exc}") from exc
553	- f"kubernetes discovery failed for explicit namespace '{namespace}': {exc}"
554	- ) from exc
555	warnings.append(f"kubernetes discovery failed: {exc}. fallback to static defaults.")	555	warnings.append(f"kubernetes discovery failed: {exc}. fallback to static defaults.")
556	mode = "fallback"	556	mode = "fallback"
557	engine_targets = []	557	engine_targets = []

Aexamples/features/observability/stack/scripts/inject-pr170-error-logs.py+203-0

@@ -0,0 +1,203 @@
	1	+# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
	2	+# MindIE is licensed under Mulan PSL v2.
	3	+# You can use this software according to the terms and conditions of the Mulan PSL v2.
	4	+# You may obtain a copy of Mulan PSL v2 at:
	5	+# http://license.coscl.org.cn/MulanPSL2
	6	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
	7	+# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
	8	+# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
	9	+# See the Mulan PSL v2 for more details.
	10	+
	11	+"""Inject PR170-style error logs (and optional OTLP traces) for observability testing."""
	12	+
	13	+from __future__ import annotations
	14	+
	15	+import argparse
	16	+import json
	17	+import secrets
	18	+import sys
	19	+import time
	20	+import urllib.error
	21	+import urllib.request
	22	+from datetime import datetime, timezone
	23	+
	24	+
	25	+def _random_trace_id() -> str:
	26	+ return secrets.token_hex(16)
	27	+
	28	+
	29	+def _random_request_id() -> str:
	30	+ return f"req-{secrets.token_hex(8)}"
	31	+
	32	+
	33	+def build_log_lines(
	34	+ *,
	35	+ service_name: str,
	36	+ trace_id: str,
	37	+ request_id: str,
	38	+ count: int,
	39	+) -> list[str]:
	40	+ lines: list[str] = []
	41	+ for i in range(count):
	42	+ lines.append(
	43	+ f"HTTP request send failed. url=http://127.0.0.1:8080/v1/chat/completions, "
	44	+ f"error=connection refused attempt={i + 1} "
	45	+ f"Possible causes: 1) engine not ready 2) network partition 3) timeout "
	46	+ f"trace_id={trace_id} x_request_id={request_id} service_name={service_name}"
	47	+ )
	48	+ lines.append(
	49	+ f"error message: upstream engine unavailable for request {request_id} "
	50	+ f"trace_id={trace_id} x_request_id={request_id}"
	51	+ )
	52	+ return lines
	53	+
	54	+
	55	+def push_loki_logs(
	56	+ *,
	57	+ loki_url: str,
	58	+ service_name: str,
	59	+ lines: list[str],
	60	+) -> None:
	61	+ # Each line needs a distinct timestamp; Loki dedupes same stream + ts + line.
	62	+ base_ts_ns = int(time.time() * 1_000_000_000)
	63	+ values = [[str(base_ts_ns + i), line] for i, line in enumerate(lines)]
	64	+ payload = {
	65	+ "streams": [
	66	+ {
	67	+ "stream": {"service_name": service_name, "job": "inject-pr170"},
	68	+ "values": values,
	69	+ }
	70	+ ]
	71	+ }
	72	+ req = urllib.request.Request(
	73	+ f"{loki_url.rstrip('/')}/loki/api/v1/push",
	74	+ data=json.dumps(payload).encode("utf-8"),
	75	+ headers={"Content-Type": "application/json"},
	76	+ method="POST",
	77	+ )
	78	+ with urllib.request.urlopen(req, timeout=10) as resp:
	79	+ if resp.status >= 300:
	80	+ raise RuntimeError(f"Loki push failed: HTTP {resp.status}")
	81	+
	82	+
	83	+def push_otlp_trace(
	84	+ *,
	85	+ otlp_http_url: str,
	86	+ service_name: str,
	87	+ trace_id: str,
	88	+ request_id: str,
	89	+) -> None:
	90	+ # Minimal OTLP/HTTP JSON trace with x_request_id + error.message attributes.
	91	+ span_id = secrets.token_hex(8)
	92	+ now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
	93	+ payload = {
	94	+ "resourceSpans": [
	95	+ {
	96	+ "resource": {
	97	+ "attributes": [
	98	+ {"key": "service.name", "value": {"stringValue": service_name}},
	99	+ ]
	100	+ },
	101	+ "scopeSpans": [
	102	+ {
	103	+ "scope": {"name": "inject-pr170"},
	104	+ "spans": [
	105	+ {
	106	+ "traceId": trace_id,
	107	+ "spanId": span_id,
	108	+ "name": "router.dispatch",
	109	+ "kind": 1,
	110	+ "startTimeUnixNano": str(int(time.time() * 1_000_000_000)),
	111	+ "endTimeUnixNano": str(int(time.time() * 1_000_000_000) + 50_000_000),
	112	+ "attributes": [
	113	+ {
	114	+ "key": "x_request_id",
	115	+ "value": {"stringValue": request_id},
	116	+ },
	117	+ {
	118	+ "key": "error.message",
	119	+ "value": {
	120	+ "stringValue": (
	121	+ f"error message: upstream engine unavailable for request {request_id}"
	122	+ )
	123	+ },
	124	+ },
	125	+ ],
	126	+ "status": {"code": 2, "message": "upstream engine unavailable"},
	127	+ }
	128	+ ],
	129	+ }
	130	+ ],
	131	+ }
	132	+ ]
	133	+ }
	134	+ req = urllib.request.Request(
	135	+ f"{otlp_http_url.rstrip('/')}/v1/traces",
	136	+ data=json.dumps(payload).encode("utf-8"),
	137	+ headers={"Content-Type": "application/json"},
	138	+ method="POST",
	139	+ )
	140	+ with urllib.request.urlopen(req, timeout=10) as resp:
	141	+ if resp.status >= 300:
	142	+ raise RuntimeError(f"OTLP trace push failed: HTTP {resp.status}")
	143	+ print(f"[inject] OTLP trace sent trace_id={trace_id} at {now}")
	144	+
	145	+
	146	+def parse_args() -> argparse.Namespace:
	147	+ parser = argparse.ArgumentParser(description=__doc__)
	148	+ parser.add_argument(
	149	+ "--mode",
	150	+ choices=("loki", "both"),
	151	+ default="loki",
	152	+ help="Push logs only, or logs + OTLP trace (default: loki)",
	153	+ )
	154	+ parser.add_argument("--loki-url", default="http://127.0.0.1:3100")
	155	+ parser.add_argument("--otlp-http-url", default="http://127.0.0.1:4318")
	156	+ parser.add_argument("--service-name", default="motor-coordinator")
	157	+ parser.add_argument("--trace-id", default="")
	158	+ parser.add_argument("--request-id", default="")
	159	+ parser.add_argument("--count", type=int, default=2, help="Number of error pairs to emit")
	160	+ return parser.parse_args()
	161	+
	162	+
	163	+def main() -> int:
	164	+ args = parse_args()
	165	+ trace_id = args.trace_id or _random_trace_id()
	166	+ request_id = args.request_id or _random_request_id()
	167	+ lines = build_log_lines(
	168	+ service_name=args.service_name,
	169	+ trace_id=trace_id,
	170	+ request_id=request_id,
	171	+ count=max(1, args.count),
	172	+ )
	173	+
	174	+ try:
	175	+ push_loki_logs(
	176	+ loki_url=args.loki_url,
	177	+ service_name=args.service_name,
	178	+ lines=lines,
	179	+ )
	180	+ except urllib.error.URLError as exc:
	181	+ print(f"[inject] Loki push failed: {exc}", file=sys.stderr)
	182	+ return 1
	183	+
	184	+ print(f"[inject] pushed {len(lines)} log lines to Loki trace_id={trace_id} x_request_id={request_id}")
	185	+ print('[inject] Explore query: {service_name=~"motor-.*"} \|= "Possible causes:"')
	186	+
	187	+ if args.mode == "both":
	188	+ try:
	189	+ push_otlp_trace(
	190	+ otlp_http_url=args.otlp_http_url,
	191	+ service_name=args.service_name,
	192	+ trace_id=trace_id,
	193	+ request_id=request_id,
	194	+ )
	195	+ except urllib.error.URLError as exc:
	196	+ print(f"[inject] OTLP trace push failed: {exc}", file=sys.stderr)
	197	+ return 1
	198	+
	199	+ return 0
	200	+
	201	+
	202	+if __name__ == "__main__":
	203	+ raise SystemExit(main())

Mexamples/features/observability/stack/start.sh+89-12

	@@ -13,7 +13,7 @@ if [[ ! -f .env ]]; then
13	fi	13	fi
14		14
15	PROFILES=""	15	PROFILES=""
16	-STACK_MODE="${OBS_STACK_MODE:-full}"	16	+STACK_MODE="${OBS_STACK_MODE:-minimal}"
17	WITH_MOCK=0	17	WITH_MOCK=0
18		18
19	while [[ $# -gt 0 ]]; do	19	while [[ $# -gt 0 ]]; do
	@@ -38,12 +38,16 @@ while [[ $# -gt 0 ]]; do
38	-h\|--help)	38	-h\|--help)
39	cat <<EOF	39	cat <<EOF
40	Usage: $0 [options]	40	Usage: $0 [options]
41	- --minimal start Prometheus/Grafana/Tempo/OTel only	41	+ --minimal start core stack with Loki (default)
42	- --full start full stack with Loki/node-exporter/cAdvisor (default)	42	+ --full add node-exporter/cAdvisor infra exporters
43	--with-mock enable mock profile when present	43	--with-mock enable mock profile when present
44	--profile <list> comma-separated profiles (default: none)	44	--profile <list> comma-separated profiles (default: none)
45	known: npu, full	45	known: npu, full
46	-h, --help show this help	46	-h, --help show this help
		47	+
		48	+Loki image:
		49	+ Pulls grafana/loki from registry first; on failure runs
		50	+ ./scripts/build-loki-image.sh to build a local scratch image.
47	EOF	51	EOF
48	exit 0	52	exit 0
49	;;	53	;;
	@@ -55,7 +59,22 @@ EOF
55	done	59	done
56		60
57	DISCOVERED_ENV="${SCRIPT_DIR}/generated/discovered.env"	61	DISCOVERED_ENV="${SCRIPT_DIR}/generated/discovered.env"
		62	+# Preserve launch.sh / caller overrides before dotenv files (see PR 252).
		63	+PRESERVE_PROMETHEUS_CONFIG="${PROMETHEUS_CONFIG_FILE:-}"
		64	+PRESERVE_OTEL_CONFIG="${OTEL_CONFIG_FILE:-}"
		65	+PRESERVE_GRAFANA_PROV="${GRAFANA_PROVISIONING_DIR:-}"
		66	+# Load .env defaults first; discovered.env (from launch.sh discovery) must win.
		67	+load_dotenv "${SCRIPT_DIR}/.env"
58	load_dotenv "${DISCOVERED_ENV}"	68	load_dotenv "${DISCOVERED_ENV}"
		69	+if [[ -n "${PRESERVE_PROMETHEUS_CONFIG}" ]]; then
		70	+ PROMETHEUS_CONFIG_FILE="${PRESERVE_PROMETHEUS_CONFIG}"
		71	+fi
		72	+if [[ -n "${PRESERVE_OTEL_CONFIG}" ]]; then
		73	+ OTEL_CONFIG_FILE="${PRESERVE_OTEL_CONFIG}"
		74	+fi
		75	+if [[ -n "${PRESERVE_GRAFANA_PROV}" ]]; then
		76	+ GRAFANA_PROVISIONING_DIR="${PRESERVE_GRAFANA_PROV}"
		77	+fi
59		78
60	prepare_minimal_provisioning() {	79	prepare_minimal_provisioning() {
61	local out_dir="${SCRIPT_DIR}/generated/grafana-provisioning-minimal"	80	local out_dir="${SCRIPT_DIR}/generated/grafana-provisioning-minimal"
	@@ -67,9 +86,20 @@ prepare_minimal_provisioning() {
67	}	86	}
68		87
69	prepare_minimal_prometheus() {	88	prepare_minimal_prometheus() {
70	- local input_file="${PROMETHEUS_CONFIG_FILE:-${SCRIPT_DIR}/generated/prometheus.yml}"	89	+ local generated_prom="${SCRIPT_DIR}/generated/prometheus.yml"
71	- if [[ ! -f "${input_file}" ]]; then	90	+ local input_file="${PROMETHEUS_CONFIG_FILE:-}"
72	- input_file="${SCRIPT_DIR}/prometheus/prometheus-minimal.yml"	91	+ if [[ -z "${input_file}" \|\| "${input_file}" == "./prometheus/prometheus.yml" ]]; then
		92	+ if [[ -f "${generated_prom}" ]]; then
		93	+ input_file="${generated_prom}"
		94	+ else
		95	+ input_file="${SCRIPT_DIR}/prometheus/prometheus-minimal.yml"
		96	+ fi
		97	+ elif [[ ! -f "${input_file}" ]]; then
		98	+ if [[ -f "${generated_prom}" ]]; then
		99	+ input_file="${generated_prom}"
		100	+ else
		101	+ input_file="${SCRIPT_DIR}/prometheus/prometheus-minimal.yml"
		102	+ fi
73	fi	103	fi
74	local output_file="${SCRIPT_DIR}/generated/prometheus-minimal.runtime.yml"	104	local output_file="${SCRIPT_DIR}/generated/prometheus-minimal.runtime.yml"
75	mkdir -p "${SCRIPT_DIR}/generated"	105	mkdir -p "${SCRIPT_DIR}/generated"
	@@ -84,6 +114,43 @@ start_host_helpers() {
84	fi	114	fi
85	}	115	}
86		116
		117	+ensure_loki_image() {
		118	+ local prefix="${REGISTRY_PREFIX:-}"
		119	+ local lv="${LOKI_VERSION:-3.3.0}"
		120	+ local loki_img="${LOKI_IMAGE:-${prefix}grafana/loki:${lv}}"
		121	+ local docker_bin="${DOCKER_BIN:-docker}"
		122	+
		123	+ if "${docker_bin}" image inspect "${loki_img}" >/dev/null 2>&1; then
		124	+ echo "[start] Loki image available: ${loki_img}"
		125	+ LOKI_IMAGE="${loki_img}"
		126	+ export LOKI_IMAGE
		127	+ return 0
		128	+ fi
		129	+
		130	+ echo "[start] pulling Loki image: ${loki_img}"
		131	+ if "${docker_bin}" pull "${loki_img}" >/dev/null 2>&1; then
		132	+ LOKI_IMAGE="${loki_img}"
		133	+ export LOKI_IMAGE
		134	+ return 0
		135	+ fi
		136	+
		137	+ if [[ -n "${prefix}" ]]; then
		138	+ local hub_img="grafana/loki:${lv}"
		139	+ echo "[start] pulling ${hub_img}..."
		140	+ if "${docker_bin}" pull "${hub_img}" >/dev/null 2>&1; then
		141	+ "${docker_bin}" tag "${hub_img}" "${loki_img}"
		142	+ LOKI_IMAGE="${loki_img}"
		143	+ export LOKI_IMAGE
		144	+ return 0
		145	+ fi
		146	+ fi
		147	+
		148	+ echo "[start] Loki pull failed; building local image via scripts/build-loki-image.sh"
		149	+ LOKI_IMAGE="${loki_img}" "${SCRIPT_DIR}/scripts/build-loki-image.sh"
		150	+ LOKI_IMAGE="${loki_img}"
		151	+ export LOKI_IMAGE
		152	+}
		153	+
87	DOCKER_BIN="${DOCKER_BIN:-docker}"	154	DOCKER_BIN="${DOCKER_BIN:-docker}"
88	if ! "${DOCKER_BIN}" compose version >/dev/null 2>&1; then	155	if ! "${DOCKER_BIN}" compose version >/dev/null 2>&1; then
89	echo "[start] error: '${DOCKER_BIN} compose' is not available. Install Docker Compose v2." >&2	156	echo "[start] error: '${DOCKER_BIN} compose' is not available. Install Docker Compose v2." >&2
	@@ -105,26 +172,32 @@ if [[ -n "${PROFILES}" ]]; then
105	done	172	done
106	fi	173	fi
107		174
		175	+ensure_loki_image
		176	+
108	if [[ "${STACK_MODE}" == "minimal" ]]; then	177	if [[ "${STACK_MODE}" == "minimal" ]]; then
109	prepare_minimal_provisioning	178	prepare_minimal_provisioning
110	prepare_minimal_prometheus	179	prepare_minimal_prometheus
111	OTEL_CONFIG_FILE="${OTEL_CONFIG_FILE:-./otel-collector/otel-collector-minimal.yaml}"	180	OTEL_CONFIG_FILE="${OTEL_CONFIG_FILE:-./otel-collector/otel-collector-minimal.yaml}"
112	else	181	else
113	PROMETHEUS_CONFIG_FILE="${PROMETHEUS_CONFIG_FILE:-./prometheus/prometheus.yml}"	182	PROMETHEUS_CONFIG_FILE="${PROMETHEUS_CONFIG_FILE:-./prometheus/prometheus.yml}"
		183	+ GRAFANA_PROVISIONING_DIR="${GRAFANA_PROVISIONING_DIR:-./grafana/provisioning}"
114	OTEL_CONFIG_FILE="${OTEL_CONFIG_FILE:-./otel-collector/otel-collector.yaml}"	184	OTEL_CONFIG_FILE="${OTEL_CONFIG_FILE:-./otel-collector/otel-collector.yaml}"
115	fi	185	fi
116	-export PROMETHEUS_CONFIG_FILE OTEL_CONFIG_FILE	186	+export PROMETHEUS_CONFIG_FILE OTEL_CONFIG_FILE GRAFANA_PROVISIONING_DIR
117		187
118	start_host_helpers	188	start_host_helpers
119		189
120	ensure_compose_images() {	190	ensure_compose_images() {
121	- # Preserve runtime paths set above (minimal/full); .env must not override them.
122	local saved_grafana_prov="${GRAFANA_PROVISIONING_DIR:-}"	191	local saved_grafana_prov="${GRAFANA_PROVISIONING_DIR:-}"
123	local saved_prom_config="${PROMETHEUS_CONFIG_FILE:-}"	192	local saved_prom_config="${PROMETHEUS_CONFIG_FILE:-}"
124	local saved_otel_config="${OTEL_CONFIG_FILE:-}"	193	local saved_otel_config="${OTEL_CONFIG_FILE:-}"
		194	+ local saved_loki_image="${LOKI_IMAGE:-}"
125	if [[ -f .env ]]; then	195	if [[ -f .env ]]; then
126	load_dotenv "${SCRIPT_DIR}/.env"	196	load_dotenv "${SCRIPT_DIR}/.env"
127	fi	197	fi
		198	+ if [[ "${STACK_MODE}" == "minimal" && -f "${DISCOVERED_ENV}" ]]; then
		199	+ load_dotenv "${DISCOVERED_ENV}"
		200	+ fi
128	if [[ -n "${saved_grafana_prov}" ]]; then	201	if [[ -n "${saved_grafana_prov}" ]]; then
129	GRAFANA_PROVISIONING_DIR="${saved_grafana_prov}"	202	GRAFANA_PROVISIONING_DIR="${saved_grafana_prov}"
130	export GRAFANA_PROVISIONING_DIR	203	export GRAFANA_PROVISIONING_DIR
	@@ -137,6 +210,10 @@ ensure_compose_images() {
137	OTEL_CONFIG_FILE="${saved_otel_config}"	210	OTEL_CONFIG_FILE="${saved_otel_config}"
138	export OTEL_CONFIG_FILE	211	export OTEL_CONFIG_FILE
139	fi	212	fi
		213	+ if [[ -n "${saved_loki_image}" ]]; then
		214	+ LOKI_IMAGE="${saved_loki_image}"
		215	+ export LOKI_IMAGE
		216	+ fi
140	local prefix="${REGISTRY_PREFIX:-}"	217	local prefix="${REGISTRY_PREFIX:-}"
141	local gv="${GRAFANA_VERSION:-11.3.0}"	218	local gv="${GRAFANA_VERSION:-11.3.0}"
142	local grafana_img="${prefix}grafana/grafana:${gv}"	219	local grafana_img="${prefix}grafana/grafana:${gv}"
	@@ -156,9 +233,6 @@ ensure_compose_images() {
156		233
157	ensure_compose_images	234	ensure_compose_images
158		235
159	-# missing: 本地已有镜像则不拉，缺失时才 pull（与 docker-compose pull_policy: if_not_present 一致）。
160	-# 代理环境：首次 pull 继承当前 shell 的 HTTP_PROXY，需拉镜像时可先 source proxy；详见 SERVICE_GUIDE.md §2.2。
161	-# 可覆盖：OBS_COMPOSE_PULL=never\|always\|missing
162	COMPOSE_PULL="${OBS_COMPOSE_PULL:-missing}"	236	COMPOSE_PULL="${OBS_COMPOSE_PULL:-missing}"
163	COMPOSE_UP_ARGS=(up -d --pull "${COMPOSE_PULL}")	237	COMPOSE_UP_ARGS=(up -d --pull "${COMPOSE_PULL}")
164	if [[ "${OBS_COMPOSE_BUILD:-0}" == "1" ]]; then	238	if [[ "${OBS_COMPOSE_BUILD:-0}" == "1" ]]; then
	@@ -167,11 +241,12 @@ else
167	COMPOSE_UP_ARGS+=(--no-build)	241	COMPOSE_UP_ARGS+=(--no-build)
168	fi	242	fi
169		243
170	-echo "[start] starting Docker Compose stack mode=${STACK_MODE} profiles: ${PROFILES:-<none>}"	244	+echo "[start] starting Docker Compose stack mode=${STACK_MODE} loki_image=${LOKI_IMAGE} profiles: ${PROFILES:-<none>}"
171	"${DOCKER_BIN}" compose "${PROFILE_ARGS[@]}" "${COMPOSE_UP_ARGS[@]}"	245	"${DOCKER_BIN}" compose "${PROFILE_ARGS[@]}" "${COMPOSE_UP_ARGS[@]}"
172		246
173	GRAFANA_PORT="${GRAFANA_PORT:-3000}"	247	GRAFANA_PORT="${GRAFANA_PORT:-3000}"
174	PROMETHEUS_PORT="${PROMETHEUS_PORT:-9090}"	248	PROMETHEUS_PORT="${PROMETHEUS_PORT:-9090}"
		249	+LOKI_PORT="${LOKI_PORT:-3100}"
175		250
176	if [[ "${STACK_MODE}" == "minimal" ]] && command -v curl >/dev/null 2>&1; then	251	if [[ "${STACK_MODE}" == "minimal" ]] && command -v curl >/dev/null 2>&1; then
177	curl -fsS -X POST "http://localhost:${PROMETHEUS_PORT}/-/reload" >/dev/null 2>&1 \|\| true	252	curl -fsS -X POST "http://localhost:${PROMETHEUS_PORT}/-/reload" >/dev/null 2>&1 \|\| true
	@@ -185,6 +260,7 @@ pyMotor observability stack is up.
185	Grafana http://localhost:${GRAFANA_PORT} (user: motor / pass: motor)	260	Grafana http://localhost:${GRAFANA_PORT} (user: motor / pass: motor)
186	Prometheus http://localhost:${PROMETHEUS_PORT}	261	Prometheus http://localhost:${PROMETHEUS_PORT}
187	Tempo http://localhost:${TEMPO_QUERY_PORT:-3200}	262	Tempo http://localhost:${TEMPO_QUERY_PORT:-3200}
		263	+ Loki http://localhost:${LOKI_PORT} (image: ${LOKI_IMAGE})
188	OTel OTLP localhost:${OTEL_GRPC_PORT:-4317} (gRPC) / ${OTEL_HTTP_PORT:-4318} (HTTP)	264	OTel OTLP localhost:${OTEL_GRPC_PORT:-4317} (gRPC) / ${OTEL_HTTP_PORT:-4318} (HTTP)
189		265
190	Mode: ${STACK_MODE}	266	Mode: ${STACK_MODE}
	@@ -194,5 +270,6 @@ Tips:
194	* 推荐入口: ./launch.sh （自动发现 + 自动生成 Prometheus 配置）	270	* 推荐入口: ./launch.sh （自动发现 + 自动生成 Prometheus 配置）
195	* 当前 Prometheus 配置: ${PROMETHEUS_CONFIG_FILE:-./prometheus/prometheus.yml}	271	* 当前 Prometheus 配置: ${PROMETHEUS_CONFIG_FILE:-./prometheus/prometheus.yml}
196	* Verify tracing: ./scripts/verify-tracing.sh (OTLP → Tempo)	272	* Verify tracing: ./scripts/verify-tracing.sh (OTLP → Tempo)
		273	+ * Build Loki locally: LOKI_DOWNLOAD_INSECURE=1 ./scripts/build-loki-image.sh
197	================================================================	274	================================================================
198	EOF	275	EOF

Mexamples/features/observability/stack/stop.sh+1-2

	@@ -18,7 +18,7 @@ if [[ -f .env ]]; then
18	load_dotenv .env	18	load_dotenv .env
19	fi	19	fi
20		20
21	-STACK_MODE="${OBS_STACK_MODE:-full}"	21	+STACK_MODE="${OBS_STACK_MODE:-minimal}"
22	WITH_MOCK="${OBS_WITH_MOCK:-0}"	22	WITH_MOCK="${OBS_WITH_MOCK:-0}"
23	PROFILES="${OBS_COMPOSE_PROFILES:-}"	23	PROFILES="${OBS_COMPOSE_PROFILES:-}"
24		24
	@@ -38,7 +38,6 @@ if "${DOCKER_BIN}" compose version >/dev/null 2>&1; then
38	PROFILE_ARGS+=(--profile "${p}")	38	PROFILE_ARGS+=(--profile "${p}")
39	done	39	done
40	fi	40	fi
41	- # Ascend NPU exporter uses profile npu; include it so down matches typical up.
42	PROFILE_ARGS+=(--profile npu)	41	PROFILE_ARGS+=(--profile npu)
43		42
44	ARGS=(compose "${PROFILE_ARGS[@]}" down)	43	ARGS=(compose "${PROFILE_ARGS[@]}" down)