#!/bin/bash
# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.
# Licensed under Mulan PSL v2.
#
# Run ALL runnable examples in sequence.
#
# Usage:
#   bash script/run_all_examples.sh [options]
#
# Mode (default: run Python + C++ + Bench). Each flag is a mutually
# exclusive selector that runs ONLY that section and disables the others.
#   --python              Only run Python examples
#   --cpp                 Only run C++ examples
#   --run-bench           Only run performance benchmarks
#   (no mode flag)        Run all three sections
#
# Options:
#   --meta-config PATH    MetaService config path (default: config/mmc-meta.conf)
#   --local-config PATH   LocalService config path (default: config/mmc-local.conf)
#   --example-timeout N   Kill hanging examples after N seconds (default: 180)
#   --dry-run             Print what would be run without executing
#   --verbose             Print example stdout/stderr (default: hide, show only pass/fail)

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
CONFIG_DIR="$PROJECT_DIR/config"
EXAMPLES_DIR="$PROJECT_DIR/example"

# ── Debug + cleanup trap ──────────────────────────────────────────────────────
DEBUG_LINE="start"
META_PID=""
META_STARTED_BY_US=false

cleanup_meta() {
    if $META_STARTED_BY_US && [[ -n "$META_PID" ]] && kill -0 "$META_PID" 2>/dev/null; then
        echo -e "  \e[36m[INFO]\e[0m  Stopping MetaService (PID $META_PID)..."
        kill "$META_PID" 2>/dev/null || true
        wait "$META_PID" 2>/dev/null || true
        META_PID=""
    fi
}

on_exit() {
    local code=$?
    if [[ $code -ne 0 && $code -ne 130 ]]; then
        echo "[DEBUG] Exited at line: $DEBUG_LINE (code=$code)" >&2
    fi
    cleanup_meta
    [[ -n "${config_tmp:-}" && -f "$config_tmp" ]] && rm -f "$config_tmp" 2>/dev/null || true
}
trap on_exit EXIT
trace() { DEBUG_LINE="$*"; }

# ── Defaults ──────────────────────────────────────────────────────────────────
META_CONFIG_PATH="$CONFIG_DIR/mmc-meta.conf"
LOCAL_CONFIG_PATH="$CONFIG_DIR/mmc-local.conf"
EXAMPLE_TIMEOUT=180
RUN_PYTHON=true
RUN_CPP=true
RUN_BENCH=true
DRY_RUN=false
VERBOSE=false

# ── Parse args ────────────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
    case "$1" in
        --python)          RUN_PYTHON=true;  RUN_CPP=false; RUN_BENCH=false; shift ;;
        --cpp)             RUN_PYTHON=false; RUN_CPP=true;  RUN_BENCH=false; shift ;;
        --run-bench)       RUN_PYTHON=false; RUN_CPP=false; RUN_BENCH=true;  shift ;;
        --meta-config)     META_CONFIG_PATH="$2";  shift 2 ;;
        --local-config)    LOCAL_CONFIG_PATH="$2";  shift 2 ;;
        --example-timeout) EXAMPLE_TIMEOUT="$2";   shift 2 ;;
        --dry-run)         DRY_RUN=true;           shift ;;
        --verbose)         VERBOSE=true;           shift ;;
        --help)
            head -20 "$0" | sed -n '/^#/p' | sed 's/^#//'
            exit 0
            ;;
        *)
            echo "[ERROR] Unknown option: $1"
            exit 1
            ;;
    esac
done

# ── Helpers ───────────────────────────────────────────────────────────────────
PASS=0
FAIL=0
SKIP=0
FAILED_NAMES=()

info()  { echo -e "  \e[36m[INFO]\e[0m  $*"; }
pass()  { echo -e "  \e[32m[PASS]\e[0m  $*"; PASS=$((PASS + 1)); }
fail()  { echo -e "  \e[31m[FAIL]\e[0m  $*"; FAIL=$((FAIL + 1)); FAILED_NAMES+=("$1"); }
skip()  { echo -e "  \e[33m[SKIP]\e[0m  $*"; SKIP=$((SKIP + 1)); }
header(){ echo -e "\n\e[1;34m━━━ $* ━━━\e[0m"; }
sub()   { echo -e "  \e[90m$ $*\e[0m"; }

devices_idle() {
    local ids="${NPU_IDLE_IDS:-,}"
    local d
    for d in "$@"; do
        if [[ ",$ids," != *",$d,"* ]]; then
            return 1
        fi
    done
    return 0
}

run_example() {
    local name="$1"
    local dir="$2"
    local cmd="$3"
    shift 3

    if $DRY_RUN; then
        echo -e "  \e[90m[DRY-RUN] cd $dir && $cmd\e[0m"
        return
    fi

    trace "cd $dir"
    cd "$dir"

    local timeout_cmd
    if command -v timeout &>/dev/null; then
        timeout_cmd="timeout $EXAMPLE_TIMEOUT"
    else
        timeout_cmd=""
    fi

    local output
    local ret=0
    if $VERBOSE; then
        echo ""
        sub "$cmd"
        bash -c "$timeout_cmd $cmd" || ret=$?
    else
        output=$(bash -c "$timeout_cmd $cmd" 2>&1) || ret=$?
    fi

    if [[ $ret -eq 0 ]]; then
        pass "$name"
    else
        if [[ $ret -eq 124 ]]; then
            fail "$name (TIMEOUT after ${EXAMPLE_TIMEOUT}s)"
        else
            fail "$name"
        fi
        if ! $VERBOSE && [[ -n "${output:-}" ]]; then
            echo -e "  \e[90m$(echo "$output" | tail -5 | sed 's/^/  | /')\e[0m"
        fi
    fi
    trace "cd $PROJECT_DIR"
    cd "$PROJECT_DIR"
}

run_python_example() {
    local name="$1"
    local dir="$2"
    local script="$3"
    shift 3
    run_example "$name" "$dir" "python3 $script $*"
}

detect_npu_info() {
    local total=0 idle=0 idle_ids=""
    total=$(ls -d /dev/davinci[0-9]* 2>/dev/null | wc -l || true)
    if [[ "$total" -eq 0 ]]; then
        echo "0 0"
        return
    fi
    if command -v npu-smi &>/dev/null; then
        local output
        output=$(npu-smi info 2>/dev/null) || { echo "$total 0"; return; }
        if echo | grep -P "" >/dev/null 2>&1; then
            idle_ids=$(echo "$output" | grep -oP "No running processes found in NPU \K[0-9]+" | tr '\n' ',' | sed 's/,$//' || true)
        else
            idle_ids=$(echo "$output" | sed -n 's/.*No running processes found in NPU \([0-9][0-9]*\).*/\1/p' | paste -sd ',' || true)
        fi
        if [[ -n "$idle_ids" ]]; then
            idle=$(echo "$idle_ids" | tr ',' '\n' | wc -l)
        fi
    fi
    echo "$total $idle $idle_ids"
}

check_python_package() {
    python3 -c "import $1" 2>/dev/null && return 0 || return 1
}

# ── Auto-build helpers ────────────────────────────────────────────────────────
find_whl() {
    ls -1 "$PROJECT_DIR/output/memcache/wheel"/memcache_hybrid-*.whl 2>/dev/null | head -1
}

find_run_pkg() {
    ls -1 "$PROJECT_DIR/output"/memcache_hybrid-*.run 2>/dev/null | head -1
}

find_set_env() {
    for p in \
        "/usr/local/memcache_hybrid/set_env.sh" \
        "/usr/local/memcache_hybrid/latest/set_env.sh"
    do
        [[ -f "$p" ]] && { echo "$p"; return 0; }
    done
    return 1
}

find_mf_set_env() {
    for p in \
        "/usr/local/memfabric_hybrid/set_env.sh" \
        "/usr/local/memfabric_hybrid/latest/set_env.sh"
    do
        [[ -f "$p" ]] && { echo "$p"; return 0; }
    done
    return 1
}

# Find memfabric_hybrid installed via pip (no set_env.sh in pip package).
find_mf_pip_dir() {
    python3 -c "import memfabric_hybrid, os; print(os.path.dirname(memfabric_hybrid.__file__))" 2>/dev/null
}

# Tracks whether source_all_cpp_env has already been called in this run,
# to suppress duplicate "Sourcing ..." log messages on repeated invocations.
_SOURCED_CPP_ENV=false

source_all_cpp_env() {
    local mf_env
    if mf_env=$(find_mf_set_env); then
        if ! $_SOURCED_CPP_ENV; then
            info "Sourcing memfabric env: $mf_env"
        fi
        source "$mf_env"
    else
        local mf_pip
        if mf_pip=$(find_mf_pip_dir) && [[ -n "$mf_pip" ]] && [[ -d "$mf_pip/include" ]]; then
            export MEMFABRIC_HYBRID_HOME_PATH="$mf_pip"
            if [[ ":${LD_LIBRARY_PATH:-}:" != *":$mf_pip/lib:"* ]]; then
                export LD_LIBRARY_PATH="$mf_pip/lib:${LD_LIBRARY_PATH:-}"
            fi
            # Workaround: pip-installed memfabric_hybrid wheel ships libs as
            # `libfoo.so` (no SONAME link), but binaries link against versioned
            # SONAMEs (e.g. `libhcom.so.0`). Create the missing symlink in-place.
            if [[ -d "$mf_pip/lib" ]]; then
                local so so_base
                for so in "$mf_pip/lib"/*.so; do
                    [[ -f "$so" ]] || continue
                    so_base=$(basename "$so" .so)
                    if [[ ! -e "$mf_pip/lib/${so_base}.so.0" ]]; then
                        ln -sf "$(basename "$so")" "$mf_pip/lib/${so_base}.so.0"
                    fi
                done
            fi
            if ! $_SOURCED_CPP_ENV; then
                info "memfabric_hybrid not at /usr/local; using pip install at $mf_pip"
            fi
        fi
    fi
    local mc_env
    if mc_env=$(find_set_env); then
        if ! $_SOURCED_CPP_ENV; then
            info "Sourcing memcache env: $mc_env"
        fi
        source "$mc_env"
    fi
    if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then
        for p in \
            "/usr/local/Ascend/ascend-toolkit/latest" \
            "/usr/local/Ascend/ascend-toolkit" \
            "/usr/local/Ascend/latest" \
            "/usr/local/Ascend"
        do
            if [[ -d "$p" && -d "$p/lib64" ]]; then
                export ASCEND_HOME_PATH="$p"
                if ! $_SOURCED_CPP_ENV; then
                    info "ASCEND_HOME_PATH auto-detected: $ASCEND_HOME_PATH"
                fi
                break
            fi
        done
        # Fallback: pick the highest-versioned cann-* directory
        if [[ -z "${ASCEND_HOME_PATH:-}" ]] && [[ -d /usr/local/Ascend ]]; then
            local detected
            detected=$(ls -d /usr/local/Ascend/cann-* 2>/dev/null | sort -V | tail -1)
            if [[ -n "$detected" && -d "$detected/lib64" ]]; then
                export ASCEND_HOME_PATH="$detected"
                if ! $_SOURCED_CPP_ENV; then
                    info "ASCEND_HOME_PATH auto-detected: $ASCEND_HOME_PATH"
                fi
            fi
        fi
    fi
    # Add Ascend toolkit runtime libs (libhcom.so, libhccl.so, ...) to LD_LIBRARY_PATH
    if [[ -n "${ASCEND_HOME_PATH:-}" ]] && [[ -d "$ASCEND_HOME_PATH/lib64" ]]; then
        if [[ ":${LD_LIBRARY_PATH:-}:" != *":$ASCEND_HOME_PATH/lib64:"* ]]; then
            export LD_LIBRARY_PATH="$ASCEND_HOME_PATH/lib64:${LD_LIBRARY_PATH:-}"
        fi
    fi
    _SOURCED_CPP_ENV=true
}

build_project() {
    info "memcache_hybrid not found, building from source (this may take a while)..."
    trace "git submodule update --init 3rdparty/"
    git submodule update --init 3rdparty/ 2>/dev/null || true
    trace "build_and_pack_run.sh"
    bash "$SCRIPT_DIR/build_and_pack_run.sh" --build_mode RELEASE --build_test OFF
}

ensure_python_deps() {
    if check_python_package "memcache_hybrid"; then
        return 0
    fi
    build_project
    local whl
    whl=$(find_whl)
    if [[ -z "$whl" ]]; then
        info "Wheel not found after build, trying pip install from source"
        pip install "$PROJECT_DIR/src/memcache/python" 2>/dev/null && return 0
        return 1
    fi
    info "Installing wheel: $(basename "$whl")"
    pip install "$whl" 2>/dev/null || return 1
    check_python_package "memcache_hybrid" && return 0
    return 1
}

ensure_cpp_deps() {
    # 1. Already set?
    if [[ -n "${MEMCACHE_HYBRID_HOME_PATH:-}" ]] && \
       [[ -d "$MEMCACHE_HYBRID_HOME_PATH/include" ]] && \
       [[ -n "${MEMFABRIC_HYBRID_HOME_PATH:-}" ]]; then
        info "MEMCACHE_HYBRID_HOME_PATH: $MEMCACHE_HYBRID_HOME_PATH"
        info "MEMFABRIC_HYBRID_HOME_PATH: $MEMFABRIC_HYBRID_HOME_PATH"
        source_all_cpp_env
        return 0
    fi

    # 2. Installed at default location?
    if find_set_env >/dev/null || find_mf_set_env >/dev/null; then
        source_all_cpp_env
        if [[ -n "${MEMFABRIC_HYBRID_HOME_PATH:-}" ]]; then
            return 0
        fi
    fi

    # 3. Build → install
    build_project
    local run_pkg
    run_pkg=$(find_run_pkg)
    if [[ -z "$run_pkg" ]]; then
        return 1
    fi
    info "Installing run package: $(basename "$run_pkg")"
    bash "$run_pkg" --no-check || return 1
    source_all_cpp_env
    if [[ -n "${MEMFABRIC_HYBRID_HOME_PATH:-}" ]]; then
        return 0
    fi
    return 1
}

# ── Detect environment ────────────────────────────────────────────────────────
echo -e "\e[1mMemCache Hybrid — Run All Examples\e[0m"
echo -e "  Project: $PROJECT_DIR"
echo -e "  Meta config: $META_CONFIG_PATH"
echo -e "  Local config: $LOCAL_CONFIG_PATH"
if $RUN_PYTHON && $RUN_CPP && $RUN_BENCH; then
    echo -e "  Mode: all (Python + C++ + Bench)"
elif $RUN_PYTHON && $RUN_CPP; then
    echo -e "  Mode: Python + C++"
elif $RUN_PYTHON && $RUN_BENCH; then
    echo -e "  Mode: Python + Bench"
elif $RUN_CPP && $RUN_BENCH; then
    echo -e "  Mode: C++ + Bench"
elif $RUN_PYTHON; then
    echo -e "  Mode: Python only"
elif $RUN_CPP; then
    echo -e "  Mode: C++ only"
else
    echo -e "  Mode: Bench only"
fi
echo ""

NPU_TOTAL=0
NPU_IDLE=0
NPU_IDLE_IDS=""
read -r NPU_TOTAL NPU_IDLE NPU_IDLE_IDS < <(detect_npu_info)
info "NPU: ${NPU_TOTAL} total, ${NPU_IDLE} idle (IDs: ${NPU_IDLE_IDS:-none})"

# ── Auto-build Python deps if needed ──────────────────────────────────────────
if $RUN_PYTHON && ! $DRY_RUN; then
    if ! ensure_python_deps; then
        echo -e "  \e[31m[ERROR] memcache_hybrid Python package not available. Run 'bash script/build_and_pack_run.sh' or install the wheel manually.\e[0m"
        skip "All Python examples (Python package not available)"
        RUN_PYTHON=false
    fi
fi

HAS_MEMCACHE_PKG=false
if check_python_package "memcache_hybrid"; then
    HAS_MEMCACHE_PKG=true
    info "Python package memcache_hybrid: found"
fi

HAS_TORCH_NPU=false
if check_python_package "torch_npu"; then
    HAS_TORCH_NPU=true
    info "torch_npu: found"
else
    info "torch_npu: not found (will skip layers examples)"
fi

HAS_TORCH=false
if check_python_package "torch"; then
    HAS_TORCH=true
    info "torch: found"
else
    info "torch: not found (will skip bench)"
fi

HAS_ACL=false
if check_python_package "acl"; then
    HAS_ACL=true
    info "acl: found"
fi

# ── MetaService auto-detect and start ─────────────────────────────────────────
detect_meta_port() {
    if [[ -f "$META_CONFIG_PATH" ]]; then
        grep -oP 'tcp://[^:]+:\K[0-9]+' "$META_CONFIG_PATH" 2>/dev/null | head -1
    fi
}

is_port_listening() {
    local port="$1"
    if command -v ss &>/dev/null; then
        ss -tlnp 2>/dev/null | grep -q ":$port "
    elif command -v netstat &>/dev/null; then
        netstat -tlnp 2>/dev/null | grep -q ":$port "
    else
        # fallback: try connecting
        timeout 2 bash -c "echo >/dev/tcp/127.0.0.1/$port" 2>/dev/null
    fi
}

start_meta_service() {
    if ! $HAS_MEMCACHE_PKG; then
        info "Cannot start MetaService: memcache_hybrid package not installed"
        return 1
    fi
    info "Starting MetaService with config: $META_CONFIG_PATH ..."
    export MMC_META_CONFIG_PATH="$META_CONFIG_PATH"
    export MMC_LOCAL_CONFIG_PATH="$LOCAL_CONFIG_PATH"
    local meta_script
    meta_script=$(mktemp)
    cat > "$meta_script" << 'PYEOF'
from memcache_hybrid import MetaService
MetaService.main()
PYEOF
    python3 "$meta_script" &
    META_PID=$!
    META_STARTED_BY_US=true
    # Wait up to 15s for startup
    local waited=0
    while [[ $waited -lt 15 ]]; do
        sleep 1
        if kill -0 "$META_PID" 2>/dev/null && is_port_listening "${META_PORT:-5000}"; then
            info "MetaService started (PID: $META_PID)"
            return 0
        fi
        waited=$((waited + 1))
    done
    info "MetaService failed to start (PID $META_PID)"
    return 1
}

META_PORT=$(detect_meta_port)
META_PORT="${META_PORT:-5000}"

# ── Auto-downgrade protocol for single-node loopback ──────────────────────────
# RDMA/URMA/TCP protocols fail on 127.0.0.1; fall back to host_shm.
detect_local_protocol() {
    if [[ -f "$LOCAL_CONFIG_PATH" ]]; then
        grep -oP 'ock\.mmc\.local_service\.protocol\s*=\s*\K\S+' "$LOCAL_CONFIG_PATH" 2>/dev/null | head -1
    fi
}

LOCAL_PROTOCOL=$(detect_local_protocol)
LOCAL_PROTOCOL="${LOCAL_PROTOCOL:-host_rdma}"
meta_host=$(grep -oP 'tcp://\K[^:]+' "$META_CONFIG_PATH" 2>/dev/null | head -1)
meta_host="${meta_host:-127.0.0.1}"
if [[ "$LOCAL_PROTOCOL" != "host_shm" ]] && [[ "$meta_host" == "127.0.0.1" || "$meta_host" == "localhost" ]]; then
    config_tmp=$(mktemp)
    cp "$LOCAL_CONFIG_PATH" "$config_tmp"
    sed -i "s/^[[:space:]]*ock\.mmc\.local_service\.protocol[[:space:]]*=.*/ock.mmc.local_service.protocol = host_shm/" "$config_tmp"
    info "Single-node loopback detected; protocol downgraded from $LOCAL_PROTOCOL to host_shm"
    LOCAL_CONFIG_PATH="$config_tmp"
fi

if { $RUN_PYTHON || $RUN_CPP || $RUN_BENCH; } && ! $DRY_RUN; then
    if is_port_listening "$META_PORT"; then
        info "MetaService already running on port $META_PORT, skipping start"
    else
        start_meta_service || { echo -e "  \e[31m[ERROR] MetaService failed to start. Check $META_CONFIG_PATH\e[0m"; exit 1; }
    fi
fi

# ──────────────────────────────────────────────────────────────────────────────
#  Python examples
# ──────────────────────────────────────────────────────────────────────────────

if $RUN_PYTHON; then

header "example/python — Basic CRUD"

if ! $HAS_MEMCACHE_PKG || ! $HAS_ACL; then
    skip "all python examples (need memcache_hybrid + acl packages)"
elif [[ $NPU_IDLE -lt 1 ]]; then
    skip "all python examples (no NPU available, need >=1)"
else
    export MMC_META_CONFIG_PATH="$META_CONFIG_PATH"
    export MMC_LOCAL_CONFIG_PATH="$LOCAL_CONFIG_PATH"

    d="$EXAMPLES_DIR/python"

    if [[ -f "$d/test_mmc_demo.py" ]]; then
        trace "about to run: test_mmc_demo"
        run_python_example "test_mmc_demo" "$d" "test_mmc_demo.py"
        trace "completed: test_mmc_demo"
    else
        skip "test_mmc_demo (not found)"
    fi

    if [[ -f "$d/test_mmc_batch.py" ]]; then
        run_python_example "test_mmc_batch" "$d" "test_mmc_batch.py"
    else
        skip "test_mmc_batch (not found)"
    fi

    skip "interactive_app.py (interactive CLI, skip for batch mode)"

    # test_mmc_start_meta_service_and_simple_test: self-contained, starts MetaService in subprocess
    # NOTE: hardcodes protocol=device_rdma, skip on single-node loopback
    if [[ "$meta_host" == "127.0.0.1" || "$meta_host" == "localhost" ]]; then
        skip "test_mmc_start_meta_service_and_simple_test (hardcodes device_rdma, needs multi-node RDMA)"
    elif [[ -f "$d/test_mmc_start_meta_service_and_simple_test.py" ]]; then
        run_python_example "test_mmc_start_meta_service_and_simple_test" "$d" "test_mmc_start_meta_service_and_simple_test.py"
    else
        skip "test_mmc_start_meta_service_and_simple_test (not found)"
    fi

    skip "test_mmc_meta_service.py (MetaService bootstrap only, not a test)"
fi

header "example/python — Layers (torch_npu)"

if ! $HAS_MEMCACHE_PKG || ! $HAS_ACL || ! $HAS_TORCH_NPU; then
    skip "layers examples (need memcache_hybrid + acl + torch_npu)"
elif [[ $NPU_IDLE -lt 1 ]]; then
    skip "layers examples (no NPU available, need >=1)"
else
    d="$EXAMPLES_DIR/python"

    if [[ -f "$d/test_mmc_layers.py" ]]; then
        run_python_example "test_mmc_layers" "$d" "test_mmc_layers.py"
    else
        skip "test_mmc_layers (not found)"
    fi

    if [[ -f "$d/test_mmc_layers_batch.py" ]]; then
        run_python_example "test_mmc_layers_batch" "$d" "test_mmc_layers_batch.py"
    else
        skip "test_mmc_layers_batch (not found)"
    fi
fi

fi  # end RUN_PYTHON

# ──────────────────────────────────────────────────────────────────────────────
#  C++ example
# ──────────────────────────────────────────────────────────────────────────────
run_cpp_example() {
    local d="$EXAMPLES_DIR/cpp"
    [[ -f "$d/CMakeLists.txt" ]] || { skip "cpp example (CMakeLists.txt not found)"; return; }
    source_all_cpp_env
    if $DRY_RUN; then
        echo -e "  \e[90m[DRY-RUN] cd $d && mkdir -p build && cmake -B build && make -C build\e[0m"
        echo -e "  \e[90m[DRY-RUN] cd $d/build && ./memcache_cpp_test\e[0m"
        return
    fi
    export MMC_META_CONFIG_PATH="$META_CONFIG_PATH"
    export MMC_LOCAL_CONFIG_PATH="$LOCAL_CONFIG_PATH"
    cd "$d" || return
    if [[ -f "build/memcache_cpp_test" ]]; then
        info "C++ binary already exists, skipping build"
        pass "cpp build (cached)"
    else
        info "Building C++ example ..."
        mkdir -p build
        if ! cmake -B build >/dev/null 2>&1; then
            fail "cpp build (cmake failed)"
            cd "$PROJECT_DIR"
            return
        fi
        if ! make -C build >/dev/null 2>&1; then
            fail "cpp build (make failed)"
            cd "$PROJECT_DIR"
            return
        fi
        pass "cpp build"
    fi
    test_input="put k1 v1\nget k1\nremove k1\nexit\n"
    tc=""
    command -v timeout &>/dev/null && tc="timeout 30"
    cpp_log=$(mktemp)
    cpp_exit=0
    echo -e "$test_input" | $tc ./build/memcache_cpp_test >"$cpp_log" 2>&1 || cpp_exit=$?
    # The C++ binary triggers a glibc heap-corruption abort (SIGABRT, exit
    # 134; sometimes SIGSEGV, exit 139) inside the Ascend CANN runtime
    # DSO destructor during process exit, AFTER all store operations and
    # g_store->TearDown() have completed. When the log shows the run reached
    # "Exiting program.", the test itself succeeded, so tolerate the
    # exit-time crash instead of reporting a false failure.
    if [[ $cpp_exit -eq 0 ]]; then
        pass "cpp run"
    elif [[ $cpp_exit -ge 128 ]] && grep -q "Exiting program." "$cpp_log"; then
        pass "cpp run (exit=$cpp_exit tolerated: CANN at-exit crash after TearDown)"
    else
        fail "cpp run (exit=$cpp_exit)"
        if $VERBOSE; then
            echo -e "    \e[90m--- cpp test output ---\e[0m"
            sed 's/^/    /' "$cpp_log" | head -30
            echo -e "    \e[90m--- end ---\e[0m"
        fi
    fi
    rm -f "$cpp_log"
    cd "$PROJECT_DIR"
}

if ! $RUN_CPP; then
    :  # C++ not selected — no header, no skip count
elif ! $DRY_RUN && ! ensure_cpp_deps; then
    header "example/cpp — C++ Interactive Test"
    skip "cpp example (failed to prepare run package)"
elif [[ $NPU_IDLE -lt 1 ]]; then
    header "example/cpp — C++ Interactive Test"
    skip "cpp example (no NPU available, need >=1)"
else
    header "example/cpp — C++ Interactive Test"
    run_cpp_example
fi

# ──────────────────────────────────────────────────────────────────────────────
#  example/benchmark — Performance Benchmark
# ──────────────────────────────────────────────────────────────────────────────

run_bench() {
    local d="$EXAMPLES_DIR/benchmark"
    [[ -f "$d/bench_start.sh" ]] || { skip "benchmark (bench_start.sh not found)"; return; }
    source_all_cpp_env
    if $DRY_RUN; then
        echo -e "  \e[90m[DRY-RUN] cd $d && bash bench_start.sh -t write -p 1 -b 1 -s 4096 -n 10 -d 1 -e memcache -l npu\e[0m"
        echo -e "  \e[90m[DRY-RUN] cd $d && bash bench_start.sh -t read  -p 1 -b 1 -s 4096 -n 10 -d 1 -e memcache -l npu\e[0m"
        return
    fi
    export MMC_META_CONFIG_PATH="$META_CONFIG_PATH"
    export MMC_LOCAL_CONFIG_PATH="$LOCAL_CONFIG_PATH"
    info "Running benchmark smoke test (write + read, batch=1, 4KB, 10x, 1 process)..."
    cd "$d" || return
    local cmd=(bash bench_start.sh -p 1 -b 1 -s 4096 -n 10 -d 1 -e memcache -l npu)
    local write_log read_log write_pid waited=0 max_wait=60
    write_log=$(mktemp)
    read_log=$(mktemp)

    # Write must run in background: write_worker sleeps 30min after writing
    # to keep store data alive for the reader.
    # PYTHONUNBUFFERED=1 forces immediate flush so we can detect completion
    # by polling the log for the write_total_size marker.
    PYTHONUNBUFFERED=1 "${cmd[@]}" -t write >"$write_log" 2>&1 &
    write_pid=$!
    info "Benchmark write started (PID: $write_pid), waiting for data ..."
    while [[ $waited -lt $max_wait ]]; do
        if grep -q 'write_total_size\|write finish' "$write_log" 2>/dev/null; then
            break
        fi
        sleep 2
        waited=$((waited + 2))
    done
    if [[ $waited -ge $max_wait ]]; then
        fail "benchmark write (timeout after ${max_wait}s, no data written)"
        if $VERBOSE; then
            echo -e "    \e[90m--- bench write output ---\e[0m"
            sed 's/^/    /' "$write_log" | head -30
            echo -e "    \e[90m--- end ---\e[0m"
        fi
        kill "$write_pid" 2>/dev/null || true
        pkill -f "run_mutil_process.py write" 2>/dev/null || true
        wait "$write_pid" 2>/dev/null || true
        rm -f "$write_log" "$read_log"
        cd "$PROJECT_DIR"
        return
    fi
    pass "benchmark write (data written in ${waited}s)"

    # Read: reads data written by the write worker, then exits after ~10s sleep.
    if PYTHONUNBUFFERED=1 "${cmd[@]}" -t read >"$read_log" 2>&1; then
        pass "benchmark read"
    else
        fail "benchmark read"
        if $VERBOSE; then
            echo -e "    \e[90m--- bench read output ---\e[0m"
            sed 's/^/    /' "$read_log" | head -30
            echo -e "    \e[90m--- end ---\e[0m"
        fi
    fi

    # Kill the lingering write process (sleeping 30min to keep store alive).
    kill "$write_pid" 2>/dev/null || true
    pkill -f "run_mutil_process.py write" 2>/dev/null || true
    wait "$write_pid" 2>/dev/null || true

    if $VERBOSE; then
        echo -e "    \e[90m--- bench write output ---\e[0m"
        sed 's/^/    /' "$write_log" | head -30
        echo -e "    \e[90m--- end ---\e[0m"
    fi
    rm -f "$write_log" "$read_log"
    cd "$PROJECT_DIR"
}

if ! $RUN_BENCH; then
    :  # Bench not selected — no header, no skip count
elif ! $HAS_MEMCACHE_PKG || ! $HAS_ACL || ! $HAS_TORCH || ! $HAS_TORCH_NPU; then
    header "example/benchmark — Performance Benchmark"
    skip "benchmark (need memcache_hybrid + acl + torch + torch_npu)"
elif [[ $NPU_IDLE -lt 1 ]]; then
    header "example/benchmark — Performance Benchmark"
    skip "benchmark (no NPU available, need >=1)"
else
    header "example/benchmark — Performance Benchmark"
    run_bench
fi

# ──────────────────────────────────────────────────────────────────────────────
#  example/metrics — Grafana dashboards
# ──────────────────────────────────────────────────────────────────────────────
header "example/metrics — Grafana Dashboards"

skip "memcache_dashboard_graph.json (Grafana dashboard template, import manually)"
skip "memcache_dashboard_math.json (Grafana dashboard template, import manually)"

# ──────────────────────────────────────────────────────────────────────────────
#  Summary
# ──────────────────────────────────────────────────────────────────────────────
echo ""
echo -e "\e[1;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\e[0m"
echo -e "  \e[1mSummary:\e[0m"
echo -e "  \e[32mPASS:  $PASS\e[0m"
echo -e "  \e[31mFAIL:  $FAIL\e[0m"
echo -e "  \e[33mSKIP:  $SKIP\e[0m"
if [[ ${#FAILED_NAMES[@]} -gt 0 ]]; then
    echo -e "  \e[31mFailed: ${FAILED_NAMES[*]}\e[0m"
fi
echo -e "\e[1;34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\e[0m"

if [[ $FAIL -gt 0 ]]; then
    exit 1
fi