#!/bin/bash
#
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
# openUBMC is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#         http://license.coscl.org.cn/MulanPSL2
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
#
# 循环运行测试脚本(使用 meson test)
#
# 功能:
#   - 循环运行测试,自动重建项目
#   - 使用 meson test 运行所有测试套件
#   - 当测试超时时自动重试(timeout-multiplier=3)
#   - 保存测试日志和 meson testlog 到 test_failure_logs 目录
#   - 支持错误分类:Timeout/SIGABRT/Assertion/SIGSEGV/MemoryLeak/Sanitizer/TestFailure
#
# 使用方法:
#   ./scripts/run_tests_loop.sh [选项]
#
# 选项:
#   -n, --no-build         跳过构建,直接运行测试(需要 builddir 已存在)
#   -t, --timeout SEC      设置测试超时时间(秒,默认: 300)
#   -i, --iterations N     设置循环次数(默认: 100)
#   -f, --filter FILTER    设置 gtest 过滤器(例如: "-ModuleManagerTest.*:-mMicroComponentTest*")
#   -u, --unidev[=BOOL]    是否启用 unidev(默认: false,控制 csr1/connector 驱动)
#   -s, --stop-on-failure  遇到失败即停止测试
#   -v, --verbose          详细输出模式
#   -h, --help             显示帮助信息
#
# 环境变量:
#   MAX_ITERATIONS: 最大循环次数(默认: 100)
#   TEST_TIMEOUT: 测试超时时间,单位秒(默认: 300)
#   GTEST_FILTER: 传递给 gtest 的过滤器(默认: 空,不过滤)
#   BUILD_JOBS: 编译并发数(默认按 cgroup/内存估算,避免 -j$(nproc) OOM)
#
# 示例:
#   # 完整构建并测试
#   ./scripts/run_tests_loop.sh
#
#   # 跳过构建,直接测试(需要先构建)
#   ./scripts/run_tests_loop.sh --no-build
#
#   # 自定义超时和循环次数
#   ./scripts/run_tests_loop.sh --timeout 600 --iterations 10
#
#   # 只测 csr1 套件
#   ./scripts/run_tests_loop.sh -n -i 50 --suite csr1
#

set -uo pipefail

# 脚本配置
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
# setup.sh 在 scripts/;成功后会在仓库根目录生成 build.sh 辅助脚本
SETUP_SH="${SCRIPT_DIR}/setup.sh"
BUILD_SH="${PROJECT_ROOT}/build.sh"
BUILD_DIR="${PROJECT_ROOT}/builddir"
LOG_DIR="${PROJECT_ROOT}/test_failure_logs"
MAX_ITERATIONS="${MAX_ITERATIONS:-100}"

# 测试超时时间(秒),可通过环境变量覆盖
# 例如: TEST_TIMEOUT=600 ./scripts/run_tests_loop.sh
TEST_TIMEOUT="${TEST_TIMEOUT:-300}"

# gtest 过滤器,默认不过滤(可通过命令行参数或环境变量设置)
GTEST_FILTER="${GTEST_FILTER:-}"

# Meson test suite 过滤(默认全部套件)
TEST_SUITE="${TEST_SUITE:-}"

# unidev 构建选项(默认关闭,与 meson 原生默认一致;--unidev 可开启)
UNIDEV="${UNIDEV:-false}"

# 颜色输出(在使用前定义,避免 set -u 报未绑定变量)
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# 计算安全编译并发:与 smart_build.sh 一致,优先尊重 cgroup memory.max,
# 避免容器内 nproc=64 且内存配额仅 ~40GiB 时 unity 编译被 OOM killer 杀掉。
calculate_build_jobs() {
    local cpu_cores
    cpu_cores=$(nproc)
    if [ -r /sys/fs/cgroup/cpu.max ]; then
        local _cpu_max _quota _period
        _cpu_max=$(cat /sys/fs/cgroup/cpu.max 2>/dev/null || true)
        _quota=$(echo "${_cpu_max}" | awk '{print $1}')
        _period=$(echo "${_cpu_max}" | awk '{print $2}')
        if [ "${_quota:-}" != "max" ] && [ -n "${_quota:-}" ] && [ -n "${_period:-}" ] && [ "${_period}" -gt 0 ] 2>/dev/null; then
            local _effective=$(( (_quota + _period - 1) / _period ))
            if [ "${_effective}" -gt 0 ] && [ "${_effective}" -lt "${cpu_cores}" ]; then
                cpu_cores=${_effective}
            fi
        fi
    fi

    local available_mem_kb mem_budget_gb
    available_mem_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
    mem_budget_gb=$(( available_mem_kb / 1024 / 1024 ))
    if [ -r /sys/fs/cgroup/memory.max ]; then
        local _mem_max
        _mem_max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || true)
        if [ "${_mem_max:-}" != "max" ] && [ -n "${_mem_max:-}" ]; then
            local _cgroup_gb=$(( _mem_max / 1024 / 1024 / 1024 ))
            if [ "${_cgroup_gb}" -gt 0 ] && [ "${_cgroup_gb}" -lt "${mem_budget_gb}" ]; then
                mem_budget_gb=${_cgroup_gb}
            fi
        fi
    fi
    # 默认硬上限 40GiB,与 smart_build 一致
    if [ "${mem_budget_gb}" -gt 40 ]; then
        mem_budget_gb=40
    fi

    local mem_per_job_mb="${SMART_BUILD_MEM_PER_JOB_MB:-1200}"
    local mem_based_jobs=$(( mem_budget_gb * 1024 / mem_per_job_mb ))
    local swap_total_kb
    swap_total_kb=$(awk '/SwapTotal/ {print $2}' /proc/meminfo)
    if [ "${swap_total_kb:-0}" -eq 0 ]; then
        mem_based_jobs=$(( mem_based_jobs * 9 / 10 ))
    fi

    local cpu_based_jobs
    if [ "${cpu_cores}" -le 16 ]; then
        cpu_based_jobs=${cpu_cores}
    else
        cpu_based_jobs=$(( cpu_cores * 3 / 4 ))
    fi

    local jobs=${mem_based_jobs}
    if [ "${cpu_based_jobs}" -lt "${jobs}" ]; then
        jobs=${cpu_based_jobs}
    fi
    jobs=$(( jobs < 1 ? 1 : jobs ))
    jobs=$(( jobs > 64 ? 64 : jobs ))
    echo "${jobs}"
}

BUILD_JOBS="${BUILD_JOBS:-$(calculate_build_jobs)}"

# LD_LIBRARY_PATH:测试运行时需要加载的共享库路径
# 与 tests/meson.build 中 test_runtime_lib_dirs 保持一致;并追加 bingo temp(liblmc_binding/liblua)
TEST_LIB_DIRS=(
    "${PROJECT_ROOT}/.libmcpp_prebuilt/src"
    "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcbase/src"
    "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcengine/src"
    "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcexpr/src"
    "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcdbus/src"
    "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcapp/src"
    "${BUILD_DIR}/opt/bmc/drivers"
    "${BUILD_DIR}/usr/lib64"
    "${BUILD_DIR}/gen"
    "${BUILD_DIR}/drivers/internal"
    "${BUILD_DIR}/subprojects/libmcpp/src"
    "${BUILD_DIR}/libraries/protocol"
    "${BUILD_DIR}/libraries/smbus"
    # 仅暴露 lua/lmc 运行库的隔离目录(见 prepare_lua_runtime_libs)
    "${PROJECT_ROOT}/.lua_runtime_libs"
)

# 勿把整个 temp/usr/lib64 加入 LD_LIBRARY_PATH:同目录旧 libmcdbus 等会经
# $ORIGIN 抢在 prebuilt 之前加载(symbol lookup / exit 127)。
prepare_lua_runtime_libs() {
    local src="${PROJECT_ROOT}/temp/usr/lib64"
    local dst="${PROJECT_ROOT}/.lua_runtime_libs"
    mkdir -p "${dst}"
    local lib
    for lib in liblua.so liblua.so.0 liblmc_binding.so liblmc_binding.so.0 liblmc_binding.so.0.1.0 libsecurec.so; do
        if [ -e "${src}/${lib}" ]; then
            ln -sfn "${src}/${lib}" "${dst}/${lib}"
        fi
    done
}

build_ld_library_path() {
    prepare_lua_runtime_libs
    local path=""
    for d in "${TEST_LIB_DIRS[@]}"; do
        if [ -d "${d}" ]; then
            path="${path}${path:+:}${d}"
        fi
    done
    # 追加系统 LD_LIBRARY_PATH
    if [ -n "${LD_LIBRARY_PATH:-}" ]; then
        path="${path}:${LD_LIBRARY_PATH}"
    fi
    echo "${path}"
}

# bootstrap.lua 硬编码 /opt/bmc/{mcskynet,skynet,luaclib,...};本地 meson 测试时
# 将 bingo temp 产物符号链接过去(仅补缺失项,不覆盖已有目录)。
prepare_opt_bmc_runtime() {
    local temp_bmc="${PROJECT_ROOT}/temp/opt/bmc"
    if [ ! -d "${temp_bmc}" ]; then
        echo -e "${YELLOW}  [警告] 未找到 ${temp_bmc},lua_engine 相关用例可能失败(需先 bingo 拉依赖)${NC}"
        return 0
    fi
    mkdir -p /opt/bmc
    local name
    for name in mcskynet skynet luaclib lualib; do
        if [ -e "${temp_bmc}/${name}" ] && [ ! -e "/opt/bmc/${name}" ]; then
            ln -sfn "${temp_bmc}/${name}" "/opt/bmc/${name}"
            echo "  链接 /opt/bmc/${name} -> ${temp_bmc}/${name}"
        fi
    done
}

# 生成 meson test --wrapper(对齐 smart_build.sh):
# meson 部分 ABI/csr1 用例会 set LD_LIBRARY_PATH 覆盖外层环境,导致找不到
# libsecurec.so(exit 127)。wrapper 在进程内重排路径。
# lua_engine 相关用例依赖 bingo temp 的 mc.so(std::string_view ABI),需将
# temp/usr/lib64 前置;其余用例 prebuilt 前置、temp 垫后,避免旧 so 混链。
generate_meson_test_wrapper() {
    local wrapper="${BUILD_DIR}/meson_test_wrapper.sh"
    local prebuilt=""
    local d
    for d in \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/src" \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcbase/src" \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcengine/src" \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcexpr/src" \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcdbus/src" \
        "${PROJECT_ROOT}/.libmcpp_prebuilt/libraries/mcapp/src"
    do
        if [ -d "${d}" ]; then
            prebuilt="${prebuilt:+${prebuilt}:}${d}"
        fi
    done
    local temp_lib="${PROJECT_ROOT}/temp/usr/lib64"
    [ -d "${temp_lib}" ] || temp_lib=""

    cat > "${wrapper}" <<EOF
#!/bin/bash
_prebuilt="${prebuilt}"
_temp="${temp_lib}"
_ld="\${LD_LIBRARY_PATH:-}"
_path_in_ld() {
    local _needle="\$1" _haystack="\$2" _entry _oifs="\$IFS"
    IFS=':'
    for _entry in \$_haystack; do
        if [ "\$_entry" = "\$_needle" ]; then
            IFS="\$_oifs"
            return 0
        fi
    done
    IFS="\$_oifs"
    return 1
}
_prepend_paths() {
    local _paths="\$1" _d _keep="" _oifs="\$IFS"
    IFS=':'
    for _d in \$_paths; do
        [ -z "\$_d" ] && continue
        IFS="\$_oifs"
        if ! _path_in_ld "\$_d" "\$_ld"; then
            _keep="\${_keep:+\${_keep}:}\${_d}"
        fi
        IFS=':'
    done
    IFS="\$_oifs"
    if [ -n "\$_keep" ]; then
        _ld="\${_keep}\${_ld:+:\${_ld}}"
    fi
}
_append_path() {
    local _p="\$1"
    [ -n "\$_p" ] || return 0
    if ! _path_in_ld "\$_p" "\$_ld"; then
        _ld="\${_ld:+\${_ld}:}\${_p}"
    fi
}
# 判断是否为依赖 bingo temp mc.so 的 lua_engine 用例
_is_lua_engine_test=false
case " \$* " in
    *test_lua_plugin*|*test_lua_seri*|*test_chip_base*)
        _is_lua_engine_test=true ;;
esac
if [ "\${_is_lua_engine_test}" = true ] && [ -n "\${_temp}" ]; then
    # temp 必须最终排在最前(后 prepend 的在前):mc.so 与 libmcengine 同 ABI
    _prepend_paths "\${_prebuilt}"
    _prepend_paths "\${_temp}"
else
    # 常规:prebuilt 优先,temp 仅作 securec/lua 兜底
    _prepend_paths "\${_prebuilt}"
    _append_path "\${_temp}"
fi
export LD_LIBRARY_PATH="\${_ld}"
exec "\$@"
EOF
    chmod +x "${wrapper}"
    echo "${wrapper}"
}

# 解析命令行参数
SKIP_BUILD=false
SHOW_HELP=false
STOP_REQUESTED=false
STOP_ON_FAILURE=false
VERBOSE=false
test_child_pid=""

handle_sigint() {
    echo ""
    echo "收到中断信号,准备退出..."
    STOP_REQUESTED=true
    if [ -n "${test_child_pid}" ] && kill -0 "${test_child_pid}" 2>/dev/null; then
        kill -INT "${test_child_pid}" 2>/dev/null || true
    fi
}

trap handle_sigint INT TERM

while [[ $# -gt 0 ]]; do
    case $1 in
        -n|--no-build)
            SKIP_BUILD=true
            shift
            ;;
        -n=true|--no-build=true)
            SKIP_BUILD=true
            shift
            ;;
        -n=false|--no-build=false)
            SKIP_BUILD=false
            shift
            ;;
        -t|--timeout)
            TEST_TIMEOUT="$2"
            shift 2
            ;;
        -i|--iterations)
            MAX_ITERATIONS="$2"
            shift 2
            ;;
        -f|--filter)
            GTEST_FILTER="$2"
            shift 2
            ;;
        -u|--unidev)
            UNIDEV=true
            shift
            ;;
        -u=true|--unidev=true)
            UNIDEV=true
            shift
            ;;
        -u=false|--unidev=false)
            UNIDEV=false
            shift
            ;;
        --suite)
            TEST_SUITE="$2"
            shift 2
            ;;
        -s|--stop-on-failure)
            STOP_ON_FAILURE=true
            shift
            ;;
        -v|--verbose)
            VERBOSE=true
            shift
            ;;
        -h|--help)
            SHOW_HELP=true
            shift
            ;;
        *)
            echo -e "${RED}错误: 未知参数: $1${NC}"
            echo "使用 --help 查看帮助信息"
            exit 1
            ;;
    esac
done

# 显示帮助信息
if [ "${SHOW_HELP}" = true ]; then
    cat << EOF
循环运行测试脚本(使用 meson test)

用法:
    $0 [选项]

选项:
    -n, --no-build            跳过构建,直接运行测试(需要 builddir 已存在)
    -n=false                  显式关闭跳过构建(与默认相同)
    -u, --unidev              启用 unidev(与 --unidev=true 相同)
    -u=false, --unidev=false  关闭 unidev(默认,不编译 csr1/connector 相关驱动)
    -t, --timeout SEC         设置测试超时时间(秒,默认: 300)
    -i, --iterations N        设置循环次数(默认: 100)
    -f, --filter FILTER       设置 gtest 过滤器(例如: "Csr1PerfTest.*" 或 "-BusTest.*:ChipTest.*")
    --suite SUITE             只运行指定 meson 测试套件(如 "csr1",默认全部)
    -s, --stop-on-failure     遇到失败即停止测试
    -v, --verbose             详细输出模式
    -h, --help                显示此帮助信息

环境变量:
    MAX_ITERATIONS            最大循环次数(默认: 100)
    TEST_TIMEOUT              测试超时时间,单位秒(默认: 300)
    GTEST_FILTER              gtest 过滤器(默认: 空,不过滤)

测试发现:
    脚本使用 "meson test --list" 从 builddir 自动发现所有注册的测试目标。

示例:
    # 完整构建并测试
    $0

    # 跳过构建,直接测试(需要先构建)
    $0 --no-build

    # 自定义超时和循环次数
    $0 --timeout 600 --iterations 10

    # 只测 csr1 性能测试用例
    $0 -n -i 100 -f "Csr1PerfTest.*"

    # 只测 csr1 套件
    $0 -n --suite csr1

    # 遇到失败即停止
    $0 --stop-on-failure

    # 详细输出模式
    $0 --verbose --iterations 10
EOF
    exit 0
fi

# 创建日志目录
mkdir -p "${LOG_DIR}"

# 统计信息
success_count=0
failure_count=0

echo "=========================================="
echo "开始循环测试:共 ${MAX_ITERATIONS} 次"
if [ "${SKIP_BUILD}" = true ]; then
    echo "模式: 跳过构建(直接运行测试)"
else
    echo "模式: 完整构建并测试(setup.sh + build.sh)"
    echo "编译并发: ${BUILD_JOBS}"
fi
echo "unidev: ${UNIDEV}"
echo "超时时间: ${TEST_TIMEOUT} 秒"
if [ -n "${GTEST_FILTER}" ]; then
    echo "gtest 过滤器: ${GTEST_FILTER}"
fi
if [ -n "${TEST_SUITE}" ]; then
    echo "meson 套件过滤: ${TEST_SUITE}"
fi
if [ "${STOP_ON_FAILURE}" = true ]; then
    echo "遇到失败即停止: 是"
fi
echo "=========================================="
echo ""

for iteration in $(seq 1 ${MAX_ITERATIONS}); do
    if [ "${STOP_REQUESTED}" = true ]; then
        echo "检测到中断请求,停止循环。"
        break
    fi
    echo "[${iteration}/${MAX_ITERATIONS}] 开始第 ${iteration} 次测试..."

    # 根据参数决定是否构建
    if [ "${SKIP_BUILD}" = false ]; then
        # 清理共享内存
        echo "  清理 /dev/shm/*..."
        rm -rf /dev/shm/* 2>/dev/null || true

        # 步骤 1: meson setup(通过 setup.sh)
        echo "  运行 setup.sh(meson setup)..."
        # unidev 默认关闭,--unidev 可开启(编译 csr1/connector 相关驱动)
        SETUP_ARGS=(-Dtests=true -Dunidev=${UNIDEV})
        if ! bash "${SETUP_SH}" "${SETUP_ARGS[@]}" > "${LOG_DIR}/setup_${iteration}.log" 2>&1; then
            echo -e "${RED}  [失败] setup.sh (meson setup) 失败${NC}"
            echo -e "${YELLOW}  日志: ${LOG_DIR}/setup_${iteration}.log${NC}"
            failure_count=$((failure_count + 1))
            if [ "${STOP_ON_FAILURE}" = true ]; then
                echo -e "${YELLOW}检测到构建失败,停止后续测试 (--stop-on-failure)${NC}"
                break
            fi
            continue
        fi

        # 步骤 2: meson compile(通过 build.sh;传入并发数,避免默认 -j$(nproc) OOM)
        echo "  运行 build.sh(meson compile, -j ${BUILD_JOBS})..."
        if ! bash "${BUILD_SH}" "${BUILD_JOBS}" > "${LOG_DIR}/build_${iteration}.log" 2>&1; then
            echo -e "${RED}  [失败] build.sh (meson compile) 失败${NC}"
            echo -e "${YELLOW}  日志: ${LOG_DIR}/build_${iteration}.log${NC}"
            failure_count=$((failure_count + 1))
            if [ "${STOP_ON_FAILURE}" = true ]; then
                echo -e "${YELLOW}检测到构建失败,停止后续测试 (--stop-on-failure)${NC}"
                break
            fi
            continue
        fi

        # 清理构建日志(成功后)
        rm -f "${LOG_DIR}/setup_${iteration}.log" "${LOG_DIR}/build_${iteration}.log"
    else
        # 跳过构建,检查 builddir 是否存在
        if [ ! -d "${BUILD_DIR}" ]; then
            echo -e "${RED}  [失败] builddir 不存在,无法跳过构建${NC}"
            echo -e "${YELLOW}  提示: 请先运行完整构建,或移除 --no-build 参数${NC}"
            failure_count=$((failure_count + 1))
            continue
        fi
        echo "  跳过构建,使用现有的 builddir"
    fi

    # 准备 /opt/bmc Lua 运行时(mcskynet/skynet/luaclib)
    prepare_opt_bmc_runtime

    # 自动发现所有测试目标(如果 builddir 存在)
    mapfile -t ALL_TESTS < <(meson test -C "${BUILD_DIR}" --list 2>/dev/null | awk '{print $1}')
    if [ ${#ALL_TESTS[@]} -eq 0 ]; then
        echo -e "${RED}  [失败] 未发现任何 meson 测试目标${NC}"
        failure_count=$((failure_count + 1))
        continue
    fi
    echo "  发现 ${#ALL_TESTS[@]} 个测试目标"

    # 运行测试(使用 meson test,串行执行)
    echo "  运行测试(meson test)..."
    test_log="${LOG_DIR}/test_${iteration}.log"
    full_log="${LOG_DIR}/test_${iteration}_full.log"
    test_exit_code=0
    test_failed=false
    test_timeout=false

    # 构建 meson test 命令
    # --num-processes 1: 串行执行,避免资源竞争导致假失败
    # --print-errorlogs: 失败时打印错误日志
    # --wrapper: 绕过 meson env 对 LD_LIBRARY_PATH 的覆盖(见 smart_build.sh)
    test_wrapper="$(generate_meson_test_wrapper)"
    MESON_TEST_CMD=(
        meson test -C "${BUILD_DIR}"
        --num-processes 1
        --print-errorlogs
        --wrapper "${test_wrapper}"
    )

    # 如果指定了 suite,添加 --suite 参数
    if [ -n "${TEST_SUITE}" ]; then
        MESON_TEST_CMD+=(--suite "${TEST_SUITE}")
    fi

    # 使用 timeout 命令包装 meson test
    test_child_pid=""
    LD_LIBRARY_PATH="$(build_ld_library_path)" \
        timeout "${TEST_TIMEOUT}" "${MESON_TEST_CMD[@]}" > "${test_log}" 2>&1 &
    test_child_pid=$!
    wait "${test_child_pid}"
    test_exit_code=$?
    test_child_pid=""

    # 检查是否超时(timeout 命令在超时时返回 124)
    if [ ${test_exit_code} -eq 124 ]; then
        test_timeout=true
        test_failed=true
        echo ""
        echo "=========================================="
        echo "测试超时(${TEST_TIMEOUT} 秒)"
        echo "=========================================="
    elif [ ${test_exit_code} -ne 0 ]; then
        test_failed=true
    fi

    # 保存 meson testlog(meson 测试框架的完整日志)
    full_log_src="${BUILD_DIR}/meson-logs/testlog.txt"
    if [ -f "${full_log_src}" ]; then
        cp "${full_log_src}" "${full_log}"
    fi

    # 如果测试超时,尝试超时重试
    if [ "${test_timeout}" = true ]; then
        if [ -f "${full_log}" ] && grep -q "result:[[:space:]]*killed by signal 15 SIGTERM" "${full_log}"; then
            echo "检测到测试超时(SIGTERM),自动重试(timeout-multiplier=3)..."
            MESON_RETRY_CMD=(
                meson test -C "${BUILD_DIR}"
                --num-processes 1
                --timeout-multiplier 3
                --print-errorlogs
                --wrapper "${test_wrapper}"
            )
            if [ -n "${TEST_SUITE}" ]; then
                MESON_RETRY_CMD+=(--suite "${TEST_SUITE}")
            fi
            if LD_LIBRARY_PATH="$(build_ld_library_path)" \
                timeout "${TEST_TIMEOUT}" "${MESON_RETRY_CMD[@]}" > "${test_log}" 2>&1; then
                test_failed=false
                test_timeout=false
                test_exit_code=0
                echo "超时重试后测试通过"
                # 更新 testlog
                if [ -f "${full_log_src}" ]; then
                    cp "${full_log_src}" "${full_log}"
                fi
            fi
        fi
    fi

    if [ "${test_failed}" = false ]; then
        # 测试成功,删除日志避免占用空间
        rm -f "${test_log}" "${full_log}"
        success_count=$((success_count + 1))
        echo -e "${GREEN}  [成功] 第 ${iteration} 次测试通过${NC}"
        if [ "${VERBOSE}" = true ]; then
            echo "  退出码: ${test_exit_code}"
        fi
    else
        # 测试失败,分析错误类型
        failure_count=$((failure_count + 1))

        # 分类错误类型(优先从 "killed by signal" 精确提取,避免被其他测试输出噪音误导)
        error_type="TestFailure"
        error_reason="测试失败"

        if [ "${test_timeout}" = true ]; then
            error_type="Timeout"
            error_reason="TIMEOUT (测试超时 ${TEST_TIMEOUT} 秒)"
        elif [ -f "${full_log}" ]; then
            # 优先匹配 "killed by signal N SIGxxx" 精确提取信号类型
            killed_line=$(grep "killed by signal" "${full_log}" 2>/dev/null | head -1)
            if [ -n "${killed_line}" ]; then
                if echo "${killed_line}" | grep -q "SIGSEGV"; then
                    error_type="SIGSEGV"
                    error_reason="SIGSEGV (段错误)"
                elif echo "${killed_line}" | grep -q "SIGABRT"; then
                    if grep -q "assertion failed" "${full_log}" 2>/dev/null; then
                        error_type="Assertion"
                        error_reason="SIGABRT 断言失败"
                        assertion_info=$(grep "assertion failed" "${full_log}" | head -1 | sed 's/^[[:space:]]*//')
                        error_reason="${error_reason}: ${assertion_info}"
                    else
                        error_type="SIGABRT"
                        error_reason="SIGABRT (程序异常终止)"
                    fi
                elif echo "${killed_line}" | grep -q "SIGTERM"; then
                    error_type="SIGTERM"
                    error_reason="SIGTERM (程序被终止)"
                elif echo "${killed_line}" | grep -q "SIGBUS"; then
                    error_type="SIGBUS"
                    error_reason="SIGBUS (总线错误)"
                elif echo "${killed_line}" | grep -q "SIGFPE"; then
                    error_type="SIGFPE"
                    error_reason="SIGFPE (算术异常)"
                elif echo "${killed_line}" | grep -q "SIGILL"; then
                    error_type="SIGILL"
                    error_reason="SIGILL (非法指令)"
                fi
            # 没有 "killed by signal" 行时,回退到全量搜索
            elif grep -q "memory leak\|leaked" "${full_log}" 2>/dev/null; then
                error_type="MemoryLeak"
                error_reason="MemoryLeak (检测到内存泄漏)"
            elif grep -q "ASAN\|MSAN\|UBSAN" "${full_log}" 2>/dev/null; then
                error_type="Sanitizer"
                error_reason="Sanitizer (消毒器检测到问题)"
            fi
        fi

        # 提取失败的测试用例
        failed_tests=""
        if [ -f "${test_log}" ]; then
            failed_tests=$(grep -E "FAIL|TIMEOUT|killed by signal" "${test_log}" 2>/dev/null | head -20 || true)
        fi
        if [ -z "${failed_tests}" ] && [ -f "${full_log}" ]; then
            # testlog.txt 格式: "1/10 suite:name  FAIL  12.34s" 或 "1/10 suite:name  OK  12.34s"
            failed_tests=$(grep -E "[[:space:]]+FAIL[[:space:]]+" "${full_log}" 2>/dev/null | head -20 || true)
        fi

        # 输出失败信息
        echo -e "${RED}  [失败] 第 ${iteration} 次测试失败${NC}"
        echo -e "${YELLOW}  错误类型: ${error_type}${NC}"
        echo -e "${YELLOW}  错误原因: ${error_reason}${NC}"
        if [ -n "${failed_tests}" ]; then
            echo -e "${YELLOW}  失败的测试:${NC}"
            echo "${failed_tests}" | head -10 | sed 's/^/    /'
            if [ "$(echo "${failed_tests}" | wc -l)" -gt 10 ]; then
                echo "    ... (还有更多失败信息)"
            fi
        fi
        echo -e "${YELLOW}  日志文件:${NC}"
        if [ -f "${test_log}" ] && [ -s "${test_log}" ]; then
            echo "    - 测试输出: ${test_log}"
        fi
        if [ -f "${full_log}" ] && [ -s "${full_log}" ]; then
            echo "    - Meson testlog: ${full_log} (完整日志,优先查看)"
        fi
    fi

    # 遇到失败时停止
    if [ "${test_failed}" = true ] && [ "${STOP_ON_FAILURE}" = true ]; then
        echo ""
        echo -e "${YELLOW}检测到失败,停止后续测试 (--stop-on-failure)${NC}"
        break
    fi

    echo ""
done

# 输出统计信息
echo "=========================================="
echo "测试循环完成"
echo "=========================================="
echo "总次数: ${MAX_ITERATIONS}"
echo -e "成功: ${GREEN}${success_count}${NC}"
echo -e "失败: ${RED}${failure_count}${NC}"
echo ""
if [ ${failure_count} -gt 0 ]; then
    echo -e "${YELLOW}失败日志保存在: ${LOG_DIR}${NC}"
    echo ""
    echo "失败的测试迭代:"
    # 收集所有失败迭代的编号(从所有可能的日志文件)
    failed_iterations=""
    for log_file in "${LOG_DIR}"/test_*.log; do
        if [ -f "${log_file}" ]; then
            iter_num=$(basename "${log_file}" | sed -E 's/test_([0-9]+)_.*\.log/\1/' | head -1)
            if [ -n "${iter_num}" ] && [ "${iter_num}" != "*" ]; then
                if ! echo "${failed_iterations}" | grep -q "^${iter_num}$\|^${iter_num} \| ${iter_num}$\| ${iter_num} "; then
                    failed_iterations="${failed_iterations} ${iter_num}"
                fi
            fi
        fi
    done

    # 对迭代号排序并去重
    for iter_num in $(echo "${failed_iterations}" | tr ' ' '\n' | sort -n | uniq); do
        if [ -n "${iter_num}" ]; then
            echo ""
            echo "  迭代 ${iter_num}:"

            # 从 testlog.txt(full_log)分析错误类型
            full_file="${LOG_DIR}/test_${iter_num}_full.log"

            if [ -f "${full_file}" ] && [ -s "${full_file}" ]; then
                # 优先从 "killed by signal N SIGxxx" 精确提取信号类型
                killed_line=$(grep "killed by signal" "${full_file}" 2>/dev/null | head -1)
                if [ -n "${killed_line}" ]; then
                    if echo "${killed_line}" | grep -q "SIGSEGV"; then
                        echo "    错误: SIGSEGV (段错误)"
                    elif echo "${killed_line}" | grep -q "SIGABRT"; then
                        if grep -q "assertion failed" "${full_file}" 2>/dev/null; then
                            echo "    错误: Assertion (断言失败)"
                            assertion_info=$(grep "assertion failed" "${full_file}" | head -1 | sed 's/^/      /')
                            echo "${assertion_info}"
                        else
                            echo "    错误: SIGABRT (程序异常终止)"
                        fi
                    elif echo "${killed_line}" | grep -q "SIGTERM"; then
                        echo "    错误: SIGTERM (程序被终止)"
                    elif echo "${killed_line}" | grep -q "SIGBUS"; then
                        echo "    错误: SIGBUS (总线错误)"
                    elif echo "${killed_line}" | grep -q "SIGFPE"; then
                        echo "    错误: SIGFPE (算术异常)"
                    elif echo "${killed_line}" | grep -q "SIGILL"; then
                        echo "    错误: SIGILL (非法指令)"
                    else
                        signal_info=$(echo "${killed_line}" | sed 's/^[[:space:]]*result:[[:space:]]*//;s/^[[:space:]]*//')
                        echo "    错误: ${signal_info}"
                    fi
                # 无 killed_by_signal 时,回退到特征匹配
                elif grep -q "assertion failed" "${full_file}" 2>/dev/null; then
                    echo "    错误: Assertion (断言失败)"
                    assertion_info=$(grep "assertion failed" "${full_file}" | head -1 | sed 's/^/      /')
                    echo "${assertion_info}"
                elif grep -q "memory leak\|leaked" "${full_file}" 2>/dev/null; then
                    echo "    错误: MemoryLeak (内存泄漏)"
                elif grep -q "ASAN\|MSAN\|UBSAN" "${full_file}" 2>/dev/null; then
                    echo "    错误: Sanitizer (消毒器检测到问题)"
                elif grep -q "FAIL" "${full_file}" 2>/dev/null; then
                    echo "    错误: 测试用例失败"
                else
                    echo "    错误: 未知错误"
                fi

                # 列出失败的测试套件
                failed_suites=$(grep -E "[[:space:]]+FAIL[[:space:]]+" "${full_file}" 2>/dev/null | head -10 || true)
                if [ -n "${failed_suites}" ]; then
                    echo "    失败的测试:"
                    echo "${failed_suites}" | sed 's/^/      /'
                fi
            fi

            # 列出相关日志文件
            echo "    日志:"
            test_file="${LOG_DIR}/test_${iter_num}.log"
            if [ -f "${test_file}" ] && [ -s "${test_file}" ]; then
                echo "      - ${test_file} (测试输出)"
            fi
            if [ -f "${full_file}" ] && [ -s "${full_file}" ]; then
                echo "      - ${full_file} (Meson testlog,优先查看)"
            fi
        fi
    done
else
    echo -e "${GREEN}所有测试都通过了!${NC}"
    # 清理日志目录(如果所有测试都成功)
    rm -rf "${LOG_DIR}"
fi