-- Copyright (c) 2026 Huawei Technologies Co., Ltd.
-- openUBMC is licensed under Mulan PSL v2.
-- You can use this software according to the terms and conditions of the Mulan PSL v2.
-- You may obtain a copy of Mulan PSL v2 at:
--          http://license.coscl.org.cn/MulanPSL2
-- THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
-- EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
-- MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
-- See the Mulan PSL v2 for more details.

local skynet = require('skynet')
require('skynet.manager')
local coroutine = require('skynet.coroutine')
local logging = require('mc.logging')
local c_tasks = require('mc.orm.tasks')
local utils = require('mc.utils')
local utils_core = require('utils.core')
local RETRY_MAX_TIMES <const> = skynet.getenv('TEST_DATA_DIR') and 4 or 10 -- 最大重试次数
local WORK_RESTART_MAX_TIMES <const> = 2 -- work服务重拉最大尝试次数,超过后报健康状态检查失败

logging:set_log_module_name('hwproxy')

local monitor = {}
monitor.work_service = {}
monitor.position_count = {}
monitor.position_count_back = {}
monitor.monitor_service = {}
monitor.monitor_status = {}
monitor.retry_times = {}
monitor.bus_name_to_merge_key = {}
monitor.merge_key_to_bus_names = {}
monitor.merge_key_to_restart_cnt = {} -- 分组共享的重启计数, 保持组级"1小时内10次"健康保护语义
local HEART_BEAT_WAIT_TIME <const> = skynet.getenv('HEART_BEAT_WAIT_TIME') or 6000 -- 6000: 超时检测时间1分钟
local HEART_BEAT_INTERVAL <const> = skynet.getenv('HEART_BEAT_INTERVAL') or 3000 -- 3000: 每30s检测一轮心跳
local HEART_BEAT_RESTART_TIME <const> = 3600 -- 1个小时内,10次重拉失败则重启进程
local NEED_SET_RESTART_HEALTH <const> = 1
local NEED_RESTART_WORK_SERVICE <const> = 2

-- 设置可取消的定时器
-- @return: function_handler 返回的是函数句柄,函数的作用是将f置为nil,在调用时取消定时器
local function set_cancelable_timeout(ti, func)
    local function cb()
        if func then
            func()
        end
    end
    local function cancel()
        func = nil
    end
    skynet.timeout(ti, cb)
    return cancel
end

-- 校验协程是否异常终止或正常结束,是否还存在
-- @param running 协程
-- @return:true boolean 协程存在
-- @return false boolean 协程不存在
local function is_running_valid(running)
    return running and coroutine.status(running) ~= 'dead'
end

local function check_timeout(timeout, func, address)
    local co = coroutine.running()
    local fork_done = false -- 标志fork协程是否执行完成
    local timer_triggered = false -- 标志是否超时

    local cancel_timer = set_cancelable_timeout(timeout, function()
        if not fork_done and is_running_valid(co) then -- fork的协程未执行完成,已超时,主协程还在,唤醒主协程
            timer_triggered = true
            skynet.wakeup(co)
        end
    end)

    local handler = skynet.fork(function()
        local ok, err = pcall(func)
        if not ok then
            logging:error('Call service address: %s heart_beat function failed. error: %s', address, err)
        elseif not timer_triggered then -- fork的协程执行完成,未超时
            fork_done = true
        end
        cancel_timer() -- 取消定时器
        if is_running_valid(co) then -- 唤醒主协程
            skynet.wakeup(co)
        end
    end)

    skynet.wait()

    if fork_done then
        return false
    end
    skynet.killthread(handler)
    return true
end

local function check_restart(work_obj)
    local restart_queue = work_obj.restart_cnt
    local uptime = utils_core.get_bmc_uptime()
    table.insert(restart_queue, uptime)
    if #restart_queue >= RETRY_MAX_TIMES then
        if uptime - restart_queue[1] < HEART_BEAT_RESTART_TIME then
            return true
        else
            table.remove(restart_queue, 1)
        end
    end
    return false
end

local function set_group_restart_flag(bus_name, flag)
    local merge_key = monitor.bus_name_to_merge_key[bus_name]
    if not merge_key then
        return
    end
    for _, other_bus_name in pairs(monitor.merge_key_to_bus_names[merge_key] or {}) do
        if monitor.work_service[other_bus_name] then
            monitor.work_service[other_bus_name].need_restart = flag
        end
    end
end

function monitor.send_restart_message(work_obj, bus_name)
    if work_obj.need_restart then
        return
    end
    if check_restart(work_obj) then
        work_obj.need_restart = true
        logging:error('set the hwproxy health status to need restart now, bus_name: %s', bus_name)
        skynet.call('.hwproxy', 'lua', 'set_health_status_abn')
        return
    end
    set_group_restart_flag(bus_name, true)
    skynet.kill(work_obj.address)
    local ok, new_addr = pcall(skynet.call, '.hwproxy', 'lua', 'restart_work_service', bus_name)
    if not ok or not new_addr then
        logging:error('restart work service failed, bus_name: %s, err: %s', bus_name, new_addr)
        set_group_restart_flag(bus_name, false)
        return
    end
    work_obj.address = new_addr
    work_obj.count = 0
    work_obj.need_restart = false -- 重启成功后清除触发通道的并发锁标记
    local merge_key = monitor.bus_name_to_merge_key[bus_name]
    if merge_key then
        for _, other_bus_name in pairs(monitor.merge_key_to_bus_names[merge_key] or {}) do
            if other_bus_name ~= bus_name and monitor.work_service[other_bus_name] then
                monitor.work_service[other_bus_name].address = new_addr
                monitor.work_service[other_bus_name].count = 0
                monitor.work_service[other_bus_name].need_restart = false
            end
        end
    end
end

local function process_after_abnormal(error, service_bus_name)
    if error == NEED_SET_RESTART_HEALTH then
        logging:error('set the hwproxy health status to need restart now, bus_name: %s', service_bus_name)
        skynet.call('.hwproxy', 'lua', 'set_health_status_abn')
    elseif error == NEED_RESTART_WORK_SERVICE then
        logging:error('start to restart work service, bus_name: %s', service_bus_name)
        monitor.send_restart_message(monitor.work_service[service_bus_name], service_bus_name)
    else
        logging:error('monitor return error, value: %s, bus_name: %s', error, service_bus_name)
    end
end

local function monitor_all_work(bus_name, service_bus_name)
    local position_count = monitor.position_count[bus_name]
    monitor.position_count_back[bus_name] = {}
    monitor.position_count_back[bus_name] = utils.table_copy(position_count)
    for position, count in pairs(monitor.position_count_back[bus_name]) do
        if count >= RETRY_MAX_TIMES and not monitor.monitor_status[bus_name][position] then -- 当检测次数到达10次后仍未收到消息则重拉服务
            monitor.position_count[bus_name] = {} -- 清空整条bus上的所有检测, 随后停止该监听task
            monitor.retry_times[service_bus_name] = monitor.retry_times[service_bus_name] or 0
            monitor.retry_times[service_bus_name] = monitor.retry_times[service_bus_name] + 1
            if monitor.retry_times[service_bus_name] > WORK_RESTART_MAX_TIMES then
                return false, NEED_SET_RESTART_HEALTH
            else
                return false, NEED_RESTART_WORK_SERVICE
            end
        end
        position_count[position] = position_count[position] + 1
        if monitor.monitor_status[bus_name][position] then -- 代表接收到消息了,不需要再检测该postion
            monitor.position_count[bus_name][position] = nil
        elseif position_count[position] > 2 then
            logging:notice(
                'main service no recevice from work service, bus_name: %s, position: %s, count:%s',
                bus_name,
                position,
                position_count[position]
            ) -- 由于定时任务第一次遍历position时不能保证是发送后2s, 因此第一次扫描也正常, 不打印
        end
    end
    return true
end

local function process_monitoring_errors(bus_name, error, task, keep_monitor)
    if not keep_monitor then
        process_after_abnormal(error, bus_name)
        if error == NEED_SET_RESTART_HEALTH then
            task:stop()
        end
    end
end

local function create_standard_monitor_task(bus_name)
    monitor.position_count[bus_name] = {}
    monitor.monitor_status[bus_name] = {}
    -- 共享服务组按分组统计重启次数, 避免组内各通道独立计数稀释重启上限保护
    local retry_key = monitor.bus_name_to_merge_key[bus_name] or bus_name
    local tasks = c_tasks.get_instance()
    local task = tasks:new_task('work_monitor_' .. bus_name):set_timeout_ms(2000)

    task:loop(function()
        local keep_monitor, error = monitor_all_work(bus_name, retry_key)
        process_monitoring_errors(bus_name, error, task, keep_monitor)
    end)
    return task
end

function monitor.work_service_register(bus_name, addr, merge_key)
    logging:notice('start work monitor register, bus_name : %s', bus_name)
    local restart_cnt = {}
    if merge_key then
        monitor.bus_name_to_merge_key[bus_name] = merge_key
        monitor.merge_key_to_bus_names[merge_key] = monitor.merge_key_to_bus_names[merge_key] or {}
        table.insert(monitor.merge_key_to_bus_names[merge_key], bus_name)
        -- 组内所有通道共用同一份重启计数, 心跳路径的"1小时内10次"上限按分组生效
        monitor.merge_key_to_restart_cnt[merge_key] = monitor.merge_key_to_restart_cnt[merge_key] or {}
        restart_cnt = monitor.merge_key_to_restart_cnt[merge_key]
    end
    monitor.work_service[bus_name] = { address = addr, count = 0, restart_cnt = restart_cnt, need_restart = false }
    skynet.fork_loop({ count = 0 }, monitor.work_service_heart_beat, bus_name, monitor.work_service[bus_name])
    create_standard_monitor_task(bus_name)
end

function monitor.work_service_heart_beat(bus_name, work_obj)
    local max_counts = 3 -- 默认心跳3次丢失认为异常
    if string.find(bus_name, 'I3cOverLocalBus') then
        max_counts = 10 -- i3c升级npu场景下插件占用时间较长, 增加心跳检查次数为10次
    end
    while true do
        local is_timeout = check_timeout(HEART_BEAT_WAIT_TIME, function()
            skynet.call(work_obj.address, 'lua', 'heart_beat')
        end, work_obj.address)

        if is_timeout then
            work_obj.count = work_obj.count + 1
        else
            work_obj.count = 0
        end
        if work_obj.count >= max_counts then -- 连续3次丢失心跳执行重启自愈
            logging:warn('%s work service heart beat lost, restart work service', bus_name)
            monitor.send_restart_message(work_obj, bus_name)
        end
        skynet.sleep(HEART_BEAT_INTERVAL)
    end
end

function monitor.add_work_monitor(bus_name, position)
    logging:info('start to add monitor work on bus: %s, position: %s', bus_name, position)
    if not monitor.monitor_status[bus_name] or not monitor.position_count[bus_name] then
        logging:error('the bus: %s is not registered', bus_name)
        return
    end
    monitor.position_count[bus_name][position] = 1
    monitor.monitor_status[bus_name][position] = false
end

function monitor.set_work_monitor_status(bus_name, position)
    logging:info('the bus: %s, position: %s successfully receives', bus_name, position)
    local monitor_status = monitor.monitor_status[bus_name]
    if monitor_status then
        monitor_status[position] = true
    else
        logging:error('an unexpected message is received from work service: %s, position: %s', bus_name, position)
    end
end

local CMD = {}

function CMD.register(bus_name, addr, merge_key)
    monitor.work_service_register(bus_name, addr, merge_key)
end

function CMD.add_work_monitor(bus_name, position) -- 添加work检测
    monitor.add_work_monitor(bus_name, position)
end

function CMD.set_work_monitor_status(bus_name, position) -- 设置监测成功标志
    monitor.set_work_monitor_status(bus_name, position)
end

function CMD.set_debug_log_level(value)
    logging:setLevel(value)
end

skynet.start(function()
    skynet.dispatch('lua', function(_, _, cmd, ...)
        local f = CMD[cmd]
        if f == nil then
            skynet.ret(skynet.pack(nil, 'Invalid command ' .. cmd))
            return
        end
        skynet.ret(skynet.pack(f(...)))
    end)
end)