local skynet = require('skynet')
require('skynet.manager')
local coroutine = require('skynet.coroutine')
local logging = require('mc.logging')
local c_tasks = require('mc.orm.tasks')
local utils = require('mc.utils')
local utils_core = require('utils.core')
local RETRY_MAX_TIMES <const> = skynet.getenv('TEST_DATA_DIR') and 4 or 10
local WORK_RESTART_MAX_TIMES <const> = 2
logging:set_log_module_name('hwproxy')
local monitor = {}
monitor.work_service = {}
monitor.position_count = {}
monitor.position_count_back = {}
monitor.monitor_service = {}
monitor.monitor_status = {}
monitor.retry_times = {}
monitor.bus_name_to_merge_key = {}
monitor.merge_key_to_bus_names = {}
monitor.merge_key_to_restart_cnt = {}
local HEART_BEAT_WAIT_TIME <const> = skynet.getenv('HEART_BEAT_WAIT_TIME') or 6000
local HEART_BEAT_INTERVAL <const> = skynet.getenv('HEART_BEAT_INTERVAL') or 3000
local HEART_BEAT_RESTART_TIME <const> = 3600
local NEED_SET_RESTART_HEALTH <const> = 1
local NEED_RESTART_WORK_SERVICE <const> = 2
local function set_cancelable_timeout(ti, func)
local function cb()
if func then
func()
end
end
local function cancel()
func = nil
end
skynet.timeout(ti, cb)
return cancel
end
local function is_running_valid(running)
return running and coroutine.status(running) ~= 'dead'
end
local function check_timeout(timeout, func, address)
local co = coroutine.running()
local fork_done = false
local timer_triggered = false
local cancel_timer = set_cancelable_timeout(timeout, function()
if not fork_done and is_running_valid(co) then
timer_triggered = true
skynet.wakeup(co)
end
end)
local handler = skynet.fork(function()
local ok, err = pcall(func)
if not ok then
logging:error('Call service address: %s heart_beat function failed. error: %s', address, err)
elseif not timer_triggered then
fork_done = true
end
cancel_timer()
if is_running_valid(co) then
skynet.wakeup(co)
end
end)
skynet.wait()
if fork_done then
return false
end
skynet.killthread(handler)
return true
end
local function check_restart(work_obj)
local restart_queue = work_obj.restart_cnt
local uptime = utils_core.get_bmc_uptime()
table.insert(restart_queue, uptime)
if #restart_queue >= RETRY_MAX_TIMES then
if uptime - restart_queue[1] < HEART_BEAT_RESTART_TIME then
return true
else
table.remove(restart_queue, 1)
end
end
return false
end
local function set_group_restart_flag(bus_name, flag)
local merge_key = monitor.bus_name_to_merge_key[bus_name]
if not merge_key then
return
end
for _, other_bus_name in pairs(monitor.merge_key_to_bus_names[merge_key] or {}) do
if monitor.work_service[other_bus_name] then
monitor.work_service[other_bus_name].need_restart = flag
end
end
end
function monitor.send_restart_message(work_obj, bus_name)
if work_obj.need_restart then
return
end
if check_restart(work_obj) then
work_obj.need_restart = true
logging:error('set the hwproxy health status to need restart now, bus_name: %s', bus_name)
skynet.call('.hwproxy', 'lua', 'set_health_status_abn')
return
end
set_group_restart_flag(bus_name, true)
skynet.kill(work_obj.address)
local ok, new_addr = pcall(skynet.call, '.hwproxy', 'lua', 'restart_work_service', bus_name)
if not ok or not new_addr then
logging:error('restart work service failed, bus_name: %s, err: %s', bus_name, new_addr)
set_group_restart_flag(bus_name, false)
return
end
work_obj.address = new_addr
work_obj.count = 0
work_obj.need_restart = false
local merge_key = monitor.bus_name_to_merge_key[bus_name]
if merge_key then
for _, other_bus_name in pairs(monitor.merge_key_to_bus_names[merge_key] or {}) do
if other_bus_name ~= bus_name and monitor.work_service[other_bus_name] then
monitor.work_service[other_bus_name].address = new_addr
monitor.work_service[other_bus_name].count = 0
monitor.work_service[other_bus_name].need_restart = false
end
end
end
end
local function process_after_abnormal(error, service_bus_name)
if error == NEED_SET_RESTART_HEALTH then
logging:error('set the hwproxy health status to need restart now, bus_name: %s', service_bus_name)
skynet.call('.hwproxy', 'lua', 'set_health_status_abn')
elseif error == NEED_RESTART_WORK_SERVICE then
logging:error('start to restart work service, bus_name: %s', service_bus_name)
monitor.send_restart_message(monitor.work_service[service_bus_name], service_bus_name)
else
logging:error('monitor return error, value: %s, bus_name: %s', error, service_bus_name)
end
end
local function monitor_all_work(bus_name, service_bus_name)
local position_count = monitor.position_count[bus_name]
monitor.position_count_back[bus_name] = {}
monitor.position_count_back[bus_name] = utils.table_copy(position_count)
for position, count in pairs(monitor.position_count_back[bus_name]) do
if count >= RETRY_MAX_TIMES and not monitor.monitor_status[bus_name][position] then
monitor.position_count[bus_name] = {}
monitor.retry_times[service_bus_name] = monitor.retry_times[service_bus_name] or 0
monitor.retry_times[service_bus_name] = monitor.retry_times[service_bus_name] + 1
if monitor.retry_times[service_bus_name] > WORK_RESTART_MAX_TIMES then
return false, NEED_SET_RESTART_HEALTH
else
return false, NEED_RESTART_WORK_SERVICE
end
end
position_count[position] = position_count[position] + 1
if monitor.monitor_status[bus_name][position] then
monitor.position_count[bus_name][position] = nil
elseif position_count[position] > 2 then
logging:notice(
'main service no recevice from work service, bus_name: %s, position: %s, count:%s',
bus_name,
position,
position_count[position]
)
end
end
return true
end
local function process_monitoring_errors(bus_name, error, task, keep_monitor)
if not keep_monitor then
process_after_abnormal(error, bus_name)
if error == NEED_SET_RESTART_HEALTH then
task:stop()
end
end
end
local function create_standard_monitor_task(bus_name)
monitor.position_count[bus_name] = {}
monitor.monitor_status[bus_name] = {}
local retry_key = monitor.bus_name_to_merge_key[bus_name] or bus_name
local tasks = c_tasks.get_instance()
local task = tasks:new_task('work_monitor_' .. bus_name):set_timeout_ms(2000)
task:loop(function()
local keep_monitor, error = monitor_all_work(bus_name, retry_key)
process_monitoring_errors(bus_name, error, task, keep_monitor)
end)
return task
end
function monitor.work_service_register(bus_name, addr, merge_key)
logging:notice('start work monitor register, bus_name : %s', bus_name)
local restart_cnt = {}
if merge_key then
monitor.bus_name_to_merge_key[bus_name] = merge_key
monitor.merge_key_to_bus_names[merge_key] = monitor.merge_key_to_bus_names[merge_key] or {}
table.insert(monitor.merge_key_to_bus_names[merge_key], bus_name)
monitor.merge_key_to_restart_cnt[merge_key] = monitor.merge_key_to_restart_cnt[merge_key] or {}
restart_cnt = monitor.merge_key_to_restart_cnt[merge_key]
end
monitor.work_service[bus_name] = { address = addr, count = 0, restart_cnt = restart_cnt, need_restart = false }
skynet.fork_loop({ count = 0 }, monitor.work_service_heart_beat, bus_name, monitor.work_service[bus_name])
create_standard_monitor_task(bus_name)
end
function monitor.work_service_heart_beat(bus_name, work_obj)
local max_counts = 3
if string.find(bus_name, 'I3cOverLocalBus') then
max_counts = 10
end
while true do
local is_timeout = check_timeout(HEART_BEAT_WAIT_TIME, function()
skynet.call(work_obj.address, 'lua', 'heart_beat')
end, work_obj.address)
if is_timeout then
work_obj.count = work_obj.count + 1
else
work_obj.count = 0
end
if work_obj.count >= max_counts then
logging:warn('%s work service heart beat lost, restart work service', bus_name)
monitor.send_restart_message(work_obj, bus_name)
end
skynet.sleep(HEART_BEAT_INTERVAL)
end
end
function monitor.add_work_monitor(bus_name, position)
logging:info('start to add monitor work on bus: %s, position: %s', bus_name, position)
if not monitor.monitor_status[bus_name] or not monitor.position_count[bus_name] then
logging:error('the bus: %s is not registered', bus_name)
return
end
monitor.position_count[bus_name][position] = 1
monitor.monitor_status[bus_name][position] = false
end
function monitor.set_work_monitor_status(bus_name, position)
logging:info('the bus: %s, position: %s successfully receives', bus_name, position)
local monitor_status = monitor.monitor_status[bus_name]
if monitor_status then
monitor_status[position] = true
else
logging:error('an unexpected message is received from work service: %s, position: %s', bus_name, position)
end
end
local CMD = {}
function CMD.register(bus_name, addr, merge_key)
monitor.work_service_register(bus_name, addr, merge_key)
end
function CMD.add_work_monitor(bus_name, position)
monitor.add_work_monitor(bus_name, position)
end
function CMD.set_work_monitor_status(bus_name, position)
monitor.set_work_monitor_status(bus_name, position)
end
function CMD.set_debug_log_level(value)
logging:setLevel(value)
end
skynet.start(function()
skynet.dispatch('lua', function(_, _, cmd, ...)
local f = CMD[cmd]
if f == nil then
skynet.ret(skynet.pack(nil, 'Invalid command ' .. cmd))
return
end
skynet.ret(skynet.pack(f(...)))
end)
end)