#!/opt/lnxall_app/bin/lua

-- Created by hongbo.he@lnxall.com
-- system monitor
-- 2025/10/30

local posix		= require "posix"
local invoker	= require "invoker"
local cjson		= require "cjson"
local logctl	= require "logctl"
local sdmmc		= require "sdmmc"
local mosquitto	= require "mosquitto"

local g_has_softirq_pending = false
local g_mqtt = nil
local g_gwsn = nil
local g_core_pattern = '/proc/sys/kernel/core_pattern'
local g_retain_cores = {}

local g_applist_file = '/app/config/statmon_config.json'
local g_logdir = '/var/log/statmon'
local g_sysload_file = g_logdir .. '/sysload.json'


local function sort_by_mtime(f0, f1)
	if type(f0) ~= "string" then return false end
	if type(f1) ~= "string" then return false end
	local st0 = posix.stat(f0)
	local st1 = posix.stat(f1)
	if type(st0) ~= "table" then return false end
	if type(st1) ~= "table" then return false end

	st0, st1 = st0['mtime'], st1['mtime']
	if type(st0) ~= "number" then st0 = 0 end
	if type(st1) ~= "number" then st1 = 0 end
	return st0 > st1
end

local function load_config()
	local output = invoker.readfile(g_applist_file)
	if not output or output == "" then
		io.stderr:write(string.format("Error, read config: %s\n", g_applist_file))
		io.stderr:flush()
		return nil
	end

	return cjson.decode(output)
end

local function parse_list(list)
	if not list then return nil end

	local rlst = {}
	for _, info in ipairs(list) do
		table.insert(rlst, info.name)
	end
	return rlst
end

--
-- total cpu usage
--
local function get_cpustat()
	local st = {}

	local output = invoker.readfile("/proc/stat")
	if type(output) ~= "string" then output = "" end

	local user, nice, system, idle, iowait, irq, softirq =
		string.match(output, "cpu%s+(%d+)%s+(%d+)%s+(%d+)%s+(%d+)%s+(%d+)%s+(%d+)%s+(%d+)")

	st.user	= tonumber(user)
    st.nice	= tonumber(nice)
    st.system	= tonumber(system)
    st.idle	= tonumber(idle)
    st.iowait	= tonumber(iowait)
	st.irq		= tonumber(irq)
    st.softirq	= tonumber(softirq)

	return st
end

local function get_cpu_usage()
	local base_st = get_cpustat()
	invoker.msleep(1000)
	local next_st = get_cpustat()

	delta_idle = next_st.idle - base_st.idle

	base_sum = base_st.user + base_st.nice + base_st.system + base_st.idle +
		base_st.iowait + base_st.irq + base_st.softirq
	next_sum = next_st.user + next_st.nice + next_st.system + next_st.idle +
		next_st.iowait + next_st.irq + next_st.softirq;
	delta_sum = next_sum - base_sum

	return 1 - delta_idle / delta_sum
end

--
-- memory info of applications: vmrss, vmhwm
--
local function applist_create(apps)
	local applist = {}

	for _, app in ipairs(apps) do
		applist[app] = { pid = 0, appn = app, restart = false, vmrss = 0, vmhwm = 0 }
	end
	return applist
end

local function iterate_process(plst)
	local plist = invoker.glob('/proc/*/exe')

	for _, pexe in ipairs(plist) do
		local pid = string.match(pexe, '^/proc/(%d+)/exe')
		if not pid then break end

		pid = tonumber(pid)

		local appn = invoker.readfile(string.format('/proc/%d/comm', pid))
		if not appn then break end

		appn = appn:gsub("[\r\n]+$", '')

		if plst and plst[appn] then
			if plst[appn].pid > 0 and pid ~= plst[appn].pid then
				plst[appn].restart = true
			else
				plst[appn].restart = false
			end
			plst[appn].pid = pid
		end
	end
	return true
end

local function get_proc_status(plst)
	if not plst then return nil end

	iterate_process(plst)

	local pstat = {}
	for appn, appinfo in pairs(plst) do
		local output = invoker.readfile(string.format('/proc/%d/status', appinfo.pid))
		if not output or #output == 0 then
			io.stderr:write("Error, failed to open 'status' of " .. appn .. "\n")
			io.stderr:flush()
		else
			local vmrss = nil
			local vm_rss = string.match(output, 'VmRSS:%s+(%d+)%s+kB')
			if vm_rss then vmrss = tonumber(vm_rss) end

			local vmhwm = nil
			local vm_hwm = string.match(output, 'VmHWM:%s+(%d+)%s+kB')
			if vm_hwm then vmhwm = tonumber(vm_hwm) end

			local appi = plst[appn]
			if vmrss and vmhwm then
				table.insert(pstat, {
					Name = appn,
					PID = appinfo.pid,
					Restart = appinfo.restart,
					VmRSS = vmrss,
					VmHWM = vmhwm
				})
				plst[appn].vmrss = vmrss
				plst[appn].vmhwm = vmhwm
			end
		end
	end

	return pstat
end

--
-- record and manage coredump event
--
local function get_coredump_event()
	if invoker.uptime() < 180 then return nil end

	local cores = invoker.glob('/var/crash/coredumps/*.core')
	if type(cores) ~= 'table' or #cores == 0 then
		return nil
	end

	if #cores > 1 then table.sort(cores, sort_by_mtime) end

	local dumpinfo = {}
	local retained_cnt = 0

	for _, core in ipairs(cores) do
		local is_retained = false

		for _, rtcore in ipairs(g_retain_cores) do
			if core == rtcore then
				is_retained = true
				retained_cnt = retained_cnt + 1
				break
			end
		end

		if not is_retained then
			local pexe, ts, pid, sig = string.match(core, '^([^-]+)%-(%d+)%-(%d+)%-(%d+)%.core$')
			local exename = string.match(pexe, '([^!]+)$')

			table.insert(dumpinfo, {
				Name = exename,
				UTC = ts,
				PID = pid,
				SIG = sig,
			})
		end
	end

	-- maintain 2 lastest core files
	g_retain_cores = { cores[1], cores[2] }
	for i = 3, #cores do
		io.stdout:write(string.format("Deleting core file: %s\n", cores[i]))
		io.stdout:flush()
		posix.unlink(cores[i])
	end

	return (#dumpinfo > 0 or retained_cnt < 2) and dumpinfo or nil
end

--
-- storages health: mmc, ssd
--
local function read_emmc_info(mmc)
	if not mmc or type(mmc) ~= 'string' then return nil end

	local base = '/sys/block/' .. mmc .. '/device/'
	local result = {}

	local life_time = invoker.readfile(base .. 'life_time')
	if life_time then
		local rd, wr = string.match(life_time, '0x(%x+)%s+0x(%x+)')
		if rd and wr then
			rd, wr = tonumber(rd, 16), tonumber(wr, 16)
			result.LIFE_RD = rd
			result.LIFE_WR = wr
		end
	end

	local eol = invoker.readfile(base .. 'pre_eol_info')
	if eol then
		local code = string.match(eol, '^0x(%x%x)')
		if code then code = tonumber(code, 16) end

		result.EOL = ({
			[0x00] = 'Normal',
			[0x01] = 'Warning',
			[0x02] = 'Error'
		})[code] or 'Unknown'
	end
	return next(result) and result or nil
end

local function check_softirq_pending()
	if g_has_softirq_pending then return true end

	local _, output = invoker.invoke(invoker.OUTPUT, 'bash', '-c', 'dmesg | grep local_softirq_pending')
	if output and #output > 0 then
		g_has_softirq_pending = true
		return true
	end
	return false
end

local function read_storage_nvme(device)
	if not device then return false end

	local command = string.format('nvme smart-log %s', device)
	local _, output = invoker.invoke(invoker.OUTPUT, 'bash', '-c', command)
	if type(output) ~= "string" or #output == 0 then
		return nil
	end

	local crit = string.match(output, "critical_warning%s+: (%d+)")
	local spare = string.match(output, "available_spare%s+: (%d+)%%")
	local used = string.match(output, "percentage_used%s+: (%d+)%%")

	return {
		CRIT_WARN = tonumber(crit),
		AVA_SPARE = tonumber(spare),
		USED = tonumber(used),
		SOFTIRQ_PENDING = check_softirq_pending()
	}
end

local function read_storage_usb(device)
	if not device then return false end

	local command = string.format('smartctl -H %s', device)
	local _, output = invoker.invoke(invoker.OUTPUT, 'bash', '-c', command)
	if type(output) == "string" and #output > 0 then
		local result = string.match(output, '(%u+)%s*$')
		return result
	end

	return nil
end

local function read_ssd_info(device)
	if string.find(device, 'nvme') then
		return read_storage_nvme(device)
	else
		return read_storage_usb(device)
	end
end

--
-- docker stats
--
local function get_docker_stats(applist)
	if not applist then return nil end

	local stats = {}
	for _, app in ipairs(applist) do
		local cmd = string.format('docker stats --no-stream %s --format json', app)
		local _, output = invoker.invoke(invoker.OUTPUT, 'bash', '-c', cmd)
		if type(output) == 'string' and #output > 0 then
			local stat = cjson.decode(output)
			if stat then
				table.insert(stats, {
					CONTAINER = stat.Container,
					CPU = stat.CPUPerc,
					MEM = stat.MemPerc,
					PIDS = tonumber(stat.PIDs)
				})
			end
		end
	end
	return stats
end

--
-- upload exceptions by mqtt
--
local function mqtt_session_init()
	local mqtt = mosquitto.new()

	-- connect to local broker
	local okay = mqtt:connect()
	if not okay then
		io.stderr:write("Error, failed to connect local broker\n")
		io.stderr:flush()
		return false
	end
	g_mqtt = mqtt

	-- get gateway SN
	local gwsn = nil
	local eval, output = invoker.invoke(invoker.NOSTDIO + invoker.OUTPUT, "factory", "get")
	if eval == 0 and type(output) == "string" then gwsn = string.match(output, "SN=(%x+)") end
    if not gwsn then
        output = invoker.readfile("/app/config/fac.ini")
        if type(output) == "string" then gwsn = string.match(output, "sn%s*=%s*(%x+)") end
    end
	if not gwsn then
		io.stderr:write("Error, failed to determine gateway SN!\n")
		io.stderr:flush()
		return false
    end
	g_gwsn = gwsn
    io.stdout:write(string.format("Gateway SN: %s\n", gwsn))
    io.stdout:flush()

	return true
end

local function mqtt_transmit(sysload)
	local mqtt = g_mqtt
	if not mqtt then
		io.stderr:write("Error, no internal MQTT connection found.\n")

		if not mqtt_session_init() then
			io.stderr:write("Error, try reconnected failed.\n")
			io.stderr:flush()
			return false
		end
	end

	local topic = string.format('ipc/%s/statmon/stat', g_gwsn)
	local okay, mid = mqtt:publish(topic, sysload)
	if not okay then
		io.stderr:write("Error, MQTT Publish failed.\n")
		mqtt:disconnect()
		g_mqtt = nil
		return false
	end
	return true
end


local function main_func()
	invoker.setname('STATMON')
	if not mqtt_session_init() then
		invoker.msleep(5000)
		return false
	end

	local period_ms, sys_mmc, applist = 15 * 60 * 1000, nil, nil
	local config = load_config()
	if not config then return false end

	-- logging configuration
	local logging = config.logging
	if logging then
		local max_days = logging.max_days
		if max_days and type(max_days) == 'number' then
			logctl.set(max_days)
		end
		if logging.period_minutes then
			period_ms = logging.period_minutes * 60 * 1000
		end
	end
	invoker.mkdir(g_logdir)
	logctl.open(g_sysload_file)

	-- storage configuration
	local storage = config.storage
	if storage and storage.system_mmc then
		sys_mmc = posix.basename(storage.system_mmc)
	end

	local apps = parse_list(config.applist)
	if apps then
		applist = applist_create(apps)
	end
	local containers = parse_list(config.containers)

	while true do
		local cpu_usage = get_cpu_usage()
		local sdmmc_life, ssd_info = nil, nil
		if storage then
			if storage.extended_mmc then
				sdmmc_life = select(8, sdmmc.info(storage.extended_mmc))
			end
			if storage.ssd then
				ssd_info = read_ssd_info(storage.ssd)
			end
		end

		local sysload = cjson.encode({
			UPTIME = invoker.uptime(),
			UTC = os.time(),
			CPU = tonumber(string.format("%.4f", cpu_usage)),
			PSTAT = get_proc_status(applist),
			CRASH = get_coredump_event(),
			STORAGE = {
				EMMC = read_emmc_info(sys_mmc),
				SDMMC = sdmmc_life,
				SSD = ssd_info,
			},
			CONTAINER = get_docker_stats(containers)
		})

		logctl.writeln(sysload)
		mqtt_transmit(sysload)

		logctl.try_rotate()
		invoker.msleep(period_ms)
	end
end

main_func()
os.exit(1)
