#!/bin/bash
# If QMI link is down, restart quectel-qmi. Run from cron every 1 min.
# Logs to /app/qmi-log/qmi-failover.log.

QMI_LOG_DIR="/app/qmi-log"
LOG="$QMI_LOG_DIR/qmi-failover.log"
mkdir -p "$QMI_LOG_DIR" 2>/dev/null || true
log() { echo "$(date -Iseconds) $*" >> "$LOG"; }

DEV="/dev/cdc-wdm0"
PING_TARGET="8.8.8.8"
# One stop/start per cron run; 1-min cadence replaces in-run retries.
MAX_TRIES_BEFORE_RESET=1
RESET_POLL_MAX_SEC=20
RESET_POLL_INTERVAL_SEC=5
RUN_MAX_SEC=50
QMICLI_TIMEOUT_SEC=10
QMICLI_CLASSIFY_TIMEOUT_SEC=5
STOP_START_STOP_TIMEOUT_SEC=15
STOP_START_START_TIMEOUT_SEC=35
UNRESPONSIVE_STOP_TIMEOUT_SEC=8
SERVICE_RESTART_MIN_BUDGET_SEC=15
STATE_DIR="/var/lib/matelex"
STATE_FILE="$STATE_DIR/qmi-failover.state"
RUN_DIR="/var/run/qmi-failover.run"
RUN_PID_FILE="$RUN_DIR/pid"
RUN_STARTED_FILE="$RUN_DIR/started"
RUN_STALE_MAX_SEC=90
RUN_HARD_MAX_SEC=55
SERVICE_RESTART_TIMEOUT_SEC=30
REBOOT_THRESHOLD=15
UNRESPONSIVE_REBOOT_THRESHOLD=5
REBOOT_DELAY_SEC=10
MODEMINIT="/usr/lib/matelex/modeminit"
DEV_WAIT_POLL_SEC=2
START_MIN_BUDGET_SEC=20
QMI_NETWORK_STATUS_TIMEOUT_SEC=5

is_our_run_pid()
{
	local p="$1"

	[ -n "$p" ] || return 1
	/bin/kill -0 "$p" 2>/dev/null || return 1
	/usr/bin/grep -q 'qmi-failover' "/proc/$p/cmdline" 2>/dev/null
}

is_stale_run_dir()
{
	local started now

	started=$(/bin/cat "$RUN_STARTED_FILE" 2>/dev/null) || return 0
	case "$started" in
		''|*[!0-9]*) return 0 ;;
	esac
	now=$(/bin/date +%s)
	[ $((now - started)) -gt "$RUN_STALE_MAX_SEC" ]
}

init_run_dir()
{
	echo $$ > "$RUN_PID_FILE"
	/bin/date +%s > "$RUN_STARTED_FILE"
	trap 'rm -rf "$RUN_DIR"' EXIT
}

acquire_run_slot()
{
	if /bin/mkdir "$RUN_DIR" 2>/dev/null; then
		init_run_dir
		return 0
	fi

	local old_pid reason=""

	old_pid=$(/bin/cat "$RUN_PID_FILE" 2>/dev/null)
	if ! /bin/kill -0 "$old_pid" 2>/dev/null; then
		reason="pid-gone"
	elif ! /usr/bin/grep -q 'qmi-failover' "/proc/$old_pid/cmdline" 2>/dev/null; then
		reason="pid-not-qmi-failover"
	elif is_stale_run_dir; then
		reason="started-too-old"
	else
		log "previous run still active (pid=$old_pid); skipping this cron tick"
		exit 0
	fi

	log "stale run dir (pid=${old_pid:-none}, reason=$reason); reclaiming"
	rm -rf "$RUN_DIR"
	if ! /bin/mkdir "$RUN_DIR" 2>/dev/null; then
		log "run slot busy after reclaim; skipping this cron tick"
		exit 0
	fi
	init_run_dir
}

check_run_budget()
{
	local elapsed=$(( $(/bin/date +%s) - RUN_START ))
	if [ "$elapsed" -ge "$RUN_MAX_SEC" ]; then
		log "run budget (${RUN_MAX_SEC}s) exceeded after ${elapsed}s; exiting to release run slot"
		log "=== qmi-failover done (budget-exceeded) ==="
		exit 0
	fi
}

run_budget_remaining()
{
	local elapsed=$(( $(/bin/date +%s) - RUN_START ))
	echo $(( RUN_MAX_SEC - elapsed ))
}

run_qmicli()
{
	/usr/bin/timeout "$QMICLI_TIMEOUT_SEC" /usr/bin/qmicli -d "$DEV" "$@" 2>&1
}

run_qmicli_classify()
{
	/usr/bin/timeout "$QMICLI_CLASSIFY_TIMEOUT_SEC" /usr/bin/qmicli -d "$DEV" "$@" 2>&1
}

qmicli_output_is_ctl_timeout()
{
	local out="$1"

	echo "$out" | /usr/bin/grep -Eqi \
		'CID allocation failed|Transaction timed out|couldn'\''t create client|Operation was cancelled'
}

find_qmi_iface()
{
	local w n
	for w in /sys/class/net/wwan*; do
		[ -d "$w" ] || continue
		n="${w##*/}"
		[ -f "/sys/class/net/$n/qmi/raw_ip" ] && echo "$n" && return 0
	done
	for w in /sys/class/net/*; do
		[ -d "$w" ] || continue
		n="${w##*/}"
		[ -f "/sys/class/net/$n/qmi/raw_ip" ] && echo "$n" && return 0
	done
	return 1
}

IFACE="$(find_qmi_iface 2>/dev/null || true)"

get_qmi_network_status()
{
	/usr/bin/timeout "$QMI_NETWORK_STATUS_TIMEOUT_SEC" \
		/usr/bin/qmi-network "$DEV" status 2>/dev/null \
		| /usr/bin/awk '/Status:/ {print $2}'
}

ping_ok()
{
	[ -n "$IFACE" ] || return 1
	/usr/bin/ping -w 3 -I "$IFACE" "$PING_TARGET" >> "$LOG" 2>&1
	return $?
}

ping_on_iface()
{
	local iface="$1"
	/usr/bin/ping -w 3 -I "$iface" "$PING_TARGET" >> "$LOG" 2>&1
	return $?
}

iface_online()
{
	local iface="$1"
	local carrier operstate

	[ -d "/sys/class/net/$iface" ] || return 1

	if [ -f "/sys/class/net/$iface/operstate" ]; then
		operstate=$(/bin/cat "/sys/class/net/$iface/operstate" 2>/dev/null)
		[ "$operstate" = "up" ] || {
			log "primary check: $iface operstate=${operstate:-unknown}"
			return 1
		}
	fi

	if [ -f "/sys/class/net/$iface/carrier" ]; then
		carrier=$(/bin/cat "/sys/class/net/$iface/carrier" 2>/dev/null)
		[ "$carrier" = "1" ] || {
			log "primary check: $iface carrier=${carrier:-unknown}"
			return 1
		}
	fi

	if ping_on_iface "$iface"; then
		log "primary check: $iface can ping $PING_TARGET"
		return 0
	fi

	log "primary check: $iface cannot ping $PING_TARGET"
	return 1
}

primary_network_online()
{
	if iface_online eth1; then
		return 0
	fi
	if iface_online wlan0; then
		return 0
	fi
	return 1
}

load_state()
{
	FAIL_COUNT=0
	UNRESPONSIVE_PHASE=idle
	[ -d "$STATE_DIR" ] || /bin/mkdir -p "$STATE_DIR" 2>/dev/null || true
	if [ -f "$STATE_FILE" ]; then
		FAIL_COUNT="$(/usr/bin/awk -F= '/^fail_count=/ {print $2}' "$STATE_FILE" 2>/dev/null)"
		UNRESPONSIVE_PHASE="$(/usr/bin/awk -F= '/^unresponsive_phase=/ {print $2}' "$STATE_FILE" 2>/dev/null)"
	fi
	case "$FAIL_COUNT" in
		''|*[!0-9]*) FAIL_COUNT=0 ;;
	esac
	case "$UNRESPONSIVE_PHASE" in
		idle|await_start) ;;
		*) UNRESPONSIVE_PHASE=idle ;;
	esac
}

save_state()
{
	/bin/mkdir -p "$STATE_DIR" 2>/dev/null || true
	/usr/bin/printf "fail_count=%s\nunresponsive_phase=%s\n" \
		"$FAIL_COUNT" "$UNRESPONSIVE_PHASE" > "$STATE_FILE"
}

load_fail_count()
{
	load_state
}

save_fail_count()
{
	save_state
}

set_unresponsive_phase()
{
	UNRESPONSIVE_PHASE="$1"
	save_state
}

reset_fail_count()
{
	FAIL_COUNT=0
	UNRESPONSIVE_PHASE=idle
	save_state
}

reset_fail_count_if_needed()
{
	local reason="$1"
	if [ "$FAIL_COUNT" -ne 0 ] || [ "$UNRESPONSIVE_PHASE" != "idle" ]; then
		log "resetting QMI fail count from $FAIL_COUNT phase=$UNRESPONSIVE_PHASE (reason: $reason)"
	fi
	reset_fail_count
}

record_failure_and_maybe_reboot()
{
	local reason="$1"
	local threshold="${2:-$REBOOT_THRESHOLD}"

	# Soft budget may already be tight; still count failure before optional reboot delay.
	if primary_network_online; then
		reset_fail_count_if_needed "primary network online before reboot count"
		log "skipping reboot count for '$reason' because eth1/wlan0 is online"
		return 0
	fi

	FAIL_COUNT=$((FAIL_COUNT + 1))
	UNRESPONSIVE_PHASE=idle
	save_state
	log "long-term failure count incremented to $FAIL_COUNT (reason: $reason, threshold: $threshold)"

	if [ "$FAIL_COUNT" -ge "$threshold" ]; then
		if primary_network_online; then
			reset_fail_count_if_needed "primary network online before reboot"
			log "failure threshold reached, but eth1/wlan0 is online; reboot cancelled"
			return 0
		fi
		log "failure threshold reached ($FAIL_COUNT >= $threshold); rebooting in ${REBOOT_DELAY_SEC}s"
		sleep "$REBOOT_DELAY_SEC"
		if primary_network_online; then
			reset_fail_count_if_needed "primary network online after reboot delay"
			log "reboot delay elapsed, but eth1/wlan0 is online; reboot cancelled"
			return 0
		fi
		/bin/systemctl reboot >> "$LOG" 2>&1 || true
	fi
}

do_stop_start()
{
	check_run_budget
	log "restarting quectel-qmi (stop -> start)"
	/usr/bin/timeout "$STOP_START_STOP_TIMEOUT_SEC" /usr/lib/matelex/quectel-qmi stop >> "$LOG" 2>&1 || true
	sleep 2
	check_run_budget
	/usr/bin/timeout "$STOP_START_START_TIMEOUT_SEC" /usr/lib/matelex/quectel-qmi start >> "$LOG" 2>&1
	return $?
}

refresh_qmi_once_no_reboot()
{
	local reason="$1"

	reset_fail_count_if_needed "$reason"
	log "classification '$reason': refreshing QMI service once in this cron run; no DMS reset, no reboot count"
	do_stop_start >> "$LOG" 2>&1 || true
	log "=== qmi-failover done ($reason-refresh-only) ==="
	exit 0
}

qmi_healthy()
{
	local status ping_rc

	IFACE="$(find_qmi_iface 2>/dev/null || true)"
	if [ -z "$IFACE" ]; then
		log "qmi health: no QMI interface (qmi/raw_ip)"
		return 1
	fi

	ping_ok
	ping_rc=$?
	# Ping first: on failure skip status to avoid CTL hang delaying recovery paths.
	if [ $ping_rc -ne 0 ]; then
		log "qmi health: iface=$IFACE status=skipped; ping_rc=$ping_rc"
		return 1
	fi

	status="$(get_qmi_network_status)"
	log "qmi health: iface=$IFACE status=${status:-unknown}; ping_rc=$ping_rc"

	[ "$status" = "connected" ]
}

# Early main path: ping must succeed before asking qmi-network status.
try_qmi_healthy_fast()
{
	local status

	IFACE="$(find_qmi_iface 2>/dev/null || true)"
	if [ -z "$IFACE" ]; then
		log "qmi fast health: no QMI interface (qmi/raw_ip)"
		return 1
	fi

	if ! ping_ok; then
		log "qmi fast health: iface=$IFACE ping failed; skip status, proceed to classify"
		return 1
	fi

	status="$(get_qmi_network_status)"
	log "qmi fast health: iface=$IFACE status=${status:-unknown}; ping_rc=0"
	[ "$status" = "connected" ]
}

wait_for_cdc_wdm()
{
	local max_wait="$1"
	local waited=0

	while [ $waited -lt "$max_wait" ]; do
		[ -c "$DEV" ] && return 0
		remaining=$(run_budget_remaining)
		[ "$remaining" -le 1 ] && return 1
		sleep "$DEV_WAIT_POLL_SEC"
		waited=$((waited + DEV_WAIT_POLL_SEC))
	done
	[ -c "$DEV" ]
}

do_dms_reset_then_wait()
{
	local waited=0
	local remaining

	check_run_budget
	log "triggering dms-reset after stop/start did not recover"
	run_qmicli --dms-reset >> "$LOG" 2>&1 || true

	log "polling up to ${RESET_POLL_MAX_SEC}s after reset (interval ${RESET_POLL_INTERVAL_SEC}s)"
	while [ $waited -lt $RESET_POLL_MAX_SEC ]; do
		check_run_budget
		if qmi_healthy; then
			log "recovered during post-reset polling"
			return 0
		fi
		sleep $RESET_POLL_INTERVAL_SEC
		waited=$((waited + RESET_POLL_INTERVAL_SEC))
	done

	remaining=$(run_budget_remaining)
	if [ "$remaining" -lt "$SERVICE_RESTART_MIN_BUDGET_SEC" ]; then
		log "post-reset polling timed out; skipping quectel-qmi.service restart (${remaining}s run budget left)"
		log "still not recovered after reset (service restart deferred)"
		return 1
	fi

	log "post-reset polling timed out; restarting quectel-qmi.service"
	/usr/bin/timeout "$SERVICE_RESTART_TIMEOUT_SEC" /bin/systemctl restart quectel-qmi.service >> "$LOG" 2>&1 || true
	sleep 3
	if qmi_healthy; then
		log "recovered after service restart"
		return 0
	fi

	log "still not recovered after reset+restart"
	return 1
}

classify_sim_and_registration()
{
	local card_status serving_system system_info

	check_run_budget
	card_status=$(run_qmicli_classify --uim-get-card-status || true)
	log "uim card status: $(echo "$card_status" | /usr/bin/tr '\n' ' ' | /usr/bin/sed 's/[[:space:]]\+/ /g')"

	if qmicli_output_is_ctl_timeout "$card_status"; then
		echo "qmi-unresponsive"
		return 0
	fi

	if echo "$card_status" | /usr/bin/grep -qi "Card state: 'absent'"; then
		echo "no-sim"
		return 0
	fi
	if echo "$card_status" | /usr/bin/grep -Eqi "PIN1 state: '(enabled-not-verified|blocked|permanently-blocked)'|UPIN state: '(enabled-not-verified|blocked|permanently-blocked)'"; then
		echo "sim-locked"
		return 0
	fi

	check_run_budget
	serving_system=$(run_qmicli_classify --nas-get-serving-system || true)
	log "nas serving system: $(echo "$serving_system" | /usr/bin/tr '\n' ' ' | /usr/bin/sed 's/[[:space:]]\+/ /g')"

	if qmicli_output_is_ctl_timeout "$serving_system"; then
		echo "qmi-unresponsive"
		return 0
	fi

	if echo "$serving_system" | /usr/bin/grep -q "Packet switched: 'all-calls'"; then
		echo "ps-barring"
		return 0
	fi

	if echo "$serving_system" | /usr/bin/grep -Eqi "Registration state: '(not-registered|not searching|registration-denied|unknown)'"; then
		echo "not-registered"
		return 0
	fi

	check_run_budget
	system_info=$(run_qmicli_classify --nas-get-system-info || true)
	log "nas system info: $(echo "$system_info" | /usr/bin/tr '\n' ' ' | /usr/bin/sed 's/[[:space:]]\+/ /g')"

	if qmicli_output_is_ctl_timeout "$system_info"; then
		echo "qmi-unresponsive"
		return 0
	fi

	if echo "$system_info" | /usr/bin/grep -Eqi "Service status: '(none|limited|no-service)'|Domain: 'none'"; then
		echo "not-registered"
		return 0
	fi

	echo "unknown"
}

finish_unresponsive_start()
{
	local remaining wait_budget

	remaining=$(run_budget_remaining)
	if [ "$remaining" -lt "$START_MIN_BUDGET_SEC" ]; then
		log "await_start: only ${remaining}s budget left; deferring quectel-qmi start to next cron"
		set_unresponsive_phase await_start
		log "=== qmi-failover done (qmi-unresponsive-await-start) ==="
		exit 0
	fi

	wait_budget=$((remaining - START_MIN_BUDGET_SEC))
	[ "$wait_budget" -lt 2 ] && wait_budget=2
	[ "$wait_budget" -gt 30 ] && wait_budget=30

	log "await_start: waiting up to ${wait_budget}s for $DEV"
	if ! wait_for_cdc_wdm "$wait_budget"; then
		log "await_start: $DEV still missing after GPIO reset"
		record_failure_and_maybe_reboot "qmi-unresponsive:no-device-after-gpio" "$UNRESPONSIVE_REBOOT_THRESHOLD"
		log "=== qmi-failover done (qmi-unresponsive-failed) ==="
		exit 0
	fi

	log "await_start: starting quectel-qmi after GPIO hard reset"
	/usr/bin/timeout "$STOP_START_START_TIMEOUT_SEC" /usr/lib/matelex/quectel-qmi start >> "$LOG" 2>&1 || true
	sleep 2

	if qmi_healthy; then
		reset_fail_count_if_needed "QMI recovered after GPIO hard reset"
		log "=== qmi-failover done (qmi-unresponsive-recovered) ==="
		exit 0
	fi

	if primary_network_online; then
		reset_fail_count_if_needed "primary network online after GPIO hard reset"
		log "GPIO hard reset did not restore QMI, but eth1/wlan0 is online; no reboot count"
		log "=== qmi-failover done (primary-online-after-gpio) ==="
		exit 0
	fi

	record_failure_and_maybe_reboot "qmi-unresponsive:gpio-reset-no-recovery" "$UNRESPONSIVE_REBOOT_THRESHOLD"
	log "=== qmi-failover done (qmi-unresponsive-failed) ==="
	exit 0
}

recover_qmi_unresponsive()
{
	local remaining

	log "classification 'qmi-unresponsive' (CTL/CID timeout); soft QMI/DMS recovery skipped; entering GPIO hard reset"

	if [ "$UNRESPONSIVE_PHASE" = "await_start" ]; then
		finish_unresponsive_start
	fi

	log "qmi-unresponsive: best-effort quectel-qmi stop (short timeout)"
	/usr/bin/timeout "$UNRESPONSIVE_STOP_TIMEOUT_SEC" /usr/lib/matelex/quectel-qmi stop >> "$LOG" 2>&1 || true

	if [ ! -x "$MODEMINIT" ]; then
		log "qmi-unresponsive: $MODEMINIT missing or not executable; cannot GPIO-reset"
		record_failure_and_maybe_reboot "qmi-unresponsive:no-modeminit" "$UNRESPONSIVE_REBOOT_THRESHOLD"
		log "=== qmi-failover done (qmi-unresponsive-failed) ==="
		exit 0
	fi

	remaining=$(run_budget_remaining)
	if [ "$remaining" -lt 30 ]; then
		log "qmi-unresponsive: only ${remaining}s budget left before modeminit reset; counting failure and deferring"
		record_failure_and_maybe_reboot "qmi-unresponsive:no-budget-for-gpio" "$UNRESPONSIVE_REBOOT_THRESHOLD"
		log "=== qmi-failover done (qmi-unresponsive-failed) ==="
		exit 0
	fi

	log "qmi-unresponsive: triggering GPIO hard reset via $MODEMINIT reset"
	"$MODEMINIT" reset >> "$LOG" 2>&1 || {
		log "qmi-unresponsive: modeminit reset returned non-zero"
	}

	remaining=$(run_budget_remaining)
	if [ "$remaining" -lt "$START_MIN_BUDGET_SEC" ]; then
		log "qmi-unresponsive: GPIO reset done; ${remaining}s left — deferring start to next cron (phase=await_start)"
		set_unresponsive_phase await_start
		log "=== qmi-failover done (qmi-unresponsive-await-start) ==="
		exit 0
	fi

	finish_unresponsive_start
}

recover_qmi_or_count_failure()
{
	local reason="$1"
	local tries=0

	while [ $tries -lt $MAX_TRIES_BEFORE_RESET ]; do
		check_run_budget
		tries=$((tries + 1))
		log "recoverable QMI fault '$reason': stop/start attempt $tries/$MAX_TRIES_BEFORE_RESET"
		do_stop_start
		sleep 2
		if qmi_healthy; then
			reset_fail_count_if_needed "QMI recovered after stop/start"
			log "=== qmi-failover done (recovered) ==="
			exit 0
		fi
	done

	check_run_budget
	do_dms_reset_then_wait
	if [ $? -eq 0 ]; then
		reset_fail_count_if_needed "QMI recovered after DMS reset"
		log "=== qmi-failover done (recovered) ==="
		exit 0
	fi

	check_run_budget
	if primary_network_online; then
		reset_fail_count_if_needed "primary network online after failed QMI recovery"
		log "QMI recovery failed for '$reason', but eth1/wlan0 is online; no reboot count"
		log "=== qmi-failover done (primary-online-after-recovery) ==="
		exit 0
	fi

	record_failure_and_maybe_reboot "$reason"
	log "=== qmi-failover done (failed) ==="
	exit 0
}

main()
{
	RUN_START=$(/bin/date +%s)
	log "=== qmi-failover run (pid=$$, lock=mkdir-v1) ==="

	load_state

	# Resume GPIO hard-reset flow before requiring a live cdc-wdm node.
	if [ "$UNRESPONSIVE_PHASE" = "await_start" ]; then
		log "resuming qmi-unresponsive phase=await_start"
		finish_unresponsive_start
	fi

	if [ ! -c "$DEV" ]; then
		log "no $DEV device node; cannot manage QMI, exiting"
		log "=== qmi-failover done (no-device) ==="
		return 0
	fi

	# Healthy fast path before classify (normal minutes). Ping fails → classify soon for CTL hang.
	if try_qmi_healthy_fast; then
		reset_fail_count_if_needed "QMI healthy (fast path)"
		log "=== qmi-failover done (ok) ==="
		return 0
	fi

	classification="$(classify_sim_and_registration)"
	case "$classification" in
		no-sim|sim-locked|not-registered)
			refresh_qmi_once_no_reboot "$classification"
			;;
		qmi-unresponsive)
			recover_qmi_unresponsive
			;;
		ps-barring)
			log "PS barring detected; entering QMI recovery flow"
			recover_qmi_or_count_failure "ps-barring"
			;;
		unknown)
			log "classification 'unknown'; treating as qmi-service-failure:unclassified"
			recover_qmi_or_count_failure "qmi-service-failure:unclassified"
			;;
	esac

	if [ -z "$IFACE" ]; then
		log "no QMI interface (qmi/raw_ip); classified as qmi-service-failure"
		recover_qmi_or_count_failure "qmi-service-failure:no-qmi-iface"
	fi

	# Defensive fallback if classification returned an unexpected value.
	if qmi_healthy; then
		reset_fail_count_if_needed "QMI healthy"
		log "=== qmi-failover done (ok) ==="
		return 0
	fi

	status="$(get_qmi_network_status)"
	if [ "$status" = "connected" ]; then
		recover_qmi_or_count_failure "qmi-connected-but-no-ping"
	else
		recover_qmi_or_count_failure "qmi-service-failure:status-${status:-unknown}"
	fi
}

if [ "${1:-}" = "--main" ]; then
	main
	exit 0
fi

acquire_run_slot
if ! /usr/bin/timeout -s TERM "$RUN_HARD_MAX_SEC" /bin/bash "$0" --main; then
	log "run exceeded ${RUN_HARD_MAX_SEC}s hard limit"
fi
exit 0
