=== local_identity === router-ops 2026-07-11T10:26:52Z lo UNKNOWN 127.0.0.1/8 ::1/128 eth0 UP 192.168.30.84/24 metric 100 fe80::be24:11ff:feef:ba33/64 router-ops-mgts UNKNOWN 10.250.100.4/32 default via 192.168.30.1 dev eth0 proto dhcp src 192.168.30.84 metric 100 10.71.100.0/24 dev router-ops-mgts scope link 10.200.0.0/24 dev router-ops-mgts scope link 10.201.0.0/24 dev router-ops-mgts scope link 10.250.100.0/24 dev router-ops-mgts scope link 192.168.30.0/24 dev eth0 proto kernel scope link src 192.168.30.84 metric 100 192.168.30.1 dev eth0 proto dhcp scope link src 192.168.30.84 metric 100 === pve_access_test === PVE_SSH_OK pve-wg-paid 2026-07-11T10:26:51Z -rw------- 1 root root 419 Jul 6 12:32 /root/.ssh/pve_to_openwrt_mgts_ed25519 === pve_to_vm101_access_test === VM101_SSH_OK 2026-07-11T10:26:53Z ash: hostname: not found === vm101_readonly_audit === === VM101 READONLY INSERTION AUDIT START === hostname= date_utc=2026-07-11T10:26:54Z === IDENTITY === Linux OpenWrt 6.6.93 #0 SMP Mon Jun 23 20:40:36 2025 x86_64 GNU/Linux DISTRIB_ID='OpenWrt' DISTRIB_RELEASE='24.10.2' DISTRIB_REVISION='r28739-d9340319c6' DISTRIB_TARGET='x86/64' DISTRIB_ARCH='x86_64' DISTRIB_DESCRIPTION='OpenWrt 24.10.2 r28739-d9340319c6' DISTRIB_TAINTS='' === PROCESS HEALTH / RECOVERY === 29 root 0 SW [watchdogd] 12008 root 1244 S {router-egress-h} /bin/sh /usr/local/sbin/router-egress-health-repair-watch.sh 14753 root 1204 S {router-egress-s} /bin/sh /usr/local/sbin/router-egress-slot-health.sh --dry-run 14760 root 1204 S {router-egress-s} /bin/sh /usr/local/sbin/router-egress-slot-health.sh --dry-run 30879 root 1180 S /bin/sh -c while true; do /usr/local/sbin/router-egress-slot-health.sh "--dry-run" >> "/var/log/router-eg === INIT SCRIPTS RELATED === --- init: /etc/init.d/router-egress-health-repair --- -rwxr-xr-x 1 root root 288 Jul 11 08:22 /etc/init.d/router-egress-health-repair enabled=true running=true --- init: /etc/init.d/router-egress-slot-health --- -rwxr-xr-x 1 root root 493 Jul 10 13:47 /etc/init.d/router-egress-slot-health enabled=true running=true --- init: /etc/init.d/router-egress-slots --- -rwxr-xr-x 1 root root 233 Jul 8 13:00 /etc/init.d/router-egress-slots enabled=true running=true --- init: /etc/init.d/router-egress-health-repair --- -rwxr-xr-x 1 root root 288 Jul 11 08:22 /etc/init.d/router-egress-health-repair enabled=true running=true --- init: /etc/init.d/router-egress-slot-health --- -rwxr-xr-x 1 root root 493 Jul 10 13:47 /etc/init.d/router-egress-slot-health enabled=true running=true --- init: /etc/init.d/router-egress-health-repair --- -rwxr-xr-x 1 root root 288 Jul 11 08:22 /etc/init.d/router-egress-health-repair enabled=true running=true === CRON RELATED === # STEP_029C1_DISABLED * * * * * /usr/bin/vpn-egress-manager.sh >/dev/null 2>&1 # STEP_029C1_DISABLED 20 4 * * * /root/hmn/hmn-refresh-pool-cron.sh # STEP_029C1_DISABLED 7,22,37,52 * * * * /root/hmn/hmn-pool-low-watermark-check.sh run >/dev/null 2>&1 # STEP_029C1_DISABLED 5,35 * * * * /root/hmn/hmn-refresh-retry-cron.sh # STEP_029C1_DISABLED 10 3 * * * /root/hmn/hmn-clean-old-files.sh >/dev/null 2>&1 # STEP_029C1_DISABLED * * * * * /root/hmn/hmn-vpn-user-override.sh tick >/dev/null 2>&1 20 4 * * * /root/hmn/hmn-refresh-pool-safe.sh >>/root/hmn/logs/hmn-refresh-pool-cron.log 2>&1 # STEP_049C_HMN_REFRESH_DAILY_0420_POOL_ONLY_NO_MANAGER 40 4 * * * /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh --commit --confirm REBALANCE_TOP5_DAILY >>/var/log/router-egress-rebalance-top5.log 2>&1 # STEP_049F_DAILY_TOP5_REBALANCE_AFTER_HMN_REFRESH === CORE SCRIPT MANIFEST === --- file: /usr/local/sbin/router-egress-health-repair-watch.sh --- exists=true -rwx------ 1 root root 5803 Jul 11 08:30 /usr/local/sbin/router-egress-health-repair-watch.sh 178 /usr/local/sbin/router-egress-health-repair-watch.sh d48c41014ff94cbecd6dd8a2f543b5a08c797d226c3e745756a2787340d57151 /usr/local/sbin/router-egress-health-repair-watch.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 8:MODE="${MODE:---dry-run}" 16:DISPATCHER="${DISPATCHER:-/usr/local/sbin/router-egress-recovery-dispatcher.sh}" 25: --dry-run) MODE="--dry-run"; shift ;; 26: --commit) MODE="--commit"; shift ;; 47: ping -I "$iface" -c 3 -W 2 "$target" > "$out" 2> "$err" 74: while read -r slot iface table mark dscp provider adapter rest; do 86: cooldown_file="$STATE_DIR/cooldown-${slot}" 87: fail_count="$(cat "$fail_file" 2>/dev/null || echo 0)" 88: case "$fail_count" in ''|*[!0-9]*) fail_count=0 ;; esac 90: decision="" 92: dispatcher_decision="" 93: required_confirm="" 96: fail_count=0 98: decision="healthy" 101: fail_count=$((fail_count + 1)) 102: echo "$fail_count" > "$fail_file" 104: cooldown_until="$(cat "$cooldown_file" 2>/dev/null || echo 0)" 105: case "$cooldown_until" in ''|*[!0-9]*) cooldown_until=0 ;; esac 107: if [ "$fail_count" -lt "$FAIL_THRESHOLD" ]; then 108: decision="fail_observed_below_threshold" 110: elif [ "$now" -lt "$cooldown_until" ]; then 111: decision="cooldown" 114: dry="$("$DISPATCHER" --dry-run --slot "$slot" --reason health_watch 2>/dev/null || true)" 115: required_confirm="$(printf '%s\n' "$dry" | sed -n 's/.*"required_dispatch_confirm": "\([^"]*\)".*/\1/p' | head -1)" 116: dispatcher_decision="$(printf '%s\n' "$dry" | sed -n 's/.*"decision": "\([^"]*\)".*/\1/p' | head -1)" 118: if [ "$dispatcher_decision" = "dry_run_ok" ] && [ "$MODE" = "--commit" ] && [ -n "$required_confirm" ]; then 119: commit="$("$DISPATCHER" --commit --slot "$slot" --reason health_watch --confirm "$required_confirm" 2>/dev/null || true)" 120: dispatcher_decision="$(printf '%s\n' "$commit" | sed -n 's/.*"decision": "\([^"]*\)".*/\1/p' | head -1)" 121: action="commit_dispatch" 123: echo $((now + COOLDOWN_SEC)) > "$cooldown_file" 124: [ "$dispatcher_decision" = "commit_ok" ] && echo 0 > "$fail_file" 125: decision="$dispatcher_decision" 127: action="dry_run_dispatch" 128: decision="$dispatcher_decision" 129: if [ "$MODE" = "--dry-run" ]; then 130: # Dry-run must not create real cooldown that can hide later simulation. 143: printf ' {"slot":"%s","iface":"%s","status_ok":%s,"fail_count":%s,"decision":"%s","action":"%s","required_confirm":"%s"}' \ 144: "$(json_escape "$slot")" "$(json_escape "$iface")" "$status_ok" "$fail_count" "$(json_escape "$decision")" "$(json_escape "$action")" "$(json_escape "$required_confirm")" >> "$json_slots" 146: echo "health_repair ts=$(date -Is) slot=$slot iface=$iface status_ok=$status_ok fail_count=$fail_count decision=$decision action=$action mode=$MODE run_mode=$RUN_MODE" >> "$LOG" grep_points_end --- file: /usr/local/sbin/router-egress-recovery-dispatcher.sh --- exists=true -rwx------ 1 root root 3015 Jul 11 08:17 /usr/local/sbin/router-egress-recovery-dispatcher.sh 85 /usr/local/sbin/router-egress-recovery-dispatcher.sh cfbc54d2facf4b8578654c5b3ba93901abfea02cd8a306f06e564ef15b299d85 /usr/local/sbin/router-egress-recovery-dispatcher.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 4:ADAPTER="${ADAPTER:-/usr/local/sbin/router-egress-recovery-hmn-pool-replace.sh}" 5:MODE="--dry-run" 14: --dry-run) MODE="--dry-run"; shift ;; 15: --commit) MODE="--commit"; shift ;; 16: --confirm) CONFIRM="${2:-}"; shift 2 ;; 26: echo '{"schema":"router-egress-recovery-dispatcher-v1","decision":"refuse","reason":"adapter_missing","apply_performed":false}' 30:dry_json="$("$ADAPTER" --dry-run --slot "$SLOT" 2>/dev/null || true)" 31:candidate="$(printf '%s\n' "$dry_json" | sed -n 's/.*"candidate_endpoint": "\([^"]*\)".*/\1/p' | head -1)" 33:dry_decision="$(printf '%s\n' "$dry_json" | sed -n 's/.*"decision": "\([^"]*\)".*/\1/p' | head -1)" 35:decision="dry_run_ok" 36:apply_performed=false 37:adapter_commit_json="" 39:if [ "$dry_decision" != "dry_run_ok" ] || [ -z "$candidate" ] || [ -z "$iface" ]; then 40: decision="refuse" 41: reason_out="adapter_dryrun_not_ready" 42:elif [ "$MODE" = "--dry-run" ]; then 43: decision="dry_run_ok" 44: reason_out="dispatcher_ready" 45:elif [ "$MODE" = "--commit" ]; then 46: expected="DISPATCH_${SLOT}_${candidate}" 48: decision="refuse" 49: reason_out="missing_or_wrong_dispatch_confirm" 51: adapter_confirm="APPLY_${SLOT}_${iface}_${candidate}" 52: adapter_commit_json="$("$ADAPTER" --commit --slot "$SLOT" --confirm "$adapter_confirm" 2>/dev/null || true)" 53: adapter_commit_decision="$(printf '%s\n' "$adapter_commit_json" | sed -n 's/.*"decision": "\([^"]*\)".*/\1/p' | head -1)" 54: apply_performed=true 55: if [ "$adapter_commit_decision" = "commit_ok" ]; then 56: decision="commit_ok" 57: reason_out="adapter_commit_ok" 59: decision="commit_failed" 60: reason_out="adapter_commit_failed" 64: decision="refuse" 69:echo ' "schema": "router-egress-recovery-dispatcher-v1",' 74:echo " \"candidate_endpoint\": \"$(json_escape "$candidate")\"," 75:echo " \"adapter_dryrun_decision\": \"$(json_escape "$dry_decision")\"," 76:echo " \"decision\": \"$(json_escape "$decision")\"," 78:echo " \"apply_performed\": $apply_performed," 79:echo " \"required_dispatch_confirm\": \"$(json_escape "DISPATCH_${SLOT}_${candidate}")\"," 81:echo ' "dry_run_no_uci_set": true,' 82:echo ' "dry_run_no_ifup_ifdown": true,' 83:echo ' "commit_requires_dispatch_confirm": true' grep_points_end --- file: /usr/local/sbin/router-egress-recovery-hmn-pool-replace.sh --- exists=true -rwx------ 1 root root 8847 Jul 10 18:24 /usr/local/sbin/router-egress-recovery-hmn-pool-replace.sh 245 /usr/local/sbin/router-egress-recovery-hmn-pool-replace.sh 2b4c3b0ffc5041d74619dec682666cb538dc07f226798570c8564faf6afdd1d9 /usr/local/sbin/router-egress-recovery-hmn-pool-replace.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 7:MODE="${MODE:---dry-run}" 14: --dry-run) MODE="--dry-run"; shift ;; 15: --commit) MODE="--commit"; shift ;; 16: --confirm) CONFIRM="${2:-}"; shift 2 ;; 26:PREFERRED_POOL_FILES="${PREFERRED_POOL_FILES:-ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv working-awg1-latest.tsv selected-awg1-latest.tsv}" 36:extract_endpoints() { 45:endpoint_count() { 47: extract_endpoints "$f" | wc -l 50:current_endpoint_for_iface() { 52: ep="$(uci -q get network.${iface}.hmn_endpoint 2>/dev/null || true)" 54: host="$(uci -q get network.awg_${iface}.endpoint_host 2>/dev/null || uci -q get network.amneziawg_${iface}.endpoint_host 2>/dev/null || true)" 55: port="$(uci -q get network.awg_${iface}.endpoint_port 2>/dev/null || uci -q get network.amneziawg_${iface}.endpoint_port 2>/dev/null || true)" 61:strict_ping() { 65: ping -I "$iface" -c 3 -W 2 "$target" >/tmp/hmn-recovery-ping.out 2>/tmp/hmn-recovery-ping.err 67: recv="$(grep -Eo '[0-9]+ packets received' /tmp/hmn-recovery-ping.out 2>/dev/null | awk '{print $1}' | tail -1)" 71: rm -f /tmp/hmn-recovery-ping.out /tmp/hmn-recovery-ping.err 76:iface=""; table=""; mark=""; dscp=""; provider=""; adapter=""; reason="" 85: adapter="$(echo "$slot_line" | awk '{print $7}')" 90:current_ep="$( [ -n "$iface" ] && current_endpoint_for_iface "$iface" || true )" 93:pool_file="/tmp/router-egress-hmn-pool.$$" 95:trap 'rm -f "$used_file" "$pool_file" "$unused_file"' EXIT 99: ep="$(current_endpoint_for_iface "$i")" 104:selected_pool="" 105:selected_pool_age_sec=999999999 106:selected_pool_endpoint_count=0 110: epc="$(endpoint_count "$f")" 112: selected_pool="$f" 114: selected_pool_age_sec=$((now - mt)) 115: selected_pool_endpoint_count="$epc" 119:candidate="" 120:candidate_count=0 121:pool_is_fresh=false 122:if [ -n "$selected_pool" ]; then 123: [ "$selected_pool_age_sec" -le "$MAX_POOL_AGE_SEC" ] && pool_is_fresh=true || pool_is_fresh=false 124: extract_endpoints "$selected_pool" > "$pool_file" 125: awk 'NR==FNR {used[$0]=1; next} $0 != "" && !($0 in used) {print $0}' "$used_file" "$pool_file" > "$unused_file" 126: candidate_count="$(wc -l < "$unused_file" 2>/dev/null || echo 0)" 127: [ "$pool_is_fresh" = "true" ] && candidate="$(head -1 "$unused_file" 2>/dev/null || true)" 130:decision="refuse" 131:apply_performed=false 132:apply_rc=0 133:post_strict_ok=false 134:rollback_file="" 138:elif [ "$provider" != "hidemyname" ] || [ "$adapter" != "hmn_pool_replace" ]; then 139: reason="wrong_provider_or_adapter" 140:elif [ -z "$selected_pool" ]; then 141: reason="no_pool_file" 142:elif [ "$pool_is_fresh" != "true" ]; then 143: reason="stale_pool" 144:elif [ "$candidate_count" = "0" ] || [ -z "$candidate" ]; then 145: reason="no_unused_candidate" 146:elif [ "$MODE" = "--dry-run" ]; then 147: decision="dry_run_ok" 148: reason="dry_run_candidate_selected" 149:elif [ "$MODE" = "--commit" ]; then 150: expected="APPLY_${SLOT}_${iface}_${candidate}" 152: decision="refuse" 153: reason="missing_or_wrong_confirm_token" 157: uci show network > "$backup_dir/network.uci.before" 2>/dev/null || true 159: current_before="$current_ep" 161: rollback_file="${backup_dir}/rollback-${SLOT}.sh" 162: cat > "$rollback_file" </dev/null 2>&1 || true 168:ifup ${iface} >/dev/null 2>&1 || true 170:if [ -x /usr/local/sbin/router-egress-slots-apply.sh ]; then 171: /usr/local/sbin/router-egress-slots-apply.sh >/dev/null 2>&1 || true 173:echo "rollback_done=true" 175: chmod 700 "$rollback_file" 177: uci set network.${iface}.hmn_endpoint="$candidate" 178: uci commit network 179: ifdown "$iface" >/dev/null 2>&1 || true 180: ifup "$iface" >/dev/null 2>&1 181: apply_rc=$? 184: if [ -x /usr/local/sbin/router-egress-slots-apply.sh ]; then 185: /usr/local/sbin/router-egress-slots-apply.sh >/dev/null 2>&1 || true 188: apply_performed=true 189: if [ "$apply_rc" = "0" ] && ip link show "$iface" >/dev/null 2>&1 && ip route show table "$table" 2>/dev/null | grep -q . && strict_ping "$iface"; then 190: decision="commit_ok" 191: reason="candidate_applied_and_strict_ok" 192: post_strict_ok=true 194: decision="commit_failed" 195: reason="candidate_apply_or_strict_failed" 196: post_strict_ok=false 203:out="${STATE_DIR}/hmn-pool-replace-last.json" 206: echo ' "schema": "router-egress-recovery-hmn-pool-replace-v2",' 216: echo " \"adapter\": \"$(json_escape "$adapter")\"," 217: echo " \"current_endpoint\": \"$(json_escape "$current_ep")\"," 218: echo " \"selected_pool\": \"$(json_escape "$selected_pool")\"," 219: echo " \"selected_pool_age_sec\": $selected_pool_age_sec," 220: echo " \"max_pool_age_sec\": $MAX_POOL_AGE_SEC," 221: echo " \"selected_pool_endpoint_count\": $selected_pool_endpoint_count," 222: echo " \"pool_is_fresh\": $pool_is_fresh," 223: echo " \"unused_candidate_count\": $candidate_count," 224: echo " \"candidate_endpoint\": \"$(json_escape "$candidate")\"," 225: echo " \"decision\": \"$(json_escape "$decision")\"," 227: echo " \"apply_performed\": $apply_performed," 228: echo " \"apply_rc\": $apply_rc," 229: echo " \"post_strict_ok\": $post_strict_ok," 230: echo " \"rollback_file\": \"$(json_escape "$rollback_file")\"," 232: echo " \"requires_explicit_commit\": $([ "$REQUIRE_EXPLICIT_COMMIT" = "1" ] && echo true || echo false)," 233: echo ' "dry_run_no_ifup_ifdown": true,' 234: echo ' "dry_run_no_uci_set": true,' 235: echo ' "dry_run_no_wg_set": true,' 236: echo ' "dry_run_no_route_change": true,' 237: echo ' "no_vm100_change": true' 243:echo "router-egress-recovery-hmn ts=$iso slot=$SLOT mode=$MODE decision=$decision reason=$reason candidate=$candidate apply=$apply_performed rollback=$rollback_file" >> "$LOG" grep_points_end --- file: /usr/local/sbin/router-egress-hmn-plan-top5.sh --- exists=true -rwx------ 1 root root 4875 Jul 11 08:17 /usr/local/sbin/router-egress-hmn-plan-top5.sh 134 /usr/local/sbin/router-egress-hmn-plan-top5.sh 001e099adbb23035cfdad63c7d5821fc8854f1be552cc7ff46f42e1d1ceaef47 /usr/local/sbin/router-egress-hmn-plan-top5.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 7:PREFERRED_POOL_FILES="${PREFERRED_POOL_FILES:-ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv}" 8:MODE="--dry-run" 12: --dry-run) MODE="--dry-run"; shift ;; 20:pool="" 21:pool_age=999999999 22:pool_endpoints=0 27: # strict TSV contract: rank, file, endpoint, avg_ms, ping_loss, config_path 33: pool="$f" 34: pool_age="$age" 35: pool_endpoints="$epc" 40:tmp_current="/tmp/hmn-plan-current.$$" 43:trap 'rm -f "$tmp_top" "$tmp_current" "$tmp_missing" "$tmp_plan"' EXIT 46:: > "$tmp_current" 50:if [ -n "$pool" ]; then 54: rank=$1; file=$2; ep=$3; avg=$4; loss=$5; cfg=$6 57: print rank "\t" avg "\t" ep "\t" file "\t" loss "\t" cfg 60: ' "$pool" | sort -n -k1,1 | head -5 > "$tmp_top" 63:# current slots in slot order 64:grep -Ev '^[[:space:]]*(#|$)' "$SLOTS_CONF" 2>/dev/null | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | while IFS="$(printf '\t')" read -r slot iface table mark dscp provider adapter; do 65: cur="$(uci -q get network.${iface}.hmn_endpoint 2>/dev/null || true)" 66: printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$slot" "$iface" "$table" "$mark" "$dscp" "$provider" "$adapter" "$cur" 67:done > "$tmp_current" 69:# top endpoints missing from current assignment 73:' "$tmp_current" "$tmp_top" > "$tmp_missing" 76:# - If current endpoint is already in top5, keep it on the same slot to minimize churn. 77:# - If current endpoint is not in top5, replace it with next missing top endpoint. 81: top_rank[$3]=$1 88: missing_rank[m]=$1 94: slot=$1; iface=$2; table=$3; mark=$4; dscp=$5; provider=$6; adapter=$7; cur=$8 96: target=cur; change="false"; rank=top_rank[cur]; avg=top_avg[cur]; file=top_file[cur] 99: target=missing[mi]; change="true"; rank=missing_rank[mi]; avg=missing_avg[mi]; file=missing_file[mi] 101: if (target == "") { target=cur; change="false"; rank=""; avg=""; file="" } 102: print slot "\t" iface "\t" table "\t" mark "\t" dscp "\t" provider "\t" adapter "\t" cur "\t" target "\t" change "\t" rank "\t" avg "\t" file 104:' "$tmp_top" "$tmp_missing" "$tmp_current" > "$tmp_plan" 106:changes_count="$(awk -F '\t' '$10=="true"{n++} END{print n+0}' "$tmp_plan")" 108:decision="plan_ok" 109:reason="top5_pool_ready" 110:[ -n "$pool" ] || { decision="refuse"; reason="no_fresh_pool_with_5_endpoints"; } 115:echo " \"decision\": \"$(json_escape "$decision")\"," 117:echo " \"pool\": \"$(json_escape "$pool")\"," 118:echo " \"pool_age_sec\": $pool_age," 119:echo " \"pool_endpoint_count\": $pool_endpoints," 121:echo " \"changes_count\": $changes_count," 124:while IFS="$(printf '\t')" read -r slot iface table mark dscp provider adapter cur target change rank avg file; do 127: printf ' {"slot":"%s","iface":"%s","table":"%s","mark":"%s","dscp":"%s","provider":"%s","adapter":"%s","current":"%s","target":"%s","change":%s,"target_rank":"%s","target_avg_ms":"%s","target_file":"%s"}' \ 129: "$(json_escape "$provider")" "$(json_escape "$adapter")" "$(json_escape "$cur")" "$(json_escape "$target")" "$change" "$(json_escape "$rank")" "$(json_escape "$avg")" "$(json_escape "$file")" 133:echo ' "safety": {"dry_run_no_uci_set": true, "dry_run_no_ifup_ifdown": true, "dry_run_no_route_change": true}' grep_points_end --- file: /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh --- exists=true -rwx------ 1 root root 7723 Jul 11 08:22 /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh 213 /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh 2fa6b725c5e123b7fdc3dd14747058a02415d025231054c7537d72b8edc2346f /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 7:LOG="${LOG:-/var/log/router-egress-rebalance-top5.log}" 9:PREFERRED_POOL_FILES="${PREFERRED_POOL_FILES:-ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv}" 11:MODE="--dry-run" 16: --dry-run) MODE="--dry-run"; shift ;; 17: --commit) MODE="--commit"; shift ;; 18: --confirm) CONFIRM="${2:-}"; shift 2 ;; 27:strict_ping() { 31: ping -I "$iface" -c 3 -W 2 "$target" >/tmp/rebalance-ping.out 2>/tmp/rebalance-ping.err 33: recv="$(grep -Eo '[0-9]+ packets received' /tmp/rebalance-ping.out 2>/dev/null | awk '{print $1}' | tail -1)" 37: rm -f /tmp/rebalance-ping.out /tmp/rebalance-ping.err 42:pool="" 43:pool_age=999999999 44:pool_endpoints=0 54: pool="$f" 55: pool_age="$age" 56: pool_endpoints="$epc" 60:tmp_top="/tmp/rebalance-top.$$" 61:tmp_current="/tmp/rebalance-current.$$" 62:tmp_missing="/tmp/rebalance-missing.$$" 63:tmp_plan="/tmp/rebalance-plan.$$" 64:trap 'rm -f "$tmp_top" "$tmp_current" "$tmp_missing" "$tmp_plan"' EXIT 67:: > "$tmp_current" 71:if [ -n "$pool" ]; then 75: rank=$1; file=$2; ep=$3; avg=$4; loss=$5; cfg=$6 78: print rank "\t" avg "\t" ep "\t" file "\t" loss "\t" cfg 81: ' "$pool" | sort -n -k1,1 | head -5 > "$tmp_top" 84:grep -Ev '^[[:space:]]*(#|$)' "$SLOTS_CONF" 2>/dev/null | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | while IFS="$(printf '\t')" read -r slot iface table mark dscp provider adapter; do 85: cur="$(uci -q get network.${iface}.hmn_endpoint 2>/dev/null || true)" 86: printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$slot" "$iface" "$table" "$mark" "$dscp" "$provider" "$adapter" "$cur" 87:done > "$tmp_current" 92:' "$tmp_current" "$tmp_top" > "$tmp_missing" 97: top_rank[$3]=$1 104: missing_rank[m]=$1 110: slot=$1; iface=$2; table=$3; mark=$4; dscp=$5; provider=$6; adapter=$7; cur=$8 112: target=cur; change="false"; rank=top_rank[cur]; avg=top_avg[cur]; file=top_file[cur] 115: target=missing[mi]; change="true"; rank=missing_rank[mi]; avg=missing_avg[mi]; file=missing_file[mi] 117: if (target == "") { target=cur; change="false"; rank=""; avg=""; file="" } 118: print slot "\t" iface "\t" table "\t" mark "\t" dscp "\t" provider "\t" adapter "\t" cur "\t" target "\t" change "\t" rank "\t" avg "\t" file 120:' "$tmp_top" "$tmp_missing" "$tmp_current" > "$tmp_plan" 122:changes_count="$(awk -F '\t' '$10=="true"{n++} END{print n+0}' "$tmp_plan")" 124:decision="plan_ok" 125:reason="top5_pool_ready" 126:apply_performed=false 127:apply_ok=true 129:[ -n "$pool" ] || { decision="refuse"; reason="no_fresh_pool_with_5_endpoints"; } 130:[ "$top_count" = "5" ] || { decision="refuse"; reason="top_count_not_5"; } 132:if [ "$MODE" = "--commit" ]; then 134: decision="refuse" 135: reason="missing_or_wrong_confirm" 136: elif [ "$decision" = "plan_ok" ]; then 137: if [ "$changes_count" = "0" ]; then 138: decision="noop" 140: apply_performed=false 142: decision="commit_ok" 143: reason="changes_applied" 144: while IFS="$(printf '\t')" read -r slot iface table mark dscp provider adapter cur target change rank avg file; do 145: [ "$change" = "true" ] || continue 146: apply_performed=true 147: backup_dir="${STATE_DIR}/rebalance-${slot}-$(date +%Y%m%d-%H%M%S)" 149: uci show network > "$backup_dir/network.uci.before" 2>/dev/null || true 151: cat > "$backup_dir/rollback-${slot}.sh" </dev/null 2>&1 || true 156:ifup ${iface} >/dev/null 2>&1 || true 158:[ -x /usr/local/sbin/router-egress-slots-apply.sh ] && /usr/local/sbin/router-egress-slots-apply.sh >/dev/null 2>&1 || true 159:echo rollback_done=true 161: chmod 700 "$backup_dir/rollback-${slot}.sh" 163: uci set network.${iface}.hmn_endpoint="$target" 164: uci commit network 165: ifdown "$iface" >/dev/null 2>&1 || true 166: ifup "$iface" >/dev/null 2>&1 169: [ -x /usr/local/sbin/router-egress-slots-apply.sh ] && /usr/local/sbin/router-egress-slots-apply.sh >/dev/null 2>&1 || true 171: if [ "$rc" != "0" ] || ! ip route show table "$table" 2>/dev/null | grep -q . || ! strict_ping "$iface"; then 172: apply_ok=false 173: decision="commit_failed" 174: reason="slot_apply_failed_${slot}" 175: "$backup_dir/rollback-${slot}.sh" >/dev/null 2>&1 || true 181:elif [ "$MODE" = "--dry-run" ]; then 184: decision="refuse" 189:echo ' "schema": "router-egress-hmn-rebalance-top5-v1",' 191:echo " \"decision\": \"$(json_escape "$decision")\"," 193:echo " \"pool\": \"$(json_escape "$pool")\"," 194:echo " \"pool_age_sec\": $pool_age," 195:echo " \"pool_endpoint_count\": $pool_endpoints," 197:echo " \"changes_count\": $changes_count," 198:echo " \"apply_performed\": $apply_performed," 199:echo " \"apply_ok\": $apply_ok," 202:while IFS="$(printf '\t')" read -r slot iface table mark dscp provider adapter cur target change rank avg file; do 205: printf ' {"slot":"%s","iface":"%s","table":"%s","mark":"%s","dscp":"%s","provider":"%s","adapter":"%s","current":"%s","target":"%s","change":%s,"target_rank":"%s","target_avg_ms":"%s","target_file":"%s"}' \ 207: "$(json_escape "$provider")" "$(json_escape "$adapter")" "$(json_escape "$cur")" "$(json_escape "$target")" "$change" "$(json_escape "$rank")" "$(json_escape "$avg")" "$(json_escape "$file")" 211:echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' 213:echo "rebalance_top5 ts=$(date -Is) mode=$MODE decision=$decision reason=$reason changes=$changes_count apply=$apply_performed" >> "$LOG" grep_points_end --- file: /usr/local/sbin/router-egress-slot-health.sh --- exists=true -rwx------ 1 root root 7247 Jul 10 13:47 /usr/local/sbin/router-egress-slot-health.sh 192 /usr/local/sbin/router-egress-slot-health.sh 5c32cb9c3182f522e2e3e5971443e9b891d8848c476c1957c94f0a185f05f94c /usr/local/sbin/router-egress-slot-health.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 4:MODE="${1:---dry-run}" 41:grep -Ev '^[[:space:]]*(#|$)' "$SLOTS_CONF" | while read -r slot_id iface table mark dscp provider repair_adapter health_targets strict_count strict_timeout enabled rest; do 49: endpoint="" 59: endpoint="$(wg show "$iface" endpoints 2>/dev/null | sed -E 's#([A-Za-z0-9+/=]{20,})#KEYMASK#g' | tr '\n' ';' | sed 's/;$//')" 87: ping -I "$iface" -c "$strict_count" -W "$strict_timeout" "$target" > "$out" 2> "$err" 96: if [ "$rc" != "0" ] || [ "$recv" != "$strict_count" ]; then 98: fail_reasons="${fail_reasons}${target}:rc=${rc}:recv=${recv}_expected_${strict_count};" 113: "$slot_id" "$iface" "$table" "$mark" "$dscp" "$provider" "$repair_adapter" "$health_targets" \ 114: "$strict_count" "$strict_timeout" "$iface_exists" "$operstate" "$route_ok" "$status" "$target_results" "$fail_reasons" >> "$tmp" 139: while IFS="$(printf '\t')" read -r slot_id iface table mark dscp provider repair_adapter health_targets strict_count strict_timeout iface_exists operstate route_ok status target_results fail_reasons; do 142: printf ' {"slot_id":"%s","interface":"%s","table":"%s","mark":"%s","dscp":"%s","provider":"%s","repair_adapter":"%s","health_targets":"%s","strict_count":%s,"strict_timeout":%s,"iface_exists":%s,"operstate":"%s","route_ok":%s,"status":"%s","target_results":"%s","fail_reasons":"%s"}' \ 144: "$(json_escape "$dscp")" "$(json_escape "$provider")" "$(json_escape "$repair_adapter")" "$(json_escape "$health_targets")" \ 145: "$strict_count" "$strict_timeout" "$iface_exists" "$(json_escape "$operstate")" "$route_ok" "$(json_escape "$status")" \ 153: echo ' "repair_layer": "separate_provider_adapter",' 154: echo ' "apply_performed": false,' 155: echo ' "apply_reason": "dry_run_status_only"' 175: while IFS="$(printf '\t')" read -r slot_id iface table mark dscp provider repair_adapter health_targets strict_count strict_timeout iface_exists operstate route_ok status target_results fail_reasons; do 181: echo "slot.${slot_id}.repair_adapter=$repair_adapter" 190:echo "router-egress-slot-health ts=$iso mode=$MODE overall_ok=$overall_ok good=$good_count bad=$bad_count enabled=$enabled_count apply_performed=false" >> "$LOG" grep_points_end --- file: /root/hmn/hmn-refresh-pool-safe.sh --- exists=true -rwx------ 1 root root 8670 Jul 10 17:54 /root/hmn/hmn-refresh-pool-safe.sh 308 /root/hmn/hmn-refresh-pool-safe.sh bb047e820196c4c5cb263499ed890c782190293257a2033568015c6b19230682 /root/hmn/hmn-refresh-pool-safe.sh syntax_check_begin syntax_ok=true syntax_check_end grep_points_begin 11:VALIDATE="$BASE/hmn-validate-current-pool.sh" 14:LOCK="/tmp/hmn-refresh-pool-safe.lock" 21:LOG="$LOGDIR/refresh-pool-safe-$TS-$$.log" 22:RCFILE="/tmp/hmn-refresh-pool-safe.rc.$$" 26:RETRY_FLAG="$STATE/refresh-retry-needed" 27:RETRY_REASON="$STATE/refresh-retry-reason" 30:LAST_REFRESH_POOL_SOURCE="$STATE/last-refresh-pool-source" 37: echo "$1" > "$STATE/refresh-status" 38: date -Iseconds > "$STATE/last-refresh-status-at" 60: say "rollback: no backup dir available" 64: say "rollback: restoring published pointers/tables from $BACK" 73: say "rollback: latest symlink restored to readlink target: $OLD_LINK" 77: say "rollback: latest symlink restored to realpath: $OLD_REAL" 79: say "rollback WARN: no usable old latest target" 83: ok-awg1-strict-all-latest.tsv \ 84: ok-awg1-strict-foreign-latest.tsv \ 89: say "rollback: restored cache/$BN" 96:run_validate_current_pool() { 105: say "validate_current_pool_rc=$VALIDATE_RC" 116: say "=== hmn-refresh-pool-safe start ===" 133: echo "=== lock ===" 135: say "another refresh instance is already running: $LOCK" 149: echo "=== backup current published pointers/tables ===" 150: BACK="$BACKUPDIR/refresh-safe-before-$TS-$$" 157: "$BASE/cache/ok-awg1-strict-all-latest.tsv" \ 158: "$BASE/cache/ok-awg1-strict-foreign-latest.tsv" \ 202: echo "=== validation decision ===" 205: say "validate fresh/current latest pool through vpn_test" 207: if run_validate_current_pool; then 213: say "fresh pool validation failed; rolling back to previous published pool" 216: echo "rollback-after-fresh-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 222: say "fallback: keep old configs/latest and validate current local pool through vpn_test" 224: if run_validate_current_pool; then 234: say "old/current pool validation failed; rolling back published tables" 237: echo "rollback-after-old-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 239: set_retry_needed "fresh_download_failed_and_old_pool_validation_failed_rc_$DOWNLOAD_RC" 248: echo "STEP_048M: legacy manager stage quarantined; skipping $MAN" 271: echo "=== refresh status ===" 272: cat "$STATE/refresh-status" 2>/dev/null || true 273: cat "$STATE/last-refresh-status-at" 2>/dev/null || true 274: echo "pool_source=$(cat "$LAST_REFRESH_POOL_SOURCE" 2>/dev/null || echo unknown)" 281: echo "=== strict foreign head ===" 282: head -n 12 "$BASE/cache/ok-awg1-strict-foreign-latest.tsv" 2>/dev/null || echo "missing" 290: BAD="$STATE/bad-endpoints-$(date +%Y%m%d).txt" 293: say "=== hmn-refresh-pool-safe done ===" grep_points_end === CONFIG MANIFEST === --- config: /etc/router-egress-health-repair.conf --- exists=true -rw------- 1 root root 318 Jul 11 08:22 /etc/router-egress-health-repair.conf ENABLED=1 MODE=--commit INTERVAL_SEC=60 FAIL_THRESHOLD=2 COOLDOWN_SEC=900 PING_TARGETS="1.1.1.1 8.8.8.8" STATE_DIR=/var/lib/router-egress-recovery/health-watch LOG=/var/log/router-egress-health-repair.log SLOTS_CONF=/etc/router-egress-slots.d/slots.conf DISPATCHER=/usr/local/sbin/router-egress-recovery-dispatcher.sh --- config: /etc/router-egress-recovery-hmn.conf --- exists=true -rw------- 1 root root 390 Jul 10 18:24 /etc/router-egress-recovery-hmn.conf MODE=--dry-run SLOTS_CONF=/etc/router-egress-slots.d/slots.conf HMN_CACHE_DIR=/root/hmn/cache MAX_POOL_AGE_SEC=129600 PREFERRED_POOL_FILES="ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv working-awg1-latest.tsv selected-awg1-latest.tsv" STATE_DIR=/var/lib/router-egress-recovery LOG=/var/log/router-egress-recovery-hmn.log REQUIRE_EXPLICIT_COMMIT=1 POST_APPLY_SLEEP_SEC=12 --- config: /etc/router-egress-slot-health.conf --- exists=true -rw------- 1 root root 334 Jul 10 13:47 /etc/router-egress-slot-health.conf MODE=--dry-run INTERVAL=60 SLOTS_CONF=/etc/router-egress-slots.d/slots.conf STATE_DIR=/var/lib/router-egress-slot-health STATE_JSONL=/var/lib/router-egress-slot-health/status.jsonl STATUS_KV=/var/lib/router-egress-slot-health/status.kv LAST_JSON=/var/lib/router-egress-slot-health/last.json LOG=/var/log/router-egress-slot-health.log --- config: /etc/router-egress-slots.d/slots.conf --- exists=true -rw------- 1 root root 538 Jul 10 13:47 /etc/router-egress-slots.d/slots.conf # schema=router-egress-slots-v1 # fields: # slot_id interface table mark dscp provider repair_adapter health_targets strict_count strict_timeout enabled egress1 vpn1 201 0x201 cs4 hidemyname hmn_pool_replace 1.1.1.1,8.8.8.8 3 2 1 egress2 vpn2 202 0x202 cs5 hidemyname hmn_pool_replace 1.1.1.1,8.8.8.8 3 2 1 egress3 vpn3 203 0x203 cs1 hidemyname hmn_pool_replace 1.1.1.1,8.8.8.8 3 2 1 egress4 vpn4 204 0x204 cs2 hidemyname hmn_pool_replace 1.1.1.1,8.8.8.8 3 2 1 egress5 vpn5 205 0x205 cs3 hidemyname hmn_pool_replace 1.1.1.1,8.8.8.8 3 2 1 === EXISTING RECOVERY STATE TREE === --- tree: /var/lib/router-egress-recovery --- /var/lib/router-egress-recovery/backup-egress2-20260711-091232/network.uci.before /var/lib/router-egress-recovery/backup-egress2-20260711-091232/rollback-egress2.sh /var/lib/router-egress-recovery/backup-egress2-20260711-091232/table.before /var/lib/router-egress-recovery/health-watch/cooldown-egress2 /var/lib/router-egress-recovery/health-watch/fail-egress1 /var/lib/router-egress-recovery/health-watch/fail-egress2 /var/lib/router-egress-recovery/health-watch/fail-egress3 /var/lib/router-egress-recovery/health-watch/fail-egress4 /var/lib/router-egress-recovery/health-watch/fail-egress5 /var/lib/router-egress-recovery/health-watch/last.err /var/lib/router-egress-recovery/health-watch/last.json /var/lib/router-egress-recovery/hmn-pool-replace-last.json --- statefile: /var/lib/router-egress-recovery/backup-egress2-20260711-091232/network.uci.before --- -rw-r--r-- 1 root root 10480 Jul 11 09:12 /var/lib/router-egress-recovery/backup-egress2-20260711-091232/network.uci.before network.loopback=interface network.loopback.device='lo' network.loopback.proto='static' network.loopback.ipaddr='127.0.0.1' network.loopback.netmask='255.0.0.0' network.globals=globals network.globals.packet_steering='1' network.wan=interface network.wan.device='eth0' network.wan.proto='dhcp' network.wan.peerdns='0' network.wan.dns='1.1.1.1' '8.8.8.8' network.mgmt=interface network.mgmt.proto='static' network.mgmt.device='eth2' network.mgmt.ipaddr='10.71.100.2' network.mgmt.netmask='255.255.255.0' network.direct_in=interface network.direct_in.proto='static' network.direct_in.device='eth3' network.direct_in.ipaddr='10.201.0.2' network.direct_in.netmask='255.255.255.0' network.vpn_in=interface network.vpn_in.proto='static' network.vpn_in.device='eth1' network.vpn_in.ipaddr='10.200.0.2/24' network.@route[0]=route network.@route[0].interface='mgmt' network.@route[0].target='10.250.100.0/24' network.@route[0].gateway='10.71.100.1' network.vpn1=interface network.vpn1.proto='amneziawg' network.vpn1.private_key='CF/y7ER2K59rSMWf9G90u10yQlWfKu1WiE/zYcrY7EI=' network.vpn1.awg_jc='3' network.vpn1.awg_jmin='50' network.vpn1.awg_jmax='100' network.vpn1.awg_s1='92' network.vpn1.awg_s2='90' network.vpn1.awg_h1='1128827374' network.vpn1.awg_h2='1959569870' network.vpn1.awg_h3='96441901' network.vpn1.awg_h4='2091878015' network.vpn1.auto='0' network.vpn1.disabled='0' network.vpn1.delegate='0' network.vpn1.peerdns='0' network.vpn1.defaultroute='0' network.vpn1.addresses='10.91.233.132/32' network.vpn1.dns='1.1.1.1' network.vpn1.hmn_role='active_spare_slot' network.vpn1.hmn_source_config='/root/hmn/configs/awg1/latest/150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf' network.vpn1.hmn_loaded_at='2026-07-06T17:07:53+00:00' network.vpn1.hmn_endpoint='178.215.227.13:44408' network.@amneziawg_vpn1[0]=amneziawg_vpn1 network.@amneziawg_vpn1[0].description='150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf' network.@amneziawg_vpn1[0].public_key='CAZ2FgESKM67A3XofKNoftcFcy9mtaVga8t/3qcCABI=' network.@amneziawg_vpn1[0].allowed_ips='0.0.0.0/0' network.@amneziawg_vpn1[0].route_allowed_ips='0' network.@amneziawg_vpn1[0].persistent_keepalive='23' network.@amneziawg_vpn1[0].endpoint_host='178.215.227.13' network.@amneziawg_vpn1[0].endpoint_port='44408' network.vpn_user=interface network.vpn_user.proto='amneziawg' network.vpn_user.private_key='AINDvaqL8gOQMSD5UEp1ghPgOHmmTiBj9LirFUWTCl4=' network.vpn_user.awg_jc='3' network.vpn_user.awg_jmin='50' network.vpn_user.awg_jmax='100' network.vpn_user.awg_s1='20' network.vpn_user.awg_s2='87' network.vpn_user.awg_h1='217767672' network.vpn_user.awg_h2='1805649910' network.vpn_user.awg_h3='665837737' network.vpn_user.awg_h4='1257888244' network.vpn_user.auto='0' network.vpn_user.disabled='0' network.vpn_user.delegate='0' network.vpn_user.peerdns='0' network.vpn_user.defaultroute='0' network.vpn_user.addresses='10.90.235.244/32' network.vpn_user.dns='1.1.1.1' --- statefile: /var/lib/router-egress-recovery/backup-egress2-20260711-091232/rollback-egress2.sh --- -rwx------ 1 root root 334 Jul 11 09:12 /var/lib/router-egress-recovery/backup-egress2-20260711-091232/rollback-egress2.sh #!/bin/sh set -u uci set network.vpn2.hmn_endpoint='192.121.163.235:42530' uci commit network ifdown vpn2 >/dev/null 2>&1 || true ifup vpn2 >/dev/null 2>&1 || true sleep 12 if [ -x /usr/local/sbin/router-egress-slots-apply.sh ]; then /usr/local/sbin/router-egress-slots-apply.sh >/dev/null 2>&1 || true fi echo "rollback_done=true" --- statefile: /var/lib/router-egress-recovery/backup-egress2-20260711-091232/table.before --- -rw-r--r-- 1 root root 0 Jul 11 09:12 /var/lib/router-egress-recovery/backup-egress2-20260711-091232/table.before --- statefile: /var/lib/router-egress-recovery/health-watch/cooldown-egress2 --- -rw-r--r-- 1 root root 11 Jul 11 09:12 /var/lib/router-egress-recovery/health-watch/cooldown-egress2 1783762052 --- statefile: /var/lib/router-egress-recovery/health-watch/fail-egress1 --- -rw-r--r-- 1 root root 2 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/fail-egress1 0 --- statefile: /var/lib/router-egress-recovery/health-watch/fail-egress2 --- -rw-r--r-- 1 root root 2 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/fail-egress2 0 --- statefile: /var/lib/router-egress-recovery/health-watch/fail-egress3 --- -rw-r--r-- 1 root root 2 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/fail-egress3 0 --- statefile: /var/lib/router-egress-recovery/health-watch/fail-egress4 --- -rw-r--r-- 1 root root 2 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/fail-egress4 0 --- statefile: /var/lib/router-egress-recovery/health-watch/fail-egress5 --- -rw-r--r-- 1 root root 2 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/fail-egress5 0 --- statefile: /var/lib/router-egress-recovery/health-watch/last.err --- -rw-r--r-- 1 root root 0 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/last.err --- statefile: /var/lib/router-egress-recovery/health-watch/last.json --- -rw-r--r-- 1 root root 789 Jul 11 10:26 /var/lib/router-egress-recovery/health-watch/last.json { "schema": "router-egress-health-repair-watch-v2", "mode": "--commit", "run_mode": "loop", "slots": [ {"slot":"egress1","iface":"vpn1","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress2","iface":"vpn2","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress3","iface":"vpn3","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress4","iface":"vpn4","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress5","iface":"vpn5","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""} ], "any_action": false } --- statefile: /var/lib/router-egress-recovery/hmn-pool-replace-last.json --- -rw-r--r-- 1 root root 1014 Jul 11 09:20 /var/lib/router-egress-recovery/hmn-pool-replace-last.json { "schema": "router-egress-recovery-hmn-pool-replace-v2", "mode": "--dry-run", "epoch": 1783761608, "iso": "2026-07-11T09:20:08+00:00", "slot": "egress2", "interface": "vpn2", "table": "202", "mark": "0x202", "dscp": "cs5", "provider": "hidemyname", "adapter": "hmn_pool_replace", "current_endpoint": "109.248.198.4:37129", "selected_pool": "/root/hmn/cache/ok-awg1-strict-foreign-latest.tsv", "selected_pool_age_sec": 58866, "max_pool_age_sec": 129600, "selected_pool_endpoint_count": 15, "pool_is_fresh": true, "unused_candidate_count": 10, "candidate_endpoint": "151.236.30.139:33998", "decision": "dry_run_ok", "reason": "dry_run_candidate_selected", "apply_performed": false, "apply_rc": 0, "post_strict_ok": false, "rollback_file": "", "safety": { "requires_explicit_commit": true, "dry_run_no_ifup_ifdown": true, "dry_run_no_uci_set": true, "dry_run_no_wg_set": true, "dry_run_no_route_change": true, "no_vm100_change": true } } --- tree: /var/lib/router-egress-slot-health --- /var/lib/router-egress-slot-health/last.json /var/lib/router-egress-slot-health/status.jsonl /var/lib/router-egress-slot-health/status.kv --- statefile: /var/lib/router-egress-slot-health/last.json --- -rw-r--r-- 1 root root 2445 Jul 11 10:25 /var/lib/router-egress-slot-health/last.json { "schema": "router-egress-slot-health-v1", "mode": "--dry-run", "epoch": 1783765525, "iso": "2026-07-11T10:25:25+00:00", "slots_conf": "/etc/router-egress-slots.d/slots.conf", "overall_ok": true, "enabled_count": 5, "good_count": 5, "bad_count": 0, "missing_count": 0, "slots": [ {"slot_id":"egress1","interface":"vpn1","table":"201","mark":"0x201","dscp":"cs4","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=53.822|8.8.8.8,rc=0,recv=3,avg=50.254|","fail_reasons":""}, {"slot_id":"egress2","interface":"vpn2","table":"202","mark":"0x202","dscp":"cs5","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=39.291|8.8.8.8,rc=0,recv=3,avg=39.501|","fail_reasons":""}, {"slot_id":"egress3","interface":"vpn3","table":"203","mark":"0x203","dscp":"cs1","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=50.782|8.8.8.8,rc=0,recv=3,avg=44.195|","fail_reasons":""}, {"slot_id":"egress4","interface":"vpn4","table":"204","mark":"0x204","dscp":"cs2","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=50.275|8.8.8.8,rc=0,recv=3,avg=50.054|","fail_reasons":""}, {"slot_id":"egress5","interface":"vpn5","table":"205","mark":"0x205","dscp":"cs3","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=58.971|8.8.8.8,rc=0,recv=3,avg=73.362|","fail_reasons":""} ], "summary": { "health_layer": "generic", "repair_layer": "separate_provider_adapter", "apply_performed": false, "apply_reason": "dry_run_status_only" } } --- statefile: /var/lib/router-egress-slot-health/status.jsonl --- -rw-r--r-- 1 root root 2265491 Jul 11 10:25 /var/lib/router-egress-slot-health/status.jsonl { "schema": "router-egress-slot-health-v1", "mode": "--dry-run", "epoch": 1783691233, "iso": "2026-07-10T13:47:13+00:00", "slots_conf": "/etc/router-egress-slots.d/slots.conf", "overall_ok": true, "enabled_count": 5, "good_count": 5, "bad_count": 0, "missing_count": 0, "slots": [ {"slot_id":"egress1","interface":"vpn1","table":"201","mark":"0x201","dscp":"cs4","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=53.950|8.8.8.8,rc=0,recv=3,avg=50.613|","fail_reasons":""}, {"slot_id":"egress2","interface":"vpn2","table":"202","mark":"0x202","dscp":"cs5","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=39.431|8.8.8.8,rc=0,recv=3,avg=39.204|","fail_reasons":""}, {"slot_id":"egress3","interface":"vpn3","table":"203","mark":"0x203","dscp":"cs1","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=52.144|8.8.8.8,rc=0,recv=3,avg=45.716|","fail_reasons":""}, {"slot_id":"egress4","interface":"vpn4","table":"204","mark":"0x204","dscp":"cs2","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=50.115|8.8.8.8,rc=0,recv=3,avg=50.115|","fail_reasons":""}, {"slot_id":"egress5","interface":"vpn5","table":"205","mark":"0x205","dscp":"cs3","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=58.900|8.8.8.8,rc=0,recv=3,avg=73.193|","fail_reasons":""} ], "summary": { "health_layer": "generic", "repair_layer": "separate_provider_adapter", "apply_performed": false, "apply_reason": "dry_run_status_only" } } { "schema": "router-egress-slot-health-v1", "mode": "--dry-run", "epoch": 1783691254, "iso": "2026-07-10T13:47:34+00:00", "slots_conf": "/etc/router-egress-slots.d/slots.conf", "overall_ok": true, "enabled_count": 5, "good_count": 5, "bad_count": 0, "missing_count": 0, "slots": [ {"slot_id":"egress1","interface":"vpn1","table":"201","mark":"0x201","dscp":"cs4","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=53.688|8.8.8.8,rc=0,recv=3,avg=52.987|","fail_reasons":""}, {"slot_id":"egress2","interface":"vpn2","table":"202","mark":"0x202","dscp":"cs5","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=39.207|8.8.8.8,rc=0,recv=3,avg=39.233|","fail_reasons":""}, {"slot_id":"egress3","interface":"vpn3","table":"203","mark":"0x203","dscp":"cs1","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=52.027|8.8.8.8,rc=0,recv=3,avg=45.657|","fail_reasons":""}, {"slot_id":"egress4","interface":"vpn4","table":"204","mark":"0x204","dscp":"cs2","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=49.987|8.8.8.8,rc=0,recv=3,avg=50.288|","fail_reasons":""}, {"slot_id":"egress5","interface":"vpn5","table":"205","mark":"0x205","dscp":"cs3","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=58.969|8.8.8.8,rc=0,recv=3,avg=73.332|","fail_reasons":""} ], "summary": { "health_layer": "generic", "repair_layer": "separate_provider_adapter", "apply_performed": false, "apply_reason": "dry_run_status_only" } } { "schema": "router-egress-slot-health-v1", "mode": "--dry-run", "epoch": 1783691334, "iso": "2026-07-10T13:48:54+00:00", "slots_conf": "/etc/router-egress-slots.d/slots.conf", "overall_ok": true, "enabled_count": 5, "good_count": 5, "bad_count": 0, "missing_count": 0, "slots": [ {"slot_id":"egress1","interface":"vpn1","table":"201","mark":"0x201","dscp":"cs4","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=53.920|8.8.8.8,rc=0,recv=3,avg=50.748|","fail_reasons":""}, {"slot_id":"egress2","interface":"vpn2","table":"202","mark":"0x202","dscp":"cs5","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=39.268|8.8.8.8,rc=0,recv=3,avg=39.249|","fail_reasons":""}, {"slot_id":"egress3","interface":"vpn3","table":"203","mark":"0x203","dscp":"cs1","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=52.108|8.8.8.8,rc=0,recv=3,avg=45.836|","fail_reasons":""}, {"slot_id":"egress4","interface":"vpn4","table":"204","mark":"0x204","dscp":"cs2","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=50.282|8.8.8.8,rc=0,recv=3,avg=50.175|","fail_reasons":""}, {"slot_id":"egress5","interface":"vpn5","table":"205","mark":"0x205","dscp":"cs3","provider":"hidemyname","repair_adapter":"hmn_pool_replace","health_targets":"1.1.1.1,8.8.8.8","strict_count":3,"strict_timeout":2,"iface_exists":true,"operstate":"unknown","route_ok":true,"status":"good","target_results":"1.1.1.1,rc=0,recv=3,avg=59.135|8.8.8.8,rc=0,recv=3,avg=73.263|","fail_reasons":""} ], "summary": { "health_layer": "generic", "repair_layer": "separate_provider_adapter", "apply_performed": false, "apply_reason": "dry_run_status_only" } } { "schema": "router-egress-slot-health-v1", "mode": "--dry-run", "epoch": 1783691415, "iso": "2026-07-10T13:50:15+00:00", --- statefile: /var/lib/router-egress-slot-health/status.kv --- -rw-r--r-- 1 root root 2100 Jul 11 10:25 /var/lib/router-egress-slot-health/status.kv schema=router-egress-slot-health-v1 epoch=1783765525 iso=2026-07-11T10:25:25+00:00 mode=--dry-run slots_conf=/etc/router-egress-slots.d/slots.conf overall_ok=true enabled_count=5 good_count=5 bad_count=0 missing_count=0 slot.egress1.interface=vpn1 slot.egress1.table=201 slot.egress1.mark=0x201 slot.egress1.dscp=cs4 slot.egress1.provider=hidemyname slot.egress1.repair_adapter=hmn_pool_replace slot.egress1.iface_exists=true slot.egress1.route_ok=true slot.egress1.status=good slot.egress1.target_results=1.1.1.1,rc=0,recv=3,avg=53.822|8.8.8.8,rc=0,recv=3,avg=50.254| slot.egress1.fail_reasons= slot.egress2.interface=vpn2 slot.egress2.table=202 slot.egress2.mark=0x202 slot.egress2.dscp=cs5 slot.egress2.provider=hidemyname slot.egress2.repair_adapter=hmn_pool_replace slot.egress2.iface_exists=true slot.egress2.route_ok=true slot.egress2.status=good slot.egress2.target_results=1.1.1.1,rc=0,recv=3,avg=39.291|8.8.8.8,rc=0,recv=3,avg=39.501| slot.egress2.fail_reasons= slot.egress3.interface=vpn3 slot.egress3.table=203 slot.egress3.mark=0x203 slot.egress3.dscp=cs1 slot.egress3.provider=hidemyname slot.egress3.repair_adapter=hmn_pool_replace slot.egress3.iface_exists=true slot.egress3.route_ok=true slot.egress3.status=good slot.egress3.target_results=1.1.1.1,rc=0,recv=3,avg=50.782|8.8.8.8,rc=0,recv=3,avg=44.195| slot.egress3.fail_reasons= slot.egress4.interface=vpn4 slot.egress4.table=204 slot.egress4.mark=0x204 slot.egress4.dscp=cs2 slot.egress4.provider=hidemyname slot.egress4.repair_adapter=hmn_pool_replace slot.egress4.iface_exists=true slot.egress4.route_ok=true slot.egress4.status=good slot.egress4.target_results=1.1.1.1,rc=0,recv=3,avg=50.275|8.8.8.8,rc=0,recv=3,avg=50.054| slot.egress4.fail_reasons= slot.egress5.interface=vpn5 slot.egress5.table=205 slot.egress5.mark=0x205 slot.egress5.dscp=cs3 slot.egress5.provider=hidemyname slot.egress5.repair_adapter=hmn_pool_replace slot.egress5.iface_exists=true slot.egress5.route_ok=true slot.egress5.status=good slot.egress5.target_results=1.1.1.1,rc=0,recv=3,avg=58.971|8.8.8.8,rc=0,recv=3,avg=73.362| slot.egress5.fail_reasons= === CURRENT VPN ENDPOINTS FROM UCI === --- vpn1 --- hmn_endpoint=178.215.227.13:44408 disabled=0 auto=0 --- vpn2 --- hmn_endpoint=109.248.198.4:37129 disabled=0 auto=0 --- vpn3 --- hmn_endpoint=95.211.68.119:45176 disabled=0 auto=0 --- vpn4 --- hmn_endpoint=78.31.250.16:41729 disabled=0 auto=0 --- vpn5 --- hmn_endpoint=157.173.27.69:41797 disabled=0 auto=0 === INTERFACES ROUTES STRICT === --- vpn1 table 201 --- IFACE_vpn1=OK table201: default dev vpn1 scope link ROUTE_table201=OK STRICT_vpn1=OK --- vpn2 table 202 --- IFACE_vpn2=OK table202: default dev vpn2 scope link ROUTE_table202=OK STRICT_vpn2=OK --- vpn3 table 203 --- IFACE_vpn3=OK table203: default dev vpn3 scope link ROUTE_table203=OK STRICT_vpn3=OK --- vpn4 table 204 --- IFACE_vpn4=OK table204: default dev vpn4 scope link ROUTE_table204=OK STRICT_vpn4=OK --- vpn5 table 205 --- IFACE_vpn5=OK table205: default dev vpn5 scope link ROUTE_table205=OK STRICT_vpn5=OK === POOLS === --- pool: /root/hmn/cache/ok-awg1-strict-foreign-latest.tsv --- exists=true -rw------- 1 root root 2829 Jul 10 16:59 /root/hmn/cache/ok-awg1-strict-foreign-latest.tsv rank file endpoint avg_ms ping_loss config_path 1 128-LV-Latvia-Riga-S1-192.121.163.235-awg1.conf 192.121.163.235:42530 39.008 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/128-LV-Latvia-Riga-S1-192.121.163.235-awg1.conf 2 075-NL-Netherlands-Amsterdam-H13-95.211.68.119-awg1.conf 95.211.68.119:45176 44.271 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/075-NL-Netherlands-Amsterdam-H13-95.211.68.119-awg1.conf 3 072-NL-Netherlands-Amsterdam-S8-78.31.250.16-awg1.conf 78.31.250.16:41729 50.697 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/072-NL-Netherlands-Amsterdam-S8-78.31.250.16-awg1.conf 4 150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf 178.215.227.13:44408 50.900 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf 5 049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf 157.173.27.69:41797 56.803 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf 6 149-AT-Austria-Vienna-S3-157.173.25.198-awg1.conf 157.173.25.198:34340 62.423 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/149-AT-Austria-Vienna-S3-157.173.25.198-awg1.conf 7 146-GB-United-Kingdom-London-S1-37.235.54.96-awg1.conf 37.235.54.96:38971 62.926 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/146-GB-United-Kingdom-London-S1-37.235.54.96-awg1.conf data_rows=15 --- pool: /root/hmn/cache/ok-awg1-strict-all-latest.tsv --- exists=true -rw------- 1 root root 3576 Jul 10 16:59 /root/hmn/cache/ok-awg1-strict-all-latest.tsv rank file endpoint avg_ms ping_loss config_path 1 128-LV-Latvia-Riga-S1-192.121.163.235-awg1.conf 192.121.163.235:42530 39.008 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/128-LV-Latvia-Riga-S1-192.121.163.235-awg1.conf 2 075-NL-Netherlands-Amsterdam-H13-95.211.68.119-awg1.conf 95.211.68.119:45176 44.271 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/075-NL-Netherlands-Amsterdam-H13-95.211.68.119-awg1.conf 3 072-NL-Netherlands-Amsterdam-S8-78.31.250.16-awg1.conf 78.31.250.16:41729 50.697 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/072-NL-Netherlands-Amsterdam-S8-78.31.250.16-awg1.conf 4 150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf 178.215.227.13:44408 50.900 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf 5 049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf 157.173.27.69:41797 56.803 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf 6 149-AT-Austria-Vienna-S3-157.173.25.198-awg1.conf 157.173.25.198:34340 62.423 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/149-AT-Austria-Vienna-S3-157.173.25.198-awg1.conf 7 146-GB-United-Kingdom-London-S1-37.235.54.96-awg1.conf 37.235.54.96:38971 62.926 9.9.9.9:0%,1.1.1.1:0% /root/hmn/configs/awg1/latest/146-GB-United-Kingdom-London-S1-37.235.54.96-awg1.conf data_rows=19 --- pool: /root/hmn/cache/working-awg1-latest.tsv --- exists=true -rw------- 1 root root 6294 Jul 10 16:36 /root/hmn/cache/working-awg1-latest.tsv id country name wg_ip wg_port endpoint endpoint_match address config_file 12 DK Denmark, Copenhagen S4 192.36.27.17 35500 192.36.27.17:35500 yes 10.119.249.139/32 /root/hmn/configs/awg1/20260710-163600/012-DK-Denmark-Copenhagen-S4-192.36.27.17-awg1.conf 29 UA Ukraine, Kyiv S3 38.244.192.244 45389 38.244.192.244:45389 yes 10.112.60.219/32 /root/hmn/configs/awg1/20260710-163600/029-UA-Ukraine-Kyiv-S3-38.244.192.244-awg1.conf 49 DE Germany, Berlin S7 157.173.27.69 41797 157.173.27.69:41797 yes 10.97.187.205/32 /root/hmn/configs/awg1/20260710-163600/049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf 53 FI Finland, Helsinki S4 192.71.247.18 35999 192.71.247.18:35999 yes 10.98.185.144/32 /root/hmn/configs/awg1/20260710-163600/053-FI-Finland-Helsinki-S4-192.71.247.18-awg1.conf 62 BE Belgium, Oostkamp S7 194.71.227.212 32583 194.71.227.212:32583 yes 10.100.254.15/32 /root/hmn/configs/awg1/20260710-163600/062-BE-Belgium-Oostkamp-S7-194.71.227.212-awg1.conf 63 CL Chile, Santiago 38.7.197.83 38465 38.7.197.83:38465 yes 10.101.61.41/32 /root/hmn/configs/awg1/20260710-163600/063-CL-Chile-Santiago-38.7.197.83-awg1.conf 70 CZ Czech Republic, Prague S1 193.235.207.39 38330 193.235.207.39:38330 yes 10.102.248.214/32 /root/hmn/configs/awg1/20260710-163600/070-CZ-Czech-Republic-Prague-S1-193.235.207.39-awg1.conf data_rows=35 === OPTIONAL DRY-RUNS ONLY === --- planner dry-run --- { "schema": "router-egress-hmn-top5-plan-v1", "mode": "--dry-run", "decision": "plan_ok", "reason": "top5_pool_ready", "pool": "/root/hmn/cache/ok-awg1-strict-foreign-latest.tsv", "pool_age_sec": 62872, "pool_endpoint_count": 15, "top_count": 5, "changes_count": 1, "plan": [ {"slot":"egress1","iface":"vpn1","table":"201","mark":"0x201","dscp":"cs4","provider":"hidemyname","adapter":"hmn_pool_replace","current":"178.215.227.13:44408","target":"178.215.227.13:44408","change":false,"target_rank":"4","target_avg_ms":"50.900","target_file":"150-NL-Netherlands-Amsterdam-R5-178.215.227.13-awg1.conf"}, {"slot":"egress2","iface":"vpn2","table":"202","mark":"0x202","dscp":"cs5","provider":"hidemyname","adapter":"hmn_pool_replace","current":"109.248.198.4:37129","target":"192.121.163.235:42530","change":true,"target_rank":"1","target_avg_ms":"39.008","target_file":"128-LV-Latvia-Riga-S1-192.121.163.235-awg1.conf"}, {"slot":"egress3","iface":"vpn3","table":"203","mark":"0x203","dscp":"cs1","provider":"hidemyname","adapter":"hmn_pool_replace","current":"95.211.68.119:45176","target":"95.211.68.119:45176","change":false,"target_rank":"2","target_avg_ms":"44.271","target_file":"075-NL-Netherlands-Amsterdam-H13-95.211.68.119-awg1.conf"}, {"slot":"egress4","iface":"vpn4","table":"204","mark":"0x204","dscp":"cs2","provider":"hidemyname","adapter":"hmn_pool_replace","current":"78.31.250.16:41729","target":"78.31.250.16:41729","change":false,"target_rank":"3","target_avg_ms":"50.697","target_file":"072-NL-Netherlands-Amsterdam-S8-78.31.250.16-awg1.conf"}, {"slot":"egress5","iface":"vpn5","table":"205","mark":"0x205","dscp":"cs3","provider":"hidemyname","adapter":"hmn_pool_replace","current":"157.173.27.69:41797","target":"157.173.27.69:41797","change":false,"target_rank":"5","target_avg_ms":"56.803","target_file":"049-DE-Germany-Berlin-S7-157.173.27.69-awg1.conf"} ], "safety": {"dry_run_no_uci_set": true, "dry_run_no_ifup_ifdown": true, "dry_run_no_route_change": true} } --- dispatcher egress2 dry-run --- { "schema": "router-egress-recovery-dispatcher-v1", "mode": "--dry-run", "slot": "egress2", "reason_input": "health_fail", "iface": "vpn2", "candidate_endpoint": "151.236.30.139:33998", "adapter_dryrun_decision": "dry_run_ok", "decision": "dry_run_ok", "reason": "dispatcher_ready", "apply_performed": false, "required_dispatch_confirm": "DISPATCH_egress2_151.236.30.139:33998", "safety": { "dry_run_no_uci_set": true, "dry_run_no_ifup_ifdown": true, "commit_requires_dispatch_confirm": true } } --- watcher once dry-run --- { "schema": "router-egress-health-repair-watch-v2", "mode": "--dry-run", "run_mode": "once", "slots": [ {"slot":"egress1","iface":"vpn1","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress2","iface":"vpn2","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress3","iface":"vpn3","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress4","iface":"vpn4","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""}, {"slot":"egress5","iface":"vpn5","status_ok":true,"fail_count":0,"decision":"healthy","action":"none","required_confirm":""} ], "any_action": false } === VM101 READONLY INSERTION AUDIT END === vm101_audit_rc=0