=== runner === references_refresh=true references_apply=true references_planner=true references_state_helper=true -- dry_run_refs -- 21: mode="dry-run" 35: --dry-run|--dryrun) 36: mode="dry-run" 39: mode="dry-run" 215: if [ "$mode" = "dry-run" ]; then -- commit_refs -- 9: EMERGENCY_COMMIT_ENABLED="${EMERGENCY_COMMIT_ENABLED:-false}" 41: --commit|--apply) 168: echo " \"commit_enabled\": $(json_bool "$EMERGENCY_COMMIT_ENABLED")," 229: if [ "$EMERGENCY_COMMIT_ENABLED" != "true" ]; then 273: "$EMERGENCY_REBALANCE_APPLY_CMD" --commit --confirm REBALANCE_TOP5_DAILY >> "$EMERGENCY_LOG" 2>&1 -- refresh_invocation_refs -- 13: EMERGENCY_REFRESH_CMD="${EMERGENCY_REFRESH_CMD:-/root/hmn/hmn-refresh-pool-safe.sh}" 172: echo " \"refresh_cmd\": \"$(json_s "$EMERGENCY_REFRESH_CMD")\"," -- apply_invocation_refs -- 15: EMERGENCY_REBALANCE_APPLY_CMD="${EMERGENCY_REBALANCE_APPLY_CMD:-/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh}" -- counter_reset_refs -- 105: reg_daily_repair_get 2>/dev/null || echo 0 -- success_guard_refs -- 114: if ip link show dev "$iface" >/dev/null 2>&1 && ping -I "$iface" -c 1 -W 3 1.1.1.1 >/tmp/emergency-refresh-ping-${iface}.$$ 2>/dev/null; then 260: if [ ! -x "$EMERGENCY_REFRESH_CMD" ]; then 270: if [ "$refresh_rc" = "0" ]; then 272: if [ -x "$EMERGENCY_REBALANCE_APPLY_CMD" ]; then 280: if [ "$rebalance_rc" = "0" ]; then -- lock_refs -- 18: EMERGENCY_LOCK_DIR="${EMERGENCY_LOCK_DIR:-/var/lock/router-egress-emergency-refresh.lock}" -- direct_failopen_refs -- 10: EMERGENCY_DIRECT_FAILOPEN_ENABLED="${EMERGENCY_DIRECT_FAILOPEN_ENABLED:-false}" 169: echo " \"direct_failopen_enabled\": $(json_bool "$EMERGENCY_DIRECT_FAILOPEN_ENABLED")," 170: echo " \"direct_failopen_action\": \"disabled_not_allowed_in_step050f\"," 258: printf '%s action=emergency_refresh_start count=%s threshold=%s direct_failopen=false\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date)" "$cnt" "$EMERGENCY_DAILY_FAIL_THRESHOLD" >> "$EMERGENCY_LOG" 2>/dev/null || true 290: emit_json "refresh_ok_rebalance_failed" "refresh_succeeded_but_rebalance_failed_no_direct_failopen" 298: emit_json "refresh_failed_degraded_no_direct" "refresh_failed_but_direct_failopen_disabled_in_state_machine" === refresh === -- backup_refs -- 60: say "rollback: no backup dir available" 64: say "rollback: restoring published pointers/tables from $BACK" 73: say "rollback: latest symlink restored to readlink target: $OLD_LINK" 77: say "rollback: latest symlink restored to realpath: $OLD_REAL" 79: say "rollback WARN: no usable old latest target" 89: say "rollback: restored cache/$BN" 149: echo "=== backup current published pointers/tables ===" 164: say "backup=$BACK" 216: echo "rollback-after-fresh-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 237: echo "rollback-after-old-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" -- atomic_write_refs -- 14: LOCK="/tmp/hmn-refresh-pool-safe.lock" 22: RCFILE="/tmp/hmn-refresh-pool-safe.rc.$$" -- failure_guard_refs -- 60: say "rollback: no backup dir available" 64: say "rollback: restoring published pointers/tables from $BACK" 73: say "rollback: latest symlink restored to readlink target: $OLD_LINK" 77: say "rollback: latest symlink restored to realpath: $OLD_REAL" 79: say "rollback WARN: no usable old latest target" 89: say "rollback: restored cache/$BN" 140: trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT 216: echo "rollback-after-fresh-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 237: echo "rollback-after-old-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" -- pool_refs -- 11: VALIDATE="$BASE/hmn-validate-current-pool.sh" 14: LOCK="/tmp/hmn-refresh-pool-safe.lock" 21: LOG="$LOGDIR/refresh-pool-safe-$TS-$$.log" 22: RCFILE="/tmp/hmn-refresh-pool-safe.rc.$$" 30: LAST_REFRESH_POOL_SOURCE="$STATE/last-refresh-pool-source" 83: ok-awg1-strict-all-latest.tsv \ 84: ok-awg1-strict-foreign-latest.tsv \ 85: selected-awg1-latest.tsv 116: say "=== hmn-refresh-pool-safe start ===" 157: "$BASE/cache/ok-awg1-strict-all-latest.tsv" \ 158: "$BASE/cache/ok-awg1-strict-foreign-latest.tsv" \ 159: "$BASE/cache/selected-awg1-latest.tsv" 205: say "validate fresh/current latest pool through vpn_test" 213: say "fresh pool validation failed; rolling back to previous published pool" 222: say "fallback: keep old configs/latest and validate current local pool through vpn_test" 234: say "old/current pool validation failed; rolling back published tables" 282: head -n 12 "$BASE/cache/ok-awg1-strict-foreign-latest.tsv" 2>/dev/null || echo "missing" 286: cat "$BASE/cache/selected-awg1-latest.tsv" 2>/dev/null || echo "missing" 293: say "=== hmn-refresh-pool-safe done ===" -- provider_fetch_refs -- 5: BASE="/root/hmn" 10: DOWNLOADER="$BASE/hmn-download-all-awg.sh" 11: VALIDATE="$BASE/hmn-validate-current-pool.sh" 14: LOCK="/tmp/hmn-refresh-pool-safe.lock" 22: RCFILE="/tmp/hmn-refresh-pool-safe.rc.$$" 116: say "=== hmn-refresh-pool-safe start ===" 173: say "trying fresh HMN download via: $DOWNLOADER" 188: say "simulating HMN fresh download failure" 293: say "=== hmn-refresh-pool-safe done ===" === apply === -- backup_refs -- 151: cat > "$backup_dir/rollback-${slot}.sh" </dev/null 2>&1 || true 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' -- rollback_refs -- 64: trap 'rm -f "$tmp_top" "$tmp_current" "$tmp_missing" "$tmp_plan"' EXIT 151: cat > "$backup_dir/rollback-${slot}.sh" </dev/null 2>&1 || true 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' -- validation_refs -- 9: PREFERRED_POOL_FILES="${PREFERRED_POOL_FILES:-ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv}" 27: strict_ping() { 31: ping -I "$iface" -c 3 -W 2 "$target" >/tmp/rebalance-ping.out 2>/tmp/rebalance-ping.err 33: recv="$(grep -Eo '[0-9]+ packets received' /tmp/rebalance-ping.out 2>/dev/null | awk '{print $1}' | tail -1)" 37: rm -f /tmp/rebalance-ping.out /tmp/rebalance-ping.err 150: ip route show table "$table" > "$backup_dir/table.before" 2>/dev/null || true 171: if [ "$rc" != "0" ] || ! ip route show table "$table" 2>/dev/null | grep -q . || ! strict_ping "$iface"; then 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' -- slot_refs -- -- planner_refs -- === planner === -- quarantine_refs -- 78: QUARANTINE_ENABLED=true 80: QUARANTINE_ENABLED=false 96: quarantine_excluded=0 110: if [ "$QUARANTINE_ENABLED" = "true" ] && reg_endpoint_quarantined_for_pool "$endpoint" "$POOL"; then 111: quarantine_excluded=$((quarantine_excluded + 1)) 172: echo " \"quarantine_enabled\": $QUARANTINE_ENABLED," 173: echo " \"quarantine_excluded_count\": $quarantine_excluded," -- five_slot_refs -- 7: WORK="/tmp/router-egress-hmn-plan-top5.$$" 8: SLOTS="egress1:vpn1 egress2:vpn2 egress3:vpn3 egress4:vpn4 egress5:vpn5" 71: echo "{\"schema\":\"router-egress-hmn-plan-top5-v2\",\"decision\":\"missing_pool\",\"pool\":\"$(json_s "$POOL")\",\"changes_count\":0,\"plan\":[]}" 168: echo " \"schema\": \"router-egress-hmn-plan-top5-v2\"," -- json_refs -- 13: json_s() { 71: echo "{\"schema\":\"router-egress-hmn-plan-top5-v2\",\"decision\":\"missing_pool\",\"pool\":\"$(json_s "$POOL")\",\"changes_count\":0,\"plan\":[]}" 115: printf '%s\t%s\t%s\n' "$endpoint" "${rank:-0}" "${avg_ms:-}" >> "$CAND" 138: rank="$(printf '%s\n' "$row" | awk -F '\t' '{print $2}')" 139: avg="$(printf '%s\n' "$row" | awk -F '\t' '{print $3}')" 143: target="$(printf '%s\n' "$row" | awk -F '\t' '{print $1}')" 144: rank="$(printf '%s\n' "$row" | awk -F '\t' '{print $2}')" 145: avg="$(printf '%s\n' "$row" | awk -F '\t' '{print $3}')" 169: echo " \"decision\": \"$(json_s "$decision")\"," 170: echo " \"pool\": \"$(json_s "$POOL")\"," 181: printf ' {"slot":"%s","iface":"%s","current":"%s","target":"%s","change":%s,"target_rank":"%s","target_avg_ms":"%s"}\n' \ 182: "$(json_s "$slot")" \ 183: "$(json_s "$iface")" \ 184: "$(json_s "$cur")" \ 185: "$(json_s "$target")" \ 187: "$(json_s "$rank")" \ 188: "$(json_s "$avg")" === helper === -- counter_get_refs -- 177: reg_daily_repair_get() { -- counter_reset_refs -- -- state_dir_refs -- 3: REG_STATE_DIR="${REG_STATE_DIR:-/var/lib/router-egress-recovery}" 4: REG_QUARANTINE_TSV="${REG_QUARANTINE_TSV:-$REG_STATE_DIR/quarantine.tsv}" 5: REG_COUNTER_DIR="${REG_COUNTER_DIR:-$REG_STATE_DIR/fail-counter}" 6: REG_STATE_KV="${REG_STATE_KV:-$REG_STATE_DIR/state.kv}" 7: REG_LOCK_DIR="${REG_LOCK_DIR:-$REG_STATE_DIR/locks}" 30: mkdir -p "$REG_STATE_DIR" "$REG_COUNTER_DIR" "$REG_LOCK_DIR" 2>/dev/null || return 1 186: old_state_dir="$REG_STATE_DIR" 193: REG_STATE_DIR="$test_root/state" 194: REG_QUARANTINE_TSV="$REG_STATE_DIR/quarantine.tsv" 195: REG_COUNTER_DIR="$REG_STATE_DIR/fail-counter" 196: REG_STATE_KV="$REG_STATE_DIR/state.kv" 197: REG_LOCK_DIR="$REG_STATE_DIR/locks" 255: REG_STATE_DIR="$old_state_dir" === hook === -- runner_refs -- 8: RUNNER="${EMERGENCY_DECISION_RUNNER:-/usr/local/sbin/router-egress-emergency-refresh.sh}" 47: echo " \"runner\": \"$(json_s "$RUNNER")\"" 60: elif [ ! -x "$RUNNER" ]; then 64: "$RUNNER" --dry-run > "$tmp" 2> "$err" -- dry_run_refs -- 28: --dry-run) 58: emit_fallback "unsafe_config_refused" "dry_run_hook_refuses_commit_or_direct_configuration" > "$tmp" 64: "$RUNNER" --dry-run > "$tmp" 2> "$err" -- commit_refs -- === sequence === { "refresh_position": 539, "apply_position": 737, "counter_reset_position": 2459, "refresh_before_apply": true, "counter_reset_after_apply": true }