=== runner lines 220-320 === 220: emit_json "would_run_emergency_refresh" "threshold_or_force_reached_no_commit_in_dry_run" 221: fi 222: else 223: emit_json "below_threshold_noop" "daily_fail_count_below_threshold" 224: fi 225: exit 0 226: fi 227: 228: if [ "$mode" = "commit" ]; then 229: if [ "$EMERGENCY_COMMIT_ENABLED" != "true" ]; then 230: emit_json "commit_disabled_in_step050f" "skeleton_installed_but_commit_disabled_until_reviewed_followup" 231: exit 0 232: fi 233: 234: if [ "$confirm" != "$EMERGENCY_CONFIRM_TOKEN" ]; then 235: emit_json "missing_or_wrong_confirm_token" "commit_requires_exact_confirm_token" 236: exit 0 237: fi 238: 239: if [ "$force" != "true" ] && ! threshold_reached "$cnt"; then 240: emit_json "below_threshold_noop" "daily_fail_count_below_threshold" 241: exit 0 242: fi 243: 244: if [ "$cd_rem" -gt 0 ]; then 245: emit_json "cooldown_active" "emergency_refresh_cooldown_active" 246: exit 0 247: fi 248: 249: if ! acquire_lock; then 250: emit_json "lock_busy" "another_emergency_refresh_is_running" 251: exit 0 252: fi 253: 254: now="$(now_epoch)" 255: set_state mode EMERGENCY_REFRESH 256: set_state last_emergency_refresh_epoch "$now" 257: mkdir -p "$(dirname "$EMERGENCY_LOG")" 2>/dev/null || true 258: printf '%s action=emergency_refresh_start count=%s threshold=%s direct_failopen=false\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date)" "$cnt" "$EMERGENCY_DAILY_FAIL_THRESHOLD" >> "$EMERGENCY_LOG" 2>/dev/null || true 259: 260: if [ ! -x "$EMERGENCY_REFRESH_CMD" ]; then 261: set_state last_emergency_refresh_status refresh_cmd_missing 262: release_lock 263: emit_json "refresh_cmd_missing" "configured_refresh_command_missing" 264: exit 0 265: fi 266: 267: "$EMERGENCY_REFRESH_CMD" >> "$EMERGENCY_LOG" 2>&1 268: refresh_rc=$? 269: 270: if [ "$refresh_rc" = "0" ]; then 271: set_state last_emergency_refresh_status refresh_ok_rebalance_pending 272: if [ -x "$EMERGENCY_REBALANCE_APPLY_CMD" ]; then 273: "$EMERGENCY_REBALANCE_APPLY_CMD" --commit --confirm REBALANCE_TOP5_DAILY >> "$EMERGENCY_LOG" 2>&1 274: rebalance_rc=$? 275: set_state last_emergency_rebalance_rc "$rebalance_rc" 276: else 277: rebalance_rc=127 278: set_state last_emergency_rebalance_rc "$rebalance_rc" 279: fi 280: if [ "$rebalance_rc" = "0" ]; then 281: set_state mode NORMAL 282: set_state last_emergency_refresh_status refresh_ok_rebalance_ok 283: release_lock 284: emit_json "refresh_ok_rebalance_ok" "fresh_pool_and_top5_rebalance_completed" 285: exit 0 286: else 287: set_state mode DEGRADED_NO_FRESH_POOL 288: set_state last_emergency_refresh_status refresh_ok_rebalance_failed 289: release_lock 290: emit_json "refresh_ok_rebalance_failed" "refresh_succeeded_but_rebalance_failed_no_direct_failopen" 291: exit 0 292: fi 293: else 294: set_state mode DEGRADED_NO_FRESH_POOL 295: set_state last_emergency_refresh_status refresh_failed 296: set_state last_emergency_refresh_rc "$refresh_rc" 297: release_lock 298: emit_json "refresh_failed_degraded_no_direct" "refresh_failed_but_direct_failopen_disabled_in_state_machine" 299: exit 0 300: fi 301: fi 302: 303: emit_json "unknown_mode" "unsupported_mode" 304: exit 0 === runner counter/cooldown refs === 105: reg_daily_repair_get 2>/dev/null || echo 0 8: EMERGENCY_COOLDOWN_SECONDS="${EMERGENCY_COOLDOWN_SECONDS:-600}" 136: cooldown_remaining() { 138: last="$(get_state last_emergency_refresh_epoch 0)" 140: if [ "$elapsed" -lt "$EMERGENCY_COOLDOWN_SECONDS" ] 2>/dev/null; then 141: echo $((EMERGENCY_COOLDOWN_SECONDS - elapsed)) 151: cd_rem="$(cooldown_remaining)" 153: last_status="$(get_state last_emergency_refresh_status NONE)" 154: last_epoch="$(get_state last_emergency_refresh_epoch 0)" 165: echo " \"cooldown_seconds\": $EMERGENCY_COOLDOWN_SECONDS," 166: echo " \"cooldown_remaining\": $cd_rem," 180: echo " \"last_emergency_refresh_status\": \"$(json_s "$last_status")\"," 181: echo " \"last_emergency_refresh_epoch\": $last_epoch" 198: cd_rem="$(cooldown_remaining)" 213: cd_rem="$(cooldown_remaining)" 218: emit_json "cooldown_active" "threshold_or_force_reached_but_cooldown_active" 245: emit_json "cooldown_active" "emergency_refresh_cooldown_active" 256: set_state last_emergency_refresh_epoch "$now" 261: set_state last_emergency_refresh_status refresh_cmd_missing 271: set_state last_emergency_refresh_status refresh_ok_rebalance_pending 282: set_state last_emergency_refresh_status refresh_ok_rebalance_ok 288: set_state last_emergency_refresh_status refresh_ok_rebalance_failed 295: set_state last_emergency_refresh_status refresh_failed 138: last="$(get_state last_emergency_refresh_epoch 0)" 153: last_status="$(get_state last_emergency_refresh_status NONE)" 154: last_epoch="$(get_state last_emergency_refresh_epoch 0)" 180: echo " \"last_emergency_refresh_status\": \"$(json_s "$last_status")\"," 181: echo " \"last_emergency_refresh_epoch\": $last_epoch" 256: set_state last_emergency_refresh_epoch "$now" 261: set_state last_emergency_refresh_status refresh_cmd_missing 271: set_state last_emergency_refresh_status refresh_ok_rebalance_pending 282: set_state last_emergency_refresh_status refresh_ok_rebalance_ok 284: emit_json "refresh_ok_rebalance_ok" "fresh_pool_and_top5_rebalance_completed" 288: set_state last_emergency_refresh_status refresh_ok_rebalance_failed 295: set_state last_emergency_refresh_status refresh_failed === helper functions === 9: reg_now_epoch() 13: reg_now_utc() 17: reg_today_utc() 21: reg_key_safe() 25: reg_clean_tsv() 29: reg_init_state() 38: reg_pool_mtime_epoch() 67: reg_set_state() 77: reg_get_state() 92: reg_quarantine_endpoint() 120: reg_latest_quarantine_ts() 126: reg_endpoint_quarantined_for_pool() 143: reg_counter_file() 148: reg_counter_get() 162: reg_counter_inc() 173: reg_daily_repair_key() 177: reg_daily_repair_get() 181: reg_daily_repair_inc() 185: reg_selftest() === refresh backup/validation refs === 24: BACK="" 64: say "rollback: restoring published pointers/tables from $BACK" 149: echo "=== backup current published pointers/tables ===" 150: BACK="$BACKUPDIR/refresh-safe-before-$TS-$$" 216: echo "rollback-after-fresh-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 237: echo "rollback-after-old-validation-failed" > "$LAST_REFRESH_POOL_SOURCE" 205: say "validate fresh/current latest pool through vpn_test" 213: say "fresh pool validation failed; rolling back to previous published pool" 222: say "fallback: keep old configs/latest and validate current local pool through vpn_test" 234: say "old/current pool validation failed; rolling back published tables" === apply confirmation/rollback/validation refs === 18: --confirm) CONFIRM="${2:-}"; shift 2 ;; 133: if [ "$CONFIRM" != "REBALANCE_TOP5_DAILY" ]; then 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' 150: ip route show table "$table" > "$backup_dir/table.before" 2>/dev/null || true 151: cat > "$backup_dir/rollback-${slot}.sh" </dev/null 2>&1 || true 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' 27: strict_ping() { 150: ip route show table "$table" > "$backup_dir/table.before" 2>/dev/null || true 171: if [ "$rc" != "0" ] || ! ip route show table "$table" 2>/dev/null | grep -q . || ! strict_ping "$iface"; then 211: echo ' "safety": {"commit_requires_confirm": true, "per_slot_strict_check": true, "per_slot_rollback": true}' === exact contract === { "schema": "vm101-m07-exact-source-contract-v2", "contract": { "runner_commit_gate": true, "runner_real_refresh_call": true, "runner_real_apply_call_with_confirmation": true, "runner_concurrency_lock": true, "refresh_backup_and_rollback": true, "refresh_validates_new_or_old_pool": true, "apply_requires_confirmation": true, "apply_has_per_slot_rollback": true, "apply_has_route_and_strict_validation": true, "planner_enforces_quarantine": true, "automatic_hook_calls_runner_dry_run": true, "automatic_hook_has_no_commit_path": true, "direct_failopen_disabled_in_runner": true, "post_success_suppression_proven": true }, "source_contract_ready": true, "blocking_gaps": [], "post_success_suppression": { "counter_reset_proven": false, "cooldown_state_proven": true, "mode": "COOLDOWN_STATE" }, "evidence": { "runner_refresh_calls": [ { "line": 267, "text": " \"$EMERGENCY_REFRESH_CMD\" >> \"$EMERGENCY_LOG\" 2>&1" } ], "runner_apply_calls": [ { "line": 273, "text": " \"$EMERGENCY_REBALANCE_APPLY_CMD\" --commit --confirm REBALANCE_TOP5_DAILY >> \"$EMERGENCY_LOG\" 2>&1" } ], "runner_counter_refs": [ { "line": 105, "text": " reg_daily_repair_get 2>/dev/null || echo 0" } ], "explicit_counter_reset_refs": [], "helper_functions": [ { "name": "reg_now_epoch", "line": 9 }, { "name": "reg_now_utc", "line": 13 }, { "name": "reg_today_utc", "line": 17 }, { "name": "reg_key_safe", "line": 21 }, { "name": "reg_clean_tsv", "line": 25 }, { "name": "reg_init_state", "line": 29 }, { "name": "reg_pool_mtime_epoch", "line": 38 }, { "name": "reg_set_state", "line": 67 }, { "name": "reg_get_state", "line": 77 }, { "name": "reg_quarantine_endpoint", "line": 92 }, { "name": "reg_latest_quarantine_ts", "line": 120 }, { "name": "reg_endpoint_quarantined_for_pool", "line": 126 }, { "name": "reg_counter_file", "line": 143 }, { "name": "reg_counter_get", "line": 148 }, { "name": "reg_counter_inc", "line": 162 }, { "name": "reg_daily_repair_key", "line": 173 }, { "name": "reg_daily_repair_get", "line": 177 }, { "name": "reg_daily_repair_inc", "line": 181 }, { "name": "reg_selftest", "line": 185 } ], "helper_counter_reset_functions": [], "runner_cooldown_refs": [ { "line": 8, "text": "EMERGENCY_COOLDOWN_SECONDS=\"${EMERGENCY_COOLDOWN_SECONDS:-600}\"" }, { "line": 136, "text": "cooldown_remaining() {" }, { "line": 138, "text": " last=\"$(get_state last_emergency_refresh_epoch 0)\"" }, { "line": 140, "text": " if [ \"$elapsed\" -lt \"$EMERGENCY_COOLDOWN_SECONDS\" ] 2>/dev/null; then" }, { "line": 141, "text": " echo $((EMERGENCY_COOLDOWN_SECONDS - elapsed))" }, { "line": 151, "text": " cd_rem=\"$(cooldown_remaining)\"" }, { "line": 153, "text": " last_status=\"$(get_state last_emergency_refresh_status NONE)\"" }, { "line": 154, "text": " last_epoch=\"$(get_state last_emergency_refresh_epoch 0)\"" }, { "line": 165, "text": " echo \" \\\"cooldown_seconds\\\": $EMERGENCY_COOLDOWN_SECONDS,\"" }, { "line": 166, "text": " echo \" \\\"cooldown_remaining\\\": $cd_rem,\"" }, { "line": 180, "text": " echo \" \\\"last_emergency_refresh_status\\\": \\\"$(json_s \"$last_status\")\\\",\"" }, { "line": 181, "text": " echo \" \\\"last_emergency_refresh_epoch\\\": $last_epoch\"" }, { "line": 198, "text": " cd_rem=\"$(cooldown_remaining)\"" }, { "line": 213, "text": "cd_rem=\"$(cooldown_remaining)\"" }, { "line": 218, "text": " emit_json \"cooldown_active\" \"threshold_or_force_reached_but_cooldown_active\"" }, { "line": 245, "text": " emit_json \"cooldown_active\" \"emergency_refresh_cooldown_active\"" }, { "line": 256, "text": " set_state last_emergency_refresh_epoch \"$now\"" }, { "line": 261, "text": " set_state last_emergency_refresh_status refresh_cmd_missing" }, { "line": 271, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_pending" }, { "line": 282, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_ok" }, { "line": 288, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_failed" }, { "line": 295, "text": " set_state last_emergency_refresh_status refresh_failed" } ], "runner_success_state_refs": [ { "line": 138, "text": " last=\"$(get_state last_emergency_refresh_epoch 0)\"" }, { "line": 153, "text": " last_status=\"$(get_state last_emergency_refresh_status NONE)\"" }, { "line": 154, "text": " last_epoch=\"$(get_state last_emergency_refresh_epoch 0)\"" }, { "line": 180, "text": " echo \" \\\"last_emergency_refresh_status\\\": \\\"$(json_s \"$last_status\")\\\",\"" }, { "line": 181, "text": " echo \" \\\"last_emergency_refresh_epoch\\\": $last_epoch\"" }, { "line": 256, "text": " set_state last_emergency_refresh_epoch \"$now\"" }, { "line": 261, "text": " set_state last_emergency_refresh_status refresh_cmd_missing" }, { "line": 271, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_pending" }, { "line": 282, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_ok" }, { "line": 284, "text": " emit_json \"refresh_ok_rebalance_ok\" \"fresh_pool_and_top5_rebalance_completed\"" }, { "line": 288, "text": " set_state last_emergency_refresh_status refresh_ok_rebalance_failed" }, { "line": 295, "text": " set_state last_emergency_refresh_status refresh_failed" } ], "runner_lock_refs": [ { "line": 18, "text": "EMERGENCY_LOCK_DIR=\"${EMERGENCY_LOCK_DIR:-/var/lock/router-egress-emergency-refresh.lock}\"" }, { "line": 186, "text": " if mkdir \"$EMERGENCY_LOCK_DIR\" 2>/dev/null; then" }, { "line": 193, "text": " rmdir \"$EMERGENCY_LOCK_DIR\" 2>/dev/null || true" } ], "refresh_backup_refs": [ { "line": 24, "text": "BACK=\"\"" }, { "line": 64, "text": " say \"rollback: restoring published pointers/tables from $BACK\"" }, { "line": 149, "text": " echo \"=== backup current published pointers/tables ===\"" }, { "line": 150, "text": " BACK=\"$BACKUPDIR/refresh-safe-before-$TS-$$\"" }, { "line": 216, "text": " echo \"rollback-after-fresh-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"" }, { "line": 237, "text": " echo \"rollback-after-old-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"" } ], "refresh_validation_refs": [ { "line": 205, "text": " say \"validate fresh/current latest pool through vpn_test\"" }, { "line": 213, "text": " say \"fresh pool validation failed; rolling back to previous published pool\"" }, { "line": 222, "text": " say \"fallback: keep old configs/latest and validate current local pool through vpn_test\"" }, { "line": 234, "text": " say \"old/current pool validation failed; rolling back published tables\"" } ], "apply_confirmation_refs": [ { "line": 18, "text": " --confirm) CONFIRM=\"${2:-}\"; shift 2 ;;" }, { "line": 133, "text": " if [ \"$CONFIRM\" != \"REBALANCE_TOP5_DAILY\" ]; then" }, { "line": 211, "text": "echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'" } ], "apply_rollback_refs": [ { "line": 150, "text": " ip route show table \"$table\" > \"$backup_dir/table.before\" 2>/dev/null || true" }, { "line": 151, "text": " cat > \"$backup_dir/rollback-${slot}.sh\" </dev/null 2>&1 || true" }, { "line": 211, "text": "echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'" } ], "apply_validation_refs": [ { "line": 27, "text": "strict_ping() {" }, { "line": 150, "text": " ip route show table \"$table\" > \"$backup_dir/table.before\" 2>/dev/null || true" }, { "line": 171, "text": " if [ \"$rc\" != \"0\" ] || ! ip route show table \"$table\" 2>/dev/null | grep -q . || ! strict_ping \"$iface\"; then" }, { "line": 211, "text": "echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'" } ], "planner_quarantine_refs": [ { "line": 78, "text": " QUARANTINE_ENABLED=true" }, { "line": 80, "text": " QUARANTINE_ENABLED=false" }, { "line": 96, "text": "quarantine_excluded=0" }, { "line": 110, "text": " if [ \"$QUARANTINE_ENABLED\" = \"true\" ] && reg_endpoint_quarantined_for_pool \"$endpoint\" \"$POOL\"; then" }, { "line": 111, "text": " quarantine_excluded=$((quarantine_excluded + 1))" }, { "line": 172, "text": "echo \" \\\"quarantine_enabled\\\": $QUARANTINE_ENABLED,\"" }, { "line": 173, "text": "echo \" \\\"quarantine_excluded_count\\\": $quarantine_excluded,\"" } ], "hook_dry_run_refs": [ { "line": 64, "text": " \"$RUNNER\" --dry-run > \"$tmp\" 2> \"$err\"" } ], "hook_commit_refs": [], "direct_failopen_refs": [ { "line": 10, "text": "EMERGENCY_DIRECT_FAILOPEN_ENABLED=\"${EMERGENCY_DIRECT_FAILOPEN_ENABLED:-false}\"" }, { "line": 169, "text": " echo \" \\\"direct_failopen_enabled\\\": $(json_bool \"$EMERGENCY_DIRECT_FAILOPEN_ENABLED\"),\"" }, { "line": 170, "text": " echo \" \\\"direct_failopen_action\\\": \\\"disabled_not_allowed_in_step050f\\\",\"" }, { "line": 258, "text": " printf '%s action=emergency_refresh_start count=%s threshold=%s direct_failopen=false\\n' \"$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date)\" \"$cnt\" \"$EMERGENCY_DAILY_FAIL_THRESHOLD\" >> \"$EMERGENCY_LOG\" 2>/dev/null || true" }, { "line": 290, "text": " emit_json \"refresh_ok_rebalance_failed\" \"refresh_succeeded_but_rebalance_failed_no_direct_failopen\"" }, { "line": 298, "text": " emit_json \"refresh_failed_degraded_no_direct\" \"refresh_failed_but_direct_failopen_disabled_in_state_machine\"" } ] } }