#!/usr/bin/env bash set -Eeuo pipefail umask 077 [ "$(id -un)" = "ops" ] || { echo "STOP: запускать на router-ops пользователем ops" exit 1 } STEP="STEP_050M07C4_TEMPORARY_TABLE200_BOOTSTRAP_FRESH_REFRESH_LIVE" PASS_DECISION="PASS_${STEP}" PLAN_ID="vm101-hmn-autonomous-egress-recovery" TOKEN="e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4" PUBLIC_BASE="https://helena-background-beam-harry.trycloudflare.com/r/${TOKEN}" PREVIOUS_REPORT="${PUBLIC_BASE}/20260711-200114_step050m07c3_salvage_bootstrap_and_fresh_refresh_live_stream/" PREVIOUS_ARCHITECTURE_PLAN="${PUBLIC_BASE}/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/" XS_MAP="${PUBLIC_BASE}/20260711-120734_xs_map_audit_repair_publish/" GLOBAL_PROJECT_PLAN="${PUBLIC_BASE}/20260711-123348_global_project_plan_wg_paid/" ROOT="/opt/router-ops" STATE_ROOT="${ROOT}/state" PUBROOT="${ROOT}/public/r/${TOKEN}" PLAN_STATE_DIR="${STATE_ROOT}/local-plans/${PLAN_ID}" C3_DIR="${PUBROOT}/20260711-200114_step050m07c3_salvage_bootstrap_and_fresh_refresh_live_stream" M06A_DIR="${PUBROOT}/20260711-173308_step050m06a_compare_live_toolchain_to_frozen_map" STATUS_TOOL="${ROOT}/bin/vm101-local-plan-status" REPUBLISH_TOOL="${ROOT}/bin/vm101-local-plan-republish" TS="$(date -u +%Y%m%d-%H%M%S)" REPORT_SLUG="${TS}_step050m07c4_temporary_table200_bootstrap_fresh_refresh_live" REPORT_DIR="${PUBROOT}/${REPORT_SLUG}" TRYCF_REPORT="${PUBLIC_BASE}/${REPORT_SLUG}/" REPORT_TXT="${TRYCF_REPORT}report.txt" FACTS_JSON="${TRYCF_REPORT}facts.json" ARCHITECTURE_PLAN="$PREVIOUS_ARCHITECTURE_PLAN" mkdir -p \ "$REPORT_DIR" \ "$REPORT_DIR/sources" # Точный пользовательский STEP сохраняется до любых проверок. cp -a "$0" "$REPORT_DIR/step.sh" chmod 600 "$REPORT_DIR/step.sh" PROGRESS_LOG="$REPORT_DIR/progress.log" : > "$PROGRESS_LOG" CURRENT_STAGE="initialization" LAST_SUCCESS="step_saved" HASH_CHECK_RC="NOT_RUN" VM101_RC="NOT_RUN" POSTCHECK_RC="NOT_RUN" ROLLBACK_RC="NOT_NEEDED" ROLLBACK_ARMED=false VM101_ROLLBACK="" PRODUCTION_MODIFIED=false PLAN_UPDATED=false stage() { CURRENT_STAGE="$1" echo echo ">>> [$1] $2" | tee -a "$PROGRESS_LOG" date -u '+ utc=%Y-%m-%dT%H:%M:%SZ' | tee -a "$PROGRESS_LOG" } mark_success() { LAST_SUCCESS="$1" echo "last_success=$LAST_SUCCESS" >> "$PROGRESS_LOG" } print_links() { echo echo "TRYCF_REPORT=$TRYCF_REPORT" echo "REPORT_TXT=$REPORT_TXT" echo "FACTS_JSON=$FACTS_JSON" echo "ARCHITECTURE_PLAN=$ARCHITECTURE_PLAN" echo "XS_MAP=$XS_MAP" echo "GLOBAL_PROJECT_PLAN=$GLOBAL_PROJECT_PLAN" } remote_vm101() { ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ '$1'" } rollback_vm101() { if [ "$ROLLBACK_ARMED" != true ]; then ROLLBACK_RC="NOT_NEEDED" return 0 fi if [ -z "$VM101_ROLLBACK" ]; then ROLLBACK_RC="ROLLBACK_PATH_MISSING" PRODUCTION_MODIFIED=unknown return 1 fi echo echo ">>> Запускаю внешний rollback VM101" if remote_vm101 "sh '$VM101_ROLLBACK'" \ > >(tee "$REPORT_DIR/vm101-rollback.txt") \ 2> >(tee "$REPORT_DIR/vm101-rollback.stderr" >&2) then ROLLBACK_RC=0 ROLLBACK_ARMED=false PRODUCTION_MODIFIED=false return 0 fi ROLLBACK_RC=$? PRODUCTION_MODIFIED=unknown return "$ROLLBACK_RC" } write_stop_artifacts() { local reason="$1" local rc="$2" local line="$3" python3 - \ "$REPORT_DIR" \ "$STEP" \ "$reason" \ "$rc" \ "$line" \ "$CURRENT_STAGE" \ "$LAST_SUCCESS" \ "$HASH_CHECK_RC" \ "$VM101_RC" \ "$POSTCHECK_RC" \ "$ROLLBACK_RC" \ "$PRODUCTION_MODIFIED" \ > "$REPORT_DIR/diagnostic.json" <<'PY' import json import sys from pathlib import Path ( report_dir, step, reason, rc, line, stage, last_success, hash_rc, vm101_rc, postcheck_rc, rollback_rc, modified, ) = sys.argv[1:] root = Path(report_dir) def read_tail(path: Path, limit: int = 80000): if not path.exists() or not path.is_file(): return None return path.read_text( encoding="utf-8", errors="replace", )[-limit:] streams = {} for pattern in ("*.txt", "*.stderr", "*.log"): for path in sorted(root.glob(pattern)): if path.name in {"report.txt", "progress.log"}: continue streams[path.name] = { "size_bytes": path.stat().st_size, "tail": read_tail(path), } combined = "\n".join( item["tail"] or "" for item in streams.values() ) if "bootstrap_code_test_failed" in combined: classification = "HMN_ACCESS_FAILED_WITH_PROVEN_TABLE200_BOOTSTRAP" elif "fresh_download_failed_old_pool_fallback" in combined: classification = "FRESH_DOWNLOAD_FAILED_OLD_POOL_FALLBACK" elif "fresh_download_not_proven" in combined: classification = "FRESH_DOWNLOAD_NOT_PROVEN" elif "runner_timeout" in combined: classification = "HMN_REFRESH_TIMEOUT" elif "runner_decision_not_success" in combined: classification = "RUNNER_DECISION_NOT_SUCCESS" elif "state_status_not_success" in combined: classification = "SUCCESS_STATE_NOT_WRITTEN" elif "planner_not_converged" in combined: classification = "POST_REFRESH_PLANNER_NOT_CONVERGED" elif "table200_restore_failed" in combined: classification = "TEMPORARY_BOOTSTRAP_ROUTE_RESTORE_FAILED" elif rollback_rc not in { "NOT_NEEDED", "0", "REMOTE_AUTO_ROLLBACK_OK", }: classification = "ROLLBACK_VALIDATION_FAILED" elif vm101_rc not in {"NOT_RUN", "0"}: classification = "VM101_CONTROLLED_REFRESH_FAILED" elif postcheck_rc not in {"NOT_RUN", "0"}: classification = "PERSISTED_POSTCHECK_FAILED" elif hash_rc not in {"NOT_RUN", "0"}: classification = "FROZEN_TOOLCHAIN_CHECK_FAILED" else: classification = "LOCAL_VALIDATION_OR_PLAN_UPDATE_FAILURE" print(json.dumps({ "schema": "router-step-inline-diagnostic-v7", "step": step, "failure": { "reason": reason, "rc": int(rc), "line": int(line), "stage": stage, "last_success": last_success, }, "command_results": { "hash_check_rc": hash_rc, "vm101_rc": vm101_rc, "postcheck_rc": postcheck_rc, "rollback_rc": rollback_rc, }, "automatic_classification": classification, "production_modified_after_rollback": modified, "live_streaming": { "runner_stdout": True, "runner_stderr": True, "emergency_log": True, "heartbeat_seconds": 15, }, "captured_streams": streams, "recommended_next_step": ( "Использовать automatic_classification, live stream, " "stdout/stderr и rollback-факты этого отчёта. " "Отдельный диагностический STEP не требуется." ), }, ensure_ascii=False, indent=2)) PY cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( diagnostic_path, step, reason, rc, line, stage, last_success, modified, rollback, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] with open(diagnostic_path, encoding="utf-8") as source: diagnostic = json.load(source) production_modified = { "true": True, "false": False, }.get(modified, "unknown") print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "assessment": { "decision": f"STOP_{step}_{reason}", "all_ok": False, "error_rc": int(rc), "error_line": int(line), "failed_stage": stage, "last_success": last_success, }, "inline_diagnostic": diagnostic, "safety": { "production_modified": production_modified, "rollback_vm101": rollback or None, "direct_failopen_changed": False, }, "plan": { "current_milestone": "M07", "milestone_completed": False, }, "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY cat > "$REPORT_DIR/index.html" < ${STEP}

${STEP}

EOF find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" } fatal() { local reason="$1" local rc="${2:-1}" local line="${3:-$LINENO}" trap - ERR rollback_vm101 || true write_stop_artifacts "$reason" "$rc" "$line" print_links exit "$rc" } on_error() { local rc="$?" local line="$1" fatal "UNEXPECTED_ERROR" "$rc" "$line" } trap 'on_error "$LINENO"' ERR stage "01/08" "Проверяю C3 STOP, безопасность и текущий M07" for required in \ "$C3_DIR/report.txt" \ "$C3_DIR/facts.json" \ "$C3_DIR/diagnostic.json" \ "$C3_DIR/vm101-precheck.txt" \ "$C3_DIR/vm101-precheck.stderr" \ "$C3_DIR/step.sh" \ "$M06A_DIR/vm101.sh" \ "$STATUS_TOOL" \ "$REPUBLISH_TOOL" \ "$PLAN_STATE_DIR/milestones.json" \ "$STATE_ROOT/current-local-architecture-plan-url.txt" do [ -s "$required" ] || { echo "MISSING_REQUIRED=$required" fatal "REQUIRED_ARTIFACT_MISSING" 2 "$LINENO" } done grep -Fq \ "decision=STOP_STEP_050M07C3_SALVAGE_BOOTSTRAP_AND_FRESH_REFRESH_LIVE_STREAM_HMN_CODE_OR_LIVE_PRECHECK_FAILED" \ "$C3_DIR/report.txt" || fatal "PREVIOUS_C3_STOP_NOT_PROVEN" 3 "$LINENO" grep -Fq \ "production_modified=false" \ "$C3_DIR/report.txt" || fatal "PREVIOUS_C3_SAFETY_NOT_PROVEN" 4 "$LINENO" grep -Fq \ "__FACT__ healthy_vpn_slots=5" \ "$C3_DIR/vm101-precheck.txt" || fatal "PREVIOUS_FIVE_HEALTHY_SLOTS_NOT_PROVEN" 5 "$LINENO" grep -Fq \ "ERROR: не нашёл active VPN interface в table 200." \ "$C3_DIR/vm101-precheck.txt" || fatal "PREVIOUS_TABLE200_FAILURE_NOT_PROVEN" 6 "$LINENO" grep -Fq \ "Unexpected token" \ "$C3_DIR/vm101-precheck.stderr" || fatal "PREVIOUS_BUSYBOX_AWK_FAILURE_NOT_PROVEN" 7 "$LINENO" CURRENT_PLAN="$( tr -d '\r\n' \ < "$STATE_ROOT/current-local-architecture-plan-url.txt" )" [ "$CURRENT_PLAN" = "$PREVIOUS_ARCHITECTURE_PLAN" ] || fatal "CURRENT_PLAN_POINTER_MISMATCH" 8 "$LINENO" python3 - "$PLAN_STATE_DIR/milestones.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) items = { item["id"]: item for item in data["milestones"] } assert data["current_milestone"] == "M07" assert items["M06"]["status"] == "done" assert items["M07"]["status"] == "in_progress" assert items["M08"]["status"] == "pending" PY cp -a \ "$C3_DIR/diagnostic.json" \ "$REPORT_DIR/sources/step050m07c3-diagnostic.json" cp -a \ "$C3_DIR/vm101-precheck.txt" \ "$REPORT_DIR/sources/step050m07c3-precheck.txt" cp -a \ "$C3_DIR/vm101-precheck.stderr" \ "$REPORT_DIR/sources/step050m07c3-precheck.stderr" mark_success "c3_stop_and_plan_verified" stage "02/08" "Публикую exact hash-check и controlled VM101 script" cp -a \ "$M06A_DIR/vm101.sh" \ "$REPORT_DIR/vm101-tool-hash-check.sh" cat > "$REPORT_DIR/vm101.sh" <<'VM101' #!/bin/sh set -u umask 077 CONF="/etc/router-egress-emergency-refresh.conf" RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh" HOOK="/usr/local/sbin/router-egress-emergency-decision-hook.sh" PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh" HELPER="/usr/local/lib/router-egress-recovery-state.sh" CODE_TEST="/root/hmn/hmn-code-test.sh" HOOK_INIT="/etc/init.d/router-egress-emergency-decision" WATCHER_INIT="/etc/init.d/router-egress-health-repair" SLOTS_INIT="/etc/init.d/router-egress-slots" MAPPER_INIT="/etc/init.d/router-egress-mapper" CACHE_DIR="/root/hmn/cache" POOL="${CACHE_DIR}/ok-awg1-strict-foreign-latest.tsv" STATE_DIR="/var/lib/router-egress-recovery" RUN_ID="$(date -u +%Y%m%d-%H%M%S)-$$" BACKUP_ROOT="/root/step050m07c4-backup-${RUN_ID}" ROLLBACK="/root/rollback-step050m07c4-${RUN_ID}.sh" RUNNER_OUT="/tmp/step050m07c4-runner-${RUN_ID}.out" RUNNER_ERR="/tmp/step050m07c4-runner-${RUN_ID}.err" LOG_DELTA="/tmp/step050m07c4-log-${RUN_ID}.delta" CODE_OUT="/tmp/step050m07c4-code-test-${RUN_ID}.out" ROLLBACK_READY=false ROLLBACK_DONE=false HOOK_WAS_RUNNING=false WATCHER_WAS_RUNNING=false STREAM_NEW_OFFSET=0 fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } block() { echo "__BLOCK_BEGIN__ $1" printf '%s\n' "$2" echo "__BLOCK_END__ $1" } json_block() { echo "__JSON_BEGIN__ $1" printf '%s\n' "$2" echo "__JSON_END__ $1" } bool_cmd() { if "$@" >/dev/null 2>&1; then printf true else printf false fi } state_value() { key="$1" fallback="$2" ( unset REG_STATE_DIR . "$HELPER" reg_get_state "$key" "$fallback" ) } repair_counter() { ( unset REG_STATE_DIR . "$HELPER" reg_daily_repair_get ) } strict_iface() { interface="$1" attempt=1 while [ "$attempt" -le 3 ]; do if ping \ -I "$interface" \ -c 1 \ -W 3 \ 1.1.1.1 \ >/dev/null 2>&1 then return 0 fi attempt=$((attempt + 1)) sleep 1 done return 1 } strict_all() { for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do strict_iface "$interface" || return 1 done return 0 } routes_all() { for table in 201 202 203 204 205; do ip route show table "$table" 2>/dev/null | grep -q '^default ' || return 1 done return 0 } choose_bootstrap() { for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$interface"; then printf '%s' "$interface" return 0 fi done return 1 } table200_hash() { ip route show table 200 2>/dev/null | sort | sha256sum | sed 's/[[:space:]].*$//' } restore_table200() { source_file="$1" restore_rc=0 ip route flush table 200 >/dev/null 2>&1 || restore_rc=1 while IFS= read -r route_line; do [ -n "$route_line" ] || continue set -- $route_line ip route add table 200 "$@" >/dev/null 2>&1 || restore_rc=1 done < "$source_file" return "$restore_rc" } restore_services() { if [ "$HOOK_WAS_RUNNING" = true ]; then "$HOOK_INIT" start >/dev/null 2>&1 || true fi if [ "$WATCHER_WAS_RUNNING" = true ]; then "$WATCHER_INIT" start >/dev/null 2>&1 || true fi } stream_file() { file="$1" old_offset="$2" destination="$3" label="$4" append_file="$5" if [ -f "$file" ]; then current_size="$( wc -c < "$file" 2>/dev/null | tr -d ' ' || echo 0 )" else current_size=0 fi case "$current_size" in ''|*[!0-9]*) current_size=0 ;; esac if [ "$current_size" -gt "$old_offset" ]; then count=$((current_size - old_offset)) chunk="/tmp/step050m07c4-chunk-${RUN_ID}-$$" dd \ if="$file" \ bs=1 \ skip="$old_offset" \ count="$count" \ 2>/dev/null \ > "$chunk" if [ "$destination" = stderr ]; then echo "__LIVE_CHUNK__ source=${label} bytes=${count}" >&2 cat "$chunk" >&2 else echo "__LIVE_CHUNK__ source=${label} bytes=${count}" cat "$chunk" fi if [ "$append_file" != "-" ]; then cat "$chunk" >> "$append_file" fi rm -f "$chunk" fi STREAM_NEW_OFFSET="$current_size" } detect_stage() { if [ ! -s "$LOG_DELTA" ]; then echo "runner_starting" return fi tail_text="$( tail -n 160 "$LOG_DELTA" 2>/dev/null || true )" if printf '%s\n' "$tail_text" | grep -q '=== hmn-refresh-pool-safe done ===' then echo "refresh_finished_rebalance_running" elif printf '%s\n' "$tail_text" | grep -q '=== run manager once after validation ===' then echo "manager_after_validation" elif printf '%s\n' "$tail_text" | grep -q '=== done validate-current-pool ===' then echo "pool_validation_completed" elif printf '%s\n' "$tail_text" | grep -q 'strict foreign head' then echo "strict_foreign_pool_validation" elif printf '%s\n' "$tail_text" | grep -Ei \ 'download|serverlist|fresh_download' \ >/dev/null then echo "provider_download" elif printf '%s\n' "$tail_text" | grep -Ei \ 'validate|testing|strict' \ >/dev/null then echo "tunnel_validation" else echo "hmn_refresh_running" fi } auto_rollback() { exit_rc="$?" trap - EXIT if [ "$exit_rc" -ne 0 ] && [ "$ROLLBACK_READY" = true ] && [ "$ROLLBACK_DONE" != true ] && [ -x "$ROLLBACK" ] then echo echo "=== REMOTE AUTO ROLLBACK START ===" if sh "$ROLLBACK"; then ROLLBACK_DONE=true fact auto_rollback true else fact auto_rollback false fi echo "=== REMOTE AUTO ROLLBACK END ===" elif [ "$exit_rc" -ne 0 ]; then restore_services fi rm -f \ "$RUNNER_OUT" \ "$RUNNER_ERR" \ "$LOG_DELTA" \ "$CODE_OUT" exit "$exit_rc" } trap auto_rollback EXIT for required in \ "$CONF" \ "$RUNNER" \ "$HOOK" \ "$PLANNER" \ "$HELPER" \ "$CODE_TEST" \ "$HOOK_INIT" \ "$WATCHER_INIT" \ "$CACHE_DIR" \ "$POOL" \ "$STATE_DIR" \ /etc/config/network do [ -e "$required" ] || { echo "__ERROR__ missing=$required" exit 21 } done for command_name in \ sh \ sed \ grep \ find \ sort \ cp \ mv \ rm \ mkdir \ date \ wc \ sha256sum \ dd \ cat \ tail \ kill \ tee \ uci \ wg \ ip \ ping do command -v "$command_name" >/dev/null 2>&1 || { echo "__ERROR__ command_missing=$command_name" exit 22 } done CONFIG_RAW_PRE="$( ( . "$CONF" printf '%s' "${EMERGENCY_COMMIT_ENABLED:-UNSET}" ) )" CONFIRM_TOKEN="$( ( . "$CONF" printf '%s' "${EMERGENCY_CONFIRM_TOKEN:-}" ) )" EMERGENCY_LOG="$( ( . "$CONF" printf '%s' "${EMERGENCY_LOG:-/var/log/router-egress-emergency-refresh.log}" ) )" fact commit_raw_pre "$CONFIG_RAW_PRE" fact emergency_log "$EMERGENCY_LOG" case "$CONFIG_RAW_PRE" in 1|true) ;; *) echo "__ERROR__ unsupported_commit_raw=$CONFIG_RAW_PRE" exit 23 ;; esac [ -n "$CONFIRM_TOKEN" ] || { echo "__ERROR__ confirm_token_empty" exit 24 } RUNNER_PRE="$("$RUNNER" --dry-run)" PLANNER_PRE="$("$PLANNER")" STATE_EPOCH_PRE="$( state_value last_emergency_refresh_epoch 0 )" STATE_STATUS_PRE="$( state_value last_emergency_refresh_status NONE )" COUNTER_PRE="$(repair_counter)" POOL_HASH_PRE="$( sha256sum "$POOL" | sed 's/[[:space:]].*$//' )" NETWORK_HASH_PRE="$( sha256sum /etc/config/network | sed 's/[[:space:]].*$//' )" TABLE200_HASH_PRE="$(table200_hash)" json_block runner_pre "$RUNNER_PRE" json_block planner_pre "$PLANNER_PRE" fact state_epoch_pre "$STATE_EPOCH_PRE" fact state_status_pre "$STATE_STATUS_PRE" fact repair_counter_pre "$COUNTER_PRE" fact pool_hash_pre "$POOL_HASH_PRE" fact network_hash_pre "$NETWORK_HASH_PRE" fact table200_hash_pre "$TABLE200_HASH_PRE" TABLE200_PRE="$( ip route show table 200 2>&1 || true )" block table200_before "$TABLE200_PRE" HEALTHY_PRE=0 for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$interface"; then value=true HEALTHY_PRE=$((HEALTHY_PRE + 1)) else value=false fi fact "pre.strict.${interface}" "$value" done fact healthy_slots_pre "$HEALTHY_PRE" fact routes_all_pre "$(bool_cmd routes_all)" [ "$HEALTHY_PRE" -ge 4 ] || { echo "__ERROR__ fewer_than_four_healthy_slots" exit 25 } [ "$(bool_cmd routes_all)" = true ] || { echo "__ERROR__ routes_201_205_not_ready" exit 26 } mkdir -p "$BACKUP_ROOT" cp -a "$CONF" "$BACKUP_ROOT/config.before" cp -a "$CACHE_DIR" "$BACKUP_ROOT/cache.before" cp -a "$STATE_DIR" "$BACKUP_ROOT/state.before" cp -a /etc/config/network "$BACKUP_ROOT/network.before" ip route show table 200 2>/dev/null | sort \ > "$BACKUP_ROOT/table200.before" find "$STATE_DIR" \ -maxdepth 4 \ -type f \ -name 'rollback-egress*.sh' \ 2>/dev/null | sort \ > "$BACKUP_ROOT/slot-rollbacks.before" HOOK_WAS_RUNNING="$(bool_cmd "$HOOK_INIT" running)" WATCHER_WAS_RUNNING="$(bool_cmd "$WATCHER_INIT" running)" fact hook_was_running "$HOOK_WAS_RUNNING" fact watcher_was_running "$WATCHER_WAS_RUNNING" cat > "$ROLLBACK" </dev/null 2>&1 || true "\$WATCHER_INIT" stop >/dev/null 2>&1 || true find "\$STATE_DIR" \ -maxdepth 4 \ -type f \ -name 'rollback-egress*.sh' \ 2>/dev/null | sort -r \ > "\$BACKUP_ROOT/slot-rollbacks.after" while IFS= read -r rollback_path; do [ -n "\$rollback_path" ] || continue if ! grep -Fxq \ "\$rollback_path" \ "\$BACKUP_ROOT/slot-rollbacks.before" then sh "\$rollback_path" || command_errors=\$((command_errors + 1)) fi done < "\$BACKUP_ROOT/slot-rollbacks.after" cp -a \ "\$BACKUP_ROOT/config.before" \ "\$CONF" || command_errors=\$((command_errors + 1)) rm -rf "\$CACHE_DIR" cp -a \ "\$BACKUP_ROOT/cache.before" \ "\$CACHE_DIR" || command_errors=\$((command_errors + 1)) rm -rf "\$STATE_DIR" cp -a \ "\$BACKUP_ROOT/state.before" \ "\$STATE_DIR" || command_errors=\$((command_errors + 1)) cp -a \ "\$BACKUP_ROOT/network.before" \ /etc/config/network || command_errors=\$((command_errors + 1)) rm -rf \ /var/lock/router-egress-emergency-refresh.lock \ /tmp/hmn-refresh-pool-safe.lock \ 2>/dev/null || true /etc/init.d/network reload >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) sleep 8 [ ! -x "\$SLOTS_INIT" ] || "\$SLOTS_INIT" restart >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) [ ! -x "\$MAPPER_INIT" ] || "\$MAPPER_INIT" restart >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) ip route flush table 200 >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) while IFS= read -r route_line; do [ -n "\$route_line" ] || continue set -- \$route_line ip route add table 200 "\$@" >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) done < "\$BACKUP_ROOT/table200.before" if [ "\$HOOK_WAS_RUNNING" = true ]; then "\$HOOK_INIT" start >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) fi if [ "\$WATCHER_WAS_RUNNING" = true ]; then "\$WATCHER_INIT" start >/dev/null 2>&1 || command_errors=\$((command_errors + 1)) fi config_expected="\$( sha256sum "\$BACKUP_ROOT/config.before" | sed 's/[[:space:]].*\$//' )" config_live="\$( sha256sum "\$CONF" | sed 's/[[:space:]].*\$//' )" pool_expected="\$( sha256sum \ "\$BACKUP_ROOT/cache.before/ok-awg1-strict-foreign-latest.tsv" | sed 's/[[:space:]].*\$//' )" pool_live="\$( sha256sum \ "\$CACHE_DIR/ok-awg1-strict-foreign-latest.tsv" | sed 's/[[:space:]].*\$//' )" network_expected="\$( sha256sum "\$BACKUP_ROOT/network.before" | sed 's/[[:space:]].*\$//' )" network_live="\$( sha256sum /etc/config/network | sed 's/[[:space:]].*\$//' )" table200_expected="\$( sort "\$BACKUP_ROOT/table200.before" | sha256sum | sed 's/[[:space:]].*\$//' )" table200_live="\$( ip route show table 200 2>/dev/null | sort | sha256sum | sed 's/[[:space:]].*\$//' )" strict=true for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do ping -I "\$interface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 || strict=false done routes=true for table in 201 202 203 204 205; do ip route show table "\$table" | grep -q '^default ' || routes=false done hook_restored=true watcher_restored=true if [ "\$HOOK_WAS_RUNNING" = true ]; then "\$HOOK_INIT" running >/dev/null 2>&1 || hook_restored=false fi if [ "\$WATCHER_WAS_RUNNING" = true ]; then "\$WATCHER_INIT" running >/dev/null 2>&1 || watcher_restored=false fi config_match=false pool_match=false network_match=false table200_match=false [ "\$config_expected" = "\$config_live" ] && config_match=true [ "\$pool_expected" = "\$pool_live" ] && pool_match=true [ "\$network_expected" = "\$network_live" ] && network_match=true [ "\$table200_expected" = "\$table200_live" ] && table200_match=true echo "rollback_command_errors=\$command_errors" echo "rollback_config_match=\$config_match" echo "rollback_pool_match=\$pool_match" echo "rollback_network_match=\$network_match" echo "rollback_table200_match=\$table200_match" echo "rollback_strict_all=\$strict" echo "rollback_routes_all=\$routes" echo "rollback_hook_restored=\$hook_restored" echo "rollback_watcher_restored=\$watcher_restored" # Командные предупреждения публикуются, но результат определяется # конечными инвариантами, а не промежуточным ненулевым rc. if [ "\$config_match" = true ] && [ "\$pool_match" = true ] && [ "\$network_match" = true ] && [ "\$table200_match" = true ] && [ "\$strict" = true ] && [ "\$routes" = true ] && [ "\$hook_restored" = true ] && [ "\$watcher_restored" = true ] then echo "rollback_validated=true" exit 0 fi echo "rollback_validated=false" exit 1 EOF chmod 700 "$ROLLBACK" ROLLBACK_READY=true fact rollback "$ROLLBACK" fact rollback_exists "$(bool_cmd test -x "$ROLLBACK")" fact backup_root "$BACKUP_ROOT" # С этого момента любое завершение с ошибкой обязано откатиться. fact mutation_started true "$WATCHER_INIT" stop >/dev/null 2>&1 || true "$HOOK_INIT" stop >/dev/null 2>&1 || true sleep 2 fact watcher_stopped "$( if "$WATCHER_INIT" running >/dev/null 2>&1; then echo false else echo true fi )" fact hook_stopped "$( if "$HOOK_INIT" running >/dev/null 2>&1; then echo false else echo true fi )" BOOTSTRAP_IFACE="$( choose_bootstrap || true )" [ -n "$BOOTSTRAP_IFACE" ] || { echo "__ERROR__ no_healthy_bootstrap_iface" exit 27 } fact bootstrap_iface "$BOOTSTRAP_IFACE" ip route replace \ default \ dev "$BOOTSTRAP_IFACE" \ table 200 || { echo "__ERROR__ temporary_table200_route_install_failed" exit 28 } TABLE200_TEMP="$( ip route show table 200 2>&1 || true )" block table200_with_temporary_bootstrap "$TABLE200_TEMP" ip route show table 200 | grep -q "^default dev ${BOOTSTRAP_IFACE}\([[:space:]]\|$\)" || { echo "__ERROR__ temporary_table200_route_not_visible" exit 29 } fact temporary_table200_bootstrap_installed true echo echo "=== LIVE HMN CODE TEST START ===" set +e "$CODE_TEST" > "$CODE_OUT" 2>&1 CODE_RC=$? set -e cat "$CODE_OUT" echo "=== LIVE HMN CODE TEST END ===" echo fact code_test_rc "$CODE_RC" if [ "$CODE_RC" -ne 0 ]; then echo "__ERROR__ bootstrap_code_test_failed" exit 30 fi fact code_test_passed_with_temporary_bootstrap true if [ "$CONFIG_RAW_PRE" = "1" ]; then TEMP_CONF="${CONF}.step050m07c4.$$.tmp" sed \ 's/^[[:space:]]*EMERGENCY_COMMIT_ENABLED[[:space:]]*=.*/EMERGENCY_COMMIT_ENABLED=true/' \ "$CONF" \ > "$TEMP_CONF" [ "$( grep -c \ '^[[:space:]]*EMERGENCY_COMMIT_ENABLED[[:space:]]*=true[[:space:]]*$' \ "$TEMP_CONF" )" -eq 1 ] || { rm -f "$TEMP_CONF" echo "__ERROR__ commit_boolean_normalization_failed" exit 31 } chmod 600 "$TEMP_CONF" chown 0:0 "$TEMP_CONF" mv "$TEMP_CONF" "$CONF" fi CONFIG_RAW_POST="$( ( . "$CONF" printf '%s' "${EMERGENCY_COMMIT_ENABLED:-UNSET}" ) )" fact commit_raw_post "$CONFIG_RAW_POST" [ "$CONFIG_RAW_POST" = "true" ] || { echo "__ERROR__ commit_value_not_true_after_normalization" exit 32 } LOG_SIZE_PRE="$( if [ -f "$EMERGENCY_LOG" ]; then wc -c < "$EMERGENCY_LOG" | tr -d ' ' else echo 0 fi )" fact emergency_log_size_pre "$LOG_SIZE_PRE" : > "$RUNNER_OUT" : > "$RUNNER_ERR" : > "$LOG_DELTA" "$RUNNER" \ --commit \ --confirm "$CONFIRM_TOKEN" \ > "$RUNNER_OUT" \ 2> "$RUNNER_ERR" & RUNNER_PID=$! START_EPOCH="$(date +%s)" LAST_HEARTBEAT=0 OUT_OFFSET=0 ERR_OFFSET=0 LOG_OFFSET="$LOG_SIZE_PRE" echo echo "=== LIVE HMN REFRESH START ===" echo "__HEARTBEAT__ stage=runner_starting elapsed=0s runner_pid=${RUNNER_PID}" echo while kill -0 "$RUNNER_PID" >/dev/null 2>&1; do stream_file \ "$RUNNER_OUT" \ "$OUT_OFFSET" \ stdout \ runner_stdout \ - OUT_OFFSET="$STREAM_NEW_OFFSET" stream_file \ "$RUNNER_ERR" \ "$ERR_OFFSET" \ stderr \ runner_stderr \ - ERR_OFFSET="$STREAM_NEW_OFFSET" stream_file \ "$EMERGENCY_LOG" \ "$LOG_OFFSET" \ stdout \ emergency_log \ "$LOG_DELTA" LOG_OFFSET="$STREAM_NEW_OFFSET" NOW_EPOCH="$(date +%s)" ELAPSED=$((NOW_EPOCH - START_EPOCH)) if [ $((ELAPSED - LAST_HEARTBEAT)) -ge 15 ]; then CURRENT_REFRESH_STAGE="$(detect_stage)" echo echo "__HEARTBEAT__ stage=${CURRENT_REFRESH_STAGE} elapsed=${ELAPSED}s runner_pid=${RUNNER_PID}" echo LAST_HEARTBEAT="$ELAPSED" fi if [ "$ELAPSED" -ge 1800 ]; then echo "__ERROR__ runner_timeout_after=${ELAPSED}s" kill "$RUNNER_PID" >/dev/null 2>&1 || true sleep 2 kill -9 "$RUNNER_PID" >/dev/null 2>&1 || true wait "$RUNNER_PID" >/dev/null 2>&1 || true exit 33 fi sleep 2 done set +e wait "$RUNNER_PID" RUNNER_RC=$? set -e stream_file \ "$RUNNER_OUT" \ "$OUT_OFFSET" \ stdout \ runner_stdout_final \ - OUT_OFFSET="$STREAM_NEW_OFFSET" stream_file \ "$RUNNER_ERR" \ "$ERR_OFFSET" \ stderr \ runner_stderr_final \ - ERR_OFFSET="$STREAM_NEW_OFFSET" stream_file \ "$EMERGENCY_LOG" \ "$LOG_OFFSET" \ stdout \ emergency_log_final \ "$LOG_DELTA" LOG_OFFSET="$STREAM_NEW_OFFSET" FINISH_EPOCH="$(date +%s)" RUNNER_ELAPSED=$((FINISH_EPOCH - START_EPOCH)) echo echo "=== LIVE HMN REFRESH END ===" echo "__HEARTBEAT__ stage=runner_finished elapsed=${RUNNER_ELAPSED}s runner_pid=${RUNNER_PID}" echo RUNNER_OUTPUT="$( cat "$RUNNER_OUT" 2>/dev/null || true )" RUNNER_STDERR="$( cat "$RUNNER_ERR" 2>/dev/null || true )" fact runner_commit_rc "$RUNNER_RC" fact runner_elapsed_seconds "$RUNNER_ELAPSED" json_block runner_commit "$RUNNER_OUTPUT" block runner_commit_stderr_final "$RUNNER_STDERR" OLD_POOL_FALLBACK=false FRESH_DOWNLOAD_PROVEN=false if grep -q \ 'pool_source=old-after-download-fail' \ "$LOG_DELTA" then OLD_POOL_FALLBACK=true fi if grep -Eq \ 'pool_source=fresh|last_fresh_download_rc=0' \ "$LOG_DELTA" then FRESH_DOWNLOAD_PROVEN=true fi fact old_pool_fallback "$OLD_POOL_FALLBACK" fact fresh_download_proven "$FRESH_DOWNLOAD_PROVEN" [ "$RUNNER_RC" -eq 0 ] || { echo "__ERROR__ runner_rc=$RUNNER_RC" exit 34 } printf '%s\n' "$RUNNER_OUTPUT" | grep -q \ '"decision"[[:space:]]*:[[:space:]]*"refresh_ok_rebalance_ok"' || { echo "__ERROR__ runner_decision_not_success" exit 35 } [ "$OLD_POOL_FALLBACK" = false ] || { echo "__ERROR__ fresh_download_failed_old_pool_fallback" exit 36 } [ "$FRESH_DOWNLOAD_PROVEN" = true ] || { echo "__ERROR__ fresh_download_not_proven" exit 37 } # Временный bootstrap route больше не нужен. restore_table200 "$BACKUP_ROOT/table200.before" || { echo "__ERROR__ table200_restore_failed" exit 38 } TABLE200_HASH_POST="$(table200_hash)" fact table200_hash_post "$TABLE200_HASH_POST" [ "$TABLE200_HASH_POST" = "$TABLE200_HASH_PRE" ] || { echo "__ERROR__ table200_hash_mismatch_after_restore" exit 39 } fact table200_restored true restore_services sleep 2 RUNNER_POST="$("$RUNNER" --dry-run)" HOOK_POST="$("$HOOK")" PLANNER_POST="$("$PLANNER")" STATE_MODE_POST="$(state_value mode UNKNOWN)" STATE_STATUS_POST="$( state_value last_emergency_refresh_status UNKNOWN )" STATE_EPOCH_POST="$( state_value last_emergency_refresh_epoch 0 )" COUNTER_POST="$(repair_counter)" json_block runner_post "$RUNNER_POST" json_block hook_post "$HOOK_POST" json_block planner_post "$PLANNER_POST" fact state_mode_post "$STATE_MODE_POST" fact state_status_post "$STATE_STATUS_POST" fact state_epoch_post "$STATE_EPOCH_POST" fact repair_counter_post "$COUNTER_POST" fact hook_running_post "$(bool_cmd "$HOOK_INIT" running)" fact hook_enabled_post "$(bool_cmd "$HOOK_INIT" enabled)" fact watcher_running_post "$(bool_cmd "$WATCHER_INIT" running)" fact watcher_enabled_post "$(bool_cmd "$WATCHER_INIT" enabled)" fact strict_all_post "$(bool_cmd strict_all)" fact routes_all_post "$(bool_cmd routes_all)" fact emergency_lock_present "$( bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock )" fact refresh_lock_present "$( bool_cmd test -e /tmp/hmn-refresh-pool-safe.lock )" fact direct_failopen_changed false [ "$STATE_MODE_POST" = "NORMAL" ] || { echo "__ERROR__ state_mode_not_normal" exit 40 } [ "$STATE_STATUS_POST" = "refresh_ok_rebalance_ok" ] || { echo "__ERROR__ state_status_not_success" exit 41 } [ "$STATE_EPOCH_POST" -gt "$STATE_EPOCH_PRE" ] || { echo "__ERROR__ success_epoch_not_advanced" exit 42 } printf '%s\n' "$PLANNER_POST" | grep -q \ '"changes_count"[[:space:]]*:[[:space:]]*0' || { echo "__ERROR__ planner_not_converged" exit 43 } [ "$(bool_cmd strict_all)" = true ] || { echo "__ERROR__ strict_not_all" exit 44 } [ "$(bool_cmd routes_all)" = true ] || { echo "__ERROR__ routes_not_all" exit 45 } [ "$(bool_cmd "$HOOK_INIT" running)" = true ] || { echo "__ERROR__ hook_not_restored" exit 46 } [ "$(bool_cmd "$WATCHER_INIT" running)" = true ] || { echo "__ERROR__ watcher_not_restored" exit 47 } [ "$(bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock)" = false ] || { echo "__ERROR__ emergency_lock_remains" exit 48 } [ "$(bool_cmd test -e /tmp/hmn-refresh-pool-safe.lock)" = false ] || { echo "__ERROR__ refresh_lock_remains" exit 49 } echo "__TRACE__ stage=complete" trap - EXIT rm -f \ "$RUNNER_OUT" \ "$RUNNER_ERR" \ "$LOG_DELTA" \ "$CODE_OUT" exit 0 VM101 cat > "$REPORT_DIR/vm101-postcheck.sh" <<'VM101POST' #!/bin/sh set -eu umask 077 CONF="/etc/router-egress-emergency-refresh.conf" RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh" PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh" HELPER="/usr/local/lib/router-egress-recovery-state.sh" HOOK_INIT="/etc/init.d/router-egress-emergency-decision" WATCHER_INIT="/etc/init.d/router-egress-health-repair" fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } json_block() { echo "__JSON_BEGIN__ $1" printf '%s\n' "$2" echo "__JSON_END__ $1" } bool_cmd() { if "$@" >/dev/null 2>&1; then printf true else printf false fi } state_value() { key="$1" fallback="$2" ( unset REG_STATE_DIR . "$HELPER" reg_get_state "$key" "$fallback" ) } strict_all() { for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do ping -I "$interface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 || return 1 done return 0 } routes_all() { for table in 201 202 203 204 205; do ip route show table "$table" | grep -q '^default ' || return 1 done return 0 } RAW="$( ( . "$CONF" printf '%s' "${EMERGENCY_COMMIT_ENABLED:-UNSET}" ) )" RUNNER_JSON="$("$RUNNER" --dry-run)" PLANNER_JSON="$("$PLANNER")" fact commit_raw "$RAW" fact state_mode "$(state_value mode UNKNOWN)" fact state_status "$( state_value last_emergency_refresh_status UNKNOWN )" fact state_epoch "$( state_value last_emergency_refresh_epoch 0 )" fact hook_running "$(bool_cmd "$HOOK_INIT" running)" fact hook_enabled "$(bool_cmd "$HOOK_INIT" enabled)" fact watcher_running "$(bool_cmd "$WATCHER_INIT" running)" fact watcher_enabled "$(bool_cmd "$WATCHER_INIT" enabled)" fact strict_all "$(bool_cmd strict_all)" fact routes_all "$(bool_cmd routes_all)" fact emergency_lock_present "$( bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock )" fact refresh_lock_present "$( bool_cmd test -e /tmp/hmn-refresh-pool-safe.lock )" json_block runner "$RUNNER_JSON" json_block planner "$PLANNER_JSON" fact production_modified false VM101POST chmod 600 \ "$REPORT_DIR/vm101-tool-hash-check.sh" \ "$REPORT_DIR/vm101.sh" \ "$REPORT_DIR/vm101-postcheck.sh" sh -n "$REPORT_DIR/vm101-tool-hash-check.sh" sh -n "$REPORT_DIR/vm101.sh" sh -n "$REPORT_DIR/vm101-postcheck.sh" mark_success "remote_scripts_published" stage "03/08" "Проверяю frozen toolchain" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101-tool-hash-check.sh" \ > "$REPORT_DIR/vm101-tool-hash-check.txt" \ 2> "$REPORT_DIR/vm101-tool-hash-check.stderr" then HASH_CHECK_RC=0 else HASH_CHECK_RC=$? fi echo "hash_check_rc=$HASH_CHECK_RC" | tee -a "$PROGRESS_LOG" [ "$HASH_CHECK_RC" -eq 0 ] || fatal "FROZEN_TOOLCHAIN_CHECK_FAILED" "$HASH_CHECK_RC" "$LINENO" grep -Fq \ "__SUMMARY__ checked=10 errors=0" \ "$REPORT_DIR/vm101-tool-hash-check.txt" || fatal "LIVE_TOOLCHAIN_DRIFT_DETECTED" 9 "$LINENO" mark_success "frozen_toolchain_verified" stage "04/08" "Запускаю temporary bootstrap и полный HMN live-stream" echo echo "Нормальная продолжительность refresh — около 15 минут." echo "В терминале будут полный emergency log и heartbeat каждые 15 секунд." echo if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101.sh" \ > >(tee "$REPORT_DIR/vm101.txt") \ 2> >(tee "$REPORT_DIR/vm101.stderr" >&2) then VM101_RC=0 else VM101_RC=$? fi echo "vm101_rc=$VM101_RC" | tee -a "$PROGRESS_LOG" VM101_ROLLBACK="$( grep '^__FACT__ rollback=' \ "$REPORT_DIR/vm101.txt" | tail -n1 | cut -d= -f2- || true )" if grep -Fq \ '__FACT__ mutation_started=true' \ "$REPORT_DIR/vm101.txt" then ROLLBACK_ARMED=true PRODUCTION_MODIFIED=true fi if grep -Fq \ '__FACT__ auto_rollback=true' \ "$REPORT_DIR/vm101.txt" then ROLLBACK_ARMED=false PRODUCTION_MODIFIED=false ROLLBACK_RC="REMOTE_AUTO_ROLLBACK_OK" fi [ "$VM101_RC" -eq 0 ] || fatal "VM101_CONTROLLED_REFRESH_FAILED" "$VM101_RC" "$LINENO" [ -n "$VM101_ROLLBACK" ] || fatal "ROLLBACK_PATH_MISSING" 10 "$LINENO" grep -Fq \ '__FACT__ code_test_passed_with_temporary_bootstrap=true' \ "$REPORT_DIR/vm101.txt" || fatal "BOOTSTRAP_CODE_TEST_PASS_NOT_PROVEN" 11 "$LINENO" grep -Fq \ '__FACT__ fresh_download_proven=true' \ "$REPORT_DIR/vm101.txt" || fatal "FRESH_DOWNLOAD_NOT_PROVEN" 12 "$LINENO" grep -Fq \ '__FACT__ old_pool_fallback=false' \ "$REPORT_DIR/vm101.txt" || fatal "OLD_POOL_FALLBACK_DETECTED" 13 "$LINENO" grep -Fq \ '__FACT__ table200_restored=true' \ "$REPORT_DIR/vm101.txt" || fatal "TABLE200_RESTORE_NOT_PROVEN" 14 "$LINENO" grep -Fq \ '__TRACE__ stage=complete' \ "$REPORT_DIR/vm101.txt" || fatal "REMOTE_COMPLETION_MARKER_MISSING" 15 "$LINENO" mark_success "fresh_refresh_remote_passed" stage "05/08" "Проверяю persisted state и динамический cooldown contract" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101-postcheck.sh" \ > >(tee "$REPORT_DIR/vm101-postcheck.txt") \ 2> >(tee "$REPORT_DIR/vm101-postcheck.stderr" >&2) then POSTCHECK_RC=0 else POSTCHECK_RC=$? fi echo "postcheck_rc=$POSTCHECK_RC" | tee -a "$PROGRESS_LOG" [ "$POSTCHECK_RC" -eq 0 ] || fatal "PERSISTED_POSTCHECK_REMOTE_FAILED" "$POSTCHECK_RC" "$LINENO" python3 - \ "$REPORT_DIR/vm101-postcheck.txt" \ "$REPORT_DIR/vm101.txt" \ "$REPORT_DIR/assessment.json" <<'PY' import json import sys from pathlib import Path postcheck_path, operation_path, output_path = sys.argv[1:] post_text = Path(postcheck_path).read_text( encoding="utf-8", errors="replace", ) operation_text = Path(operation_path).read_text( encoding="utf-8", errors="replace", ) def parse_stream(text): facts = {} blocks = {} current = None lines = [] for line in text.splitlines(): if line.startswith("__FACT__ "): payload = line[len("__FACT__ "):] if "=" in payload: key, value = payload.split("=", 1) facts[key] = value elif line.startswith("__JSON_BEGIN__ "): current = line[len("__JSON_BEGIN__ "):] lines = [] elif line.startswith("__JSON_END__ "): name = line[len("__JSON_END__ "):] if name == current: blocks[name] = json.loads( "\n".join(lines) ) current = None lines = [] elif current is not None: lines.append(line) return facts, blocks post_facts, post_blocks = parse_stream(post_text) operation_facts, operation_blocks = parse_stream(operation_text) runner = post_blocks["runner"] planner = post_blocks["planner"] remaining = int(runner["cooldown_remaining"]) decision = runner["decision"] if remaining > 0: cooldown_mode = "ACTIVE" cooldown_valid = decision == "cooldown_active" else: cooldown_mode = "EXPIRED_DURING_LONG_REFRESH" cooldown_valid = decision in { "would_run_emergency_refresh", "below_threshold_noop", } checks = { "commit_normalized": post_facts["commit_raw"] == "true", "state_normal": post_facts["state_mode"] == "NORMAL", "state_success": post_facts["state_status"] == "refresh_ok_rebalance_ok", "state_epoch_present": int(post_facts["state_epoch"]) > 0, "services_running_enabled": post_facts["hook_running"] == "true" and post_facts["hook_enabled"] == "true" and post_facts["watcher_running"] == "true" and post_facts["watcher_enabled"] == "true", "strict_all": post_facts["strict_all"] == "true", "routes_all": post_facts["routes_all"] == "true", "locks_released": post_facts["emergency_lock_present"] == "false" and post_facts["refresh_lock_present"] == "false", "runner_success_state": runner["last_emergency_refresh_status"] == "refresh_ok_rebalance_ok", "direct_disabled": runner["direct_failopen_enabled"] is False, "cooldown_contract_valid": cooldown_valid, "planner_converged": planner["decision"] == "plan_ok" and planner["changes_count"] == 0 and len(planner["plan"]) == 5, "temporary_bootstrap_proven": operation_facts.get( "code_test_passed_with_temporary_bootstrap" ) == "true", "fresh_download_proven": operation_facts.get( "fresh_download_proven" ) == "true", "old_pool_not_used": operation_facts.get( "old_pool_fallback" ) == "false", "table200_restored": operation_facts.get( "table200_restored" ) == "true", "live_stream_completed": "__LIVE_CHUNK__ source=emergency_log" in operation_text and "__HEARTBEAT__ stage=runner_finished" in operation_text, "read_only_postcheck": post_facts["production_modified"] == "false", } all_ok = all(checks.values()) assessment = { "all_ok": all_ok, "decision": ( "PASS_STEP_050M07C4_TEMPORARY_TABLE200_BOOTSTRAP_FRESH_REFRESH_LIVE" if all_ok else "STOP_STEP_050M07C4_TEMPORARY_TABLE200_BOOTSTRAP_FRESH_REFRESH_LIVE" ), "checks": checks, "failed_checks": [ name for name, value in checks.items() if not value ], "cooldown": { "decision": decision, "remaining_seconds": remaining, "mode": cooldown_mode, "contract_valid": cooldown_valid, }, "operation": { "bootstrap_iface": operation_facts.get("bootstrap_iface"), "runner_elapsed_seconds": int( operation_facts.get( "runner_elapsed_seconds", "0", ) ), "repair_counter_before": int( operation_facts.get( "repair_counter_pre", "0", ) ), "repair_counter_after": int( operation_facts.get( "repair_counter_post", "0", ) ), "fresh_download_proven": True, "old_pool_fallback": False, "table200_restored": True, "state_status": post_facts["state_status"], }, "safety": { "direct_failopen_changed": False, "rollback_available": True, }, "plan_transition": { "completed": "M07", "next": "M08", }, } Path(output_path).write_text( json.dumps( assessment, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) if not all_ok: raise SystemExit( "failed checks: " + ",".join(assessment["failed_checks"]) ) PY mark_success "persisted_state_validated" # После успешной независимой postcheck rollback больше не запускаем # из-за возможной ошибки публикации плана. ROLLBACK_ARMED=false PRODUCTION_MODIFIED=true stage "06/08" "Закрываю M07 и переиздаю локальный план" "$STATUS_TOOL" \ M07 done \ --evidence "$TRYCF_REPORT" \ --note "Fresh HMN download доказан через временный table-200 bootstrap. Полный refresh выводился live, top-5 rebalance завершён, state NORMAL/refresh_ok_rebalance_ok, planner converged, пять strict-слотов и маршруты 201–205 исправны. Table 200 возвращена в исходное состояние; cooldown проверен с учётом длительности refresh." \ > "$REPORT_DIR/plan-status-m07.txt" \ 2> "$REPORT_DIR/plan-status-m07.stderr" PLAN_UPDATED=true REPUBLISH_OUTPUT="$( "$REPUBLISH_TOOL" \ "$STEP" \ "$TRYCF_REPORT" )" printf '%s\n' "$REPUBLISH_OUTPUT" \ > "$REPORT_DIR/plan-republish.txt" NEW_ARCHITECTURE_PLAN="$( printf '%s\n' "$REPUBLISH_OUTPUT" | sed -n 's/^ARCHITECTURE_PLAN=//p' | tail -n1 )" [ -n "$NEW_ARCHITECTURE_PLAN" ] || fatal "PLAN_REPUBLISH_URL_MISSING" 16 "$LINENO" ARCHITECTURE_PLAN="$NEW_ARCHITECTURE_PLAN" python3 - "$PLAN_STATE_DIR/milestones.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) items = { item["id"]: item for item in data["milestones"] } assert items["M07"]["status"] == "done" assert items["M08"]["status"] == "in_progress" assert data["current_milestone"] == "M08" PY mark_success "plan_updated_to_m08" stage "07/08" "Публикую PASS report и facts" RUNNER_ELAPSED="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["operation"]["runner_elapsed_seconds"]) PY )" BOOTSTRAP_IFACE="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["operation"]["bootstrap_iface"]) PY )" COOLDOWN_MODE="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["cooldown"]["mode"]) PY )" COOLDOWN_REMAINING="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["cooldown"]["remaining_seconds"]) PY )" cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( assessment_path, step, timestamp, rollback, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] with open(assessment_path, encoding="utf-8") as source: assessment = json.load(source) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "generated_at_utc": timestamp, "assessment": assessment, "operation": { "hmn_code_test_passed": True, "fresh_download_proven": True, "old_pool_fallback": False, "runner_decision": "refresh_ok_rebalance_ok", "real_refresh_ran": True, "rebalance_apply_ran": True, "state_mode": "NORMAL", "state_status": "refresh_ok_rebalance_ok", "planner_converged": True, "temporary_table200_bootstrap": True, "table200_restored": True, }, "live_streaming": { "runner_stdout": True, "runner_stderr": True, "emergency_log": True, "heartbeat_seconds": 15, "terminal_output_complete": True, }, "safety": { "production_modified": True, "direct_failopen_changed": False, "rollback_vm101": rollback, "rollback_available": True, }, "inline_diagnostics": { "enabled": True, "live_stream_captured": True, "automatic_classification_present": True, "rollback_on_stop": True, "separate_diagnostic_step_required": False, }, "plan": { "milestone_completed": "M07", "current_milestone": "M08", "plan_republished": True, }, "next_step": "M08_DEGRADED_POOL_CONTROLLED_SIMULATION", "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY cat > "$REPORT_DIR/index.html" < ${STEP}

${STEP}

Live stream

Скрипты

Результаты

EOF find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" cat > "$STATE_ROOT/current-project-links.env" <