#!/usr/bin/env bash set -Eeuo pipefail umask 077 [ "$(id -un)" = "ops" ] || { echo "STOP: запускать на router-ops пользователем ops" exit 1 } STEP="STEP_050M07E_ATOMIC_REPAIR_REFRESH_PLANNER_EXIT_CONTRACTS" PASS_DECISION="PASS_${STEP}" TOKEN="e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4" PUBLIC_BASE="https://helena-background-beam-harry.trycloudflare.com/r/${TOKEN}" ROOT="/opt/router-ops" PUBROOT="${ROOT}/public/r/${TOKEN}" D7_DIR="${PUBROOT}/20260712-071324_step050m07d7_audit_disk_active_iface_and_runtime_divergence" ARCHITECTURE_PLAN="${PUBLIC_BASE}/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/" XS_MAP="${PUBLIC_BASE}/20260711-120734_xs_map_audit_repair_publish/" GLOBAL_PROJECT_PLAN="${PUBLIC_BASE}/20260711-123348_global_project_plan_wg_paid/" TS="$(date -u +%Y%m%d-%H%M%S)" SLUG="${TS}_step050m07e_atomic_repair_refresh_planner_exit_contracts" REPORT_DIR="${PUBROOT}/${SLUG}" TRYCF_REPORT="${PUBLIC_BASE}/${SLUG}/" REPORT_TXT="${TRYCF_REPORT}report.txt" FACTS_JSON="${TRYCF_REPORT}facts.json" SOURCE_DIR="${REPORT_DIR}/sources-live" PATCHED_DIR="${REPORT_DIR}/patched" mkdir -p "$REPORT_DIR" "$SOURCE_DIR" "$PATCHED_DIR" # Верхнеуровневый STEP сохраняется до любых проверок. cp -a "$0" "$REPORT_DIR/step.sh" chmod 600 "$REPORT_DIR/step.sh" CURRENT_STAGE="initialization" LAST_SUCCESS="step_saved" PRECHECK_RC="NOT_RUN" INSTALL_RC="NOT_RUN" POSTCHECK_RC="NOT_RUN" ROLLBACK_RC="NOT_NEEDED" PRODUCTION_MODIFIED=false ROLLBACK_ARMED=false VM101_BACKUP_DIR="" VM101_ROLLBACK="" stage() { CURRENT_STAGE="$1" echo echo ">>> [$1] $2" date -u '+ utc=%Y-%m-%dT%H:%M:%SZ' } mark_success() { LAST_SUCCESS="$1" } print_links() { echo echo "TRYCF_REPORT=$TRYCF_REPORT" echo "REPORT_TXT=$REPORT_TXT" echo "FACTS_JSON=$FACTS_JSON" echo "ARCHITECTURE_PLAN=$ARCHITECTURE_PLAN" echo "XS_MAP=$XS_MAP" echo "GLOBAL_PROJECT_PLAN=$GLOBAL_PROJECT_PLAN" } remote_vm101() { ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ '$1'" } rollback_vm101() { if [ "$ROLLBACK_ARMED" != true ]; then ROLLBACK_RC="NOT_NEEDED" return 0 fi if [ -z "$VM101_ROLLBACK" ]; then ROLLBACK_RC="ROLLBACK_PATH_MISSING" return 1 fi echo echo ">>> Автоматический rollback VM101" if remote_vm101 "sh '$VM101_ROLLBACK'" \ > >(tee "$REPORT_DIR/vm101-rollback.txt") \ 2> >(tee "$REPORT_DIR/vm101-rollback.stderr" >&2) then ROLLBACK_RC=0 ROLLBACK_ARMED=false PRODUCTION_MODIFIED=false return 0 fi ROLLBACK_RC=$? return "$ROLLBACK_RC" } create_index() { cat > "$REPORT_DIR/index.html" < ${STEP}

${STEP}

EOF } write_stop() { local reason="$1" local rc="$2" local line="$3" python3 - \ "$REPORT_DIR" \ "$STEP" \ "$reason" \ "$rc" \ "$line" \ "$CURRENT_STAGE" \ "$LAST_SUCCESS" \ "$PRECHECK_RC" \ "$INSTALL_RC" \ "$POSTCHECK_RC" \ "$ROLLBACK_RC" \ "$PRODUCTION_MODIFIED" \ > "$REPORT_DIR/diagnostic.json" <<'PY' import json import sys from pathlib import Path ( report_dir, step, reason, rc, line, stage, last_success, precheck_rc, install_rc, postcheck_rc, rollback_rc, modified, ) = sys.argv[1:] root = Path(report_dir) def tail(path: Path, limit: int = 100000): if not path.exists() or not path.is_file(): return None return path.read_text( encoding="utf-8", errors="replace", )[-limit:] streams = {} for pattern in ("*.txt", "*.stderr", "*.log"): for path in sorted(root.glob(pattern)): if path.name == "report.txt": continue streams[path.name] = { "size_bytes": path.stat().st_size, "tail": tail(path), } combined = "\n".join( item["tail"] or "" for item in streams.values() ) if "PATCH_STOP" in combined: classification = "LOCAL_SOURCE_PATCH_ASSERTION_FAILED" elif "code_test_rc=" in combined and "code_test_rc=0" not in combined: classification = "ACTIVE_INTERFACE_REPAIR_NOT_PROVEN" elif "planner_runtime_match=false" in combined: classification = "PLANNER_RUNTIME_REPAIR_NOT_PROVEN" elif "apply_contract_ok=false" in combined: classification = "APPLY_EXIT_CONTRACT_NOT_PROVEN" elif "runner_contract_ok=false" in combined: classification = "RUNNER_JSON_CONTRACT_NOT_PROVEN" elif rollback_rc not in {"NOT_NEEDED", "0"}: classification = "ROLLBACK_FAILED" elif install_rc not in {"NOT_RUN", "0"}: classification = "VM101_INSTALL_FAILED" elif postcheck_rc not in {"NOT_RUN", "0"}: classification = "VM101_POSTCHECK_FAILED" else: classification = "LOCAL_VALIDATION_OR_PUBLICATION_FAILURE" print(json.dumps({ "schema": "router-step-inline-diagnostic-v15", "step": step, "failure": { "reason": reason, "rc": int(rc), "line": int(line), "stage": stage, "last_success": last_success, }, "command_results": { "precheck_rc": precheck_rc, "install_rc": install_rc, "postcheck_rc": postcheck_rc, "rollback_rc": rollback_rc, }, "automatic_classification": classification, "production_modified_after_rollback": modified, "captured_streams": streams, }, ensure_ascii=False, indent=2)) PY cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( diagnostic_path, step, reason, rc, line, stage, last_success, modified, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] diagnostic = json.load( open(diagnostic_path, encoding="utf-8") ) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "assessment": { "decision": f"STOP_{step}_{reason}", "all_ok": False, "error_rc": int(rc), "error_line": int(line), "failed_stage": stage, "last_success": last_success, }, "diagnostic": diagnostic, "safety": { "production_modified": {"true": True, "false": False}.get( modified, "unknown", ), "full_refresh_ran": False, "rebalance_commit_ran": False, "direct_failopen_changed": False, "plan_changed": False, }, "plan": { "current_milestone": "M07", "milestone_completed": False, }, "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" } fatal() { local reason="$1" local rc="${2:-1}" local line="${3:-$LINENO}" trap - ERR rollback_vm101 || true write_stop "$reason" "$rc" "$line" print_links exit "$rc" } trap 'rc=$?; fatal "UNEXPECTED_ERROR" "$rc" "$LINENO"' ERR stage "01/08" "Проверяю D7 PASS и сохраняю его доказательства" for required in \ "$D7_DIR/report.txt" \ "$D7_DIR/facts.json" \ "$D7_DIR/assessment.json" \ "$D7_DIR/runtime-divergence.json" \ "$D7_DIR/failure-contract.txt" do [ -s "$required" ] || { echo "MISSING_REQUIRED=$required" fatal "D7_REQUIRED_ARTIFACT_MISSING" 2 "$LINENO" } done grep -Fq \ "decision=PASS_STEP_050M07D7_AUDIT_DISK_ACTIVE_IFACE_AND_RUNTIME_DIVERGENCE" \ "$D7_DIR/report.txt" || fatal "D7_PASS_NOT_PROVEN" 3 "$LINENO" grep -Fq \ '"planner_runtime": "DIVERGED_vpn1_vpn2_vpn4"' \ "$D7_DIR/facts.json" || fatal "D7_RUNTIME_DIVERGENCE_NOT_PROVEN" 4 "$LINENO" cp -a "$D7_DIR/facts.json" \ "$REPORT_DIR/sources-d7-facts.json" cp -a "$D7_DIR/runtime-divergence.json" \ "$REPORT_DIR/sources-d7-runtime-divergence.json" cp -a "$D7_DIR/failure-contract.txt" \ "$REPORT_DIR/sources-d7-failure-contract.txt" mark_success "d7_verified" stage "02/08" "Снимаю live sources и безопасный baseline VM101" cat > "$REPORT_DIR/vm101-precheck.sh" <<'VM101' #!/bin/sh set -u umask 077 AMNEZIAWG="/usr/bin/amneziawg" fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } strict_iface() { iface="$1" attempt=1 while [ "$attempt" -le 3 ]; do ping -I "$iface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 && return 0 attempt=$((attempt + 1)) sleep 1 done return 1 } routes_all() { for table in 201 202 203 204 205; do ip route show table "$table" | grep -q '^default ' || return 1 done } [ -x "$AMNEZIAWG" ] || exit 20 fact root_free_kb "$( df -Pk / | awk 'NR == 2 {print $4}' )" fact tmp_free_kb "$( df -Pk /tmp | awk 'NR == 2 {print $4}' )" fact root_free_inodes "$( df -Pi / | awk 'NR == 2 {print $4}' )" fact table200_sha256 "$( ip route show table 200 | sort | sha256sum | sed 's/[[:space:]].*$//' )" healthy=0 for iface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$iface"; then value=true healthy=$((healthy + 1)) else value=false fi fact "strict.${iface}" "$value" line="$( "$AMNEZIAWG" show "$iface" dump 2>/dev/null | sed -n '2p' )" endpoint="$( printf '%s\n' "$line" | cut -f3 )" fact "runtime_endpoint.${iface}" "${endpoint:-UNRESOLVED}" done fact healthy_slots "$healthy" fact routes_all "$( if routes_all; then echo true; else echo false; fi )" fact hook_running "$( if /etc/init.d/router-egress-emergency-decision running \ >/dev/null 2>&1 then echo true else echo false fi )" fact watcher_running "$( if /etc/init.d/router-egress-health-repair running \ >/dev/null 2>&1 then echo true else echo false fi )" fact emergency_lock_present "$( if [ -e /var/lock/router-egress-emergency-refresh.lock ] then echo true else echo false fi )" fact refresh_lock_present "$( if [ -e /tmp/hmn-refresh-pool-safe.lock ] then echo true else echo false fi )" [ "$healthy" -eq 5 ] || exit 31 routes_all || exit 32 VM101 chmod 600 "$REPORT_DIR/vm101-precheck.sh" sh -n "$REPORT_DIR/vm101-precheck.sh" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101-precheck.sh" \ > "$REPORT_DIR/vm101-precheck.txt" \ 2> "$REPORT_DIR/vm101-precheck.stderr" then PRECHECK_RC=0 else PRECHECK_RC=$? fi [ "$PRECHECK_RC" -eq 0 ] || fatal "VM101_PRECHECK_FAILED" "$PRECHECK_RC" "$LINENO" declare -a SOURCE_MAP=( "refresh|/root/hmn/hmn-refresh-pool-safe.sh" "download|/root/hmn/hmn-download-all-awg.sh" "validate|/root/hmn/hmn-validate-current-pool.sh" "code_test|/root/hmn/hmn-code-test.sh" "refresh_awg|/root/hmn/hmn-refresh-awg.sh" "planner|/usr/local/sbin/router-egress-hmn-plan-top5.sh" "apply|/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh" "runner|/usr/local/sbin/router-egress-emergency-refresh.sh" "slots_apply|/usr/local/sbin/router-egress-slots-apply.sh" ) : > "$REPORT_DIR/source-map.txt" : > "$REPORT_DIR/hashes-before.txt" for entry in "${SOURCE_MAP[@]}"; do label="${entry%%|*}" path="${entry#*|}" echo "${label}|${path}" >> "$REPORT_DIR/source-map.txt" if ! remote_vm101 "test -f '$path'"; then echo "MISSING_LIVE_SOURCE=$path" fatal "LIVE_SOURCE_MISSING" 5 "$LINENO" fi remote_vm101 "cat '$path'" \ > "$SOURCE_DIR/${label}.sh" hash="$( sha256sum "$SOURCE_DIR/${label}.sh" | cut -d' ' -f1 )" echo "${label} ${hash} ${path}" \ >> "$REPORT_DIR/hashes-before.txt" done mark_success "live_sources_captured" stage "03/08" "Строю патчи с точными assertions" python3 - \ "$SOURCE_DIR" \ "$PATCHED_DIR" \ "$REPORT_DIR/patch-summary.json" <<'PY' import json import re import sys from pathlib import Path source_dir = Path(sys.argv[1]) patched_dir = Path(sys.argv[2]) summary_path = Path(sys.argv[3]) patched_dir.mkdir(parents=True, exist_ok=True) summary = {} def inject_after_header(text: str, block: str) -> str: lines = text.splitlines() insert_at = 1 for index, line in enumerate(lines[:10]): if re.match(r"^\s*set\s+-[a-zA-Z]*u", line): insert_at = index + 1 break lines[insert_at:insert_at] = [ "", block.rstrip(), "", ] return "\n".join(lines) + "\n" ACTIVE_HELPER = r''' # M07E: table 200 is optional in the five-slot architecture. # Resolve an actually healthy AmneziaWG slot, preferring an existing # table-200 default only when it is both vpn1..vpn5 and healthy. m07e_detect_active_vpn() { candidate="$( ip -4 route show table 200 2>/dev/null | sed -n 's/^default dev $vpn[1-5]$$[[:space:]].*$\{0,1\}$/\1/p' | head -n 1 )" if [ -n "$candidate" ] && ping -I "$candidate" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 then printf '%s\n' "$candidate" return 0 fi for candidate in vpn1 vpn2 vpn3 vpn4 vpn5; do if ip link show dev "$candidate" >/dev/null 2>&1 && ping -I "$candidate" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 then printf '%s\n' "$candidate" return 0 fi done return 1 } ''' def replace_active_assignment( text: str, label: str, ) -> str: if "M07E: table 200 is optional" not in text: text = inject_after_header( text, ACTIVE_HELPER, ) lines = text.splitlines() error_indexes = [ index for index, line in enumerate(lines) if ( "active VPN interface" in line or "ACTIVE" in line and "table 200" in line ) ] candidates = [] for index, line in enumerate(lines): match = re.match( r"^\s*([A-Z][A-Z0-9_]*ACTIVE[A-Z0-9_]*|ACTIVE)\s*=", line, ) if not match: continue window = "\n".join( lines[ max(0, index - 8): min(len(lines), index + 18) ] ) if ( "table 200" in window or "active VPN interface" in window ): candidates.append( (index, match.group(1)) ) if not candidates: raise RuntimeError( f"PATCH_STOP {label}: active assignment not found" ) if error_indexes: error_index = error_indexes[0] candidates.sort( key=lambda item: abs(item[0] - error_index) ) start, variable = candidates[0] end = start if "$(" in lines[start]: balance = ( lines[start].count("$(") - lines[start].count(")") ) while balance > 0 and end + 1 < len(lines): end += 1 balance += ( lines[end].count("$(") - lines[end].count(")") ) lines[start:end + 1] = [ f'{variable}="$(m07e_detect_active_vpn || true)"' ] result = "\n".join(lines) + "\n" if "m07e_detect_active_vpn" not in result: raise RuntimeError( f"PATCH_STOP {label}: helper not installed" ) return result def patch_refresh(text: str) -> str: old_base = "/root/hmn/backups/refresh-safe-before-" new_base = "/tmp/hmn-refresh-backups/refresh-safe-before-" count = text.count(old_base) if count < 1: raise RuntimeError( "PATCH_STOP refresh: backup path not found" ) text = text.replace(old_base, new_base) backup_line = None backup_var = None for line in text.splitlines(): if new_base not in line or "=" not in line: continue match = re.match( r"^\s*([A-Za-z_][A-Za-z0-9_]*)=", line, ) if match: backup_line = line backup_var = match.group(1) break if not backup_var: raise RuntimeError( "PATCH_STOP refresh: backup variable unresolved" ) lines = text.splitlines() backup_marker = next( ( index for index, line in enumerate(lines) if "backup current published pointers/tables" in line ), None, ) fresh_marker = next( ( index for index, line in enumerate(lines) if "fresh download stage" in line ), None, ) if backup_marker is None or fresh_marker is None: raise RuntimeError( "PATCH_STOP refresh: stage markers missing" ) preflight = r''' # M07E storage preflight. Backups live in tmpfs; the fresh # generation still requires enough persistent root space. M07E_ROOT_FREE_KB="$( df -Pk / | awk 'NR == 2 {print $4}' )" M07E_ROOT_FREE_INODES="$( df -Pi / | awk 'NR == 2 {print $4}' )" M07E_TMP_FREE_KB="$( df -Pk /tmp | awk 'NR == 2 {print $4}' )" mkdir -p /tmp/hmn-refresh-backups find /tmp/hmn-refresh-backups \ -mindepth 1 \ -maxdepth 1 \ -type d \ -mtime +2 \ -exec rm -rf {} \; \ 2>/dev/null || true case "$M07E_ROOT_FREE_KB:$M07E_ROOT_FREE_INODES:$M07E_TMP_FREE_KB" in *[!0-9:]*|'') echo "ERROR: M07E storage preflight parse failed" exit 70 ;; esac if [ "$M07E_ROOT_FREE_KB" -lt 16384 ] || [ "$M07E_ROOT_FREE_INODES" -lt 128 ] || [ "$M07E_TMP_FREE_KB" -lt 32768 ] then echo "ERROR: M07E storage preflight failed root_kb=$M07E_ROOT_FREE_KB root_inodes=$M07E_ROOT_FREE_INODES tmp_kb=$M07E_TMP_FREE_KB" exit 71 fi '''.strip().splitlines() lines[backup_marker:backup_marker] = ( [""] + preflight + [""] ) # Marker shifted after the previous insertion. fresh_marker = next( index for index, line in enumerate(lines) if "fresh download stage" in line ) manifest = f''' # M07E refuses to mutate published pointers unless the rollback # manifest is complete and usable. M07E_BACKUP_OK=true for required_name in \\ configs-latest-readlink.txt \\ configs-latest-realpath.txt do [ -s "${{{backup_var}}}/$required_name" ] || M07E_BACKUP_OK=false done for required_name in \\ ok-awg1-strict-all-latest.tsv \\ ok-awg1-strict-foreign-latest.tsv \\ selected-awg1-latest.tsv do source_path="/root/hmn/cache/$required_name" if [ -e "$source_path" ]; then [ -s "${{{backup_var}}}/$required_name" ] || M07E_BACKUP_OK=false fi done if [ "$M07E_BACKUP_OK" != true ]; then echo "ERROR: M07E rollback manifest incomplete backup=${{{backup_var}}}" exit 72 fi echo "M07E_BACKUP_MANIFEST_OK backup=${{{backup_var}}}" '''.strip().splitlines() lines[fresh_marker:fresh_marker] = ( [""] + manifest + [""] ) result = "\n".join(lines) + "\n" for marker in ( "M07E storage preflight", "M07E_BACKUP_MANIFEST_OK", "/tmp/hmn-refresh-backups", ): if marker not in result: raise RuntimeError( f"PATCH_STOP refresh: marker missing {marker}" ) return result RUNTIME_HELPER = r''' # M07E: UCI hmn_endpoint may be stale after an interrupted apply. # Prefer the actual AmneziaWG peer endpoint and fall back to UCI. AMNEZIAWG_BIN="${AMNEZIAWG_BIN:-/usr/bin/amneziawg}" m07e_runtime_endpoint() { iface="$1" endpoint="" if [ -x "$AMNEZIAWG_BIN" ]; then endpoint="$( "$AMNEZIAWG_BIN" show "$iface" dump 2>/dev/null | sed -n '2p' | cut -f3 )" fi case "$endpoint" in ""|"(none)") endpoint="$( uci -q get network.${iface}.hmn_endpoint \ 2>/dev/null || true )" ;; esac printf '%s\n' "$endpoint" } ''' def patch_runtime_current( text: str, label: str, ) -> str: if "m07e_runtime_endpoint" not in text: text = inject_after_header( text, RUNTIME_HELPER, ) pattern = re.compile( r'''cur="\$$\s*uci\s+-q\s+get\s+network\.\$\{iface\}\.hmn_endpoint\s+2>/dev/null\s+\|\|\s+true\s*$"''', re.DOTALL, ) text, count = pattern.subn( 'cur="$(m07e_runtime_endpoint "$iface")"', text, count=1, ) if count != 1: raise RuntimeError( f"PATCH_STOP {label}: UCI current pattern count={count}" ) return text def find_function_end( lines: list[str], start: int, ) -> int: balance = 0 for index in range(start, len(lines)): balance += lines[index].count("{") balance -= lines[index].count("}") if index > start and balance <= 0: return index raise RuntimeError( "PATCH_STOP apply: strict_ping function end missing" ) def patch_apply(text: str) -> str: text = patch_runtime_current( text, "apply", ) if "strict_ping_once()" not in text: lines = text.splitlines() start = next( ( index for index, line in enumerate(lines) if re.search(r"\bstrict_ping\s*\{", line) ), None, ) if start is None: raise RuntimeError( "PATCH_STOP apply: strict_ping function missing" ) lines[start] = lines[start].replace( "strict_ping()", "strict_ping_once()", 1, ) end = find_function_end(lines, start) retry_wrapper = r''' # M07E: allow AmneziaWG handshake establishment after ifup. strict_ping() { iface="$1" attempt=1 while [ "$attempt" -le 3 ]; do if strict_ping_once "$iface"; then return 0 fi attempt=$((attempt + 1)) sleep 5 done return 1 } '''.strip().splitlines() lines[end + 1:end + 1] = ( [""] + retry_wrapper + [""] ) text = "\n".join(lines) + "\n" if "M07E_EXIT_CONTRACT" not in text: text = text.rstrip() + r''' # M07E_EXIT_CONTRACT: JSON failure must also be a shell failure. case "$decision" in commit_failed) exit 41 ;; refuse) if [ "$MODE" = "--commit" ]; then exit 42 fi ;; esac exit 0 ''' + "\n" return text def patch_runner(text: str) -> str: if "M07E_REBALANCE_JSON_CONTRACT" in text: return text pattern = re.compile( r''' "\$EMERGENCY_REBALANCE_APPLY_CMD" \s+--commit \s+--confirm \s+REBALANCE_TOP5_DAILY \s+>>\s*"\$EMERGENCY_LOG" \s+2>&1 [\s;]+rebalance_rc=\$\? [\s;]+set_state \s+last_emergency_rebalance_rc \s+"\$rebalance_rc" ''', re.VERBOSE | re.DOTALL, ) replacement = r''' # M07E_REBALANCE_JSON_CONTRACT rebalance_tmp="/tmp/router-egress-rebalance-contract.$$" "$EMERGENCY_REBALANCE_APPLY_CMD" \ --commit \ --confirm REBALANCE_TOP5_DAILY \ > "$rebalance_tmp" \ 2>&1 rebalance_rc=$? cat "$rebalance_tmp" \ >> "$EMERGENCY_LOG" \ 2>/dev/null || true rebalance_contract_ok=false if grep -Eq \ '"decision"[[:space:]]*:[[:space:]]*"(commit_ok|noop)"' \ "$rebalance_tmp" && grep -Eq \ '"apply_ok"[[:space:]]*:[[:space:]]*true' \ "$rebalance_tmp" then rebalance_contract_ok=true fi if [ "$rebalance_rc" = "0" ] && [ "$rebalance_contract_ok" = true ] then rebalance_rc=0 else [ "$rebalance_rc" != "0" ] || rebalance_rc=65 fi rm -f "$rebalance_tmp" set_state last_emergency_rebalance_rc "$rebalance_rc" '''.strip() text, count = pattern.subn( replacement, text, count=1, ) if count != 1: raise RuntimeError( f"PATCH_STOP runner: apply call pattern count={count}" ) return text def patch_slots_apply(text: str) -> str: pattern = re.compile( r''' ip\s+route\s+show\s+table\s+200 \s*\|\s* grep\s+-F\s+"default\s+dev\s+vpn1" \s*>/dev/null \s*\|\|\s* \{ \s*echo\s+"BLOCK:\s+legacy_table200_default_vpn1_missing" \s*exit\s+20 \s*\} ''', re.VERBOSE | re.DOTALL, ) replacement = r''' # M07E: table 200 default is optional; tables 201..205 are authoritative. if ip route show table 200 2>/dev/null | grep -q '^default ' then echo "table200_default_present_optional=1" else echo "table200_default_absent_optional=1" fi '''.strip() text, count = pattern.subn( replacement, text, count=1, ) if count != 1: raise RuntimeError( f"PATCH_STOP slots_apply: legacy gate count={count}" ) return text patchers = { "refresh": patch_refresh, "download": lambda value: replace_active_assignment( value, "download", ), "validate": lambda value: replace_active_assignment( value, "validate", ), "code_test": lambda value: replace_active_assignment( value, "code_test", ), "refresh_awg": lambda value: replace_active_assignment( value, "refresh_awg", ), "planner": lambda value: patch_runtime_current( value, "planner", ), "apply": patch_apply, "runner": patch_runner, "slots_apply": patch_slots_apply, } for label, patcher in patchers.items(): source_path = source_dir / f"{label}.sh" destination_path = patched_dir / f"{label}.sh" source = source_path.read_text( encoding="utf-8", errors="strict", ) patched = patcher(source) if patched == source: raise RuntimeError( f"PATCH_STOP {label}: no source change" ) destination_path.write_text( patched, encoding="utf-8", ) summary[label] = { "source_bytes": len(source.encode()), "patched_bytes": len(patched.encode()), "markers": [ marker for marker in ( "m07e_detect_active_vpn", "M07E_BACKUP_MANIFEST_OK", "m07e_runtime_endpoint", "M07E_EXIT_CONTRACT", "M07E_REBALANCE_JSON_CONTRACT", "table200_default_absent_optional", ) if marker in patched ], } summary_path.write_text( json.dumps( summary, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) PY for script in "$PATCHED_DIR"/*.sh; do sh -n "$script" || { echo "PATCHED_SYNTAX_FAILED=$script" fatal "PATCHED_SOURCE_SYNTAX_FAILED" 6 "$LINENO" } done : > "$REPORT_DIR/hashes-patched.txt" for script in "$PATCHED_DIR"/*.sh; do label="$(basename "$script" .sh)" hash="$(sha256sum "$script" | cut -d' ' -f1)" echo "$label $hash" >> "$REPORT_DIR/hashes-patched.txt" done mark_success "patches_built_and_syntax_checked" stage "04/08" "Создаю VM101 backup и rollback" VM101_BACKUP_DIR="/root/m07e-toolchain-backup-${TS}" VM101_ROLLBACK="/root/rollback-step050m07e-${TS}.sh" remote_vm101 "mkdir -p '$VM101_BACKUP_DIR'" while IFS='|' read -r label path; do remote_vm101 \ "cp -a '$path' '$VM101_BACKUP_DIR/${label}.before'" done < "$REPORT_DIR/source-map.txt" cat > "$REPORT_DIR/vm101-rollback.sh" <> "$REPORT_DIR/vm101-rollback.sh" <> "$REPORT_DIR/vm101-rollback.sh" <<'EOF' for service in \ /etc/init.d/router-egress-emergency-decision \ /etc/init.d/router-egress-health-repair do "$service" restart >/dev/null 2>&1 || errors=$((errors + 1)) done sleep 2 strict=true for iface in vpn1 vpn2 vpn3 vpn4 vpn5; do ping -I "$iface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 || strict=false done routes=true for table in 201 202 203 204 205; do ip route show table "$table" | grep -q '^default ' || routes=false done echo "rollback_errors=$errors" echo "rollback_strict_all=$strict" echo "rollback_routes_all=$routes" if [ "$errors" -eq 0 ] && [ "$strict" = true ] && [ "$routes" = true ] then echo "rollback_validated=true" exit 0 fi echo "rollback_validated=false" exit 1 EOF chmod 600 "$REPORT_DIR/vm101-rollback.sh" sh -n "$REPORT_DIR/vm101-rollback.sh" remote_vm101 "cat > '$VM101_ROLLBACK'" \ < "$REPORT_DIR/vm101-rollback.sh" remote_vm101 "chmod 700 '$VM101_ROLLBACK'" ROLLBACK_ARMED=true mark_success "backup_and_rollback_created" stage "05/08" "Устанавливаю patched toolchain" for entry in "${SOURCE_MAP[@]}"; do label="${entry%%|*}" path="${entry#*|}" remote_tmp="/tmp/m07e-${label}-${TS}.new" remote_vm101 "cat > '$remote_tmp'" \ < "$PATCHED_DIR/${label}.sh" remote_vm101 "sh -n '$remote_tmp'" remote_vm101 \ "cp '$remote_tmp' '$path' && chmod 700 '$path' && rm -f '$remote_tmp'" done INSTALL_RC=0 PRODUCTION_MODIFIED=true remote_vm101 \ "/etc/init.d/router-egress-emergency-decision restart >/dev/null 2>&1 && /etc/init.d/router-egress-health-repair restart >/dev/null 2>&1" mark_success "patched_toolchain_installed" stage "06/08" "Проверяю active detection, runtime planner и exit contracts" cat > "$REPORT_DIR/vm101-postcheck.sh" <<'VM101' #!/bin/sh set -u umask 077 AMNEZIAWG="/usr/bin/amneziawg" CODE_TEST="/root/hmn/hmn-code-test.sh" PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh" APPLY="/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh" RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh" fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } json_block() { echo "__JSON_BEGIN__ $1" printf '%s\n' "$2" echo "__JSON_END__ $1" } strict_iface() { iface="$1" attempt=1 while [ "$attempt" -le 3 ]; do ping -I "$iface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 && return 0 attempt=$((attempt + 1)) sleep 1 done return 1 } routes_all() { for table in 201 202 203 204 205; do ip route show table "$table" | grep -q '^default ' || return 1 done } TABLE200_PRE="$( ip route show table 200 | sort | sha256sum | sed 's/[[:space:]].*$//' )" fact table200_hash_pre "$TABLE200_PRE" set +e CODE_OUTPUT="$("$CODE_TEST" 2>&1)" CODE_RC=$? set -e fact code_test_rc "$CODE_RC" echo "__BLOCK_BEGIN__ code_test" printf '%s\n' "$CODE_OUTPUT" echo "__BLOCK_END__ code_test" PLANNER_JSON="$("$PLANNER")" set +e APPLY_JSON="$("$APPLY" --dry-run 2>&1)" APPLY_RC=$? set -e RUNNER_JSON="$("$RUNNER" --dry-run)" json_block planner "$PLANNER_JSON" json_block apply_dry_run "$APPLY_JSON" json_block runner_dry_run "$RUNNER_JSON" healthy=0 for iface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$iface"; then value=true healthy=$((healthy + 1)) else value=false fi fact "strict.${iface}" "$value" line="$( "$AMNEZIAWG" show "$iface" dump 2>/dev/null | sed -n '2p' )" endpoint="$( printf '%s\n' "$line" | cut -f3 )" fact "runtime_endpoint.${iface}" "${endpoint:-UNRESOLVED}" done fact healthy_slots "$healthy" fact routes_all "$( if routes_all; then echo true; else echo false; fi )" fact apply_dry_run_rc "$APPLY_RC" fact table200_hash_post "$( ip route show table 200 | sort | sha256sum | sed 's/[[:space:]].*$//' )" fact hook_running "$( if /etc/init.d/router-egress-emergency-decision running \ >/dev/null 2>&1 then echo true else echo false fi )" fact watcher_running "$( if /etc/init.d/router-egress-health-repair running \ >/dev/null 2>&1 then echo true else echo false fi )" fact emergency_lock_present "$( if [ -e /var/lock/router-egress-emergency-refresh.lock ] then echo true else echo false fi )" fact refresh_lock_present "$( if [ -e /tmp/hmn-refresh-pool-safe.lock ] then echo true else echo false fi )" for marker_path in \ "/root/hmn/hmn-refresh-pool-safe.sh:M07E_BACKUP_MANIFEST_OK" \ "/root/hmn/hmn-download-all-awg.sh:m07e_detect_active_vpn" \ "/root/hmn/hmn-validate-current-pool.sh:m07e_detect_active_vpn" \ "/root/hmn/hmn-code-test.sh:m07e_detect_active_vpn" \ "/root/hmn/hmn-refresh-awg.sh:m07e_detect_active_vpn" \ "/usr/local/sbin/router-egress-hmn-plan-top5.sh:m07e_runtime_endpoint" \ "/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh:M07E_EXIT_CONTRACT" \ "/usr/local/sbin/router-egress-emergency-refresh.sh:M07E_REBALANCE_JSON_CONTRACT" \ "/usr/local/sbin/router-egress-slots-apply.sh:table200_default_absent_optional" do path="${marker_path%%:*}" marker="${marker_path#*:}" if grep -Fq "$marker" "$path"; then value=true else value=false fi safe_key="$( printf '%s' "$marker" | tr -c 'A-Za-z0-9_' '_' )" fact "marker.${safe_key}" "$value" done fact full_refresh_ran false fact rebalance_commit_ran false fact direct_failopen_changed false [ "$CODE_RC" -eq 0 ] || exit 41 [ "$healthy" -eq 5 ] || exit 42 routes_all || exit 43 [ "$TABLE200_PRE" = "$( ip route show table 200 | sort | sha256sum | sed 's/[[:space:]].*$//' )" ] || exit 44 VM101 chmod 600 "$REPORT_DIR/vm101-postcheck.sh" sh -n "$REPORT_DIR/vm101-postcheck.sh" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101-postcheck.sh" \ > >(tee "$REPORT_DIR/vm101-postcheck.txt") \ 2> >(tee "$REPORT_DIR/vm101-postcheck.stderr" >&2) then POSTCHECK_RC=0 else POSTCHECK_RC=$? fi [ "$POSTCHECK_RC" -eq 0 ] || fatal "VM101_POSTCHECK_FAILED" "$POSTCHECK_RC" "$LINENO" mark_success "remote_postcheck_complete" stage "07/08" "Проверяю JSON-контракты и новые hashes" python3 - \ "$REPORT_DIR/vm101-postcheck.txt" \ "$REPORT_DIR/assessment.json" <<'PY' import json import sys from pathlib import Path text = Path(sys.argv[1]).read_text( encoding="utf-8", errors="replace", ) facts = {} blocks = {} current = None lines = [] for line in text.splitlines(): if line.startswith("__FACT__ "): payload = line[len("__FACT__ "):] if "=" in payload: key, value = payload.split("=", 1) facts[key] = value elif line.startswith("__JSON_BEGIN__ "): current = line[len("__JSON_BEGIN__ "):] lines = [] elif line.startswith("__JSON_END__ "): name = line[len("__JSON_END__ "):] if name == current: blocks[name] = json.loads( "\n".join(lines) ) current = None lines = [] elif current is not None: lines.append(line) planner = blocks["planner"] apply_dry = blocks["apply_dry_run"] runner = blocks["runner_dry_run"] interfaces = [ "vpn1", "vpn2", "vpn3", "vpn4", "vpn5", ] runtime = { iface: facts[f"runtime_endpoint.{iface}"] for iface in interfaces } planner_current = { item["iface"]: item["current"] for item in planner["plan"] } planner_runtime_match = ( set(planner_current) == set(interfaces) and all( planner_current[iface] == runtime[iface] for iface in interfaces ) ) apply_contract_ok = ( int(facts["apply_dry_run_rc"]) == 0 and apply_dry["decision"] in {"plan_ok", "noop"} and apply_dry["apply_ok"] is True ) runner_contract_ok = ( runner["direct_failopen_enabled"] is False and runner["decision"] in { "would_run_emergency_refresh", "cooldown_active", "below_threshold_noop", } ) marker_values = [ value for key, value in facts.items() if key.startswith("marker.") ] checks = { "code_test_passed": facts["code_test_rc"] == "0", "planner_runtime_match": planner_runtime_match, "apply_contract_ok": apply_contract_ok, "runner_contract_ok": runner_contract_ok, "five_strict_slots": facts["healthy_slots"] == "5" and all( facts[f"strict.{iface}"] == "true" for iface in interfaces ), "routes_201_205": facts["routes_all"] == "true", "services_running": facts["hook_running"] == "true" and facts["watcher_running"] == "true", "locks_absent": facts["emergency_lock_present"] == "false" and facts["refresh_lock_present"] == "false", "table200_unchanged": facts["table200_hash_pre"] == facts["table200_hash_post"], "all_patch_markers": marker_values and all(value == "true" for value in marker_values), "no_full_refresh": facts["full_refresh_ran"] == "false", "no_rebalance_commit": facts["rebalance_commit_ran"] == "false", "direct_unchanged": facts["direct_failopen_changed"] == "false", } all_ok = all(checks.values()) assessment = { "all_ok": all_ok, "decision": ( "PASS_STEP_050M07E_ATOMIC_REPAIR_REFRESH_PLANNER_EXIT_CONTRACTS" if all_ok else "STOP_STEP_050M07E_ATOMIC_REPAIR_REFRESH_PLANNER_EXIT_CONTRACTS" ), "checks": checks, "planner": { "decision": planner["decision"], "changes_count": planner["changes_count"], "runtime_match": planner_runtime_match, "current_endpoints": planner_current, }, "apply_dry_run": { "rc": int(facts["apply_dry_run_rc"]), "decision": apply_dry["decision"], "reason": apply_dry["reason"], "apply_ok": apply_dry["apply_ok"], "changes_count": apply_dry["changes_count"], }, "runner_dry_run": { "decision": runner["decision"], "reason": runner["reason"], "direct_failopen_enabled": runner["direct_failopen_enabled"], }, "safety": { "production_modified": True, "rollback_available": True, "full_refresh_ran": False, "rebalance_commit_ran": False, "direct_failopen_changed": False, "plan_changed": False, }, "next_step": "M07F_CONTROLLED_FRESH_REFRESH_WITH_LIVE_STREAM", } Path(sys.argv[2]).write_text( json.dumps( assessment, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) if not all_ok: print( "planner_runtime_match=" + str(planner_runtime_match).lower() ) print( "apply_contract_ok=" + str(apply_contract_ok).lower() ) print( "runner_contract_ok=" + str(runner_contract_ok).lower() ) raise SystemExit( "failed checks: " + ",".join( name for name, value in checks.items() if not value ) ) PY : > "$REPORT_DIR/hashes-after.txt" while IFS='|' read -r label path; do hash="$( remote_vm101 "sha256sum '$path'" | sed 's/[[:space:]].*$//' )" echo "${label} ${hash} ${path}" \ >> "$REPORT_DIR/hashes-after.txt" done < "$REPORT_DIR/source-map.txt" ROLLBACK_ARMED=false mark_success "all_contracts_validated" stage "08/08" "Публикую PASS" PLANNER_CHANGES="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys data = json.load(open(sys.argv[1], encoding="utf-8")) print(data["planner"]["changes_count"]) PY )" APPLY_DECISION="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys data = json.load(open(sys.argv[1], encoding="utf-8")) print(data["apply_dry_run"]["decision"]) PY )" cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( assessment_path, patch_summary_path, step, timestamp, backup_dir, rollback, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] assessment = json.load( open(assessment_path, encoding="utf-8") ) patch_summary = json.load( open(patch_summary_path, encoding="utf-8") ) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "generated_at_utc": timestamp, "assessment": assessment, "patch_summary": patch_summary, "safety": { "production_modified": True, "backup_dir": backup_dir, "rollback": rollback, "rollback_available": True, "full_refresh_ran": False, "rebalance_commit_ran": False, "direct_failopen_changed": False, "plan_changed": False, }, "plan": { "current_milestone": "M07", "milestone_completed": False, }, "next_step": "M07F_CONTROLLED_FRESH_REFRESH_WITH_LIVE_STREAM", "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" trap - ERR echo "decision=$PASS_DECISION" echo "planner_runtime_match=true" echo "planner_changes_count=$PLANNER_CHANGES" echo "apply_dry_run_decision=$APPLY_DECISION" echo "hmn_code_test_passed=true" echo "full_refresh_ran=false" echo "current_milestone=M07" print_links