#!/usr/bin/env bash set -Eeuo pipefail umask 077 [ "$(id -un)" = "ops" ] || { echo "STOP: запускать на router-ops пользователем ops" exit 1 } STEP="STEP_050M07D2_FREEZE_PRE0420_EVIDENCE" PASS_DECISION="PASS_${STEP}" TOKEN="e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4" PUBLIC_BASE="https://helena-background-beam-harry.trycloudflare.com/r/${TOKEN}" ROOT="/opt/router-ops" PUBROOT="${ROOT}/public/r/${TOKEN}" STATE_ROOT="${ROOT}/state" M07D_DIR="${PUBROOT}/20260711-210850_step050m07d_classify_false_success_and_egress1_failure" C5_DIR="${PUBROOT}/20260711-205604_step050m07c5_tmpfs_surgical_backup_fresh_refresh_live" ARCHITECTURE_PLAN="${PUBLIC_BASE}/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/" XS_MAP="${PUBLIC_BASE}/20260711-120734_xs_map_audit_repair_publish/" GLOBAL_PROJECT_PLAN="${PUBLIC_BASE}/20260711-123348_global_project_plan_wg_paid/" TS="$(date -u +%Y%m%d-%H%M%S)" REPORT_SLUG="${TS}_step050m07d2_freeze_pre0420_evidence" REPORT_DIR="${PUBROOT}/${REPORT_SLUG}" TRYCF_REPORT="${PUBLIC_BASE}/${REPORT_SLUG}/" REPORT_TXT="${TRYCF_REPORT}report.txt" FACTS_JSON="${TRYCF_REPORT}facts.json" mkdir -p \ "$REPORT_DIR" \ "$REPORT_DIR/sources" # Сохраняем точный пользовательский STEP до проверок и SSH. cp -a "$0" "$REPORT_DIR/step.sh" chmod 600 "$REPORT_DIR/step.sh" PROGRESS_LOG="$REPORT_DIR/progress.log" : > "$PROGRESS_LOG" CURRENT_STAGE="initialization" LAST_SUCCESS="step_saved" VM101_RC="NOT_RUN" stage() { CURRENT_STAGE="$1" echo echo ">>> [$1] $2" | tee -a "$PROGRESS_LOG" date -u '+ utc=%Y-%m-%dT%H:%M:%SZ' | tee -a "$PROGRESS_LOG" } mark_success() { LAST_SUCCESS="$1" echo "last_success=$LAST_SUCCESS" >> "$PROGRESS_LOG" } print_links() { echo echo "TRYCF_REPORT=$TRYCF_REPORT" echo "REPORT_TXT=$REPORT_TXT" echo "FACTS_JSON=$FACTS_JSON" echo "ARCHITECTURE_PLAN=$ARCHITECTURE_PLAN" echo "XS_MAP=$XS_MAP" echo "GLOBAL_PROJECT_PLAN=$GLOBAL_PROJECT_PLAN" } create_index() { cat > "$REPORT_DIR/index.html" < ${STEP}

${STEP}

Итог

VM101 snapshot

Замороженные доказательства

EOF } write_stop() { local reason="$1" local rc="$2" local line="$3" python3 - \ "$REPORT_DIR" \ "$STEP" \ "$reason" \ "$rc" \ "$line" \ "$CURRENT_STAGE" \ "$LAST_SUCCESS" \ "$VM101_RC" \ > "$REPORT_DIR/diagnostic.json" <<'PY' import json import sys from pathlib import Path ( report_dir, step, reason, rc, line, stage, last_success, vm101_rc, ) = sys.argv[1:] root = Path(report_dir) def tail(path: Path, limit: int = 80000): if not path.exists() or not path.is_file(): return None return path.read_text( encoding="utf-8", errors="replace", )[-limit:] streams = {} for pattern in ("*.txt", "*.stderr", "*.log"): for path in sorted(root.glob(pattern)): if path.name in {"report.txt", "progress.log"}: continue streams[path.name] = { "size_bytes": path.stat().st_size, "tail": tail(path), } combined = "\n".join( item["tail"] or "" for item in streams.values() ) if "source_missing=" in combined: classification = "REQUIRED_SOURCE_MISSING" elif "readonly_violation=" in combined: classification = "READONLY_INVARIANT_VIOLATION" elif vm101_rc not in {"NOT_RUN", "0"}: classification = "VM101_READONLY_SNAPSHOT_FAILED" else: classification = "LOCAL_PARSER_OR_PUBLICATION_FAILURE" print(json.dumps({ "schema": "router-step-inline-diagnostic-v10", "step": step, "failure": { "reason": reason, "rc": int(rc), "line": int(line), "stage": stage, "last_success": last_success, }, "command_results": { "vm101_rc": vm101_rc, }, "automatic_classification": classification, "safety": { "read_only": True, "refresh_ran": False, "rebalance_ran": False, "network_changed": False, "services_changed": False, "state_changed": False, "timer_changed": False, "plan_changed": False, "direct_failopen_changed": False, }, "captured_streams": streams, "recommended_next_step": ( "После ночного scheduled refresh выполнить read-only " "post-04:20 comparison с этим frozen snapshot." ), }, ensure_ascii=False, indent=2)) PY cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( diagnostic_path, step, reason, rc, line, stage, last_success, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] with open(diagnostic_path, encoding="utf-8") as source: diagnostic = json.load(source) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "assessment": { "decision": f"STOP_{step}_{reason}", "all_ok": False, "error_rc": int(rc), "error_line": int(line), "failed_stage": stage, "last_success": last_success, }, "inline_diagnostic": diagnostic, "safety": diagnostic["safety"], "plan": { "current_milestone": "M07", "milestone_completed": False, "milestone_changed": False, }, "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" } fatal() { local reason="$1" local rc="${2:-1}" local line="${3:-$LINENO}" trap - ERR write_stop "$reason" "$rc" "$line" print_links exit "$rc" } on_error() { local rc="$?" local line="$1" fatal "UNEXPECTED_ERROR" "$rc" "$line" } trap 'on_error "$LINENO"' ERR stage "01/06" "Проверяю M07D PASS и копирую доказательства" for required in \ "$M07D_DIR/report.txt" \ "$M07D_DIR/facts.json" \ "$M07D_DIR/assessment.json" \ "$M07D_DIR/source-contract.json" \ "$M07D_DIR/current-baseline.json" \ "$M07D_DIR/runner-source.txt" \ "$M07D_DIR/rebalance-apply-source.txt" \ "$M07D_DIR/planner-source.txt" \ "$M07D_DIR/adapter-sources.txt" \ "$M07D_DIR/failure-artifacts.txt" \ "$C5_DIR/report.txt" \ "$C5_DIR/facts.json" \ "$C5_DIR/vm101.txt" \ "$C5_DIR/vm101.stderr" do [ -s "$required" ] || { echo "MISSING_REQUIRED=$required" fatal "REQUIRED_ARTIFACT_MISSING" 2 "$LINENO" } done grep -Fq \ "decision=PASS_STEP_050M07D_CLASSIFY_FALSE_SUCCESS_AND_EGRESS1_FAILURE" \ "$M07D_DIR/report.txt" || fatal "M07D_PASS_NOT_PROVEN" 3 "$LINENO" grep -Fq \ "current_milestone=M07" \ "$M07D_DIR/report.txt" || fatal "M07_NOT_CURRENT" 4 "$LINENO" cp -a "$M07D_DIR/source-contract.json" \ "$REPORT_DIR/sources/m07d-source-contract.json" cp -a "$M07D_DIR/current-baseline.json" \ "$REPORT_DIR/sources/m07d-current-baseline.json" cp -a "$M07D_DIR/runner-source.txt" \ "$REPORT_DIR/sources/m07d-runner-source.txt" cp -a "$M07D_DIR/rebalance-apply-source.txt" \ "$REPORT_DIR/sources/m07d-rebalance-apply-source.txt" cp -a "$M07D_DIR/planner-source.txt" \ "$REPORT_DIR/sources/m07d-planner-source.txt" cp -a "$M07D_DIR/adapter-sources.txt" \ "$REPORT_DIR/sources/m07d-adapter-sources.txt" cp -a "$M07D_DIR/failure-artifacts.txt" \ "$REPORT_DIR/sources/m07d-failure-artifacts.txt" cp -a "$M07D_DIR/assessment.json" \ "$REPORT_DIR/sources/m07d-assessment.json" cp -a "$C5_DIR/report.txt" \ "$REPORT_DIR/sources/c5-report.txt" cp -a "$C5_DIR/facts.json" \ "$REPORT_DIR/sources/c5-facts.json" cp -a "$C5_DIR/vm101.txt" \ "$REPORT_DIR/sources/c5-live-stream.txt" cp -a "$C5_DIR/vm101.stderr" \ "$REPORT_DIR/sources/c5-live-stderr.txt" mark_success "existing_evidence_frozen" stage "02/06" "Публикую точный read-only pre-04:20 extractor" cat > "$REPORT_DIR/vm101.sh" <<'VM101' #!/bin/sh set -u umask 077 RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh" APPLY="/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh" PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh" HELPER="/usr/local/lib/router-egress-recovery-state.sh" CONF="/etc/router-egress-emergency-refresh.conf" HOOK_INIT="/etc/init.d/router-egress-emergency-decision" WATCHER_INIT="/etc/init.d/router-egress-health-repair" POOL="/root/hmn/cache/ok-awg1-strict-foreign-latest.tsv" fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } block() { name="$1" shift echo "__BLOCK_BEGIN__ $name" "$@" 2>&1 || true echo "__BLOCK_END__ $name" } block_file() { name="$1" path="$2" echo "__BLOCK_BEGIN__ $name" if [ -f "$path" ]; then cat "$path" else echo "FILE_NOT_PRESENT=$path" fi echo "__BLOCK_END__ $name" } bool_cmd() { if "$@" >/dev/null 2>&1; then printf true else printf false fi } state_value() { key="$1" fallback="$2" ( unset REG_STATE_DIR . "$HELPER" reg_get_state "$key" "$fallback" ) } repair_counter() { ( unset REG_STATE_DIR . "$HELPER" reg_daily_repair_get ) } strict_iface() { interface="$1" attempt=1 while [ "$attempt" -le 3 ]; do if ping \ -I "$interface" \ -c 1 \ -W 3 \ 1.1.1.1 \ >/dev/null 2>&1 then return 0 fi attempt=$((attempt + 1)) sleep 1 done return 1 } routes_all() { for table in 201 202 203 204 205; do ip route show table "$table" 2>/dev/null | grep -q '^default ' || return 1 done return 0 } for required in \ "$RUNNER" \ "$APPLY" \ "$PLANNER" \ "$HELPER" \ "$CONF" \ "$POOL" do [ -f "$required" ] || { echo "__ERROR__ source_missing=$required" exit 21 } done echo "__TRACE__ stage=clock_and_timezone" fact snapshot_epoch "$(date +%s)" fact vm101_local_time "$(date '+%Y-%m-%dT%H:%M:%S%z')" fact vm101_utc_time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" fact kernel "$(uname -a | tr ' ' '_')" fact etc_tz "$( tr '\r\n ' '_' < /etc/TZ 2>/dev/null || echo UNAVAILABLE )" fact uci_timezone "$( uci -q get system.@system[0].timezone 2>/dev/null || echo UNAVAILABLE )" fact uci_zonename "$( uci -q get system.@system[0].zonename 2>/dev/null || echo UNAVAILABLE )" block date_output date block date_utc_output date -u block uptime_output uptime echo "__TRACE__ stage=scheduler_contract" block_file root_crontab /etc/crontabs/root block crontab_l crontab -l block scheduler_processes sh -c ' ps w | grep -E "[c]rond|[c]ron|[t]imer|hmn.*refresh|refresh.*hmn" || true ' block scheduler_search sh -c ' for base in \ /etc/crontabs \ /etc/init.d \ /etc/config \ /etc/rc.local \ /root/hmn \ /usr/local/sbin do [ -e "$base" ] || continue if [ -f "$base" ]; then files="$base" else files="$( find "$base" \ -maxdepth 4 \ -type f \ 2>/dev/null | sort )" fi for path in $files; do case "$path" in *.sh|*/cron*|*/crontab*|*/rc.local|*refresh*|*timer*|*hmn*) matches="$( grep -nEi \ "(^|[[:space:]])20[[:space:]]+4([[:space:]]|$)|04:20|4:20|hmn.*refresh|refresh.*hmn" \ "$path" \ 2>/dev/null || true )" if [ -n "$matches" ]; then echo "===== $path =====" printf "%s\n" "$matches" fi ;; esac done done ' block cron_logs sh -c ' logread 2>/dev/null | grep -Ei "cron|hmn.*refresh|refresh.*hmn" | tail -n 300 || true ' echo "__TRACE__ stage=services_and_locks" fact hook_running "$(bool_cmd "$HOOK_INIT" running)" fact hook_enabled "$(bool_cmd "$HOOK_INIT" enabled)" fact watcher_running "$(bool_cmd "$WATCHER_INIT" running)" fact watcher_enabled "$(bool_cmd "$WATCHER_INIT" enabled)" fact emergency_lock_present "$( bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock )" fact refresh_lock_present "$( bool_cmd test -e /tmp/hmn-refresh-pool-safe.lock )" block relevant_processes sh -c ' ps w | grep -E \ "[h]mn-refresh|[r]outer-egress-emergency-refresh|[r]ebalance-top5|[h]ealth-repair-watch" || true ' echo "__TRACE__ stage=recovery_state" fact state_mode "$(state_value mode UNKNOWN)" fact state_status "$( state_value last_emergency_refresh_status UNKNOWN )" fact state_epoch "$( state_value last_emergency_refresh_epoch 0 )" fact repair_counter "$(repair_counter)" fact commit_raw "$( ( . "$CONF" printf '%s' "${EMERGENCY_COMMIT_ENABLED:-UNSET}" ) )" block safe_emergency_config sh -c " grep -E \ '^(EMERGENCY_(COMMIT_ENABLED|DAILY_FAIL_THRESHOLD|COOLDOWN_SECONDS|REFRESH_CMD|PLANNER_CMD|REBALANCE_APPLY_CMD|WATCHER_CMD|DIRECT_FAILOPEN_ENABLED|LOG))=' \ '$CONF' || true " block state_files sh -c ' find /var/lib/router-egress-recovery \ -maxdepth 3 \ -type f \ 2>/dev/null | sort | while IFS= read -r path; do printf "%s " "$path" sha256sum "$path" 2>/dev/null | sed "s/[[:space:]].*$//" done ' echo "__TRACE__ stage=current_endpoints" HEALTHY_COUNT=0 for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$interface"; then strict=true HEALTHY_COUNT=$((HEALTHY_COUNT + 1)) else strict=false fi fact "strict.${interface}" "$strict" endpoint="$( wg show "$interface" dump 2>/dev/null | awk ' NR > 1 && $3 != "(none)" { print $3 exit } ' )" [ -n "$endpoint" ] || endpoint="UNRESOLVED" fact "endpoint.${interface}" "$endpoint" peer_runtime="$( wg show "$interface" dump 2>/dev/null | awk ' NR > 1 { printf( "endpoint=%s latest_handshake=%s rx=%s tx=%s keepalive=%s\n", $3, $5, $6, $7, $8 ) } ' )" echo "__BLOCK_BEGIN__ runtime_${interface}" printf '%s\n' "$peer_runtime" echo "__BLOCK_END__ runtime_${interface}" done fact healthy_vpn_slots "$HEALTHY_COUNT" fact routes_201_205 "$(bool_cmd routes_all)" block table_200 ip route show table 200 block routes_201_205 sh -c ' for table in 201 202 203 204 205; do echo "===== table $table =====" ip route show table "$table" done ' block safe_network_sections sh -c ' uci show network 2>/dev/null | grep -E \ "^network\.vpn[1-5]\.(proto|device|ifname|addresses|address|endpoint_host|endpoint_port|route_allowed_ips|metric|mtu|listen_port)=" | sed \ -e "/private_key/d" \ -e "/preshared_key/d" || true ' echo "__TRACE__ stage=runner_and_planner" block runner_dry_run "$RUNNER" --dry-run block current_planner "$PLANNER" echo "__TRACE__ stage=pool_and_generation" fact pool_sha256 "$( sha256sum "$POOL" | sed 's/[[:space:]].*$//' )" fact pool_rows "$( awk 'NR > 1 {count++} END {print count + 0}' "$POOL" )" fact pool_mtime_epoch "$(date -r "$POOL" +%s)" block pool_head head -n 20 "$POOL" block pool_tail tail -n 20 "$POOL" block quarantine_files sh -c ' find /root/hmn/cache /var/lib/router-egress-recovery \ -maxdepth 4 \ -type f \ $ \ -name "*quarantine*" \ -o -name "*bad*" \ -o -name "*retry*" \ $ \ 2>/dev/null | sort | while IFS= read -r path; do echo "===== $path =====" ls -l "$path" sha256sum "$path" 2>/dev/null || true sed -n "1,200p" "$path" 2>/dev/null || true done ' LATEST_TARGET="$( readlink -f /root/hmn/configs/awg1/latest 2>/dev/null || true )" fact latest_generation "$( printf '%s' "${LATEST_TARGET:-UNRESOLVED}" | tr ' ' '_' )" block latest_generation_listing sh -c ' latest="$( readlink -f /root/hmn/configs/awg1/latest 2>/dev/null || true )" echo "latest=$latest" if [ -n "$latest" ] && [ -d "$latest" ]; then find "$latest" \ -maxdepth 1 \ -type f \ -name "*.conf" \ 2>/dev/null | sort | while IFS= read -r path; do printf "%s " "$path" sha256sum "$path" | sed "s/[[:space:]].*$//" done fi ' echo "__TRACE__ stage=sources" for source_path in \ "$RUNNER" \ "$APPLY" \ "$PLANNER" \ /usr/local/sbin/router-egress-hmn-rebalance-top5.sh \ /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh \ /usr/local/sbin/router-egress-health-repair-watch.sh \ /usr/local/sbin/router-egress-emergency-decision-hook.sh \ /root/hmn/hmn-refresh-pool-safe.sh \ /root/hmn/hmn-code-test.sh do if [ -f "$source_path" ]; then fact "source_sha256.$( printf '%s' "$source_path" | sed 's#[^A-Za-z0-9]#_#g' )" "$( sha256sum "$source_path" | sed 's/[[:space:]].*$//' )" fi done block apply_failure_contract sh -c " grep -nE \ 'commit_ok|commit_failed|slot_apply_failed|apply_ok|exit|return|adapter|rollback|strict' \ '$APPLY' || true " block runner_exit_contract sh -c " grep -nE \ 'REBALANCE_APPLY|refresh_ok_rebalance_ok|refresh_ok_rebalance_failed|apply_ok|commit_failed|exit|return' \ '$RUNNER' || true " echo "__TRACE__ stage=logs_and_failure_artifacts" block emergency_log_tail sh -c ' tail -n 500 \ /var/log/router-egress-emergency-refresh.log \ 2>/dev/null || true ' block recent_hmn_logs sh -c ' find /root/hmn/logs \ -maxdepth 2 \ -type f \ -mmin -1440 \ 2>/dev/null | sort | while IFS= read -r path; do echo "===== $path =====" ls -l "$path" sha256sum "$path" 2>/dev/null || true grep -nEi \ "download|serverlist|commit_failed|slot_apply_failed|egress1|vpn1|rollback|strict|error|failed" \ "$path" \ 2>/dev/null | tail -n 300 || true done ' block recent_recovery_artifacts sh -c ' find /var/lib/router-egress-recovery \ -maxdepth 5 \ -type f \ -mmin -1440 \ 2>/dev/null | sort | while IFS= read -r path; do echo "===== $path =====" ls -l "$path" sha256sum "$path" 2>/dev/null || true case "$path" in *.log|*.txt|*.json|*.status|*.kv) sed -n "1,500p" "$path" 2>/dev/null || true ;; *.sh) grep -nE \ "egress1|vpn1|rollback|strict|endpoint|route|uci" \ "$path" \ 2>/dev/null | head -n 300 || true ;; esac done ' echo "__TRACE__ stage=storage" block filesystem_usage df -Pk block relevant_sizes sh -c ' du -sk \ /root/hmn/cache \ /root/hmn/configs/awg1 \ /root/hmn/logs \ /var/lib/router-egress-recovery \ /tmp/step050m07* \ 2>/dev/null | sort -n ' fact read_only true fact refresh_ran false fact rebalance_ran false fact network_changed false fact services_changed false fact state_changed false fact timer_changed false fact plan_changed false fact direct_failopen_changed false echo "__TRACE__ stage=complete" exit 0 VM101 chmod 600 "$REPORT_DIR/vm101.sh" sh -n "$REPORT_DIR/vm101.sh" mark_success "readonly_extractor_published" stage "03/06" "Снимаю pre-04:20 состояние VM101" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101.sh" \ > >(tee "$REPORT_DIR/vm101.txt") \ 2> >(tee "$REPORT_DIR/vm101.stderr" >&2) then VM101_RC=0 else VM101_RC=$? fi echo "vm101_rc=$VM101_RC" | tee -a "$PROGRESS_LOG" [ "$VM101_RC" -eq 0 ] || fatal "VM101_READONLY_SNAPSHOT_FAILED" "$VM101_RC" "$LINENO" grep -Fq \ "__TRACE__ stage=complete" \ "$REPORT_DIR/vm101.txt" || fatal "REMOTE_COMPLETION_MARKER_MISSING" 5 "$LINENO" grep -Fq \ "__FACT__ read_only=true" \ "$REPORT_DIR/vm101.txt" || fatal "READONLY_FACT_MISSING" 6 "$LINENO" mark_success "vm101_snapshot_complete" stage "04/06" "Строю scheduler contract и frozen baseline" python3 - \ "$REPORT_DIR/vm101.txt" \ "$REPORT_DIR/vm101.stderr" \ "$REPORT_DIR" <<'PY' import json import re import sys from pathlib import Path runtime_path, stderr_path, report_dir = sys.argv[1:] root = Path(report_dir) runtime = Path(runtime_path).read_text( encoding="utf-8", errors="replace", ) stderr = Path(stderr_path).read_text( encoding="utf-8", errors="replace", ) def parse_facts(text): facts = {} for line in text.splitlines(): if not line.startswith("__FACT__ "): continue payload = line[len("__FACT__ "):] if "=" in payload: key, value = payload.split("=", 1) facts[key] = value return facts def block(text, name): pattern = re.compile( rf"^__BLOCK_BEGIN__ {re.escape(name)}\n" rf"(.*?)" rf"^__BLOCK_END__ {re.escape(name)}$", re.MULTILINE | re.DOTALL, ) match = pattern.search(text) if not match: return "" return match.group(1) facts = parse_facts(runtime) root_crontab = block(runtime, "root_crontab") crontab_l = block(runtime, "crontab_l") scheduler_search = block(runtime, "scheduler_search") cron_logs = block(runtime, "cron_logs") cron_0420_lines = [] for source_name, content in ( ("root_crontab", root_crontab), ("crontab_l", crontab_l), ): for raw_line in content.splitlines(): line = raw_line.strip() if not line or line.startswith("#"): continue fields = line.split() if len(fields) >= 6 and fields[0] == "20" and fields[1] == "4": cron_0420_lines.append({ "source": source_name, "line": line, }) search_0420_lines = [ line for line in scheduler_search.splitlines() if re.search(r"04:20|4:20|(^|\s)20\s+4(\s|$)", line) ] endpoints = { interface: facts.get( f"endpoint.{interface}", "UNRESOLVED", ) for interface in ( "vpn1", "vpn2", "vpn3", "vpn4", "vpn5", ) } strict = { interface: facts.get(f"strict.{interface}") == "true" for interface in endpoints } safety = { "read_only": facts.get("read_only") == "true", "refresh_ran": facts.get("refresh_ran") == "true", "rebalance_ran": facts.get("rebalance_ran") == "true", "network_changed": facts.get("network_changed") == "true", "services_changed": facts.get("services_changed") == "true", "state_changed": facts.get("state_changed") == "true", "timer_changed": facts.get("timer_changed") == "true", "plan_changed": facts.get("plan_changed") == "true", "direct_failopen_changed": facts.get("direct_failopen_changed") == "true", } readonly_invariants = { "read_only_declared": safety["read_only"], "no_refresh": not safety["refresh_ran"], "no_rebalance": not safety["rebalance_ran"], "no_network_change": not safety["network_changed"], "no_service_change": not safety["services_changed"], "no_state_change": not safety["state_changed"], "no_timer_change": not safety["timer_changed"], "no_plan_change": not safety["plan_changed"], "direct_unchanged": not safety["direct_failopen_changed"], } baseline_checks = { "five_endpoints_resolved": all(value != "UNRESOLVED" for value in endpoints.values()), "five_strict_slots": all(strict.values()) and facts.get("healthy_vpn_slots") == "5", "routes_201_205": facts.get("routes_201_205") == "true", "hook_running_enabled": facts.get("hook_running") == "true" and facts.get("hook_enabled") == "true", "watcher_running_enabled": facts.get("watcher_running") == "true" and facts.get("watcher_enabled") == "true", "locks_absent": facts.get("emergency_lock_present") == "false" and facts.get("refresh_lock_present") == "false", } unexpected_stderr = [ line for line in stderr.splitlines() if line.strip() and not line.startswith("Warning: Permanently added ") ] scheduler = { "vm101_local_time": facts.get("vm101_local_time"), "vm101_utc_time": facts.get("vm101_utc_time"), "etc_tz": facts.get("etc_tz"), "uci_timezone": facts.get("uci_timezone"), "uci_zonename": facts.get("uci_zonename"), "exact_cron_0420_matches": cron_0420_lines, "other_0420_matches": search_0420_lines, "exact_cron_0420_found": bool(cron_0420_lines), "any_0420_reference_found": bool(cron_0420_lines or search_0420_lines), "root_crontab": root_crontab, "crontab_l": crontab_l, "scheduler_search": scheduler_search, "recent_cron_logs": cron_logs, } runtime_baseline = { "captured_at": { "epoch": int(facts["snapshot_epoch"]), "vm101_local_time": facts.get("vm101_local_time"), "vm101_utc_time": facts.get("vm101_utc_time"), "timezone": { "etc_tz": facts.get("etc_tz"), "uci_timezone": facts.get("uci_timezone"), "uci_zonename": facts.get("uci_zonename"), }, }, "endpoints": endpoints, "strict": strict, "healthy_vpn_slots": int( facts.get("healthy_vpn_slots", "0") ), "routes_201_205": facts.get("routes_201_205") == "true", "services": { "hook_running": facts.get("hook_running") == "true", "hook_enabled": facts.get("hook_enabled") == "true", "watcher_running": facts.get("watcher_running") == "true", "watcher_enabled": facts.get("watcher_enabled") == "true", }, "locks": { "emergency": facts.get("emergency_lock_present") == "true", "refresh": facts.get("refresh_lock_present") == "true", }, "state": { "mode": facts.get("state_mode"), "status": facts.get("state_status"), "epoch": int( facts.get("state_epoch", "0") ), "repair_counter": int( facts.get("repair_counter", "0") ), "commit_raw": facts.get("commit_raw"), }, "pool": { "sha256": facts.get("pool_sha256"), "rows": int(facts.get("pool_rows", "0")), "mtime_epoch": int( facts.get("pool_mtime_epoch", "0") ), "latest_generation": facts.get("latest_generation"), }, "checks": baseline_checks, } all_ok = ( all(readonly_invariants.values()) and "__TRACE__ stage=complete" in runtime ) assessment = { "all_ok": all_ok, "decision": ( "PASS_STEP_050M07D2_FREEZE_PRE0420_EVIDENCE" if all_ok else "STOP_STEP_050M07D2_FREEZE_PRE0420_EVIDENCE" ), "snapshot_complete": all_ok, "readonly_invariants": readonly_invariants, "baseline": { "checks": baseline_checks, "all_ok": all(baseline_checks.values()), "warnings": [ name for name, value in baseline_checks.items() if not value ], }, "scheduler": { "exact_cron_0420_found": scheduler["exact_cron_0420_found"], "any_0420_reference_found": scheduler["any_0420_reference_found"], "exact_matches": cron_0420_lines, "other_match_count": len(search_0420_lines), "timezone": { "etc_tz": facts.get("etc_tz"), "uci_timezone": facts.get("uci_timezone"), "uci_zonename": facts.get("uci_zonename"), }, }, "stderr": { "unexpected_lines": unexpected_stderr, }, "preserved_context": { "m07d_source_contract": True, "m07d_sources": True, "m07d_failure_artifacts": True, "c5_live_stream": True, "c5_failure_facts": True, "current_vm101_runtime": True, "current_scheduler_contract": True, }, "next_action": ( "Pause changes. After the scheduled overnight refresh, " "capture a read-only post-04:20 snapshot and compare endpoints, " "pool, generation, state, counter, logs and scheduler results." ), "safety": safety, } (root / "scheduler-contract.txt").write_text( "\n".join([ "=== VM101 CLOCK/TIMEZONE ===", f"local={facts.get('vm101_local_time')}", f"utc={facts.get('vm101_utc_time')}", f"etc_tz={facts.get('etc_tz')}", f"uci_timezone={facts.get('uci_timezone')}", f"uci_zonename={facts.get('uci_zonename')}", "", "=== EXACT CRON 04:20 MATCHES ===", json.dumps( cron_0420_lines, ensure_ascii=False, indent=2, ), "", "=== OTHER 04:20 REFERENCES ===", "\n".join(search_0420_lines) if search_0420_lines else "NONE", "", "=== ROOT CRONTAB ===", root_crontab, "", "=== CRONTAB -L ===", crontab_l, "", "=== SCHEDULER SEARCH ===", scheduler_search, "", "=== RECENT CRON LOGS ===", cron_logs, "", ]) + "\n", encoding="utf-8", ) (root / "runtime-baseline.json").write_text( json.dumps( runtime_baseline, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) (root / "assessment.json").write_text( json.dumps( assessment, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) artifact_paths = sorted( str(path.relative_to(root)) for path in root.rglob("*") if path.is_file() ) evidence_index = { "schema": "pre-0420-evidence-index-v1", "snapshot_complete": all_ok, "artifact_count": len(artifact_paths), "artifacts": artifact_paths, "comparison_keys_after_0420": [ "endpoints.vpn1..vpn5", "strict.vpn1..vpn5", "state.mode", "state.status", "state.epoch", "state.repair_counter", "pool.sha256", "pool.rows", "pool.mtime_epoch", "pool.latest_generation", "emergency log delta", "scheduled job exit/result", ], } (root / "evidence-index.json").write_text( json.dumps( evidence_index, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) if not all_ok: failed = [ name for name, value in readonly_invariants.items() if not value ] raise SystemExit( "read-only invariant failure: " + ",".join(failed) ) PY mark_success "snapshot_parsed_and_indexed" stage "05/06" "Публикую frozen pre-04:20 report" BASELINE_OK="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print("true" if data["baseline"]["all_ok"] else "false") PY )" BASELINE_WARNINGS="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) warnings = data["baseline"]["warnings"] print(",".join(warnings) if warnings else "NONE") PY )" TIMER_EXACT="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print( "true" if data["scheduler"]["exact_cron_0420_found"] else "false" ) PY )" TIMER_ANY="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print( "true" if data["scheduler"]["any_0420_reference_found"] else "false" ) PY )" VM101_LOCAL_TIME="$( python3 - "$REPORT_DIR/runtime-baseline.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["captured_at"]["vm101_local_time"]) PY )" VM101_TIMEZONE="$( python3 - "$REPORT_DIR/runtime-baseline.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) tz = data["captured_at"]["timezone"] print( tz.get("uci_zonename") or tz.get("etc_tz") or tz.get("uci_timezone") or "UNRESOLVED" ) PY )" HEALTHY_SLOTS="$( python3 - "$REPORT_DIR/runtime-baseline.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["healthy_vpn_slots"]) PY )" POOL_SHA="$( python3 - "$REPORT_DIR/runtime-baseline.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["pool"]["sha256"]) PY )" LATEST_GENERATION="$( python3 - "$REPORT_DIR/runtime-baseline.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: data = json.load(source) print(data["pool"]["latest_generation"]) PY )" cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( assessment_path, baseline_path, step, timestamp, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] with open(assessment_path, encoding="utf-8") as source: assessment = json.load(source) with open(baseline_path, encoding="utf-8") as source: baseline = json.load(source) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "generated_at_utc": timestamp, "assessment": assessment, "runtime_baseline": baseline, "operation": { "pre_0420_evidence_frozen": True, "m07d_sources_preserved": True, "c5_live_stream_preserved": True, "scheduler_contract_preserved": True, }, "safety": assessment["safety"], "plan": { "current_milestone": "M07", "milestone_completed": False, "milestone_changed": False, }, "next_step": "READONLY_POST_0420_COMPARISON", "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" \ -type f \ ! -name SHA256SUMS \ -print0 | sort -z | xargs -0 sha256sum \ > "$REPORT_DIR/SHA256SUMS" mark_success "pre0420_report_published" stage "06/06" "Завершаю read-only freeze" trap - ERR echo "decision=$PASS_DECISION" | tee -a "$PROGRESS_LOG" echo "snapshot_complete=true" | tee -a "$PROGRESS_LOG" echo "vm101_local_time=$VM101_LOCAL_TIME" | tee -a "$PROGRESS_LOG" echo "vm101_timezone=$VM101_TIMEZONE" | tee -a "$PROGRESS_LOG" echo "exact_cron_0420_found=$TIMER_EXACT" | tee -a "$PROGRESS_LOG" echo "any_0420_reference_found=$TIMER_ANY" | tee -a "$PROGRESS_LOG" echo "baseline_all_ok=$BASELINE_OK" | tee -a "$PROGRESS_LOG" echo "healthy_vpn_slots=$HEALTHY_SLOTS" | tee -a "$PROGRESS_LOG" echo "pool_sha256=$POOL_SHA" | tee -a "$PROGRESS_LOG" echo "latest_generation=$LATEST_GENERATION" | tee -a "$PROGRESS_LOG" echo "current_milestone=M07" | tee -a "$PROGRESS_LOG" echo "next_action=READONLY_POST_0420_COMPARISON" | tee -a "$PROGRESS_LOG" echo "read_only=true" | tee -a "$PROGRESS_LOG" print_links