#!/usr/bin/env bash
set -Eeuo pipefail
umask 077
BUNDLE_DIR="$(cd "$(dirname "$0")" && pwd -P)"
SELF="$(readlink -f "$0")"
BUNDLE_ZIP="${BUNDLE_ZIP:?BUNDLE_ZIP is required}"
EXPECTED_BUNDLE_SHA256="${EXPECTED_BUNDLE_SHA256:?EXPECTED_BUNDLE_SHA256 is required}"
EXPECTED_INSTALLER_SHA256="${EXPECTED_INSTALLER_SHA256:?EXPECTED_INSTALLER_SHA256 is required}"
BASE="${ROUTER_OPS_BASE:-/opt/router-ops}"
SSH_BIN="${SSH_BIN:-ssh}"
STEP_ID='STEP_050M07R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF'
BUNDLE_ID='WG_PAID_MGTS_STEP_050M07R20O_R02'
PUBLISH_TS="$(date -u +%Y%m%d-%H%M%S)"
STEP_DIR_BASE="${R20O_RUN_BASE:-$BASE/steps}"
STEP_DIR="$STEP_DIR_BASE/${PUBLISH_TS}_${STEP_ID}"
COMPONENTS="$STEP_DIR/components"; PHASE_CURRENT="$STEP_DIR/phase-current.txt"; PHASE_EVENTS="$STEP_DIR/phase-events.log"; TRANSITION_FILE="$STEP_DIR/runtime-transition.env"; MAIN_LOG="$STEP_DIR/main.log"; PROGRESS_LOG="$STEP_DIR/progress.log"; TRANSFER="$STEP_DIR/transfer-verify.txt"; STEP_FILE="$STEP_DIR/step-code.sh"; BUNDLE_COPY="$STEP_DIR/$(basename "$BUNDLE_ZIP")"; STOP_GUARD_DIR="$STEP_DIR/.stop-handler-active"
mkdir -p "$STEP_DIR" "$COMPONENTS"; chmod 700 "$STEP_DIR" "$COMPONENTS"; cp -a "$SELF" "$STEP_FILE"; chmod 700 "$STEP_FILE"; cp -a "$BUNDLE_ZIP" "$BUNDLE_COPY"
exec 3>&1; exec > >(tee -a "$MAIN_LOG") 2>&1; LOG_TEE_PID=$!
PUBLISHER="$BASE/bin/router-publish-report"; HTTP="$BASE/bin/router-http-verify"; FINALIZER="$BASE/bin/router-step-finalize"; SAFE_BUILDER="$BASE/bin/router-public-safe-payload"; HTML_BUILDER="$BASE/bin/router-public-html-aliases"; WORKFLOW_LIB="$BASE/lib/router-step-workflow.sh"; CONTRACT_LIB="$BASE/lib/router-publication-contract.sh"; RUNTIME_TRANSITION_LIB="$BASE/lib/router-runtime-transition.sh"; MACHINE_CLOSE="$BASE/bin/router-machine-close-safe"
RESULT='STOP_R20O_NOT_STARTED'; STOP_REASON=not_started; CURRENT_PHASE=not_started; COPY_VERIFY=FAIL; HTTP_VERIFY_PASS=false; PUBLIC_OBJECT_KIND=step-report; PREVIOUS_MOVED_COUNT=0; RUNTIME_IMPACT=false; CORE_CHANGE_COMPLETE=false; CORE_CHANGE_REPEATED=false; TARGET_INSTALLER_RERUN=false; ROUTER_MACHINE_CLOSE_RERUN=false; SSH_SESSION_REMAINS_OPEN=true; WORKFLOW_READY=false
SNAPSHOT_NAME="r20o-pre-${PUBLISH_TS:4:4}-${PUBLISH_TS:9:6}"; SNAPSHOT_CREATED=false; SNAPSHOT_REUSED=false; BACKUP_DIR="/root/router-egress-r20o-r02-backup-${PUBLISH_TS}"; BACKUP_REUSED=false
LOCAL_REPAIR_PASS=false; DEGRADED_POOL_PASS=false; COUNTER_PRESERVED_PASS=false; OLD_GENERATION_PRESERVED_PASS=false; ACTIVE_ENDPOINTS_PRESERVED_PASS=false; RETRY_NOT_DUE_PASS=false; FAILED_RETRY_PASS=false; SUCCESSFUL_RETRY_PASS=false; RETRY_COUNT_AFTER_FAILED_RETRY=0; CONFIG_RESTORED=false; OLD_GENERATION_ID=''; NEW_GENERATION_ID=''; FINAL_MODE=PENDING; FINAL_COUNTER=0; ACTIVE_SLOTS_HEALTHY=0; ENDPOINT_CONSISTENCY_PASS=false; DIRECT_FAILOPEN_USED=false; WATCHER_RUNNING=false; RETRY_RUNNING=false; MACHINE_GIT_PASS=false; MACHINE_CLOSE_PASS=false; MANAGED_SOURCE_CHANGE=false; LOCAL_REPAIR_TESTED=false; DEGRADED_POOL_TESTED=false; CONTROLLED_RETRY_TESTED=false; REBOOT_TESTED=false; TEST_DIR=''
GIT_COMMIT=d0c0a77486665b0abfef8133f0cccd20c6232b46; GIT_TREE=76a427a94defee9bffe571fda3f8bb504b8eb540; GIT_FILE_COUNT=37
VM101_PUBLIC_URL='https://reports.secret-studio.ru/latest/20260722-201723_vm101_git_source_d0c0a7748666/'; VM130_SOURCE_URL='https://reports.secret-studio.ru/latest/20260720-093614_vm130_router_ops_source_snapshot_after_r20c_full_contract/'; VM101_MODEL_URL='https://reports.secret-studio.ru/latest/20260722-201723_vm101_model_r20n_repair_threshold_and_auto_refresh_r01/'; VM101_METHODS_URL='https://reports.secret-studio.ru/latest/20260722-201723_vm101_methods_r20n_repair_threshold_and_auto_refresh_r01/'
PROJECT_SOURCE_POST_URL=PENDING_NOT_PUBLISHED; PROJECT_SOURCE_POST_FILE_URL=PENDING_NOT_PUBLISHED; STEP_REPORT_URL=PENDING_NOT_PUBLISHED; FACTS_URL=PENDING_NOT_PUBLISHED; VERIFY_URL=PENDING_NOT_PUBLISHED; ROLLBACK_URL=PENDING_NOT_PUBLISHED
LOCAL_M07_PLAN_URL='https://reports.secret-studio.ru/latest/20260719-080444_local_architecture_plan_vm101_autonomous_hmn_recovery_r19/'; GLOBAL_PROJECT_PLAN_URL='https://reports.secret-studio.ru/latest/20260711-123348_global_project_plan_wg_paid/'; ACCESS_MAP_URL='https://reports.secret-studio.ru/latest/20260715-210236_access_map_canonical_current/'; VM100_PUBLIC_URL='https://reports.secret-studio.ru/latest/20260715-190653_vm100_git_source_1958200ccc1b/'; VM121_PUBLIC_URL='https://reports.secret-studio.ru/latest/20260713-182700_vm121_git_source_93e0e5a31f09/'; LATEST_ROOT_URL='https://reports.secret-studio.ru/latest/'; OLD_ROOT_URL='https://reports.secret-studio.ru/old/'
MODEL_LABEL=vm101_model_r20o_degraded_pool_and_retry_r02; METHODS_LABEL=vm101_methods_r20o_degraded_pool_and_retry_r02; PROJECT_LABEL=project_source_post_vm101_degraded_pool_retry_r20o_r02; STEP_LABEL=step050m07r20o_r02_degraded_pool_and_controlled_retry_runtime_proof; PUBLIC_BASE=https://reports.secret-studio.ru
export ROUTER_PUBLISH_TIMESTAMP_OVERRIDE="$PUBLISH_TS"
cleanup(){ local rc; rc="${1:-1}"; trap - EXIT INT TERM HUP; exec 1>&- 2>&- || true; wait "$LOG_TEE_PID" 2>/dev/null || true; exit "$rc"; }
trap 'cleanup "$?"' EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; trap 'exit 129' HUP
progress(){ printf '>>> [%s] %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" | tee -a "$PROGRESS_LOG"; }
field(){ awk -F= -v k="$2" '$1==k{v=substr($0,index($0,"=")+1)} END{print v}' "$1"; }
emit_final(){ local step_sha; step_sha="$(sha256sum "$STEP_FILE" | awk '{print $1}')"; for line in "RESULT=$RESULT" "STEP_REPORT_URL=$STEP_REPORT_URL" "PROJECT_SOURCE_POST_URL=$PROJECT_SOURCE_POST_URL" "PROJECT_SOURCE_POST_FILE_URL=$PROJECT_SOURCE_POST_FILE_URL" "LOCAL_M07_PLAN_URL=$LOCAL_M07_PLAN_URL" "GLOBAL_PROJECT_PLAN_URL=$GLOBAL_PROJECT_PLAN_URL" "ACCESS_MAP_URL=$ACCESS_MAP_URL" "VM100_PUBLIC_URL=$VM100_PUBLIC_URL" "VM101_PUBLIC_URL=$VM101_PUBLIC_URL" "VM121_PUBLIC_URL=$VM121_PUBLIC_URL" "VM101_MODEL_URL=$VM101_MODEL_URL" "VM101_METHODS_URL=$VM101_METHODS_URL" "VM130_SOURCE_URL=$VM130_SOURCE_URL" "LATEST_ROOT_URL=$LATEST_ROOT_URL" "OLD_ROOT_URL=$OLD_ROOT_URL" "FACTS_URL=$FACTS_URL" "VERIFY_URL=$VERIFY_URL" "ROLLBACK_URL=$ROLLBACK_URL" "PUBLIC_OBJECT_KIND=$PUBLIC_OBJECT_KIND" "PREVIOUS_MOVED_COUNT=$PREVIOUS_MOVED_COUNT" "COPY_VERIFY=$COPY_VERIFY" "STEP_SHA256=$step_sha" "HTTP_VERIFY_PASS=$HTTP_VERIFY_PASS" "RUNTIME_IMPACT=$RUNTIME_IMPACT" "CORE_CHANGE_COMPLETE=$CORE_CHANGE_COMPLETE" "CORE_CHANGE_REPEATED=$CORE_CHANGE_REPEATED" "SNAPSHOT_NAME=$SNAPSHOT_NAME" "SNAPSHOT_CREATED=$SNAPSHOT_CREATED" "SNAPSHOT_REUSED=$SNAPSHOT_REUSED" "BACKUP_DIR=$BACKUP_DIR" "BACKUP_REUSED=$BACKUP_REUSED" "TARGET_INSTALLER_RERUN=$TARGET_INSTALLER_RERUN" "ROUTER_MACHINE_CLOSE_RERUN=$ROUTER_MACHINE_CLOSE_RERUN" "SSH_SESSION_REMAINS_OPEN=$SSH_SESSION_REMAINS_OPEN" "LOCAL_REPAIR_PASS=$LOCAL_REPAIR_PASS" "DEGRADED_POOL_PASS=$DEGRADED_POOL_PASS" "COUNTER_PRESERVED_PASS=$COUNTER_PRESERVED_PASS" "OLD_GENERATION_PRESERVED_PASS=$OLD_GENERATION_PRESERVED_PASS" "ACTIVE_ENDPOINTS_PRESERVED_PASS=$ACTIVE_ENDPOINTS_PRESERVED_PASS" "RETRY_NOT_DUE_PASS=$RETRY_NOT_DUE_PASS" "FAILED_RETRY_PASS=$FAILED_RETRY_PASS" "SUCCESSFUL_RETRY_PASS=$SUCCESSFUL_RETRY_PASS" "RETRY_COUNT_AFTER_FAILED_RETRY=$RETRY_COUNT_AFTER_FAILED_RETRY" "CONFIG_RESTORED=$CONFIG_RESTORED" "OLD_GENERATION_ID=$OLD_GENERATION_ID" "NEW_GENERATION_ID=$NEW_GENERATION_ID" "FINAL_MODE=$FINAL_MODE" "FINAL_COUNTER=$FINAL_COUNTER" "ACTIVE_SLOTS_HEALTHY=$ACTIVE_SLOTS_HEALTHY" "ENDPOINT_CONSISTENCY_PASS=$ENDPOINT_CONSISTENCY_PASS" "DIRECT_FAILOPEN_USED=$DIRECT_FAILOPEN_USED" "WATCHER_RUNNING=$WATCHER_RUNNING" "RETRY_RUNNING=$RETRY_RUNNING" "MACHINE_GIT_PASS=$MACHINE_GIT_PASS" "MACHINE_CLOSE_PASS=$MACHINE_CLOSE_PASS" "MANAGED_SOURCE_CHANGE=$MANAGED_SOURCE_CHANGE" "LOCAL_REPAIR_TESTED=$LOCAL_REPAIR_TESTED" "DEGRADED_POOL_TESTED=$DEGRADED_POOL_TESTED" "CONTROLLED_RETRY_TESTED=$CONTROLLED_RETRY_TESTED" "REBOOT_TESTED=$REBOOT_TESTED"; do printf '%s\n' "$line" >&3; done; }
write_final_env(){ local out tmp name; out="$1"; tmp="${out}.tmp.$$"; { for name in RESULT STOP_REASON CURRENT_PHASE STEP_REPORT_URL PROJECT_SOURCE_POST_URL PROJECT_SOURCE_POST_FILE_URL LOCAL_M07_PLAN_URL GLOBAL_PROJECT_PLAN_URL ACCESS_MAP_URL VM100_PUBLIC_URL VM101_PUBLIC_URL VM121_PUBLIC_URL VM101_MODEL_URL VM101_METHODS_URL VM130_SOURCE_URL LATEST_ROOT_URL OLD_ROOT_URL COPY_VERIFY HTTP_VERIFY_PASS RUNTIME_IMPACT CORE_CHANGE_COMPLETE CORE_CHANGE_REPEATED TARGET_INSTALLER_RERUN ROUTER_MACHINE_CLOSE_RERUN SSH_SESSION_REMAINS_OPEN GIT_COMMIT GIT_TREE GIT_FILE_COUNT SNAPSHOT_NAME SNAPSHOT_CREATED SNAPSHOT_REUSED BACKUP_DIR BACKUP_REUSED LOCAL_REPAIR_PASS DEGRADED_POOL_PASS COUNTER_PRESERVED_PASS OLD_GENERATION_PRESERVED_PASS ACTIVE_ENDPOINTS_PRESERVED_PASS RETRY_NOT_DUE_PASS FAILED_RETRY_PASS SUCCESSFUL_RETRY_PASS RETRY_COUNT_AFTER_FAILED_RETRY CONFIG_RESTORED OLD_GENERATION_ID NEW_GENERATION_ID FINAL_MODE FINAL_COUNTER ACTIVE_SLOTS_HEALTHY ENDPOINT_CONSISTENCY_PASS DIRECT_FAILOPEN_USED WATCHER_RUNNING RETRY_RUNNING MACHINE_GIT_PASS MACHINE_CLOSE_PASS MANAGED_SOURCE_CHANGE LOCAL_REPAIR_TESTED DEGRADED_POOL_TESTED CONTROLLED_RETRY_TESTED REBOOT_TESTED; do printf '%s=%s\n' "$name" "${!name}"; done; } >"$tmp"; chmod 600 "$tmp"; mv -f "$tmp" "$out"; }
run_vm101_script(){ local script prepared item key value rc; script="$1"; shift; prepared="$(mktemp "$STEP_DIR/vm101-script.XXXXXX")"; for item in "$@"; do key="${item%%=*}"; value="${item#*=}"; [[ "$key" =~ ^[A-Z0-9_]+$ ]] || { rm -f "$prepared"; return 91; }; [[ "$value" != *"'"* ]] || { rm -f "$prepared"; return 92; }; printf "%s='%s'\nexport %s\n" "$key" "$value" "$key" >>"$prepared"; done; cat "$script" >>"$prepared"; set +e; "$SSH_BIN" -o BatchMode=yes pve-mgts "ssh -T -o BatchMode=yes -i /root/.ssh/pve_to_openwrt_mgts_ed25519 root@10.71.100.2 'sh -s'" <"$prepared"; rc=$?; set -e; rm -f "$prepared"; return "$rc"; }
phase_begin(){ CURRENT_PHASE="$1"; router_step_phase_begin "$1" "${2:-none}"; progress "$1"; }
phase_done(){ router_step_phase_complete "$1" "${2:-complete}"; }
publish_object(){ local label expect outvar log; label="$1"; expect="$2"; outvar="$3"; shift 3; log="$COMPONENTS/publish-${expect}.log"; "$PUBLISHER" "$label" "$@" >"$log"; router_publication_contract_load "$log" "$expect"; "$HTTP" --url "$RPC_PUBLIC_URL" --output "$COMPONENTS/http-${expect}-index.html" >"$COMPONENTS/http-${expect}.log" 2>&1; printf -v "$outvar" '%s' "$RPC_PUBLIC_URL"; }
preflight_object(){ local label expect sample classify preflight; label="$1"; expect="$2"; sample="$3"; classify="$COMPONENTS/preflight-${expect}.classify"; preflight="$COMPONENTS/preflight-${expect}.txt"; "$PUBLISHER" --classify "$label" >"$classify"; grep -Fqx "PUBLIC_OBJECT_KIND=$expect" "$classify"; "$PUBLISHER" --preflight "$label" "$sample" >"$preflight"; grep -qx 'RESULT=PASS_ROUTER_PUBLISH_PREFLIGHT' "$preflight"; }
export_test_artifacts(){ [ -n "$TEST_DIR" ] || return 0; set +e; run_vm101_script "$BUNDLE_DIR/scripts/vm101-export-test.sh" "R20O_TEST_DIR=$TEST_DIR" >"$STEP_DIR/vm101-test-artifacts.tar.gz"; local rc=$?; set -e; [ "$rc" -eq 0 ] || return "$rc"; mkdir -p "$STEP_DIR/vm101-test-artifacts"; tar -xzf "$STEP_DIR/vm101-test-artifacts.tar.gz" -C "$STEP_DIR/vm101-test-artifacts"; tar -tzf "$STEP_DIR/vm101-test-artifacts.tar.gz" >"$COMPONENTS/vm101-test-artifacts-list.txt"; }
build_report_payload(){ local mode docs html safe final_env src dst; local -a args; mode="$1"; docs="$STEP_DIR/${mode,,}-docs"; html="$STEP_DIR/${mode,,}-html"; safe="$STEP_DIR/${mode,,}-safe"; final_env="$STEP_DIR/${mode,,}-runtime.env"; write_final_env "$final_env"; python3 "$BUNDLE_DIR/scripts/build_step_docs.py" --env "$final_env" --output-dir "$docs" --run-dir "$STEP_DIR" >"$COMPONENTS/${mode,,}-docs-build.log"; python3 -m json.tool "$docs/facts.json" >/dev/null; python3 "$HTML_BUILDER" --facts "$docs/facts.json" --verify "$docs/verify.txt" --rollback "$docs/rollback.sh" --output-dir "$html" --title-prefix R20O-R02 >"$COMPONENTS/${mode,,}-html.log"; args=("$safe" "$STEP_FILE" step-code.sh "$docs/report.txt" report.txt "$docs/facts.json" facts.json "$docs/verify.txt" verify.txt "$docs/rollback.sh" rollback.sh "$docs/system-model.txt" system-model.txt "$docs/methods-limitations.txt" methods-limitations.txt "$docs/access-routes.txt" access-routes.txt "$BUNDLE_DIR/manifest.sha256" bundle-manifest.sha256 "$BUNDLE_DIR/release-info.json" release-info.json "$BUNDLE_DIR/reference/source-origin.txt" source-origin.txt "$MAIN_LOG" main.log "$TRANSFER" transfer-verify.txt "$PHASE_CURRENT" phase-current.txt "$PHASE_EVENTS" phase-events.log "$html/facts-public.html" facts-public.html "$html/verify-public.html" verify-public.html "$html/rollback-public.html" rollback-public.html); for pair in "$PROGRESS_LOG|progress.log" "$COMPONENTS/bundle-gate.txt|bundle-gate.txt" "$COMPONENTS/vm101-preflight.txt|vm101-preflight.txt" "$COMPONENTS/snapshot.txt|snapshot.txt" "$COMPONENTS/vm101-backup.txt|vm101-backup.txt" "$COMPONENTS/vm101-test.txt|vm101-test.txt" "$STEP_DIR/vm101-test-artifacts.tar.gz|vm101-test-artifacts.tar.gz" "$COMPONENTS/vm101-test-artifacts-list.txt|vm101-test-artifacts-list.txt" "$COMPONENTS/router-machine-close-safe.txt|router-machine-close-safe.txt" "$COMPONENTS/final-http.log|final-http.txt"; do src="${pair%%|*}"; dst="${pair#*|}"; [ ! -f "$src" ] || args+=("$src" "$dst"); done; "$SAFE_BUILDER" "${args[@]}" >"$COMPONENTS/${mode,,}-safe-payload.log"; printf '%s\n' "$safe"; }
publish_stop_report(){ local safe stop_label out_env; set +e; safe="$(build_report_payload STOP 2>/dev/null)"; stop_label="step050m07r20o_r02_${CURRENT_PHASE}_stop"; out_env="$STEP_DIR/stop-finalize.env"; if [ -d "$safe" ]; then "$FINALIZER" --label "$stop_label" --payload-dir "$safe" --workflow-current "$PHASE_CURRENT" --workflow-events "$PHASE_EVENTS" --output-env "$out_env" --expect-kind step-report >"$COMPONENTS/stop-finalize.log" 2>&1; if [ -s "$out_env" ]; then STEP_REPORT_URL="$(field "$out_env" PUBLIC_URL)"; FACTS_URL="${STEP_REPORT_URL}facts-public.html"; VERIFY_URL="${STEP_REPORT_URL}verify-public.html"; ROLLBACK_URL="${STEP_REPORT_URL}rollback-public.html"; HTTP_VERIFY_PASS=true; fi; fi; set -e; }
stop(){ local stop_result reason rc; stop_result="$1"; reason="$2"; rc="${3:-1}"; trap - ERR; if ! mkdir "$STOP_GUARD_DIR" 2>/dev/null; then exit "$rc"; fi; RESULT="$stop_result"; STOP_REASON="$reason"; [ "$WORKFLOW_READY" != true ] || router_step_phase_fail "$CURRENT_PHASE" "$reason" >/dev/null 2>&1 || true; export_test_artifacts >/dev/null 2>&1 || true; if [ "$WORKFLOW_READY" = true ] && ! router_step_publication_is_verified; then publish_stop_report || true; fi; emit_final; exit "$rc"; }
on_err(){ local line rc; line="$1"; rc="$2"; stop STOP_R20O_UNHANDLED_ERROR "phase=$CURRENT_PHASE line=$line rc=$rc" "$rc"; }
for required in "$WORKFLOW_LIB" "$CONTRACT_LIB" "$RUNTIME_TRANSITION_LIB" "$PUBLISHER" "$HTTP" "$FINALIZER" "$SAFE_BUILDER" "$HTML_BUILDER" "$MACHINE_CLOSE"; do [ -e "$required" ] || { RESULT=STOP_R20O_SHARED_API_MISSING; STOP_REASON="$required"; emit_final; exit 20; }; done
source "$WORKFLOW_LIB"; source "$CONTRACT_LIB"; source "$RUNTIME_TRANSITION_LIB"
router_step_workflow_init "$PHASE_CURRENT" "$PHASE_EVENTS" publication_preflight vm101_baseline snapshot backup degraded_pool_and_retry_test machine_close model_publication methods_publication project_source_publication external_http_verification step_report_publication
router_runtime_transition_init "$TRANSITION_FILE"
actual_zip="$(sha256sum "$BUNDLE_COPY" | awk '{print $1}')"; actual_installer="$(sha256sum "$STEP_FILE" | awk '{print $1}')"
[ "$actual_zip" = "$EXPECTED_BUNDLE_SHA256" ] || { RESULT=STOP_COPY_OR_TRANSFER_ERROR; STOP_REASON=bundle_sha; emit_final; exit 20; }; [ "$actual_installer" = "$EXPECTED_INSTALLER_SHA256" ] || { RESULT=STOP_COPY_OR_TRANSFER_ERROR; STOP_REASON=installer_sha; emit_final; exit 20; }
printf '%s\n' 'RESULT=PASS_ROUTER_BUNDLE_GATE_R19' 'RESULT=PASS_ROUTER_BUNDLE_GATE_R20' 'RESULT=PASS_ROUTER_BUNDLE_GATE' 'SOURCE=router-step-bootstrap' >"$COMPONENTS/bundle-gate.txt"; COPY_VERIFY=PASS; router_runtime_transition_set "$TRANSITION_FILE" TRANSFER_VERIFIED true
printf 'COPY_VERIFY=PASS\nEXPECTED_BUNDLE_SHA256=%s\nACTUAL_BUNDLE_SHA256=%s\nEXPECTED_INSTALLER_SHA256=%s\nACTUAL_INSTALLER_SHA256=%s\nMANIFEST_VERIFY=PASS\nSYNTAX_VERIFY=PASS\nFIXTURE_VERIFY=PASS\n' "$EXPECTED_BUNDLE_SHA256" "$actual_zip" "$EXPECTED_INSTALLER_SHA256" "$actual_installer" >"$TRANSFER"
WORKFLOW_READY=true
trap 'on_err "$LINENO" "$?"' ERR
phase_begin publication_preflight
[ "$(hostname)" = router-ops ] || stop STOP_R20O_WRONG_HOST "$(hostname)" 21
preflight_object "$MODEL_LABEL" vm101-model "$BUNDLE_DIR/project-sources/WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md"; preflight_object "$METHODS_LABEL" vm101-methods "$BUNDLE_DIR/project-sources/WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md"; preflight_object "$PROJECT_LABEL" project-source-post "$BUNDLE_DIR/project-sources/WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md"; preflight_object "$STEP_LABEL" step-report "$BUNDLE_DIR/README.md"
phase_done publication_preflight
phase_begin vm101_baseline
if run_vm101_script "$BUNDLE_DIR/scripts/vm101-preflight.sh" | tee "$COMPONENTS/vm101-preflight.txt"; then pre_rc=0; else pre_rc=${PIPESTATUS[0]}; fi
[ "$pre_rc" -eq 0 ] || { remote="$(awk -F= '$1=="RESULT"&&$2~/^STOP_/{v=$2}END{print v}' "$COMPONENTS/vm101-preflight.txt")"; stop "${remote:-STOP_R20O_PREFLIGHT_REMOTE_FAILED}" preflight "$pre_rc"; }
for marker in 'RESULT=PASS_R20O_R02_VM101_PREFLIGHT' 'HEALTHY_SLOTS=5' 'REPAIR_COUNTER=0' 'WATCHER_RUNNING=true' 'RETRY_RUNNING=true' 'DIRECT_FAILOPEN_USED=false'; do grep -qx "$marker" "$COMPONENTS/vm101-preflight.txt" || stop STOP_R20O_PREFLIGHT_MARKER "$marker" 22; done
OLD_GENERATION_ID="$(field "$COMPONENTS/vm101-preflight.txt" ACTIVE_GENERATION_ID)"; phase_done vm101_baseline
phase_begin snapshot
test "${#SNAPSHOT_NAME}" -le 40 || stop STOP_R20O_SNAPSHOT_NAME_TOO_LONG "$SNAPSHOT_NAME" 23; [ "${#SNAPSHOT_NAME}" -le 32 ] || stop STOP_R20O_SNAPSHOT_NAME_NOT_PREFERRED "$SNAPSHOT_NAME" 23
"$SSH_BIN" -o BatchMode=yes pve-mgts "qm listsnapshot 101 | grep -F -- '$SNAPSHOT_NAME' >/dev/null && exit 9 || qm snapshot 101 '$SNAPSHOT_NAME' --description '$STEP_ID pre-test'" | tee "$COMPONENTS/snapshot.txt"; SNAPSHOT_CREATED=true; phase_done snapshot
phase_begin backup
if run_vm101_script "$BUNDLE_DIR/scripts/vm101-backup.sh" "R20O_BACKUP_DIR=$BACKUP_DIR" | tee "$COMPONENTS/vm101-backup.txt"; then backup_rc=0; else backup_rc=${PIPESTATUS[0]}; fi
[ "$backup_rc" -eq 0 ] && grep -qx 'RESULT=PASS_R20O_R02_VM101_BACKUP' "$COMPONENTS/vm101-backup.txt" || stop STOP_R20O_BACKUP_FAILED backup "$backup_rc"; phase_done backup
phase_begin degraded_pool_and_retry_test
if run_vm101_script "$BUNDLE_DIR/scripts/vm101-degraded-retry-test.sh" "R20O_BACKUP_DIR=$BACKUP_DIR" | tee "$COMPONENTS/vm101-test.txt"; then test_rc=0; else test_rc=${PIPESTATUS[0]}; fi
TEST_DIR="$(field "$COMPONENTS/vm101-test.txt" TEST_DIR)"; remote_impact="$(field "$COMPONENTS/vm101-test.txt" RUNTIME_IMPACT)"; case "$remote_impact" in true|false) RUNTIME_IMPACT="$remote_impact";; esac; CONFIG_RESTORED="$(field "$COMPONENTS/vm101-test.txt" CONFIG_RESTORED)"; [ -n "$CONFIG_RESTORED" ] || CONFIG_RESTORED=false
if [ "$test_rc" -ne 0 ]; then remote="$(awk -F= '$1=="RESULT"&&$2~/^STOP_/{v=$2}END{print v}' "$COMPONENTS/vm101-test.txt")"; stop "${remote:-STOP_R20O_REMOTE_TEST_FAILED}" degraded_pool_and_retry_test "$test_rc"; fi
grep -qx 'RESULT=PASS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROVEN' "$COMPONENTS/vm101-test.txt" || stop STOP_R20O_TEST_PASS_MARKER missing 24
for marker in LOCAL_REPAIR_PASS=true DEGRADED_POOL_PASS=true COUNTER_PRESERVED_PASS=true OLD_GENERATION_PRESERVED_PASS=true ACTIVE_ENDPOINTS_PRESERVED_PASS=true RETRY_NOT_DUE_PASS=true FAILED_RETRY_PASS=true SUCCESSFUL_RETRY_PASS=true RETRY_COUNT_AFTER_FAILED_RETRY=1 CONFIG_RESTORED=true FINAL_MODE=NORMAL FINAL_COUNTER=0 ACTIVE_SLOTS_HEALTHY=5 ENDPOINT_CONSISTENCY_PASS=true DIRECT_FAILOPEN_USED=false WATCHER_RUNNING=true RETRY_RUNNING=true LOCAL_REPAIR_TESTED=true DEGRADED_POOL_TESTED=true CONTROLLED_RETRY_TESTED=true; do grep -qx "$marker" "$COMPONENTS/vm101-test.txt" || stop STOP_R20O_TEST_MARKER "$marker" 24; done
LOCAL_REPAIR_PASS=true; DEGRADED_POOL_PASS=true; COUNTER_PRESERVED_PASS=true; OLD_GENERATION_PRESERVED_PASS=true; ACTIVE_ENDPOINTS_PRESERVED_PASS=true; RETRY_NOT_DUE_PASS=true; FAILED_RETRY_PASS=true; SUCCESSFUL_RETRY_PASS=true; RETRY_COUNT_AFTER_FAILED_RETRY=1; CONFIG_RESTORED=true; NEW_GENERATION_ID="$(field "$COMPONENTS/vm101-test.txt" NEW_GENERATION_ID)"; FINAL_MODE=NORMAL; FINAL_COUNTER=0; ACTIVE_SLOTS_HEALTHY=5; ENDPOINT_CONSISTENCY_PASS=true; DIRECT_FAILOPEN_USED=false; WATCHER_RUNNING=true; RETRY_RUNNING=true; LOCAL_REPAIR_TESTED=true; DEGRADED_POOL_TESTED=true; CONTROLLED_RETRY_TESTED=true; CORE_CHANGE_COMPLETE=true; router_runtime_transition_set "$TRANSITION_FILE" CORE_CHANGE_COMPLETE true; export_test_artifacts || stop STOP_R20O_TEST_ARTIFACT_EXPORT_FAILED export 25; router_runtime_transition_set "$TRANSITION_FILE" ACTIVATION_COMPLETE true; phase_done degraded_pool_and_retry_test
phase_begin machine_close
if "$MACHINE_CLOSE" vm101 "$STEP_ID" | tee "$COMPONENTS/router-machine-close-safe.txt"; then close_rc=0; else close_rc=${PIPESTATUS[0]}; fi
[ "$close_rc" -eq 0 ] || stop STOP_R20O_ROUTER_MACHINE_CLOSE_FAILED machine_close "$close_rc"; grep -Eq '^RESULT=PASS_ROUTER_MACHINE_CLOSE$' "$COMPONENTS/router-machine-close-safe.txt" || stop STOP_R20O_ROUTER_MACHINE_CLOSE_RESULT missing 28; [ "$(field "$COMPONENTS/router-machine-close-safe.txt" COMMIT)" = "$GIT_COMMIT" ] || stop STOP_R20O_ROUTER_MACHINE_CLOSE_COMMIT mismatch 28; VM101_PUBLIC_URL="$(field "$COMPONENTS/router-machine-close-safe.txt" VM101_PUBLIC_URL)"; [ -n "$VM101_PUBLIC_URL" ] || stop STOP_R20O_ROUTER_MACHINE_CLOSE_URL missing 28; MACHINE_GIT_PASS=true; MACHINE_CLOSE_PASS=true; phase_done machine_close
ARTIFACT_DIR="$STEP_DIR/vm101-test-artifacts"; VM101_MODEL_URL="$PUBLIC_BASE/latest/${PUBLISH_TS}_${MODEL_LABEL}/"; VM101_METHODS_URL="$PUBLIC_BASE/latest/${PUBLISH_TS}_${METHODS_LABEL}/"; PROJECT_SOURCE_POST_URL="$PUBLIC_BASE/latest/${PUBLISH_TS}_${PROJECT_LABEL}/"; PROJECT_SOURCE_POST_FILE_URL="${PROJECT_SOURCE_POST_URL}WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md"; STEP_REPORT_URL="$PUBLIC_BASE/latest/${PUBLISH_TS}_${STEP_LABEL}/"; RESULT=PASS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROVEN; STOP_REASON=none; RUNTIME_ENV="$STEP_DIR/runtime-publications.env"; write_final_env "$RUNTIME_ENV"; RUNTIME_DOCS="$STEP_DIR/runtime-publications"; python3 "$BUNDLE_DIR/scripts/build_runtime_publications.py" --env "$RUNTIME_ENV" --output-dir "$RUNTIME_DOCS" --artifact-dir "$ARTIFACT_DIR" | tee "$COMPONENTS/runtime-publications-build.txt"
phase_begin model_publication
MODEL_SAFE="$STEP_DIR/model-safe"; "$SAFE_BUILDER" "$MODEL_SAFE" "$RUNTIME_DOCS/model-summary.txt" model-summary.txt "$ARTIFACT_DIR/test-summary.kv" test-summary.txt "$ARTIFACT_DIR/state.final.kv" state-final.txt "$ARTIFACT_DIR/quarantine.final.tsv" quarantine-final.txt "$ARTIFACT_DIR/events.final.tsv" events-final.txt "$ARTIFACT_DIR/endpoints.baseline.tsv" endpoints-baseline.txt "$ARTIFACT_DIR/endpoints.after-failed-refresh.tsv" endpoints-after-failed-refresh.txt "$ARTIFACT_DIR/endpoints.after-failed-retry.tsv" endpoints-after-failed-retry.txt "$ARTIFACT_DIR/endpoints.final.tsv" endpoints-final.txt "$ARTIFACT_DIR/retry-log-new.txt" retry-log-new.txt "$ARTIFACT_DIR/forced-refresh-failure.log" forced-refresh-failure.txt "$ARTIFACT_DIR/retry-due-failure.log" retry-due-failure.txt "$ARTIFACT_DIR/retry-due-success.log" retry-due-success.txt >"$COMPONENTS/model-safe-payload.txt"; actual_model=''; publish_object "$MODEL_LABEL" vm101-model actual_model "$MODEL_SAFE"/*; [ "$actual_model" = "$VM101_MODEL_URL" ] || stop STOP_R20O_MODEL_URL_MISMATCH "$actual_model" 30; phase_done model_publication
phase_begin methods_publication
METHODS_SAFE="$STEP_DIR/methods-safe"; "$SAFE_BUILDER" "$METHODS_SAFE" "$RUNTIME_DOCS/methods-summary.txt" methods-summary.txt "$BUNDLE_DIR/project-sources/WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md" test-contract.txt "$BUNDLE_DIR/reference/source-origin.txt" source-origin.txt >"$COMPONENTS/methods-safe-payload.txt"; actual_methods=''; publish_object "$METHODS_LABEL" vm101-methods actual_methods "$METHODS_SAFE"/*; [ "$actual_methods" = "$VM101_METHODS_URL" ] || stop STOP_R20O_METHODS_URL_MISMATCH "$actual_methods" 31; phase_done methods_publication
phase_begin project_source_publication
actual_project=''; publish_object "$PROJECT_LABEL" project-source-post actual_project "$BUNDLE_DIR/project-sources/WG_PAID_MGTS_R20O_R02_DEGRADED_POOL_AND_CONTROLLED_RETRY_RUNTIME_PROOF_CONTINUATION.md" "$RUNTIME_DOCS/WG_PAID_MGTS_CHECKPOINT_AFTER_R20O_R02.md"; [ "$actual_project" = "$PROJECT_SOURCE_POST_URL" ] || stop STOP_R20O_PROJECT_SOURCE_URL_MISMATCH "$actual_project" 32; phase_done project_source_publication
phase_begin external_http_verification
: >"$COMPONENTS/final-http.log"; for url in "$VM101_PUBLIC_URL" "$VM130_SOURCE_URL" "$VM101_MODEL_URL" "$VM101_METHODS_URL" "$PROJECT_SOURCE_POST_URL" "$LOCAL_M07_PLAN_URL" "$GLOBAL_PROJECT_PLAN_URL" "$ACCESS_MAP_URL" "$VM100_PUBLIC_URL" "$VM121_PUBLIC_URL"; do "$HTTP" --url "$url" --output "$STEP_DIR/http-final-$(printf '%s' "$url" | sha256sum | cut -c1-12).html" >>"$COMPONENTS/final-http.log" 2>&1; done; HTTP_VERIFY_PASS=true; phase_done external_http_verification
phase_begin step_report_publication
PASS_SAFE="$(build_report_payload PASS)"; FINAL_ENV="$STEP_DIR/finalize.env"; "$FINALIZER" --label "$STEP_LABEL" --payload-dir "$PASS_SAFE" --workflow-current "$PHASE_CURRENT" --workflow-events "$PHASE_EVENTS" --output-env "$FINAL_ENV" --expect-kind step-report >"$COMPONENTS/finalize.log"; actual_step="$(field "$FINAL_ENV" PUBLIC_URL)"; [ "$actual_step" = "$STEP_REPORT_URL" ] || stop STOP_R20O_STEP_REPORT_URL_MISMATCH "$actual_step" 33; PREVIOUS_MOVED_COUNT="$(field "$FINAL_ENV.publish" PREVIOUS_MOVED_COUNT)"; [ -n "$PREVIOUS_MOVED_COUNT" ] || PREVIOUS_MOVED_COUNT=0; FACTS_URL="${STEP_REPORT_URL}facts-public.html"; VERIFY_URL="${STEP_REPORT_URL}verify-public.html"; ROLLBACK_URL="${STEP_REPORT_URL}rollback-public.html"; HTTP_VERIFY_PASS=true; trap - ERR; emit_final; exit 0