#!/usr/bin/env bash set -Eeuo pipefail umask 077 BASE="${ROUTER_OPS_BASE:-/opt/router-ops}" ROOT="${BUNDLE_ROOT:?}" RUN_DIR="${ROUTER_DETACHED_RUN_DIR:?}" CHILD_RESULT="${ROUTER_DETACHED_CHILD_RESULT:?}" INCOMING="${ROUTER_STEP_INCOMING:-/home/ops/incoming}" SELFTEST="${ROUTER_EXACT_WORKER_SELFTEST:-false}" CASE=pass if [[ "$SELFTEST" == true ]]; then case "${ROUTER_PUBLIC_ROOT:-}" in */fail-pre/public) CASE=fail-pre ;; */fail-core/public) CASE=fail-core ;; */fail-post/public) CASE=fail-post ;; esac fi STEP=STEP_050M07P23B_R01_VM121_DOMAIN_V2_LIVE_ACTIVATION_AND_MODEL_COMPATIBILITY VM121_BASE_COMMIT=f9cc1cdfa0619ba5d6d28f359462061d97b3a558 VM121_BASE_TREE=ea0c9f14684a00a3cb6ffb2c0eb75265a5877558 VM121_BASE_COUNT=43 VM100_COMMIT=e865282f4837747b1babfac4ac199680c6b43d04 VM100_TREE=8f04afbbc77006fe117bf1228b1f4fc977be9a47 VM100_COUNT=33 MIGRATION_SHA=56c1fe6fd85d4d8a7a048ccf4efc807c8f3b20f2c98b24d6f21e4230153a9d8d NEW_CORE_SHA=d267b4340fd0f61e3da9d37a19ac43d5099d673ac7da1d6f2b35997436ffcd6b NEW_INIT_SHA=4a5f7b75ceda608f7493c3b409dbcf662b883b4cc9899bc981384780b583b9e8 NEW_DEV_SHA=44d9cdb0454c70e1339568d34a60cba5f603e858edb759c043a7b63f793d8b46 CURRENT="$RUN_DIR/workflow-current.txt"; EVENTS="$RUN_DIR/workflow-events.tsv"; EVIDENCE="$RUN_DIR/evidence"; REPORT="$RUN_DIR/report"; PAYLOAD="$RUN_DIR/public-payload"; FINAL_ENV="$RUN_DIR/final-publication.env"; TARGET_DIR="$RUN_DIR/target" mkdir -p "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" "$INCOMING"; chmod 700 "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" source "$BASE/lib/router-step-workflow.sh" source <("$BASE/bin/router-release-contract" env --root "$ROOT") router_step_workflow_init "$CURRENT" "$EVENTS" preflight live_activation machine_close postcheck report_publish archive CORE_CHANGE_COMPLETE=false CORE_CHANGE_REPEATED=false ROLLBACK_STATUS=not_required SOURCE_ROLLBACK_STATUS=not_required RUNTIME_RECOVERY_STATUS=not_required STOP_DETAIL=unknown STOP_HANDLED=false STEP_REPORT_URL=unknown LATEST_ARCHIVE=not_created LATEST_ARCHIVE_SHA256=unknown REMOTE_STAGE=unknown REMOTE_PKG=unknown VM121_NEW_COMMIT=unknown VM121_NEW_TREE=unknown VM121_NEW_FILE_COUNT=unknown VM121_NEW_PUBLIC_URL=unknown VALIDATION_BACKUP_PATH=unknown VALIDATION_BACKUP_SHA256=unknown ROLLBACK_BACKUP_PATH=unknown ROLLBACK_BACKUP_SHA256=unknown DELTA_SQL_SHA256=unknown field(){ local key="$1" file="$2"; awk -F= -v k="$key" '$1==k{print substr($0,index($0,"=")+1); exit}' "$file"; } write_child(){ local result="$1" rc="$2" tmp="${CHILD_RESULT}.tmp.$$"; { echo "RESULT=$result"; echo "WORKER_RC=$rc"; echo "CORE_CHANGE_COMPLETE=$CORE_CHANGE_COMPLETE"; echo "CORE_CHANGE_REPEATED=$CORE_CHANGE_REPEATED"; echo "ROLLBACK_STATUS=$ROLLBACK_STATUS"; echo "SOURCE_ROLLBACK_STATUS=$SOURCE_ROLLBACK_STATUS"; echo "RUNTIME_RECOVERY_STATUS=$RUNTIME_RECOVERY_STATUS"; echo "STOP_DETAIL=$STOP_DETAIL"; echo "VM121_NEW_COMMIT=$VM121_NEW_COMMIT"; echo "VM121_NEW_TREE=$VM121_NEW_TREE"; echo "VM121_NEW_FILE_COUNT=$VM121_NEW_FILE_COUNT"; echo "VM121_NEW_PUBLIC_URL=$VM121_NEW_PUBLIC_URL"; echo "VALIDATION_BACKUP_PATH=$VALIDATION_BACKUP_PATH"; echo "VALIDATION_BACKUP_SHA256=$VALIDATION_BACKUP_SHA256"; echo "ROLLBACK_BACKUP_PATH=$ROLLBACK_BACKUP_PATH"; echo "ROLLBACK_BACKUP_SHA256=$ROLLBACK_BACKUP_SHA256"; echo "STEP_REPORT_URL=$STEP_REPORT_URL"; echo "LATEST_ARCHIVE=$LATEST_ARCHIVE"; echo "LATEST_ARCHIVE_SHA256=$LATEST_ARCHIVE_SHA256"; } > "$tmp"; chmod 600 "$tmp"; mv "$tmp" "$CHILD_RESULT"; } archive_latest(){ local log="$EVIDENCE/archive.log"; router_step_archive_latest_python "$BASE" "$INCOMING" "$(date -u +%Y%m%d-%H%M%S)" | tee "$log"; LATEST_ARCHIVE="$(field LATEST_ARCHIVE "$log")"; LATEST_ARCHIVE_SHA256="$(field LATEST_ARCHIVE_SHA256 "$log")"; } cleanup_remote(){ [[ "$SELFTEST" == false && "$REMOTE_STAGE" != unknown ]] || return 0; set +e; ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE' '$REMOTE_PKG'" >/dev/null 2>&1; set -e; } write_stop_report(){ local rc="$1" line="$2" cmd="$3"; cat > "$REPORT/report.txt" <> "$REPORT/report.txt"; cp "$REPORT/report.txt" "$REPORT/verify.txt"; cp "$ROOT/scripts/run-step.sh" "$REPORT/executed-step.sh"; cp "$ROOT/release-info.json" "$REPORT/release-info.json"; cp "$ROOT/rollback/README.md" "$REPORT/rollback.txt"; } stop_handler(){ local rc="$1" line="$2" cmd="$3"; trap - ERR; [[ "$STOP_HANDLED" == false ]] || return "$rc"; STOP_HANDLED=true; cleanup_remote; write_stop_report "$rc" "$line" "$cmd"; set +e; if [[ "$SELFTEST" == true ]]; then DEST="${ROUTER_PUBLIC_ROOT:?}/latest/p23b-r01-$CASE-selftest"; rm -rf "$DEST"; mkdir -p "$DEST"; cp -a "$REPORT"/. "$DEST"/; STEP_REPORT_URL="selftest://p23b-r01-$CASE/"; archive_latest >/dev/null 2>&1; else args=(); while IFS= read -r file; do args+=("$file" "$(basename "$file")"); done < <(find "$REPORT" -maxdepth 1 -type f -print | LC_ALL=C sort); router_step_publish_payload step050m07p23b_r01_vm121_domain_v2_live_activation_and_model_compatibility_stop "$REPORT" "$PAYLOAD" "$CURRENT" "$EVENTS" "$FINAL_ENV" "${args[@]}" >/dev/null 2>&1 || true; [[ -f "$FINAL_ENV" ]] && STEP_REPORT_URL="$(field PUBLIC_URL "$FINAL_ENV")"; archive_latest >/dev/null 2>&1; fi; set -e; write_child STOP_P23B_VM121_DOMAIN_V2_LIVE_ACTIVATION_AND_MODEL_COMPATIBILITY "$rc"; return "$rc"; } trap 'rc=$?; stop_handler "$rc" "$LINENO" "$BASH_COMMAND"; exit "$rc"' ERR write_vm121_source_pre(){ cat < "$EVIDENCE/pre-vm121-source.txt"; else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/pre-vm121-source.txt" 2>&1 <<'VM121' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check VM121 fi; "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/pre-vm121-source.txt" "$VM121_BASE_COMMIT" "$VM121_BASE_TREE" "$VM121_BASE_COUNT" > "$EVIDENCE/pre-vm121-source-validation.txt"; } collect_vm100_zero(){ local stage="$1"; if [[ "$SELFTEST" == true ]]; then write_vm100_zero > "$EVIDENCE/${stage}-vm100-zero.txt"; else ssh -T -o BatchMode=yes -o ConnectTimeout=20 pve-mgts /bin/bash -s > "$EVIDENCE/${stage}-vm100-zero.txt" 2>&1 <<'PVE' set -euo pipefail ssh -T -o BatchMode=yes -o ConnectTimeout=20 -i /root/.ssh/pve_to_openwrt_mgts_ed25519 root@10.71.100.1 /bin/sh -s <<'VM100' set -eu /usr/local/sbin/router-vm100-git-publish.sh --check status="$(/usr/local/sbin/router-wgpay-peer-lifecycle.sh --status)" get_status(){ printf '%s\n' "$status" | awk -F= -v k="$1" '$1==k{print substr($0,index($0,"=")+1); exit}'; } echo "LIFECYCLE_RESULT=$(get_status RESULT)" echo "LIFECYCLE_INITIALIZED=$(get_status initialized)" echo "LIFECYCLE_REGISTRY_VALID=$(get_status registry_valid)" echo "LIFECYCLE_PROFILE_COUNT=$(get_status profile_count)" echo "LIFECYCLE_WIREGUARD_COUNT=$(get_status wireguard_count)" echo "LIFECYCLE_AMNEZIAWG_COUNT=$(get_status amneziawg_count)" echo "LIFECYCLE_SELECTOR_REGISTRY_CONSISTENT=$(get_status selector_registry_consistent)" . /etc/router-wgpay-peer-lifecycle.conf echo "WG_PAID_RUNTIME_PEER_COUNT=$(wg show "$PEER_WIREGUARD_INTERFACE" peers 2>/dev/null | awk 'NF{n++} END{print n+0}')" echo "LIFECYCLE_ACTIVE_SELECTOR_ROW_COUNT=$(awk '$0!~/^[[:space:]]*#/ && NF{n++} END{print n+0}' "$PEER_ACTIVE_SELECTOR_FILE")" echo "LIFECYCLE_CANONICAL_SELECTOR_ROW_COUNT=$(awk '$0!~/^[[:space:]]*#/ && NF{n++} END{print n+0}' "$PEER_CANONICAL_SELECTOR_FILE")" VM100 PVE fi; "$ROOT/scripts/source-check-validate.sh" vm100 "$EVIDENCE/${stage}-vm100-zero.txt" "$VM100_COMMIT" "$VM100_TREE" "$VM100_COUNT" > "$EVIDENCE/${stage}-vm100-source-validation.txt"; "$ROOT/scripts/vm100-zero-validate.sh" "$EVIDENCE/${stage}-vm100-zero.txt" > "$EVIDENCE/${stage}-vm100-zero-validation.txt"; } write_selftest_activation(){ case "$CASE" in pass|fail-post) cat < "$EVIDENCE/target-extract.txt" router_step_phase_complete preflight exact_sources_vm100_zero_and_target_verified router_step_phase_begin live_activation activation_rc=0 if [[ "$SELFTEST" == true ]]; then if write_selftest_activation > "$EVIDENCE/live-activation.txt"; then activation_rc=0; else activation_rc=$?; fi else token="$(basename "$RUN_DIR" | tr -cd 'A-Za-z0-9._-')"; REMOTE_STAGE="/root/router-step-p23b-$token"; REMOTE_PKG="/root/router-step-p23b-$token.tar.gz" if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "umask 077; cat > '$REMOTE_PKG'" < "$PKG"; then STOP_DETAIL=remote_package_transfer; false; fi if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE'; install -d -m 0700 '$REMOTE_STAGE'; tar -xzf '$REMOTE_PKG' -C '$REMOTE_STAGE'" > "$EVIDENCE/remote-stage.txt" 2>&1; then STOP_DETAIL=remote_package_extract; false; fi if ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "/bin/bash '$REMOTE_STAGE/vm121-p23b-live-activate.sh' '$REMOTE_STAGE/candidate'" > "$EVIDENCE/live-activation.txt" 2>&1; then activation_rc=0; else activation_rc=$?; fi fi remote_core_db="$(field CORE_DATABASE_CHANGE_COMPLETE "$EVIDENCE/live-activation.txt")"; remote_core_src="$(field CORE_SOURCE_CHANGE_COMPLETE "$EVIDENCE/live-activation.txt")" [[ "$remote_core_db" == true && "$remote_core_src" == true ]] && CORE_CHANGE_COMPLETE=true || CORE_CHANGE_COMPLETE=false ROLLBACK_STATUS="$(field ROLLBACK_STATUS "$EVIDENCE/live-activation.txt")"; [[ -n "$ROLLBACK_STATUS" ]] || ROLLBACK_STATUS=unknown SOURCE_ROLLBACK_STATUS="$(field SOURCE_ROLLBACK_STATUS "$EVIDENCE/live-activation.txt")"; [[ -n "$SOURCE_ROLLBACK_STATUS" ]] || SOURCE_ROLLBACK_STATUS=unknown RUNTIME_RECOVERY_STATUS="$(field RUNTIME_RECOVERY_STATUS "$EVIDENCE/live-activation.txt")"; [[ -n "$RUNTIME_RECOVERY_STATUS" ]] || RUNTIME_RECOVERY_STATUS=unknown VALIDATION_BACKUP_PATH="$(field VALIDATION_BACKUP_PATH "$EVIDENCE/live-activation.txt")"; VALIDATION_BACKUP_SHA256="$(field VALIDATION_BACKUP_SHA256 "$EVIDENCE/live-activation.txt")"; ROLLBACK_BACKUP_PATH="$(field ROLLBACK_BACKUP_PATH "$EVIDENCE/live-activation.txt")"; ROLLBACK_BACKUP_SHA256="$(field ROLLBACK_BACKUP_SHA256 "$EVIDENCE/live-activation.txt")"; DELTA_SQL_SHA256="$(field DELTA_SQL_SHA256 "$EVIDENCE/live-activation.txt")" if [[ "$activation_rc" -ne 0 ]]; then STOP_DETAIL="live_activation_$(field FAILED_STAGE "$EVIDENCE/live-activation.txt")"; false; fi "$ROOT/scripts/domain-v2-live-validate.sh" "$EVIDENCE/live-activation.txt" > "$EVIDENCE/live-activation-validation.txt" || { STOP_DETAIL=live_activation_semantics; false; } CORE_CHANGE_COMPLETE=true cleanup_remote router_step_phase_complete live_activation backups_fixtures_offline_sql_live_0003_source_compatibility_and_runtime_restore_proven router_step_phase_begin machine_close close_rc=0 if [[ "$SELFTEST" == true ]]; then if write_selftest_machine_close > "$EVIDENCE/vm121-machine-close.txt"; then close_rc=0; else close_rc=$?; fi else if "$BASE/bin/router-machine-close-safe" vm121 "$STEP: activate 0003 Domain V2 schema; install compatible ORM; retire legacy dev writer" > "$EVIDENCE/vm121-machine-close.txt" 2>&1; then close_rc=0; else close_rc=$?; fi; fi if [[ "$close_rc" -ne 0 ]]; then STOP_DETAIL=machine_close_after_core; false; fi "$ROOT/scripts/machine-close-validate.sh" "$EVIDENCE/vm121-machine-close.txt" > "$EVIDENCE/vm121-machine-close-validation.txt" || { STOP_DETAIL=machine_close_semantics; false; } VM121_NEW_COMMIT="$(field VM121_NEW_COMMIT "$EVIDENCE/vm121-machine-close-validation.txt")"; VM121_NEW_TREE="$(field VM121_NEW_TREE "$EVIDENCE/vm121-machine-close-validation.txt")"; VM121_NEW_FILE_COUNT="$(field VM121_NEW_FILE_COUNT "$EVIDENCE/vm121-machine-close-validation.txt")"; VM121_NEW_PUBLIC_URL="$(field VM121_NEW_PUBLIC_URL "$EVIDENCE/vm121-machine-close-validation.txt")" router_step_phase_complete machine_close vm121_source_closed_and_published router_step_phase_begin postcheck if [[ "$SELFTEST" == true ]]; then write_selftest_post > "$EVIDENCE/post-vm121.txt"; else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/post-vm121.txt" 2>&1 <<'VM121POST' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check PG=wgaccess-postgres; BACKEND=wgaccess-backend PGUSER="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_USER"')"; PGDB="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_DB"')" q(){ docker exec "$PG" psql -v ON_ERROR_STOP=1 -U "$PGUSER" -d "$PGDB" -Atqc "$1"; } echo "ALEMBIC_VERSION=$(q 'SELECT version_num FROM alembic_version')" echo "MIGRATION_SHA256=$(sha256sum /opt/wg-access/backend/alembic/versions/0003_domain_v2_schema.py | awk '{print $1}')" echo "CORE_MODEL_SHA256=$(sha256sum /opt/wg-access/backend/app/models/core.py | awk '{print $1}')" echo "MODELS_INIT_SHA256=$(sha256sum /opt/wg-access/backend/app/models/__init__.py | awk '{print $1}')" echo "DEV_API_SHA256=$(sha256sum /opt/wg-access/backend/app/api/dev.py | awk '{print $1}')" echo "LIVE_DOMAIN_V2_TABLE_COUNT=$(q \"SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name IN ('plans','access_grants','access_grant_protocol_limits','invites','invite_redemptions','magic_link_tokens','auth_sessions','connection_profiles','peer_credentials','audit_events')\")" echo "LEGACY_BUSINESS_ROWS=$(q \"SELECT (SELECT count(*) FROM users)+(SELECT count(*) FROM invite_codes)+(SELECT count(*) FROM orders)+(SELECT count(*) FROM subscriptions)+(SELECT count(*) FROM peers)+(SELECT count(*) FROM provisioning_jobs)+(SELECT count(*) FROM peer_egress_leases)\")" echo "EGRESS_TARGET_COUNT=$(q 'SELECT count(*) FROM egress_targets')" echo "EGRESS_ALLOCATOR_COUNT=$(q 'SELECT count(*) FROM egress_allocator_settings')" grep -Fq 'status_code=410' /opt/wg-access/backend/app/api/dev.py && echo LEGACY_DEV_WRITER_RETIRED=true || { echo LEGACY_DEV_WRITER_RETIRED=false; exit 1; } if docker exec -i "$BACKEND" python - <<'PY' >/dev/null 2>&1 import urllib.request for u in ('http://127.0.0.1:8000/health','http://127.0.0.1:8000/health/db'): with urllib.request.urlopen(u,timeout=5) as r: assert r.status==200 PY then echo BACKEND_HEALTH=PASS; else echo BACKEND_HEALTH=FAIL; exit 1; fi VM121POST fi "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/post-vm121.txt" "$VM121_NEW_COMMIT" "$VM121_NEW_TREE" "$VM121_NEW_FILE_COUNT" > "$EVIDENCE/post-vm121-source-validation.txt" || { STOP_DETAIL=post_source; false; } [[ "$(field ALEMBIC_VERSION "$EVIDENCE/post-vm121.txt")" == 0003_domain_v2_schema ]] || { STOP_DETAIL=post_alembic; false; } [[ "$(field MIGRATION_SHA256 "$EVIDENCE/post-vm121.txt")" == "$MIGRATION_SHA" ]] || { STOP_DETAIL=post_migration_sha; false; } [[ "$(field CORE_MODEL_SHA256 "$EVIDENCE/post-vm121.txt")" == "$NEW_CORE_SHA" ]] || { STOP_DETAIL=post_core_model_sha; false; } [[ "$(field MODELS_INIT_SHA256 "$EVIDENCE/post-vm121.txt")" == "$NEW_INIT_SHA" ]] || { STOP_DETAIL=post_models_init_sha; false; } [[ "$(field DEV_API_SHA256 "$EVIDENCE/post-vm121.txt")" == "$NEW_DEV_SHA" ]] || { STOP_DETAIL=post_dev_sha; false; } [[ "$(field LIVE_DOMAIN_V2_TABLE_COUNT "$EVIDENCE/post-vm121.txt")" == 10 ]] || { STOP_DETAIL=post_domain_tables; false; } [[ "$(field LEGACY_BUSINESS_ROWS "$EVIDENCE/post-vm121.txt")" == 0 ]] || { STOP_DETAIL=post_business_rows; false; } [[ "$(field EGRESS_TARGET_COUNT "$EVIDENCE/post-vm121.txt")" == 5 ]] || { STOP_DETAIL=post_egress_targets; false; } [[ "$(field EGRESS_ALLOCATOR_COUNT "$EVIDENCE/post-vm121.txt")" == 14 ]] || { STOP_DETAIL=post_egress_allocator; false; } [[ "$(field BACKEND_HEALTH "$EVIDENCE/post-vm121.txt")" == PASS ]] || { STOP_DETAIL=post_backend_health; false; } [[ "$(field LEGACY_DEV_WRITER_RETIRED "$EVIDENCE/post-vm121.txt")" == true ]] || { STOP_DETAIL=post_legacy_writer; false; } collect_vm100_zero post || { STOP_DETAIL=post_vm100_zero; false; } router_step_phase_complete postcheck published_source_live_0003_clean_baseline_backend_and_vm100_zero_verified cat > "$REPORT/report.txt" <