#!/usr/bin/env bash set -Eeuo pipefail umask 077 BASE="${ROUTER_OPS_BASE:-/opt/router-ops}" ROOT="${BUNDLE_ROOT:?}" RUN_DIR="${ROUTER_DETACHED_RUN_DIR:?}" CHILD_RESULT="${ROUTER_DETACHED_CHILD_RESULT:?}" INCOMING="${ROUTER_STEP_INCOMING:-/home/ops/incoming}" SELFTEST="${ROUTER_EXACT_WORKER_SELFTEST:-false}" STEP=STEP_050M07P21B_R01_VM121_LIVE_DRIFT_BRIDGE_MIGRATION_FIXTURES BASE_COMMIT=25d1656c1ea023b526c7aee0a75327447359de21 BASE_TREE=0ad9296bfc5f2543335cf1d3e082814fc09fcd4a BASE_COUNT=41 EXPECTED_MIGRATION_SHA256=7c10777d7416f15935f55f385f9e233d4334e633972604ad1ee551fc0d1091de CURRENT="$RUN_DIR/workflow-current.txt" EVENTS="$RUN_DIR/workflow-events.tsv" EVIDENCE="$RUN_DIR/evidence" REPORT="$RUN_DIR/report" PAYLOAD="$RUN_DIR/public-payload" FINAL_ENV="$RUN_DIR/final-publication.env" TARGET_DIR="$RUN_DIR/target" mkdir -p "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" "$INCOMING" chmod 700 "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" source "$BASE/lib/router-step-workflow.sh" router_step_workflow_init "$CURRENT" "$EVENTS" preflight bridge_fixture postcheck source_publish report_publish archive CORE_CHANGE_COMPLETE=false CORE_CHANGE_REPEATED=false SOURCE_APPLY_STARTED=false ROLLBACK_STATUS=not_required STOP_DETAIL=unknown STOP_HANDLED=false LATEST_ARCHIVE=not_created LATEST_ARCHIVE_SHA256=unknown STEP_REPORT_URL=unknown BACKUP_CREATED=false BACKUP_PATH=unknown BACKUP_SHA256=unknown DATABASE_SCHEMA_CHANGED=false LIVE_DATABASE_WRITTEN=false LIVE_CLONE_UPGRADE=unknown CLEAN_INSTALL_MIGRATION=unknown VM121_PUBLIC_URL=unknown VM121_NEW_COMMIT=unknown VM121_NEW_TREE=unknown VM121_NEW_FILE_COUNT=unknown REMOTE_STAGE=unknown REMOTE_PKG=unknown field() { local key="$1" file="$2" awk -F= -v k="$key" '$1==k{print substr($0,index($0,"=")+1); exit}' "$file" } write_child() { local result="$1" rc="$2" tmp="${CHILD_RESULT}.tmp.$$" { echo RESULT="$result" echo WORKER_RC="$rc" echo CORE_CHANGE_COMPLETE="$CORE_CHANGE_COMPLETE" echo CORE_CHANGE_REPEATED="$CORE_CHANGE_REPEATED" echo ROLLBACK_STATUS="$ROLLBACK_STATUS" echo STOP_DETAIL="$STOP_DETAIL" echo BACKUP_CREATED="$BACKUP_CREATED" echo BACKUP_PATH="$BACKUP_PATH" echo BACKUP_SHA256="$BACKUP_SHA256" echo DATABASE_SCHEMA_CHANGED="$DATABASE_SCHEMA_CHANGED" echo LIVE_DATABASE_WRITTEN="$LIVE_DATABASE_WRITTEN" echo LIVE_CLONE_UPGRADE="$LIVE_CLONE_UPGRADE" echo CLEAN_INSTALL_MIGRATION="$CLEAN_INSTALL_MIGRATION" echo VM121_PUBLIC_URL="$VM121_PUBLIC_URL" echo VM121_NEW_COMMIT="$VM121_NEW_COMMIT" echo VM121_NEW_TREE="$VM121_NEW_TREE" echo VM121_NEW_FILE_COUNT="$VM121_NEW_FILE_COUNT" echo STEP_REPORT_URL="$STEP_REPORT_URL" echo LATEST_ARCHIVE="$LATEST_ARCHIVE" echo LATEST_ARCHIVE_SHA256="$LATEST_ARCHIVE_SHA256" } > "$tmp" chmod 600 "$tmp" mv "$tmp" "$CHILD_RESULT" } archive_latest() { local log="$EVIDENCE/archive.log" router_step_archive_latest_python "$BASE" "$INCOMING" "$(date -u +%Y%m%d-%H%M%S)" | tee "$log" LATEST_ARCHIVE="$(field LATEST_ARCHIVE "$log")" LATEST_ARCHIVE_SHA256="$(field LATEST_ARCHIVE_SHA256 "$log")" } write_selftest_source() { local head="$1" tree="$2" count="$3" cat < "$EVIDENCE/${stage}-vm121-source.txt" else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s >"$EVIDENCE/${stage}-vm121-source.txt" 2>&1 <<'VM121SRC' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check VM121SRC fi "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/${stage}-vm121-source.txt" \ "$head" "$tree" "$count" > "$EVIDENCE/${stage}-vm121-source-validation.txt" 2>&1 } rollback_source_if_needed() { [[ "$CORE_CHANGE_COMPLETE" == false && "$SOURCE_APPLY_STARTED" == true ]] || return 0 set +e if [[ "$SELFTEST" == true ]]; then write_selftest_source "$BASE_COMMIT" "$BASE_TREE" "$BASE_COUNT" > "$EVIDENCE/rollback-vm121-source.txt" else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/rollback-remove.txt" 2>&1 <<'VM121ROLL' set -euo pipefail rm -f /opt/wg-access/backend/alembic/versions/0002_live_drift_bridge.py VM121ROLL ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/rollback-vm121-source.txt" 2>&1 <<'VM121SRC' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check VM121SRC fi if "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/rollback-vm121-source.txt" \ "$BASE_COMMIT" "$BASE_TREE" "$BASE_COUNT" > "$EVIDENCE/rollback-vm121-source-validation.txt" 2>&1; then ROLLBACK_STATUS=PASS else ROLLBACK_STATUS=FAIL fi set -e } cleanup_remote_stage() { [[ "$SELFTEST" == false && "$REMOTE_STAGE" != unknown ]] || return 0 set +e ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE' '$REMOTE_PKG'" >/dev/null 2>&1 set -e } stop_handler() { local rc="$1" line="$2" cmd="$3" trap - ERR [[ "$STOP_HANDLED" == false ]] || exit "$rc" STOP_HANDLED=true rollback_source_if_needed cleanup_remote_stage { echo RESULT=STOP_P21B_VM121_LIVE_DRIFT_BRIDGE_MIGRATION_FIXTURES echo STOP_RC="$rc" echo STOP_LINE="$line" printf 'STOP_COMMAND=%q\n' "$cmd" echo CORE_CHANGE_COMPLETE="$CORE_CHANGE_COMPLETE" echo CORE_CHANGE_REPEATED="$CORE_CHANGE_REPEATED" echo ROLLBACK_STATUS="$ROLLBACK_STATUS" echo STOP_DETAIL="$STOP_DETAIL" echo BACKUP_CREATED="$BACKUP_CREATED" echo BACKUP_PATH="$BACKUP_PATH" echo BACKUP_SHA256="$BACKUP_SHA256" echo DATABASE_SCHEMA_CHANGED="$DATABASE_SCHEMA_CHANGED" echo LIVE_DATABASE_WRITTEN="$LIVE_DATABASE_WRITTEN" } | tee "$REPORT/stop.txt" "$EVIDENCE/stop.txt" set +e archive_latest >/dev/null 2>&1 set -e write_child STOP_P21B_VM121_LIVE_DRIFT_BRIDGE_MIGRATION_FIXTURES "$rc" exit "$rc" } trap 'rc=$?; stop_handler "$rc" "$LINENO" "$BASH_COMMAND"' ERR write_selftest_bridge_pass() { cat <<'EOF' ALEMBIC_VERSION_BEFORE=0001_initial_schema MIGRATION_REVISION=0002_live_drift_bridge PENDING_JOB_COUNT=0 BACKUP_CREATED=true BACKUP_ENCRYPTED=true BACKUP_KEY_CREATED=false BACKUP_KEY_PUBLISHED=false BACKUP_PATH=/root/router-db-backups/wg-access-p21b-selftest.dump.enc BACKUP_SHA256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa BACKUP_MODE=600 MIGRATION_SHA256=7c10777d7416f15935f55f385f9e233d4334e633972604ad1ee551fc0d1091de SOURCE_FILE_INSTALLED=true LIVE_SCHEMA_PROVENANCE=PASS LIVE_CLONE_RESTORE=PASS LIVE_CLONE_UPGRADE=PASS LIVE_CLONE_DATA_UNCHANGED=true CLEAN_INSTALL_MIGRATION=PASS CLEAN_SCHEMA_MATCHES_CAPTURED_LIVE=true LIVE_SCHEMA_UNCHANGED=true LIVE_ALEMBIC_VERSION_AFTER=0001_initial_schema DATABASE_SCHEMA_CHANGED=false LIVE_DATABASE_WRITTEN=false ROLLBACK_STATUS=not_required PRIVATE_RUN_DIR=/root/router-db-bridge-tests/selftest RESULT=PASS_P21B_VM121_LIVE_DRIFT_BRIDGE_MIGRATION_FIXTURES EOF } write_selftest_bridge_fail() { cat <<'EOF' FAILED_STAGE=live_clone_upgrade ROLLBACK_STATUS=PASS SOURCE_FILE_PRESENT_AFTER_FAILURE=false DATABASE_SCHEMA_CHANGED=false LIVE_DATABASE_WRITTEN=false EOF } router_step_phase_begin preflight if ! collect_source_check_expected pre "$BASE_COMMIT" "$BASE_TREE" "$BASE_COUNT"; then STOP_DETAIL=pre_vm121_git_semantics false fi PKG="$ROOT/$(python3 - "$ROOT/release-info.json" <<'PY' import json,sys print(json.load(open(sys.argv[1],encoding='utf-8'))['target_package_path']) PY )" PKG_SHA="$(awk '{print $1}' "$ROOT/reference/target-package.sha256")" "$ROOT/scripts/target-apply.sh" "$PKG_SHA" "$PKG" "$TARGET_DIR" > "$EVIDENCE/target-extract.txt" [[ "$(sha256sum "$TARGET_DIR/0002_live_drift_bridge.py" | awk '{print $1}')" == "$EXPECTED_MIGRATION_SHA256" ]] router_step_phase_complete preflight vm121_source_and_bridge_package_verified router_step_phase_begin bridge_fixture SOURCE_APPLY_STARTED=true bridge_rc=0 if [[ "$SELFTEST" == true ]]; then if [[ "${ROUTER_PUBLIC_ROOT:-}" == */fail/public ]]; then write_selftest_bridge_fail > "$EVIDENCE/bridge-fixture.txt" bridge_rc=1 else write_selftest_bridge_pass > "$EVIDENCE/bridge-fixture.txt" fi else token="$(basename "$RUN_DIR" | tr -cd 'A-Za-z0-9._-')" REMOTE_STAGE="/root/router-step-p21b-$token" REMOTE_PKG="/root/router-step-p21b-$token.tar.gz" if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "umask 077; cat > '$REMOTE_PKG'" < "$PKG"; then STOP_DETAIL=remote_package_transfer false fi if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE'; install -d -m 0700 '$REMOTE_STAGE'; tar -xzf '$REMOTE_PKG' -C '$REMOTE_STAGE'" > "$EVIDENCE/remote-stage.txt" 2>&1; then STOP_DETAIL=remote_package_extract false fi set +e ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "/bin/bash '$REMOTE_STAGE/vm121-p21b-bridge-fixtures.sh'" > "$EVIDENCE/bridge-fixture.txt" 2>&1 bridge_rc=$? set -e fi BACKUP_PATH="$(field BACKUP_PATH "$EVIDENCE/bridge-fixture.txt")" BACKUP_SHA256="$(field BACKUP_SHA256 "$EVIDENCE/bridge-fixture.txt")" [[ -n "$BACKUP_PATH" && "$BACKUP_SHA256" =~ ^[0-9a-f]{64}$ ]] && BACKUP_CREATED=true || true if [[ "$bridge_rc" -ne 0 ]]; then ROLLBACK_STATUS="$(field ROLLBACK_STATUS "$EVIDENCE/bridge-fixture.txt")" [[ -n "$ROLLBACK_STATUS" ]] || ROLLBACK_STATUS=unknown STOP_DETAIL="bridge_fixture_$(field FAILED_STAGE "$EVIDENCE/bridge-fixture.txt")" false fi if ! "$ROOT/scripts/db-bridge-validate.sh" "$EVIDENCE/bridge-fixture.txt" > "$EVIDENCE/bridge-validation.txt" 2>&1; then STOP_DETAIL=bridge_semantics false fi LIVE_CLONE_UPGRADE="$(field LIVE_CLONE_UPGRADE "$EVIDENCE/bridge-fixture.txt")" CLEAN_INSTALL_MIGRATION="$(field CLEAN_INSTALL_MIGRATION "$EVIDENCE/bridge-fixture.txt")" DATABASE_SCHEMA_CHANGED="$(field DATABASE_SCHEMA_CHANGED "$EVIDENCE/bridge-fixture.txt")" LIVE_DATABASE_WRITTEN="$(field LIVE_DATABASE_WRITTEN "$EVIDENCE/bridge-fixture.txt")" ROLLBACK_STATUS=not_required cleanup_remote_stage router_step_phase_complete bridge_fixture encrypted_backup_live_clone_upgrade_and_clean_install_proven router_step_phase_begin postcheck if [[ "$SELFTEST" == true ]]; then cat > "$EVIDENCE/vm121-postcheck.txt" < "$EVIDENCE/vm121-postcheck.txt" 2>&1 <<'VM121POST' set -euo pipefail M=/opt/wg-access/backend/alembic/versions/0002_live_drift_bridge.py [[ -f "$M" ]] echo "MIGRATION_SHA256=$(sha256sum "$M" | awk '{print $1}')" PG=wgaccess-postgres PGUSER="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_USER"')" PGDB="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_DB"')" echo "LIVE_ALEMBIC_VERSION=$(docker exec "$PG" psql -v ON_ERROR_STOP=1 -U "$PGUSER" -d "$PGDB" -Atqc 'SELECT version_num FROM alembic_version')" VM121POST fi [[ "$(field MIGRATION_SHA256 "$EVIDENCE/vm121-postcheck.txt")" == "$EXPECTED_MIGRATION_SHA256" ]] || { STOP_DETAIL=post_migration_sha; false; } [[ "$(field LIVE_ALEMBIC_VERSION "$EVIDENCE/vm121-postcheck.txt")" == 0001_initial_schema ]] || { STOP_DETAIL=post_live_alembic_changed; false; } CORE_CHANGE_COMPLETE=true router_step_phase_complete postcheck migration_source_installed_live_database_unchanged router_step_phase_begin source_publish if [[ "$SELFTEST" == true ]]; then cat > "$EVIDENCE/vm121-machine-close.txt" <<'EOF' RESULT=PASS_ROUTER_MACHINE_CLOSE COMMIT=1111111111111111111111111111111111111111 TREE=2222222222222222222222222222222222222222 GIT_FILE_COUNT=42 VM121_PUBLIC_URL=selftest://vm121-source/ EOF else "$BASE/bin/router-machine-close-safe" vm121 'STEP_050M07P21B_R01: add tested 0002 live-drift bridge migration; live DB remains at 0001' > "$EVIDENCE/vm121-machine-close.txt" 2>&1 fi grep -Fqx 'RESULT=PASS_ROUTER_MACHINE_CLOSE' "$EVIDENCE/vm121-machine-close.txt" || { STOP_DETAIL=vm121_machine_close; false; } VM121_NEW_COMMIT="$(field COMMIT "$EVIDENCE/vm121-machine-close.txt")" VM121_NEW_TREE="$(field TREE "$EVIDENCE/vm121-machine-close.txt")" VM121_NEW_FILE_COUNT="$(field GIT_FILE_COUNT "$EVIDENCE/vm121-machine-close.txt")" VM121_PUBLIC_URL="$(field VM121_PUBLIC_URL "$EVIDENCE/vm121-machine-close.txt")" [[ "$VM121_NEW_COMMIT" =~ ^[0-9a-f]{40}$ && "$VM121_NEW_COMMIT" != "$BASE_COMMIT" ]] || { STOP_DETAIL=vm121_close_commit; false; } [[ "$VM121_NEW_TREE" =~ ^[0-9a-f]{40}$ ]] || { STOP_DETAIL=vm121_close_tree; false; } [[ "$VM121_NEW_FILE_COUNT" == 42 ]] || { STOP_DETAIL=vm121_close_file_count; false; } if ! collect_source_check_expected postclose "$VM121_NEW_COMMIT" "$VM121_NEW_TREE" "$VM121_NEW_FILE_COUNT"; then STOP_DETAIL=postclose_vm121_git_semantics false fi router_step_phase_complete source_publish vm121_machine_git_closed_and_published cat > "$REPORT/report.txt" <