#!/bin/sh
set -u
umask 077

CONF="/etc/router-egress-emergency-refresh.conf"
RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh"
HOOK="/usr/local/sbin/router-egress-emergency-decision-hook.sh"
PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh"
HELPER="/usr/local/lib/router-egress-recovery-state.sh"

HOOK_INIT="/etc/init.d/router-egress-emergency-decision"
WATCHER_INIT="/etc/init.d/router-egress-health-repair"
SLOTS_INIT="/etc/init.d/router-egress-slots"
MAPPER_INIT="/etc/init.d/router-egress-mapper"

CACHE_DIR="/root/hmn/cache"
POOL="${CACHE_DIR}/ok-awg1-strict-foreign-latest.tsv"
STATE_DIR="/var/lib/router-egress-recovery"

RUN_ID="$(date -u +%Y%m%d-%H%M%S)-$$"
BACKUP_ROOT="/root/step050m07b2-backup-${RUN_ID}"
ROLLBACK="/root/rollback-step050m07b2-${RUN_ID}.sh"

ROLLBACK_READY=false
ROLLBACK_DONE=false
MUTATION_STARTED=false

HOOK_WAS_RUNNING=false
WATCHER_WAS_RUNNING=false

trace() {
  printf '__TRACE__ stage=%s\n' "$1"
}

fact() {
  printf '__FACT__ %s=%s\n' "$1" "$2"
}

block() {
  echo "__BLOCK_BEGIN__ $1"
  printf '%s\n' "$2"
  echo "__BLOCK_END__ $1"
}

json_block() {
  echo "__JSON_BEGIN__ $1"
  printf '%s\n' "$2"
  echo "__JSON_END__ $1"
}

bool_cmd() {
  if "$@" >/dev/null 2>&1; then
    printf true
  else
    printf false
  fi
}

config_raw() {
  (
    . "$CONF"
    printf '%s' "${EMERGENCY_COMMIT_ENABLED:-UNSET}"
  )
}

config_value() {
  name="$1"
  fallback="$2"

  (
    . "$CONF"
    eval "value=\${$name:-\$fallback}"
    printf '%s' "$value"
  )
}

repair_counter() {
  (
    unset REG_STATE_DIR
    . "$HELPER"
    reg_daily_repair_get
  )
}

state_value() {
  key="$1"
  fallback="$2"

  (
    unset REG_STATE_DIR
    . "$HELPER"
    reg_get_state "$key" "$fallback"
  )
}

strict_ping() {
  interface="$1"
  attempt=1

  while [ "$attempt" -le 3 ]; do
    if ping \
      -I "$interface" \
      -c 1 \
      -W 4 \
      1.1.1.1 \
      >/dev/null 2>&1
    then
      return 0
    fi

    attempt=$((attempt + 1))
    sleep 1
  done

  return 1
}

collect_slots() {
  prefix="$1"
  strict_all=true
  routes_all=true

  for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do
    if strict_ping "$interface"; then
      strict=true
    else
      strict=false
      strict_all=false
    fi

    fact "${prefix}.strict.${interface}" "$strict"
  done

  fact "${prefix}.strict_all" "$strict_all"

  for table in 201 202 203 204 205; do
    route_text="$(
      ip route show table "$table" 2>&1 ||
      true
    )"

    block "${prefix}_route_${table}" "$route_text"

    if printf '%s\n' "$route_text" |
      grep -q '^default '
    then
      route=true
    else
      route=false
      routes_all=false
    fi

    fact "${prefix}.route.${table}" "$route"
  done

  fact "${prefix}.routes_all" "$routes_all"
}

restore_services() {
  if [ "$HOOK_WAS_RUNNING" = true ]; then
    "$HOOK_INIT" start >/dev/null 2>&1 || true
  fi

  if [ "$WATCHER_WAS_RUNNING" = true ]; then
    "$WATCHER_INIT" start >/dev/null 2>&1 || true
  fi
}

auto_rollback_on_exit() {
  rc="$?"

  trap - EXIT

  if [ "$rc" -ne 0 ] &&
     [ "$ROLLBACK_READY" = true ] &&
     [ "$ROLLBACK_DONE" != true ] &&
     [ -x "$ROLLBACK" ]
  then
    if sh "$ROLLBACK"; then
      ROLLBACK_DONE=true
      fact auto_rollback true
    else
      fact auto_rollback false
    fi
  elif [ "$rc" -ne 0 ]; then
    restore_services
    fact auto_rollback false
  fi

  exit "$rc"
}

trap auto_rollback_on_exit EXIT

trace required_contract

for path in \
  "$CONF" \
  "$RUNNER" \
  "$HOOK" \
  "$PLANNER" \
  "$HELPER" \
  "$HOOK_INIT" \
  "$WATCHER_INIT" \
  "$CACHE_DIR" \
  "$POOL" \
  "$STATE_DIR"
do
  [ -e "$path" ] || {
    echo "__ERROR__ missing=$path"
    exit 21
  }
done

RAW_PRE="$(config_raw)"
fact commit_raw_pre "$RAW_PRE"

[ "$RAW_PRE" = "1" ] || {
  echo "__ERROR__ expected_commit_raw_1_actual=$RAW_PRE"
  exit 22
}

CONFIRM_TOKEN="$(
  config_value EMERGENCY_CONFIRM_TOKEN ''
)"

EMERGENCY_LOG="$(
  config_value \
    EMERGENCY_LOG \
    /var/log/router-egress-emergency-refresh.log
)"

[ -n "$CONFIRM_TOKEN" ] || {
  echo "__ERROR__ confirm_token_empty"
  exit 23
}

trace preflight

COUNTER_PRE="$(repair_counter)"
RUNNER_PRE="$("$RUNNER" --dry-run)"
HOOK_PRE="$("$HOOK")"
PLANNER_PRE="$("$PLANNER")"

POOL_HASH_PRE="$(
  sha256sum "$POOL" |
    sed 's/[[:space:]].*$//'
)"

POOL_MTIME_PRE="$(
  date -r "$POOL" +%s
)"

NETWORK_HASH_PRE="$(
  uci export network |
    sha256sum |
    sed 's/[[:space:]].*$//'
)"

STATE_MODE_PRE="$(state_value mode NORMAL)"
STATE_STATUS_PRE="$(
  state_value last_emergency_refresh_status NONE
)"
STATE_EPOCH_PRE="$(
  state_value last_emergency_refresh_epoch 0
)"

fact repair_counter_pre "$COUNTER_PRE"
fact pool_hash_pre "$POOL_HASH_PRE"
fact pool_mtime_pre "$POOL_MTIME_PRE"
fact network_hash_pre "$NETWORK_HASH_PRE"
fact state_mode_pre "$STATE_MODE_PRE"
fact state_status_pre "$STATE_STATUS_PRE"
fact state_epoch_pre "$STATE_EPOCH_PRE"

json_block runner_pre "$RUNNER_PRE"
json_block hook_pre "$HOOK_PRE"
json_block planner_pre "$PLANNER_PRE"

collect_slots pre

trace backup

mkdir -p "$BACKUP_ROOT"

cp -a "$CONF" "$BACKUP_ROOT/emergency-refresh.conf.before"
cp -a "$CACHE_DIR" "$BACKUP_ROOT/cache.before"
cp -a "$STATE_DIR" "$BACKUP_ROOT/state.before"
cp -a /etc/config/network "$BACKUP_ROOT/network.before"

find "$STATE_DIR" \
  -maxdepth 4 \
  -type f \
  -name 'rollback-egress*.sh' \
  2>/dev/null |
  sort \
  > "$BACKUP_ROOT/slot-rollbacks.before"

HOOK_WAS_RUNNING="$(bool_cmd "$HOOK_INIT" running)"
WATCHER_WAS_RUNNING="$(bool_cmd "$WATCHER_INIT" running)"

fact hook_was_running "$HOOK_WAS_RUNNING"
fact watcher_was_running "$WATCHER_WAS_RUNNING"

cat > "$ROLLBACK" <<EOF
#!/bin/sh
set -u
umask 077

CONF='$CONF'
CACHE_DIR='$CACHE_DIR'
STATE_DIR='$STATE_DIR'
BACKUP_ROOT='$BACKUP_ROOT'
HOOK_INIT='$HOOK_INIT'
WATCHER_INIT='$WATCHER_INIT'
SLOTS_INIT='$SLOTS_INIT'
MAPPER_INIT='$MAPPER_INIT'
HOOK_WAS_RUNNING='$HOOK_WAS_RUNNING'
WATCHER_WAS_RUNNING='$WATCHER_WAS_RUNNING'

rc=0

"\$HOOK_INIT" stop >/dev/null 2>&1 || true
"\$WATCHER_INIT" stop >/dev/null 2>&1 || true

find "\$STATE_DIR" \
  -maxdepth 4 \
  -type f \
  -name 'rollback-egress*.sh' \
  2>/dev/null |
  sort -r \
  > "\$BACKUP_ROOT/slot-rollbacks.after"

while IFS= read -r rollback_path; do
  [ -n "\$rollback_path" ] || continue

  if ! grep -Fxq \
    "\$rollback_path" \
    "\$BACKUP_ROOT/slot-rollbacks.before"
  then
    sh "\$rollback_path" || rc=1
  fi
done < "\$BACKUP_ROOT/slot-rollbacks.after"

cp -a \
  "\$BACKUP_ROOT/emergency-refresh.conf.before" \
  "\$CONF" ||
rc=1

rm -rf "\$CACHE_DIR"
cp -a "\$BACKUP_ROOT/cache.before" "\$CACHE_DIR" ||
rc=1

rm -rf "\$STATE_DIR"
cp -a "\$BACKUP_ROOT/state.before" "\$STATE_DIR" ||
rc=1

cp -a \
  "\$BACKUP_ROOT/network.before" \
  /etc/config/network ||
rc=1

rm -rf \
  /var/lock/router-egress-emergency-refresh.lock \
  /tmp/hmn-refresh-pool-safe.lock \
  2>/dev/null ||
true

/etc/init.d/network reload >/dev/null 2>&1 || rc=1
sleep 8

[ ! -x "\$SLOTS_INIT" ] ||
  "\$SLOTS_INIT" restart >/dev/null 2>&1 ||
  rc=1

[ ! -x "\$MAPPER_INIT" ] ||
  "\$MAPPER_INIT" restart >/dev/null 2>&1 ||
  rc=1

if [ "\$HOOK_WAS_RUNNING" = true ]; then
  "\$HOOK_INIT" start >/dev/null 2>&1 || rc=1
fi

if [ "\$WATCHER_WAS_RUNNING" = true ]; then
  "\$WATCHER_INIT" start >/dev/null 2>&1 || rc=1
fi

echo "STEP_050M07B2 rollback rc=\$rc"
exit "\$rc"
EOF

chmod 700 "$ROLLBACK"
ROLLBACK_READY=true

fact rollback "$ROLLBACK"
fact backup_root "$BACKUP_ROOT"
fact rollback_exists "$(bool_cmd test -x "$ROLLBACK")"

trace isolate_services

"$WATCHER_INIT" stop >/dev/null 2>&1 || true
"$HOOK_INIT" stop >/dev/null 2>&1 || true
sleep 2

fact watcher_stopped "$(
  if "$WATCHER_INIT" running >/dev/null 2>&1; then
    echo false
  else
    echo true
  fi
)"

fact hook_stopped "$(
  if "$HOOK_INIT" running >/dev/null 2>&1; then
    echo false
  else
    echo true
  fi
)"

trace normalize_boolean

TEMP="${CONF}.step050m07b2.$$.tmp"

awk '
  BEGIN {
    count=0
  }

  /^[[:space:]]*EMERGENCY_COMMIT_ENABLED[[:space:]]*=/ {
    print "EMERGENCY_COMMIT_ENABLED=true"
    count++
    next
  }

  {
    print
  }

  END {
    if (count != 1) {
      exit 42
    }
  }
' "$CONF" > "$TEMP"

patch_rc="$?"

[ "$patch_rc" -eq 0 ] || {
  rm -f "$TEMP"
  echo "__ERROR__ config_patch_rc=$patch_rc"
  exit 24
}

chmod 600 "$TEMP"
chown 0:0 "$TEMP"
mv "$TEMP" "$CONF"

MUTATION_STARTED=true
fact mutation_started true

RAW_POST="$(config_raw)"
fact commit_raw_post "$RAW_POST"

[ "$RAW_POST" = "true" ] || {
  echo "__ERROR__ normalized_value_not_true"
  exit 25
}

trace execute_real_runner

LOG_SIZE_PRE="$(
  if [ -f "$EMERGENCY_LOG" ]; then
    wc -c < "$EMERGENCY_LOG" |
      tr -d ' '
  else
    echo 0
  fi
)"

fact emergency_log "$EMERGENCY_LOG"
fact emergency_log_size_pre "$LOG_SIZE_PRE"

RUNNER_ERR="/tmp/step050m07b2-runner-${RUN_ID}.stderr"

set +e

RUNNER_COMMIT="$(
  "$RUNNER" \
    --commit \
    --confirm "$CONFIRM_TOKEN" \
    2> "$RUNNER_ERR"
)"

RUNNER_RC="$?"

set -e

RUNNER_STDERR="$(
  cat "$RUNNER_ERR" 2>/dev/null ||
  true
)"

rm -f "$RUNNER_ERR"

fact runner_commit_rc "$RUNNER_RC"
json_block runner_commit "$RUNNER_COMMIT"
block runner_commit_stderr "$RUNNER_STDERR"

if printf '%s\n' "$RUNNER_COMMIT" |
  grep -q '"decision"[[:space:]]*:[[:space:]]*"refresh_ok_rebalance_ok"'
then
  fact refresh_ran true
  fact rebalance_apply_ran true
else
  fact refresh_ran false
  fact rebalance_apply_ran false

  echo "__ERROR__ runner_decision_not_success"
  exit 31
fi

[ "$RUNNER_RC" -eq 0 ] || {
  echo "__ERROR__ runner_commit_rc=$RUNNER_RC"
  exit 32
}

trace restore_services

restore_services
sleep 2

fact hook_restored "$(bool_cmd "$HOOK_INIT" running)"
fact watcher_restored "$(bool_cmd "$WATCHER_INIT" running)"
fact hook_enabled "$(bool_cmd "$HOOK_INIT" enabled)"
fact watcher_enabled "$(bool_cmd "$WATCHER_INIT" enabled)"

trace post_state

COUNTER_POST="$(repair_counter)"
RUNNER_POST="$("$RUNNER" --dry-run)"
HOOK_POST="$("$HOOK")"
PLANNER_POST="$("$PLANNER")"

POOL_HASH_POST="$(
  sha256sum "$POOL" |
    sed 's/[[:space:]].*$//'
)"

POOL_MTIME_POST="$(
  date -r "$POOL" +%s
)"

NETWORK_HASH_POST="$(
  uci export network |
    sha256sum |
    sed 's/[[:space:]].*$//'
)"

STATE_MODE_POST="$(state_value mode UNKNOWN)"
STATE_STATUS_POST="$(
  state_value last_emergency_refresh_status UNKNOWN
)"
STATE_EPOCH_POST="$(
  state_value last_emergency_refresh_epoch 0
)"

fact repair_counter_post "$COUNTER_POST"
fact pool_hash_post "$POOL_HASH_POST"
fact pool_mtime_post "$POOL_MTIME_POST"
fact network_hash_post "$NETWORK_HASH_POST"
fact state_mode_post "$STATE_MODE_POST"
fact state_status_post "$STATE_STATUS_POST"
fact state_epoch_post "$STATE_EPOCH_POST"

json_block runner_post "$RUNNER_POST"
json_block hook_post "$HOOK_POST"
json_block planner_post "$PLANNER_POST"

collect_slots post

LOG_SIZE_POST="$(
  if [ -f "$EMERGENCY_LOG" ]; then
    wc -c < "$EMERGENCY_LOG" |
      tr -d ' '
  else
    echo 0
  fi
)"

LOG_TAIL="$(
  tail -n 260 "$EMERGENCY_LOG" 2>/dev/null ||
  true
)"

fact emergency_log_size_post "$LOG_SIZE_POST"
block emergency_log_tail "$LOG_TAIL"

fact emergency_lock_present "$(
  bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock
)"

fact hmn_refresh_lock_present "$(
  bool_cmd test -e /tmp/hmn-refresh-pool-safe.lock
)"

fact direct_failopen_changed false

printf '%s\n' "$PLANNER_POST" |
  grep -q '"changes_count"[[:space:]]*:[[:space:]]*0' || {
    echo "__ERROR__ planner_not_converged"
    exit 33
  }

printf '%s\n' "$RUNNER_POST" |
  grep -q '"decision"[[:space:]]*:[[:space:]]*"cooldown_active"' || {
    echo "__ERROR__ cooldown_not_active"
    exit 34
  }

[ "$STATE_MODE_POST" = "NORMAL" ] || {
  echo "__ERROR__ state_mode_not_normal=$STATE_MODE_POST"
  exit 35
}

[ "$STATE_STATUS_POST" = "refresh_ok_rebalance_ok" ] || {
  echo "__ERROR__ state_status_not_success=$STATE_STATUS_POST"
  exit 36
}

[ "$(
  bool_cmd "$HOOK_INIT" running
)" = true ] || {
  echo "__ERROR__ hook_not_running_after"
  exit 37
}

[ "$(
  bool_cmd "$WATCHER_INIT" running
)" = true ] || {
  echo "__ERROR__ watcher_not_running_after"
  exit 38
}

[ "$(
  bool_cmd test -e /var/lock/router-egress-emergency-refresh.lock
)" = false ] || {
  echo "__ERROR__ emergency_lock_remains"
  exit 39
}

trace complete

trap - EXIT
exit 0