#!/usr/bin/env bash
set +e
set +u
umask 077

STEP="STEP_050M07R09D_INODE_RELIEF_OLD_TEST_RUNS"

TOKEN="e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4"
ROOT="/opt/router-ops"
PUBROOT="$ROOT/public/r/$TOKEN"
PUBLIC_BASE="https://helena-background-beam-harry.trycloudflare.com/r/$TOKEN"

EXPECTED_SOURCE_SHA="bb047e820196c4c5cb263499ed890c782190293257a2033568015c6b19230682"

TS="$(date -u +%Y%m%d-%H%M%S)"
SLUG="${TS}_step050m07r09d_inode_relief_old_test_runs"
REPORT_DIR="$PUBROOT/$SLUG"

TRYCF_REPORT="$PUBLIC_BASE/$SLUG/"
REPORT_TXT="${TRYCF_REPORT}report.txt"
FACTS_JSON="${TRYCF_REPORT}facts.json"

REMOTE_OUT="$REPORT_DIR/vm101-inode-relief.env"
SSH_ERR="$REPORT_DIR/ssh.stderr"

mkdir -p "$REPORT_DIR"
cp -a "$0" "$REPORT_DIR/step.sh"
chmod 600 "$REPORT_DIR/step.sh"

ssh -T \
  -o BatchMode=yes \
  -o ConnectTimeout=10 \
  pve-mgts \
  "ssh -T \
    -o BatchMode=yes \
    -o ConnectTimeout=10 \
    -o StrictHostKeyChecking=yes \
    -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \
    root@10.71.100.2 \
    'sh -s -- $EXPECTED_SOURCE_SHA'" \
  >"$REMOTE_OUT" \
  2>"$SSH_ERR" <<'REMOTE'
set -u

EXPECTED_SOURCE_SHA="$1"

TARGET="/root/hmn/hmn-refresh-pool-safe.sh"
LIB="/usr/local/lib/router-egress-vm101-runtime.sh"
TEST_RUNS="/root/hmn/test-runs"
BACKUPS="/root/hmn/backups"

STAMP="$(date -u +%Y%m%d-%H%M%S 2>/dev/null || date +%Y%m%d-%H%M%S)"
RELIEF="/tmp/m07-inode-relief-${STAMP}-$$"

cleanup_performed=false
archive_created=false
archive_verified=false
archive_path="NONE"
archive_sha256="NONE"
removed_directory_count=0
failure_reason="NONE"

hash_file() {
  sha256sum "$1" 2>/dev/null |
    awk '{print $1}'
}

filesystem_stats() {
  prefix="$1"
  output="$RELIEF/stat-${prefix}.txt"

  if stat -f -c '%b %f %a %c %d' / \
       >"$output" 2>/dev/null
  then
    read blocks_total blocks_free blocks_available \
      inodes_total inodes_free <"$output"

    echo "${prefix}_stat_supported=true"
    echo "${prefix}_blocks_total=$blocks_total"
    echo "${prefix}_blocks_free=$blocks_free"
    echo "${prefix}_blocks_available=$blocks_available"
    echo "${prefix}_inodes_total=$inodes_total"
    echo "${prefix}_inodes_free=$inodes_free"
  else
    echo "${prefix}_stat_supported=false"
  fi
}

root_create_probe() {
  probe="$BACKUPS/.m07-create-probe-$$"
  error_file="$RELIEF/create-probe.err"

  rm -rf "$probe"
  : >"$error_file"

  if mkdir "$probe" 2>"$error_file"
  then
    rmdir "$probe"
    ROOT_PROBE_RESULT=true
    ROOT_PROBE_ERROR="NONE"
  else
    ROOT_PROBE_RESULT=false
    ROOT_PROBE_ERROR="$(
      tr '\n' ' ' <"$error_file" |
        sed 's/[[:space:]][[:space:]]*/ /g'
    )"
  fi
}

vpn_health() {
  healthy=0
  routes=0
  slot=1

  while [ "$slot" -le 5 ]
  do
    iface="vpn$slot"
    table="$((200 + slot))"

    if ip route show table "$table" 2>/dev/null |
       grep -E "^default .*dev ${iface}([[:space:]]|$)" \
       >/dev/null
    then
      route_ok=true
      routes="$((routes + 1))"
    else
      route_ok=false
    fi

    probe_ok=false

    for probe_ip in 1.1.1.1 8.8.8.8 9.9.9.9
    do
      if ping -I "$iface" -c 1 -W 2 "$probe_ip" \
           >/dev/null 2>&1
      then
        probe_ok=true
        break
      fi
    done

    if [ "$probe_ok" = "true" ]; then
      healthy="$((healthy + 1))"
    fi

    echo "${iface}_route_ok=$route_ok"
    echo "${iface}_healthy=$probe_ok"

    slot="$((slot + 1))"
  done

  HEALTHY_COUNT="$healthy"
  ROUTE_COUNT="$routes"
}

mkdir -p "$RELIEF"

echo "host=$(hostname)"
echo "date_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)"

TARGET_SHA="$(hash_file "$TARGET")"

if [ -e "$LIB" ]; then
  LIB_STATE="PRESENT"
  LIB_SHA="$(hash_file "$LIB")"
else
  LIB_STATE="ABSENT"
  LIB_SHA="ABSENT"
fi

echo "target_sha256=$TARGET_SHA"
echo "target_is_original=$(
  [ "$TARGET_SHA" = "$EXPECTED_SOURCE_SHA" ] &&
    echo true || echo false
)"
echo "runtime_library_state=$LIB_STATE"
echo "runtime_library_sha256=$LIB_SHA"

filesystem_stats before

echo "before_root_df_available_kb=$(
  df -k / 2>/dev/null |
    awk 'NR == 2 {print $4}'
)"

echo "before_test_run_entry_count=$(
  find "$TEST_RUNS" -xdev 2>/dev/null |
    wc -l |
    tr -d '[:space:]'
)"

root_create_probe

echo "create_probe_before=$ROOT_PROBE_RESULT"
echo "create_probe_before_error=$ROOT_PROBE_ERROR"

if [ "$ROOT_PROBE_RESULT" != "true" ]; then
  ALL_LIST="$RELIEF/all-test-runs.list"
  KEEP_LIST="$RELIEF/keep-test-runs.list"
  REMOVE_LIST="$RELIEF/remove-test-runs.list"

  find "$TEST_RUNS" \
    -mindepth 1 \
    -maxdepth 1 \
    -type d 2>/dev/null |
  sort >"$ALL_LIST"

  TEST_RUN_DIRECTORY_COUNT="$(
    wc -l <"$ALL_LIST" |
      tr -d '[:space:]'
  )"

  echo "test_run_directory_count=$TEST_RUN_DIRECTORY_COUNT"

  if [ "$TEST_RUN_DIRECTORY_COUNT" -le 3 ]; then
    failure_reason="NOT_ENOUGH_OLD_TEST_RUNS_FOR_RELIEF"
  else
    tail -n 3 "$ALL_LIST" >"$KEEP_LIST"

    awk '
      NR == FNR {
        keep[$0] = 1
        next
      }
      !($0 in keep) {
        print
      }
    ' "$KEEP_LIST" "$ALL_LIST" >"$REMOVE_LIST"

    REMOVE_COUNT="$(
      wc -l <"$REMOVE_LIST" |
        tr -d '[:space:]'
    )"

    echo "selected_old_test_run_count=$REMOVE_COUNT"

    if [ "$REMOVE_COUNT" -eq 0 ]; then
      failure_reason="OLD_TEST_RUN_SELECTION_EMPTY"
    else
      set --

      while IFS= read -r directory
      do
        name="${directory##*/}"
        set -- "$@" "$name"
      done <"$REMOVE_LIST"

      archive_path="$RELIEF/old-test-runs.tar.gz"

      if tar -czf "$archive_path" \
           -C "$TEST_RUNS" "$@"
      then
        archive_created=true
      else
        failure_reason="TEST_RUN_ARCHIVE_CREATE_FAILED"
      fi

      if [ "$archive_created" = "true" ] &&
         tar -tzf "$archive_path" >/dev/null 2>&1
      then
        archive_verified=true
        archive_sha256="$(hash_file "$archive_path")"
      elif [ "$archive_created" = "true" ]; then
        failure_reason="TEST_RUN_ARCHIVE_VERIFY_FAILED"
      fi

      if [ "$archive_verified" = "true" ]; then
        delete_failed=false

        while IFS= read -r directory
        do
          if ! rm -rf "$directory"
          then
            delete_failed=true
            break
          fi

          removed_directory_count="$((removed_directory_count + 1))"
        done <"$REMOVE_LIST"

        if [ "$delete_failed" = "true" ]; then
          failure_reason="OLD_TEST_RUN_REMOVAL_FAILED"
        else
          cleanup_performed=true
          sync
        fi
      fi
    fi
  fi
fi

filesystem_stats after

echo "after_root_df_available_kb=$(
  df -k / 2>/dev/null |
    awk 'NR == 2 {print $4}'
)"

echo "after_test_run_entry_count=$(
  find "$TEST_RUNS" -xdev 2>/dev/null |
    wc -l |
    tr -d '[:space:]'
)"

root_create_probe

echo "create_probe_after=$ROOT_PROBE_RESULT"
echo "create_probe_after_error=$ROOT_PROBE_ERROR"

vpn_health

echo "route_count=$ROUTE_COUNT"
echo "healthy_count=$HEALTHY_COUNT"

TARGET_SHA_AFTER="$(hash_file "$TARGET")"

if [ -e "$LIB" ]; then
  LIB_STATE_AFTER="PRESENT"
else
  LIB_STATE_AFTER="ABSENT"
fi

echo "target_sha256_after=$TARGET_SHA_AFTER"
echo "runtime_library_state_after=$LIB_STATE_AFTER"
echo "cleanup_performed=$cleanup_performed"
echo "archive_created=$archive_created"
echo "archive_verified=$archive_verified"
echo "archive_path=$archive_path"
echo "archive_sha256=$archive_sha256"
echo "removed_directory_count=$removed_directory_count"

if [ "$TARGET_SHA_AFTER" != "$EXPECTED_SOURCE_SHA" ]; then
  failure_reason="LIVE_TARGET_CHANGED"
elif [ "$LIB_STATE_AFTER" != "ABSENT" ]; then
  failure_reason="RUNTIME_LIBRARY_UNEXPECTEDLY_PRESENT"
elif [ "$ROOT_PROBE_RESULT" != "true" ]; then
  failure_reason="ROOT_CREATE_STILL_FAILED"
elif [ "$ROUTE_COUNT" -ne 5 ] ||
     [ "$HEALTHY_COUNT" -ne 5 ]; then
  failure_reason="VPN_HEALTH_NOT_5_OF_5"
fi

if [ "$failure_reason" = "NONE" ]; then
  overall_pass=true
else
  overall_pass=false
fi

echo "failure_reason=$failure_reason"
echo "overall_pass=$overall_pass"
echo "installation_performed=false"
echo "refresh_ran=false"
echo "rebalance_ran=false"
echo "services_restarted=false"
REMOTE

SSH_RC=$?

remote_value() {
  sed -n "s/^${1}=//p" "$REMOTE_OUT" |
    tail -n1
}

OVERALL_PASS="$(remote_value overall_pass)"
FAILURE_REASON="$(remote_value failure_reason)"
CLEANUP_PERFORMED="$(remote_value cleanup_performed)"
CREATE_BEFORE="$(remote_value create_probe_before)"
CREATE_AFTER="$(remote_value create_probe_after)"
INODES_BEFORE="$(remote_value before_inodes_free)"
INODES_AFTER="$(remote_value after_inodes_free)"
ARCHIVE_PATH="$(remote_value archive_path)"
ARCHIVE_SHA="$(remote_value archive_sha256)"
REMOVED_COUNT="$(remote_value removed_directory_count)"
HEALTHY_COUNT="$(remote_value healthy_count)"
ROUTE_COUNT="$(remote_value route_count)"

[ -n "$OVERALL_PASS" ] || OVERALL_PASS=false
[ -n "$FAILURE_REASON" ] || FAILURE_REASON="SSH_OR_OUTPUT_FAILURE"
[ -n "$CLEANUP_PERFORMED" ] || CLEANUP_PERFORMED=false
[ -n "$CREATE_BEFORE" ] || CREATE_BEFORE=false
[ -n "$CREATE_AFTER" ] || CREATE_AFTER=false
[ -n "$INODES_BEFORE" ] || INODES_BEFORE="UNKNOWN"
[ -n "$INODES_AFTER" ] || INODES_AFTER="UNKNOWN"
[ -n "$ARCHIVE_PATH" ] || ARCHIVE_PATH="NONE"
[ -n "$ARCHIVE_SHA" ] || ARCHIVE_SHA="NONE"
[ -n "$REMOVED_COUNT" ] || REMOVED_COUNT=0
[ -n "$HEALTHY_COUNT" ] || HEALTHY_COUNT=0
[ -n "$ROUTE_COUNT" ] || ROUTE_COUNT=0

if [ "$SSH_RC" -ne 0 ]; then
  DECISION="STOP_${STEP}_SSH_FAILED"
  OPERATION="INODE_RELIEF_NOT_COMPLETED"
elif [ "$OVERALL_PASS" = "true" ]; then
  DECISION="PASS_${STEP}"
  OPERATION="ROOT_CREATE_CAPACITY_RESTORED_OR_CONFIRMED"
else
  DECISION="STOP_${STEP}_${FAILURE_REASON}"
  OPERATION="ROOT_CREATE_CAPACITY_NOT_RESTORED"
fi

python3 - \
  "$REPORT_DIR/facts.json" \
  "$STEP" \
  "$DECISION" \
  "$OPERATION" \
  "$SSH_RC" \
  "$OVERALL_PASS" \
  "$FAILURE_REASON" \
  "$CREATE_BEFORE" \
  "$CREATE_AFTER" \
  "$INODES_BEFORE" \
  "$INODES_AFTER" \
  "$CLEANUP_PERFORMED" \
  "$ARCHIVE_PATH" \
  "$ARCHIVE_SHA" \
  "$REMOVED_COUNT" \
  "$HEALTHY_COUNT" \
  "$ROUTE_COUNT" \
  "$TRYCF_REPORT" \
  "$REPORT_TXT" \
  "$FACTS_JSON" <<'PY'
import json
import sys
from pathlib import Path

(
    output,
    step,
    decision,
    operation,
    ssh_rc,
    overall,
    reason,
    probe_before,
    probe_after,
    inodes_before,
    inodes_after,
    cleanup,
    archive,
    archive_sha,
    removed,
    healthy,
    routes,
    report,
    report_txt,
    facts,
) = sys.argv[1:]

def maybe_int(value):
    try:
        return int(value)
    except ValueError:
        return value

data = {
    "schema": "router-step-facts-v1",
    "step": step,
    "assessment": {
        "decision": decision,
        "operation_result": operation,
        "all_ok": overall == "true",
        "failure_reason": (
            None if overall == "true" else reason
        ),
        "production_health": (
            "HEALTHY_5_OF_5"
            if healthy == "5" and routes == "5"
            else "NOT_CONFIRMED_5_OF_5"
        ),
    },
    "root_filesystem": {
        "create_probe_before":
            probe_before == "true",
        "create_probe_after":
            probe_after == "true",
        "free_inodes_before":
            maybe_int(inodes_before),
        "free_inodes_after":
            maybe_int(inodes_after),
    },
    "relief": {
        "performed": cleanup == "true",
        "method":
            "archive old test-runs to tmp then remove from root",
        "archive_path": archive,
        "archive_sha256": archive_sha,
        "removed_directory_count":
            int(removed),
    },
    "health": {
        "healthy_count": int(healthy),
        "route_count": int(routes),
    },
    "safety": {
        "installation_performed": False,
        "target_code_modified": False,
        "runtime_library_installed": False,
        "refresh_ran": False,
        "rebalance_ran": False,
        "services_restarted": False,
    },
    "next_step": (
        "RETRY_REFRESH_POOL_SAFE_INSTALL"
        if overall == "true"
        else "MANUAL_ROOT_FILESYSTEM_REPAIR"
    ),
    "transport": {
        "ssh_rc": int(ssh_rc),
    },
    "publish": {
        "trycf_report": report,
        "report_txt": report_txt,
        "facts_json": facts,
    },
}

Path(output).write_text(
    json.dumps(
        data,
        ensure_ascii=False,
        indent=2,
    ) + "\n",
    encoding="utf-8",
)
PY

cat > "$REPORT_DIR/report.txt" <<EOF
=== ${STEP} RESULT ===
step=${STEP}
decision=${DECISION}
operation_result=${OPERATION}
ssh_rc=${SSH_RC}
overall_pass=${OVERALL_PASS}
failure_reason=${FAILURE_REASON}

root_filesystem:
  create_probe_before=${CREATE_BEFORE}
  create_probe_after=${CREATE_AFTER}
  free_inodes_before=${INODES_BEFORE}
  free_inodes_after=${INODES_AFTER}

relief:
  cleanup_performed=${CLEANUP_PERFORMED}
  archive_path=${ARCHIVE_PATH}
  archive_sha256=${ARCHIVE_SHA}
  removed_directory_count=${REMOVED_COUNT}

health:
  healthy_count=${HEALTHY_COUNT}
  route_count=${ROUTE_COUNT}

safety:
  installation_performed=false
  target_code_modified=false
  runtime_library_installed=false
  refresh_ran=false
  rebalance_ran=false
  services_restarted=false

remote_details:
$(sed 's/^/  /' "$REMOTE_OUT" 2>/dev/null)

next_step=$(
  if [ "$OVERALL_PASS" = "true" ]; then
    echo "RETRY_REFRESH_POOL_SAFE_INSTALL"
  else
    echo "MANUAL_ROOT_FILESYSTEM_REPAIR"
  fi
)

TRYCF_REPORT=${TRYCF_REPORT}
REPORT_TXT=${REPORT_TXT}
FACTS_JSON=${FACTS_JSON}
EOF

cat > "$REPORT_DIR/index.html" <<EOF
<!doctype html>
<html lang="ru">
<head><meta charset="utf-8"><title>${STEP}</title></head>
<body style="font-family:system-ui;max-width:1050px;margin:40px auto">
<h1>${STEP}</h1>
<ul>
<li><a href="report.txt">report.txt</a></li>
<li><a href="facts.json">facts.json</a></li>
<li><a href="vm101-inode-relief.env">vm101-inode-relief.env</a></li>
<li><a href="ssh.stderr">ssh.stderr</a></li>
<li><a href="step.sh">step.sh</a></li>
</ul>
</body>
</html>
EOF

find "$REPORT_DIR" \
  -type f \
  ! -name SHA256SUMS \
  -print0 |
sort -z |
xargs -0 sha256sum \
  >"$REPORT_DIR/SHA256SUMS"

chmod -R a+rX "$REPORT_DIR"

echo "decision=$DECISION"
echo "operation_result=$OPERATION"
echo "create_probe_before=$CREATE_BEFORE"
echo "create_probe_after=$CREATE_AFTER"
echo "free_inodes_before=$INODES_BEFORE"
echo "free_inodes_after=$INODES_AFTER"
echo "cleanup_performed=$CLEANUP_PERFORMED"
echo "archive_path=$ARCHIVE_PATH"
echo "removed_directory_count=$REMOVED_COUNT"
echo "healthy_count=$HEALTHY_COUNT"
echo "route_count=$ROUTE_COUNT"
echo "installation_performed=false"

echo
echo "TRYCF_REPORT=$TRYCF_REPORT"
echo "REPORT_TXT=$REPORT_TXT"
echo "FACTS_JSON=$FACTS_JSON"

true