{ "assessment": { "all_ok": true, "core_ok": true, "decision": "PASS_STEP_042K_CHECKPOINT_AGENT_AUTOMATION_AND_REBOOT_RECOVERY_OK", "snapshots_ok": true, "success_checks": { "manual_agent_service_start_not_used": true, "private_material_not_published": true, "snapshot_vm100_ok": true, "snapshot_vm101_ok": true, "snapshot_vm121_ok": true, "vm100_allowed_ip_count_five": true, "vm100_ip15_absent": true, "vm100_peer_count_five": true, "vm100_selector_count_five": true, "vm100_selector_enabled": true, "vm100_selector_running": true, "vm100_uci_empty": true, "vm100_wg_paid_listen_51830": true, "vm101_egress_slots_enabled": true, "vm101_egress_slots_running": true, "vm101_legacy_table200_vpn1_ok": true, "vm101_mapper_enabled": true, "vm101_mapper_running": true, "vm101_table201_vpn1_ok": true, "vm101_table202_vpn2_ok": true, "vm101_table203_vpn3_ok": true, "vm101_table204_vpn4_ok": true, "vm101_table205_vpn5_ok": true, "vm121_backend_container_found": true, "vm121_backend_health_ok": true, "vm121_backend_helper_compile_ok": true, "vm121_backend_trigger_bind_mount_ok": true, "vm121_enabled_peer_count_five": true, "vm121_manual_agent_service_start_not_used": true, "vm121_old_agent_path_inactive": true, "vm121_pending_global_zero": true, "vm121_pg_dump_created": true, "vm121_postgres_container_found": true, "vm121_private_material_not_published": true, "vm121_reconcile_timer_active": true, "vm121_reconcile_timer_enabled": true, "vm121_watcher_active": true, "vm121_watcher_enabled": true } }, "changes": { "manual_agent_service_start_used": "NO", "private_material_published": "NO", "snapshots_attempted": "YES", "vm121_checkpoint_files_created": "YES", "vm121_db_dump_created_not_published": "YES" }, "mode": "FINAL_CHECKPOINT_AFTER_AGENT_AUTOMATION_AND_VM100_REBOOT_RECOVERY", "publish": { "local_report": "http://127.0.0.1:8099/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260709-111005_step042k_checkpoint_agent_automation_reboot_ok/", "trycf_report": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260709-111005_step042k_checkpoint_agent_automation_reboot_ok/" }, "schema": "router-step-facts-v1", "snapshot_name": "S042K_OK_20260709_1110", "snapshot_pve_ai_raw": "--- snapshot vmid=121 snap=S042K_OK_20260709_1110 ---\nsnapshotting 'drive-scsi0' (local-lvm:vm-121-disk-0)\n Logical volume \"snap_vm-121-disk-0_S042K_OK_20260709_1110\" created.\nsnapshot_rc=0 vmid=121 snap=S042K_OK_20260709_1110\n", "snapshot_pve_ai_stderr_tail": "", "snapshot_pve_mgts_raw": "--- snapshot vmid=100 snap=S042K_OK_20260709_1110 ---\nsnapshotting 'drive-sata1' (local-lvm:vm-100-disk-0)\n Logical volume \"snap_vm-100-disk-0_S042K_OK_20260709_1110\" created.\nsnapshot_rc=0 vmid=100 snap=S042K_OK_20260709_1110\n--- snapshot vmid=101 snap=S042K_OK_20260709_1110 ---\nsnapshotting 'drive-sata1' (local-lvm:vm-101-disk-0)\n Logical volume \"snap_vm-101-disk-0_S042K_OK_20260709_1110\" created.\nsnapshot_rc=0 vmid=101 snap=S042K_OK_20260709_1110\n", "snapshot_pve_mgts_stderr_tail": "", "step": "STEP_042K_CHECKPOINT_AGENT_AUTOMATION_AND_REBOOT_RECOVERY_OK", "vm100_raw": "vm100_hostname=\nvm100_uptime= 11:10:07 up 15 min, load average: 0.00, 0.00, 0.00\nvm100_wg_paid_listen_port=51830\nvm100_peer_count=5\nvm100_allowed_ip_count=5\nvm100_ip15_present_rc=1\nvm100_selector_count=5\nvm100_uci_count=0\nvm100_selector_enabled_rc=0\nvm100_selector_status_rc=0\n", "vm101_raw": "table_201_vpn1_rc=0\ntable_202_vpn2_rc=0\ntable_203_vpn3_rc=0\ntable_204_vpn4_rc=0\ntable_205_vpn5_rc=0\ntable_200_vpn1_rc=0\nvm101_egress_slots_enabled_rc=0\nvm101_egress_slots_status_rc=0\nvm101_mapper_enabled_rc=0\nvm101_mapper_status_rc=0\n", "vm121_checkpoint": { "assessment": { "all_ok": true, "success_checks": { "backend_container_found": true, "backend_health_ok": true, "backend_helper_compile_ok": true, "backend_trigger_bind_mount_ok": true, "enabled_peer_count_five": true, "manual_agent_service_start_not_used": true, "old_agent_path_inactive": true, "pending_global_zero": true, "pg_dump_created": true, "postgres_container_found": true, "private_material_not_published": true, "reconcile_timer_active": true, "reconcile_timer_enabled": true, "watcher_active": true, "watcher_enabled": true } }, "backend_container": { "helper_compile": { "rc": 0, "stderr": "", "stdout": "" }, "mounts": { "rc": 0, "stderr": "", "stdout": "[{\"Type\":\"bind\",\"Source\":\"/opt/wg-access/runtime/agent-trigger\",\"Destination\":\"/opt/wg-access/runtime/agent-trigger\",\"Mode\":\"rw\",\"RW\":true,\"Propagation\":\"rprivate\"},{\"Type\":\"bind\",\"Source\":\"/opt/wg-access/backend\",\"Destination\":\"/app\",\"Mode\":\"rw\",\"RW\":true,\"Propagation\":\"rprivate\"}]" }, "name": "wgaccess-backend", "trigger_dir_stat": { "rc": 0, "stderr": "", "stdout": "File: /opt/wg-access/runtime/agent-trigger\n Size: 4096 \tBlocks: 8 IO Block: 4096 directory\nDevice: 8,1\tInode: 131080 Links: 3\nAccess: (0775/drwxrwxr-x) Uid: ( 0/ root) Gid: ( 0/ root)\nAccess: 2026-07-09 09:53:15.640445660 +0000\nModify: 2026-07-09 10:24:32.438771049 +0000\nChange: 2026-07-09 10:45:37.269059309 +0000\n Birth: 2026-07-09 09:53:15.640445660 +0000\n File: /opt/wg-access/runtime/agent-trigger/events\n Size: 4096 \tBlocks: 8 IO Block: 4096 directory\nDevice: 8,1\tInode: 131079 Links: 2\nAccess: (0775/drwxrwxr-x) Uid: ( 0/ root) Gid: ( 0/ root)\nAccess: 2026-07-09 11:09:00.329388122 +0000\nModify: 2026-07-09 11:08:59.613404663 +0000\nChange: 2026-07-09 11:08:59.613404663 +0000\n Birth: 2026-07-09 10:24:32.438771049 +0000" } }, "changes": { "db_dump_created": "YES", "manual_agent_service_start_used": "NO", "private_material_published": "NO", "vm100_changed": "NO", "vm101_changed": "NO", "vm121_changed": "YES_CHECKPOINT_FILES_AND_DB_DUMP_ONLY" }, "checkpoint": { "manifest_path_vm121": "/opt/wg-access/status/STEP_042K_CHECKPOINT_AGENT_AUTOMATION_AND_REBOOT_RECOVERY_OK_20260709-111005.json", "manifest_sha256": "77792a543870daedae8f4f5ab03d2d682571ab78deb984e8deb207c9f3755407" }, "db": { "enabled_peer_count": 5, "pending_global": 0, "pg_container": "wgaccess-postgres", "pg_dump_command_rc": 0, "pg_dump_path_vm121": "/opt/wg-access/backups/STEP_042K_CHECKPOINT_AGENT_AUTOMATION_AND_REBOOT_RECOVERY_OK_20260709-111005/wgaccess-db-20260709-111005.sql.gz", "pg_dump_sha256": "83c51eaa5fc8acb6d2a87f103e36331945f0fd4f93ea9b3ed2c15759008ca345", "pg_dump_size_bytes": 12399, "pg_dump_stderr_tail": "", "recent_jobs_summary": { "rc": 0, "stderr": "", "stdout": "disable_peer|completed|23\nenable_peer|completed|32" } }, "errors": [], "mode": "VM121_CONTROL_PLANE_CHECKPOINT_WITH_DB_DUMP_NOT_PUBLISHED", "schema": "router-step-facts-v1", "services": { "backend_health": { "rc": 0, "stderr": "", "stdout": "{\"status\":\"ok\",\"service\":\"wg-access-backend\",\"environment\":\"dev\"}" }, "old_agent_path_active": { "rc": 3, "stderr": "", "stdout": "inactive" }, "old_agent_path_enabled": { "rc": 1, "stderr": "", "stdout": "disabled" }, "reconcile_timer_active": { "rc": 0, "stderr": "", "stdout": "active" }, "reconcile_timer_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" }, "reconcile_timer_list": { "rc": 0, "stderr": "", "stdout": "Thu 2026-07-09 11:14:55 UTC 4min 50s left Thu 2026-07-09 11:09:44 UTC 21s ago wg-access-reconcile.timer wg-access-reconcile.service" }, "watcher_active": { "rc": 0, "stderr": "", "stdout": "active" }, "watcher_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" } }, "snapshot_name": "S042K_OK_20260709_1110", "step": "STEP_042K_CHECKPOINT_AGENT_AUTOMATION_AND_REBOOT_RECOVERY_OK", "timestamp": "20260709-111005", "watcher": { "recent_journal_tail": { "rc": 0, "stderr": "", "stdout": "Jul 09 10:40:23 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:40:23Z] wg-access-trigger-watch: new_events=1 first=host-test-1783593622592192232.trigger:131091:1783593622588056744:20\nJul 09 10:40:23 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:40:23Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:40:23 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:40:23Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 10:45:53 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:45:53Z] wg-access-trigger-watch: new_events=1 first=event-1783593953692216166-cf5eb8c928e648cc8eccaa4106128b03.trigger:145790:1783593953688708840:20\nJul 09 10:45:53 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:45:53Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:45:53 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:45:53Z] wg-access-trigger-watch: wg-access-agent.service start failed rc=1 stdout='' stderr='Job for wg-access-agent.service failed because the control process exited with error code.\\nSee \"systemctl status wg-access-agent.service\" and \"journalctl -xeu wg-access-agent.service\" for details.\\n'\nJul 09 10:47:56 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:47:56Z] wg-access-trigger-watch: new_events=1 first=event-1783594076311494704-9c8e0f44d5b849fab78bd8e9f47ed4ca.trigger:145842:1783594076310062723:20\nJul 09 10:47:56 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:47:56Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:47:57 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:47:57Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 10:49:55 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:49:55Z] wg-access-trigger-watch: new_events=1 first=event-1783594194822815906-e9b98a1066fe47f6aef89a625241cf8c.trigger:145791:1783594194819464384:20\nJul 09 10:49:55 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:49:55Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:49:56 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:49:56Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 10:49:59 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:49:59Z] wg-access-trigger-watch: new_events=1 first=event-1783594199475340237-f0d88a82bafc4b7ea6d7db352214a58a.trigger:131075:1783594199471361692:20\nJul 09 10:49:59 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:49:59Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:50:00 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:50:00Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 10:53:07 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:53:07Z] wg-access-trigger-watch: new_events=1 first=event-1783594386269100623-ff56ad0e0c8a4ca7a928445cd44e3af3.trigger:131089:1783594386263202595:20\nJul 09 10:53:07 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:53:07Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:53:07 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:53:07Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 10:58:29 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:58:29Z] wg-access-trigger-watch: new_events=1 first=event-1783594709497599222-55b0727e0f1c474886ade8629e74ea6f.trigger:145846:1783594709491886037:20\nJul 09 10:58:29 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:58:29Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 10:58:30 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T10:58:30Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 11:03:45 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:03:45Z] wg-access-trigger-watch: new_events=1 first=event-1783595024561000232-1c24b484437a46d2944a901321810067.trigger:145847:1783595024556668076:20\nJul 09 11:03:45 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:03:45Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 11:03:46 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:03:46Z] wg-access-trigger-watch: wg-access-agent.service start requested OK\nJul 09 11:09:00 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:09:00Z] wg-access-trigger-watch: new_events=1 first=event-1783595339616720512-874e3aae400d41c3aed9beb57f427847.trigger:145848:1783595339613404663:20\nJul 09 11:09:00 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:09:00Z] wg-access-trigger-watch: starting wg-access-agent.service reason=new_events=1\nJul 09 11:09:01 wg-access-dev wg-access-trigger-watch.py[147081]: [2026-07-09T11:09:01Z] wg-access-trigger-watch: wg-access-agent.service start requested OK" }, "script_stat": { "rc": 0, "stderr": "", "stdout": "File: /usr/local/sbin/wg-access-trigger-watch.py\n Size: 3722 \tBlocks: 8 IO Block: 4096 regular file\nDevice: 8,1\tInode: 28173 Links: 1\nAccess: (0755/-rwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root)\nAccess: 2026-07-09 10:37:59.167454945 +0000\nModify: 2026-07-09 10:37:58.411472872 +0000\nChange: 2026-07-09 10:37:58.411472872 +0000\n Birth: 2026-07-09 10:37:58.411472872 +0000" }, "unit_cat": { "rc": 0, "stderr": "", "stdout": "# /etc/systemd/system/wg-access-trigger-watch.service\n[Unit]\nDescription=WG Access host-side trigger watcher\nAfter=network-online.target docker.service\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/local/sbin/wg-access-trigger-watch.py\nRestart=always\nRestartSec=2\nUser=root\n\n[Install]\nWantedBy=multi-user.target" } } } }