runtime/vm100-degraded-check.sh: OK runtime/vm100-normal-check-and-resume.sh: OK runtime/vm100-pause-allocator.sh: OK runtime/vm101-arm-failure.sh: OK runtime/vm101-restore-normal.sh: OK runtime/vm101-wait-degraded.sh: OK VM101_WATCHER_SYNC_SHA256=4c884dcf90cbfa03d46445a7451379d86ae5cb915ccf6e963b0ad85fb006e142 VM101_TOPOLOGY_TRANSPORT_SHA256=afd01c4d0d00cd8281330ac073a25f995ef1923d2bb28662e2ce0d67385f1c39 FALLBACK_MAPPER_LOCK_REENTRY_FIX_INSTALLED=true TOPOLOGY_TRANSPORT_ACCEPT_NEW_HOSTKEY_INSTALLED=true RESULT=PASS_R20QH_VM101_RUNTIME_FIXES_INSTALL VM100_ALLOCATOR_PAUSED=true RESULT=PASS_R20QH_VM100_ALLOCATOR_PAUSE table inet r20qh_vpn2_failure { chain output { type filter hook output priority raw; policy accept; ip daddr 192.121.163.235 udp dport 42530 counter packets 0 bytes 0 drop comment "r20qh-vpn2-endpoint-drop" } } WATCHER_STOP_PROVEN=true WATCHER_START_PROVEN=true WATCHER_PRIME_CYCLE_PROVEN=true WATCHER_PID_AFTER=1383 WATCHER_PRIME_SHA256=e33a711ef71d35c538d2b823c2b514b4f8fa029466eb5552e1267cf815bc05c4 CONTROLLED_FAILURE_METHOD=NFT_OUTPUT_DROP_EXACT_ENDPOINT CONTROLLED_NFT_TABLE=r20qh_vpn2_failure CONTROLLED_ENDPOINT=192.121.163.235:42530 VPN2_CONFIG_MUTATED=false INJECT_EPOCH=1785240558 AUTOMATIC_WATCHER_DETECTION_ARMED=true RESULT=PASS_R20QH_VM101_FAILURE_ARMED WATCH_PROGRESS_EPOCH=1785240591 WATCHER_PRESENT=true FAIL_COUNT=0 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240621 WATCHER_PRESENT=true FAIL_COUNT=0 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240651 WATCHER_PRESENT=true FAIL_COUNT=1 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240681 WATCHER_PRESENT=true FAIL_COUNT=1 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false AUTOMATIC_WATCHER_DETECTION_PROVEN=true DETECTION_LATENCY_SEC=159 CONTROLLED_NFT_DROP_PACKETS=108 CONTROLLED_NFT_DROP_PROVEN=true VPN2_CONFIG_MUTATED=false FALLBACK_MODE_DEGRADED=SLOT_EXHAUSTED FALLBACK_GENERATION_DEGRADED=000000000001 EXHAUSTED_SLOTS_DEGRADED=egress2 EFFECTIVE_CS5_SLOT_DEGRADED=egress1 EFFECTIVE_CS5_MARK_DEGRADED=0x201 ACKED_GENERATION_DEGRADED=000000000002 PENDING_PRESENT_DEGRADED=false RESULT=PASS_R20QH_VM101_DEGRADED_STATE VM100_ACCEPTED_GENERATION_DEGRADED=000000000002 VM100_MODE_DEGRADED=SLOT_EXHAUSTED VM100_MOVED_PEER_COUNT=1 VM100_MOVED_TUNNEL_IP=10.253.1.14 VM100_MOVE=cs5_to_cs1 VM100_COUNTS_DEGRADED=2,1,1,1,0 VM100_EXHAUSTED_ROWS_REMAINING=0 RESULT=PASS_R20QH_VM100_DEGRADED_APPLY UNBLOCK_PROGRESS_EPOCH=1785240718 LIVE_ENDPOINT=192.121.163.235:42530 HANDSHAKE=1785240480 ROUTE_OK=true PING1_OK=true PING2_OK=true RX_BEFORE=36568 RX_AFTER=37300 RESULT=PASS_TOPOLOGY_GENERATION_CREATED PENDING_GENERATION=000000000003 PENDING_PAYLOAD_SHA256=6eafbf1f9230a0427b9fc0ffc1dc78282770d378cd60b95db5df922def5b10f9 RESULT=PASS_WATCHER_TOPOLOGY_SYNC ACTION=bridge-clear SLOT=egress2 FALLBACK_GENERATION=000000000002 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= FIRST_HEALTHY_SLOT=egress1 OUTBOX_RC=0 CONTROLLED_NFT_DROP_REMOVED=true VPN2_CONFIG_MUTATED=false VPN2_INTERFACE_RESTARTED=false VPN2_NATURAL_RECOVERY_PROVEN=true VPN2_RECEIVE_AFTER_UNBLOCK_PROVEN=true VPN2_RX_BEFORE_UNBLOCK=36568 VPN2_RX_AFTER_UNBLOCK=37300 VPN2_RECOVERY_PROBE_TARGET=1.1.1.1 VPN2_HANDSHAKE_AFTER_UNBLOCK=1785240480 FALLBACK_NORMAL_GENERATION=000000000002 ACKED_GENERATION_NORMAL=000000000003 PENDING_PRESENT_NORMAL=false RESULT=PASS_R20QH_VM101_NORMAL_RESTORATION NO_MASS_RETURN_BEFORE_ALLOCATOR_RESUME=true VM100_ACCEPTED_GENERATION_NORMAL=000000000003 VM100_COUNTS_AFTER_NORMAL_ACK=2,1,1,1,0 VM100_ALLOCATOR_PAUSED_FOR_R20QI=true RESULT=PASS_R20QH_VM100_NORMAL_PAUSED_HANDOFF VM101_WATCHER_SYNC_SHA256=4c884dcf90cbfa03d46445a7451379d86ae5cb915ccf6e963b0ad85fb006e142 VM101_TOPOLOGY_TRANSPORT_SHA256=afd01c4d0d00cd8281330ac073a25f995ef1923d2bb28662e2ce0d67385f1c39 FALLBACK_MAPPER_LOCK_REENTRY_FIX_INSTALLED=true TOPOLOGY_TRANSPORT_ACCEPT_NEW_HOSTKEY_INSTALLED=true RESULT=PASS_R20QH_VM101_RUNTIME_FIXES_INSTALL VM100_ALLOCATOR_PAUSED=true RESULT=PASS_R20QH_VM100_ALLOCATOR_PAUSE table inet r20qh_vpn2_failure { chain output { type filter hook output priority raw; policy accept; ip daddr 192.121.163.235 udp dport 42530 counter packets 0 bytes 0 drop comment "r20qh-vpn2-endpoint-drop" } } WATCHER_STOP_PROVEN=true WATCHER_START_PROVEN=true WATCHER_PRIME_CYCLE_PROVEN=true WATCHER_PID_AFTER=1383 WATCHER_PRIME_SHA256=e33a711ef71d35c538d2b823c2b514b4f8fa029466eb5552e1267cf815bc05c4 CONTROLLED_FAILURE_METHOD=NFT_OUTPUT_DROP_EXACT_ENDPOINT CONTROLLED_NFT_TABLE=r20qh_vpn2_failure CONTROLLED_ENDPOINT=192.121.163.235:42530 VPN2_CONFIG_MUTATED=false INJECT_EPOCH=1785240558 AUTOMATIC_WATCHER_DETECTION_ARMED=true RESULT=PASS_R20QH_VM101_FAILURE_ARMED WATCH_PROGRESS_EPOCH=1785240591 WATCHER_PRESENT=true FAIL_COUNT=0 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240621 WATCHER_PRESENT=true FAIL_COUNT=0 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240651 WATCHER_PRESENT=true FAIL_COUNT=1 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false WATCH_PROGRESS_EPOCH=1785240681 WATCHER_PRESENT=true FAIL_COUNT=1 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= OUTBOX_ACK=000000000001 PENDING_PRESENT=false AUTOMATIC_WATCHER_DETECTION_PROVEN=true DETECTION_LATENCY_SEC=159 CONTROLLED_NFT_DROP_PACKETS=108 CONTROLLED_NFT_DROP_PROVEN=true VPN2_CONFIG_MUTATED=false FALLBACK_MODE_DEGRADED=SLOT_EXHAUSTED FALLBACK_GENERATION_DEGRADED=000000000001 EXHAUSTED_SLOTS_DEGRADED=egress2 EFFECTIVE_CS5_SLOT_DEGRADED=egress1 EFFECTIVE_CS5_MARK_DEGRADED=0x201 ACKED_GENERATION_DEGRADED=000000000002 PENDING_PRESENT_DEGRADED=false RESULT=PASS_R20QH_VM101_DEGRADED_STATE VM100_ACCEPTED_GENERATION_DEGRADED=000000000002 VM100_MODE_DEGRADED=SLOT_EXHAUSTED VM100_MOVED_PEER_COUNT=1 VM100_MOVED_TUNNEL_IP=10.253.1.14 VM100_MOVE=cs5_to_cs1 VM100_COUNTS_DEGRADED=2,1,1,1,0 VM100_EXHAUSTED_ROWS_REMAINING=0 RESULT=PASS_R20QH_VM100_DEGRADED_APPLY UNBLOCK_PROGRESS_EPOCH=1785240718 LIVE_ENDPOINT=192.121.163.235:42530 HANDSHAKE=1785240480 ROUTE_OK=true PING1_OK=true PING2_OK=true RX_BEFORE=36568 RX_AFTER=37300 RESULT=PASS_TOPOLOGY_GENERATION_CREATED PENDING_GENERATION=000000000003 PENDING_PAYLOAD_SHA256=6eafbf1f9230a0427b9fc0ffc1dc78282770d378cd60b95db5df922def5b10f9 RESULT=PASS_WATCHER_TOPOLOGY_SYNC ACTION=bridge-clear SLOT=egress2 FALLBACK_GENERATION=000000000002 FALLBACK_MODE=NORMAL EXHAUSTED_SLOTS= FIRST_HEALTHY_SLOT=egress1 OUTBOX_RC=0 CONTROLLED_NFT_DROP_REMOVED=true VPN2_CONFIG_MUTATED=false VPN2_INTERFACE_RESTARTED=false VPN2_NATURAL_RECOVERY_PROVEN=true VPN2_RECEIVE_AFTER_UNBLOCK_PROVEN=true VPN2_RX_BEFORE_UNBLOCK=36568 VPN2_RX_AFTER_UNBLOCK=37300 VPN2_RECOVERY_PROBE_TARGET=1.1.1.1 VPN2_HANDSHAKE_AFTER_UNBLOCK=1785240480 FALLBACK_NORMAL_GENERATION=000000000002 ACKED_GENERATION_NORMAL=000000000003 PENDING_PRESENT_NORMAL=false RESULT=PASS_R20QH_VM101_NORMAL_RESTORATION NO_MASS_RETURN_BEFORE_ALLOCATOR_RESUME=true VM100_ACCEPTED_GENERATION_NORMAL=000000000003 VM100_COUNTS_AFTER_NORMAL_ACK=2,1,1,1,0 VM100_ALLOCATOR_PAUSED_FOR_R20QI=true RESULT=PASS_R20QH_VM100_NORMAL_PAUSED_HANDOFF LIVE_SLOT_FAILURE_INJECTED=true CONTROLLED_FAILURE_METHOD=NFT_OUTPUT_DROP_EXACT_ENDPOINT CONTROLLED_NFT_DROP_PROVEN=true AUTOMATIC_WATCHER_DETECTION_PROVEN=true BRIDGE_FIRST_RUNTIME_PROVEN=true VM100_EMERGENCY_EVACUATION_PROVEN=true DEGRADED_ACKED_GENERATION=000000000002 NORMAL_ACKED_GENERATION=000000000003 CONTROLLED_NFT_DROP_REMOVED=true VPN2_CONFIG_MUTATED=false VPN2_INTERFACE_RESTARTED=false VPN2_NATURAL_RECOVERY_PROVEN=true VPN2_RECEIVE_AFTER_UNBLOCK_PROVEN=true NO_MASS_RETURN_BEFORE_ALLOCATOR_RESUME=true VM100_SELECTOR_SOURCE_CHANGE=true VM101_SOURCE_CHANGE=true VM101_LOCK_REENTRY_FIX=true VM101_TRANSPORT_ACCEPT_NEW_HOSTKEY_FIX=true VM100_ALLOCATOR_PAUSED_FOR_R20QI=true RESULT=PASS_R20QH_LIVE_END_TO_END