Graceful restart (RFC 4724, RFC 8538)

A peer that restarts keeps its routes with us, and we keep ours with our
peers while we restart, so that a restart withdraws nothing across the
network and the machine goes on forwarding. On by default for every
peer; `graceful_restart` sets the restart time we give, caps how long a
peer's routes are kept, or turns it off.

When a peer restarts: its routes are marked stale in the routing table,
and chosen as before, for at most its restart time. When it is back, what
it sends again is fresh, and the stale routes of a family go at its
End-of-RIB marker, or after the cap; at once for a family whose
forwarding it did not keep. Only a connection that failed counts as a
restart, and, with RFC 8538 agreed, an expired hold timer or a
NOTIFICATION other than a Hard Reset. The stale mark is a bit of the
route's word that selection never looks at.

When we restart: SIGUSR1 leaves the kernel's routes where they are,
closes the sessions without a word where graceful restart is agreed, and
exits with status 75, on which the systemd unit starts us again. The next
start finds the routes in the kernel, says so in its OPENs (R, and F where
forwarding was kept), and has them go once every peer has sent its
End-of-RIB. Our own End-of-RIB markers for learned routes wait for that
too. SIGTERM still stops for good, now with a Hard Reset where RFC 8538 is
agreed, as are errors of ours.

The status page and API show what was agreed with each peer, the routes
kept while a peer restarts, and which routes are stale. Tested against
FRR and BIRD in a new phase of the interoperability lab, and with routes
kept in a real kernel over a restart.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/interop/lab.sh

@@ -29,7 +29,7 @@ cd "$(dirname "$0")/../.."
 bgp="${CARGO_TARGET_DIR:-target}/release/bgp"
 work=/tmp/ferrix-lab
 pathspace=ferrixlab
-all_phases="basics roles rpki refresh flaps restart scale limits hijacks nexthops transit v6only linklocal"
+all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit v6only linklocal"
 phases="${PHASES:-$all_phases}"
 scale_routes="${SCALE_ROUTES:-200000}"
 
@@ -497,7 +497,7 @@ phase_roles() {
         || fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')"
     frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \
         || fail "FRR does not see us as its peer"
-    bird_ctl show protocols all dut4 | grep -A12 "Neighbor capabilities" | grep -q "Role: customer" \
+    bird_ctl show protocols all dut4 | grep -A40 "Neighbor capabilities" | grep -q "Role: customer" \
         || fail "BIRD does not see us as its customer"
     ok "roles agree with FRR (peer) and BIRD (provider)"
 
@@ -636,6 +636,65 @@ phase_restart() {
     ok "killed: routes stay in the kernel, and the next start clears the stale one"
 }
 
+# Graceful restart (RFC 4724): the daemon restarts for an upgrade without
+# its routes going anywhere, and keeps BIRD's while BIRD restarts.
+frr_stale() {  # frr_stale prefix: FRR keeps our route, as stale
+    vty -c "show bgp ipv4 unicast $1 json" | jq -e '.paths | any(.aspath.string == "64601" and .stale == true)'
+}
+frr_fresh() { frr_has ipv4 "$1" && ! frr_stale "$1"; }
+dut_stale() {  # dut_stale prefix peer-name: the daemon keeps the peer's route, as stale
+    api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer and .stale)'
+}
+kernel_removals() {
+    api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }'
+}
+phase_graceful() {
+    sed -i 's/graceful restart off;/graceful restart on;/' "$work/bird.conf"
+    bird_ctl configure >/dev/null
+    wait_for "BIRD to come back with graceful restart" \
+        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '[.[] | select(.name == \"bird4\" or .name == \"bird6\") | select(.graceful_restart != null)] | length == 2'"
+    converged
+
+    # The daemon restarts: everything stays where it is meanwhile.
+    dut_down USR1
+    grep -q "Leaving to be started again" "$work/dut.log" || fail "the daemon did not leave for a restart"
+    dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20" || fail "FRR's route left the kernel"
+    dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
+    dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20" || fail "BIRD's IPv6 route left the kernel"
+    wait_for "FRR to keep our route as stale" frr_stale 10.1.0.0/24
+    bird_has 10.1.0.0/24 || fail "BIRD dropped our route"
+    bird_has 2001:db8:1::/48 || fail "BIRD dropped our IPv6 route"
+    in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "packets stopped while the daemon was away"
+    dut_up
+    converged
+    wait_for "FRR to take our route afresh" frr_fresh 10.1.0.0/24
+    wait_for "nothing kept in the kernel any more" \
+        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.kept == 0'"
+    local removed
+    removed=$(kernel_removals)
+    [ "${removed:-0}" -eq 0 ] || fail "$removed routes were taken out of the kernel over the restart"
+    ok "restarted on SIGUSR1: kernel routes kept, FRR and BIRD kept ours, nothing withdrawn"
+
+    # BIRD restarts: the daemon keeps its routes meanwhile, in the table
+    # and in the kernel, and takes them afresh when it is back.
+    bird_ctl graceful restart >/dev/null || true
+    while [ -e "/proc/$(sudo cat "$work/bird.pid" 2>/dev/null || echo 0)" ]; do sleep 0.05; done
+    wait_for "the daemon to keep BIRD's route as stale" dut_stale 10.3.0.0/24 bird4
+    dut_stale 2001:db8:3::/48 bird6 >/dev/null || fail "BIRD's IPv6 route was not kept"
+    dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
+    peer_is bird4 '.stale_routes > 0' >/dev/null || fail "the daemon does not say it keeps BIRD's routes: $(peer_is bird4 .)"
+    sudo rm -f "$work/bird.ctl"
+    bird_up
+    converged
+    wait_for "BIRD's routes to be fresh" \
+        bash -c "! sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?peer=bird4' | jq -e '.routes | any(.stale)' >/dev/null"
+    ok "BIRD restarted gracefully: its routes were kept, and taken afresh"
+
+    sed -i 's/graceful restart on;/graceful restart off;/' "$work/bird.conf"
+    bird_ctl configure >/dev/null
+    converged
+}
+
 # A full table's worth of routes from BIRD, into the table and the kernel.
 phase_scale() {
     # Out of 100.0.0.0/6 and up: as many /24s as asked for.