Add an interoperability lab with FRR, BIRD and Routinator

scripts/interop/lab.sh runs the daemon against FRR, BIRD and Routinator,
each in a network namespace on one exchange LAN, and checks what every
side ends up with: sessions and routes over IPv4 and IPv6 with their
attributes, four-octet AS numbers, ADD-PATH, RFC 9234 roles and Only to
Customer, RTR version 2 and invalid routes, route refresh, session resets,
killed peers, a cut wire, the daemon stopped and killed, a table of a
million routes and a restart under it, role mismatch and the prefix limit.
All of it passes against FRR 10.5.1, BIRD 2.18 and Routinator 0.15.2.

scripts/interop/real-rpki.sh checks the RTR client against a Routinator
with the real RPKI: 713,258 IPv4 and 187,479 IPv6 VRPs and 3,379 ASPAs,
the same numbers on both sides.

What the lab found, fixed here:

- A daemon restarted with its routes in the kernel removed them all at
  once, its table being empty until the peers had sent theirs again, and
  then put them back: no forwarding meanwhile. Routes of an earlier run
  are now kept for `stale_seconds` (120 by default) and only those nobody
  announced again are removed. A restart under a million routes takes
  1.1 s to be in step, with nothing removed.
- FRR ends a session it loses to a collision with a state machine error
  without a subcode, which was shown as "error code 5 subcode 0".

CI gets a job that runs the lab.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

crates/fib/src/kernel.rs

@@ -354,6 +354,10 @@ struct Task {
     synced: bool,
     /// When the routes were last worked out again for a refusal.
     mended_at: Option<Instant>,
+    /// Whether routes of an earlier run are still being kept; counting
+    /// them means looking at every route, and is not done once there are
+    /// none to find.
+    keeping: bool,
 }
 
 impl Task {
@@ -448,7 +452,11 @@ impl Task {
     }
 
     /// Finds the routes of ours that the kernel has from before.
-    async fn adopt_routes(&mut self, nexthops: &HashMap<u32, Target>) -> Result<(), FibError> {
+    async fn adopt_routes(
+        &mut self,
+        nexthops: &HashMap<u32, Target>,
+        keep: bool,
+    ) -> Result<(), FibError> {
         for route in self.kernel_routes().await? {
             if !self.is_ours(&route) {
                 continue;
@@ -464,7 +472,7 @@ impl Task {
                 }
                 _ => None,
             };
-            self.plan.adopt(route.prefix, target);
+            self.plan.adopt(route.prefix, target, keep);
         }
         Ok(())
     }
@@ -745,7 +753,7 @@ impl Task {
         self.synced = false;
         self.plan = Plan::default();
         let nexthops = self.kernel_nexthops().await?;
-        self.adopt_routes(&nexthops).await?;
+        self.adopt_routes(&nexthops, false).await?;
         self.use_nexthops(&nexthops);
         Ok(())
     }
@@ -793,7 +801,7 @@ impl Task {
             // What is left, if anything, is routes from an earlier run
             // that carry their own gateway.
             self.plan = Plan::default();
-            self.adopt_routes(&HashMap::new()).await?;
+            self.adopt_routes(&HashMap::new(), false).await?;
         }
         let ops = self.plan.clear();
         self.send(ops, false).await?;
@@ -834,6 +842,7 @@ impl Task {
         self.fib.update(|status| {
             status.synced = self.synced;
             status.installed = self.plan.installed();
+            status.kept = if self.keeping { self.plan.kept() } else { 0 };
             status.refused = self.plan.refused();
             status.unresolved = self
                 .plan
@@ -879,18 +888,26 @@ pub(crate) async fn run(
         marks: 0,
         synced: false,
         mended_at: None,
+        keeping: false,
     };
     // The socket for news was bound before this: what changes while the
     // interfaces are being learned is told there.
     task.learn_interfaces().await?;
     let nexthops = task.kernel_nexthops().await?;
-    task.adopt_routes(&nexthops).await?;
+    // Kept for a while, if there is to be any keeping: our table is empty
+    // until the peers have sent theirs again.
+    let keep = !config.stale_time.is_zero();
+    task.adopt_routes(&nexthops, keep).await?;
     task.use_nexthops(&nexthops);
+    let mut release_at = None;
     if task.plan.installed() > 0 {
         info!(
             routes = task.plan.installed(),
+            kept_for = ?config.stale_time,
             "Found routes of ours in the kernel from an earlier run"
         );
+        release_at = keep.then(|| Instant::now() + config.stale_time);
+        task.keeping = keep;
     }
     task.publish(true);
     info!(
@@ -963,6 +980,19 @@ pub(crate) async fn run(
                 task.relearn(&mut heard).await?;
                 reader = task.rib.reader();
             }
+            () = async { sleep_until(release_at.expect("checked by the guard")).await },
+                if release_at.is_some() =>
+            {
+                release_at = None;
+                task.keeping = false;
+                let stale = task.plan.release();
+                info!(
+                    removed = stale.len(),
+                    "Routes of an earlier run that were not announced again are removed"
+                );
+                task.send(stale, false).await?;
+                task.publish(false);
+            }
             ended = &mut task.writer => {
                 return Err(match ended {
                     Ok(Err(e)) => e,

crates/fib/src/lib.rs

@@ -39,6 +39,7 @@ pub mod plan;
 
 use std::net::IpAddr;
 use std::sync::{Arc, Mutex, PoisonError};
+use std::time::Duration;
 
 use bgp_rib::table::RibTable;
 use tokio::sync::watch;
@@ -49,6 +50,10 @@ use tokio::task::JoinHandle;
 /// against static and connected routes as an operator expects.
 pub const DEFAULT_METRIC: u32 = 20;
 
+/// How long routes left by an earlier run are kept unless configured
+/// otherwise: time for sessions to come up and tables to be sent.
+pub const DEFAULT_STALE_TIME: Duration = Duration::from_secs(120);
+
 /// Where our routes go in the kernel.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct FibConfig {
@@ -60,6 +65,12 @@ pub struct FibConfig {
     /// 5.3 and later), which makes installing many routes a good deal
     /// faster. Without, every route carries its own gateway.
     pub nexthop_objects: bool,
+    /// How long routes of ours that an earlier run left in the kernel are
+    /// kept after a start without the routing table having them. The
+    /// table is empty until the peers have sent theirs again; removing
+    /// everything at once would stop the machine forwarding meanwhile.
+    /// After this long, what has not been announced again goes.
+    pub stale_time: Duration,
 }
 
 impl Default for FibConfig {
@@ -68,6 +79,7 @@ impl Default for FibConfig {
             table: netlink::TABLE_MAIN,
             metric: DEFAULT_METRIC,
             nexthop_objects: true,
+            stale_time: DEFAULT_STALE_TIME,
         }
     }
 }
@@ -108,6 +120,9 @@ pub struct Status {
     pub synced: bool,
     /// How many routes of ours the kernel has.
     pub installed: usize,
+    /// How many of them are kept from an earlier run though the routing
+    /// table does not have them (yet); see [`FibConfig::stale_time`].
+    pub kept: usize,
     /// Prefixes whose best route is not in the kernel because its next
     /// hop is not usable, by why.
     pub unresolved: Vec<(&'static str, usize)>,

crates/fib/src/plan.rs

@@ -142,6 +142,9 @@ struct Entry {
     target: Option<Target>,
     /// Whether the prefix was heard of since the last reset.
     confirmed: bool,
+    /// Whether the route is one an earlier run left that is being kept
+    /// for now though not confirmed: see [`Plan::adopt`].
+    kept: bool,
 }
 
 /// The routes of ours in the kernel, kept in step with the best routes.
@@ -209,18 +212,49 @@ impl Plan {
         self.installed.keys()
     }
 
-    /// Notes a route of ours that the kernel already has, from before this
-    /// run. It stays only if the routing table turns out to have the same.
-    pub fn adopt(&mut self, prefix: IpPrefix, target: Option<Target>) {
+    /// Notes a route of ours that the kernel already has. It stays only if
+    /// the routing table turns out to have the same.
+    ///
+    /// With `keep` it is not removed for being absent from a whole table,
+    /// only by [`Plan::release`]. That is for the routes an earlier run
+    /// left: when the daemon starts, its table is empty until its peers
+    /// have sent theirs again, and to remove everything meanwhile would
+    /// stop the machine forwarding for as long as that takes.
+    pub fn adopt(&mut self, prefix: IpPrefix, target: Option<Target>, keep: bool) {
         self.installed.insert(
             prefix,
             Entry {
                 target,
                 confirmed: false,
+                kept: keep,
             },
         );
     }
 
+    /// How many routes are being kept from an earlier run without the
+    /// routing table having confirmed them.
+    pub fn kept(&self) -> usize {
+        self.installed
+            .values()
+            .filter(|entry| entry.kept && !entry.confirmed)
+            .count()
+    }
+
+    /// The time for keeping unconfirmed routes of an earlier run is over:
+    /// those the routing table still does not have go.
+    pub fn release(&mut self) -> Vec<Op> {
+        let mut stale = Vec::new();
+        self.installed.retain(|prefix, entry| {
+            let goes = entry.kept && !entry.confirmed;
+            entry.kept = false;
+            if goes {
+                stale.push(Op::Remove { prefix: *prefix });
+            }
+            !goes
+        });
+        stale
+    }
+
     /// The whole table follows: nothing is known to be wanted until it is
     /// heard of again.
     pub fn reset(&mut self) {
@@ -246,6 +280,7 @@ impl Plan {
                 let entry = self.installed.entry(prefix).or_insert(Entry {
                     target: None,
                     confirmed: true,
+                    kept: false,
                 });
                 entry.confirmed = true;
                 if entry.target == Some(target) {
@@ -279,7 +314,7 @@ impl Plan {
         let stale: Vec<IpPrefix> = self
             .installed
             .iter()
-            .filter(|(_, entry)| !entry.confirmed)
+            .filter(|(_, entry)| !entry.confirmed && !entry.kept)
             .map(|(prefix, _)| *prefix)
             .collect();
         stale
@@ -493,10 +528,10 @@ mod tests {
         );
         let a = target("192.0.2.2", 2);
         // Left in the kernel by an earlier run.
-        plan.adopt(kept, Some(a));
-        plan.adopt(changed, Some(a));
-        plan.adopt(stale, Some(a));
-        plan.adopt(odd, None);
+        plan.adopt(kept, Some(a), false);
+        plan.adopt(changed, Some(a), false);
+        plan.adopt(stale, Some(a), false);
+        plan.adopt(odd, None, false);
         assert_eq!(plan.installed(), 4);
 
         plan.reset();
@@ -520,6 +555,58 @@ mod tests {
         assert_eq!(plan.synced(), []);
     }
 
+    /// When the daemon starts its table is empty, and stays so until its
+    /// peers have sent theirs. What an earlier run left in the kernel is
+    /// kept meanwhile.
+    #[test]
+    fn routes_of_an_earlier_run_are_kept_until_released() {
+        let mut plan = Plan::default();
+        let (still_right, moved, gone) = (
+            prefix("203.0.113.0/24"),
+            prefix("198.18.0.0/15"),
+            prefix("192.88.99.0/24"),
+        );
+        let (a, b) = (target("192.0.2.2", 2), target("192.0.2.3", 2));
+        for p in [still_right, moved, gone] {
+            plan.adopt(p, Some(a), true);
+        }
+        // The first whole table is the empty one: nothing goes.
+        plan.reset();
+        assert_eq!(plan.synced(), []);
+        assert_eq!((plan.installed(), plan.kept()), (3, 3));
+
+        // The peers come back.
+        assert_eq!(plan.best(still_right, Ok(a)), None);
+        assert_eq!(
+            plan.best(moved, Ok(b)),
+            Some(Op::Install {
+                prefix: moved,
+                target: b
+            })
+        );
+        assert_eq!(plan.kept(), 1);
+        // Time is up: what nobody announced again goes, and only that.
+        assert_eq!(plan.release(), [Op::Remove { prefix: gone }]);
+        assert_eq!((plan.installed(), plan.kept()), (2, 0));
+        assert_eq!(plan.release(), []);
+
+        // From then on a whole table decides as always.
+        plan.reset();
+        plan.best(moved, Ok(b));
+        assert_eq!(
+            plan.synced(),
+            [Op::Remove {
+                prefix: still_right
+            }]
+        );
+
+        // A route that is withdrawn while kept goes at once.
+        let mut plan = Plan::default();
+        plan.adopt(gone, Some(a), true);
+        plan.best(gone, Ok(a));
+        assert_eq!(plan.withdrawn(gone), Some(Op::Remove { prefix: gone }));
+    }
+
     #[test]
     fn a_route_the_kernel_refuses_is_not_counted_as_there() {
         let mut plan = Plan::default();
@@ -578,7 +665,7 @@ mod tests {
             let mut plan = Plan::default();
             for (prefix, gateway) in left_behind {
                 kernel.insert(p(prefix), t(gateway));
-                plan.adopt(p(prefix), Some(t(gateway)));
+                plan.adopt(p(prefix), Some(t(gateway)), false);
             }
             // The best routes: the prefix's next hop, if it resolves.
             let mut table: HashMap<u8, Option<u8>> = HashMap::default();

crates/fib/tests/kernel.rs

@@ -53,6 +53,9 @@ struct Machine {
     /// Whether routes name nexthop objects, which is how it is unless a
     /// test says otherwise.
     nexthop_objects: bool,
+    /// How long routes of an earlier run are kept: not at all, unless a
+    /// test says otherwise.
+    stale_time: Duration,
 }
 
 impl Machine {
@@ -61,6 +64,7 @@ impl Machine {
             n,
             rib: Arc::new(RibTable::new(4)),
             nexthop_objects: true,
+            stale_time: Duration::ZERO,
         };
         ip(&format!("link add {} type dummy", machine.interface()));
         machine.add_addresses();
@@ -104,6 +108,7 @@ impl Machine {
                 table: self.table(),
                 metric: 20,
                 nexthop_objects: self.nexthop_objects,
+                stale_time: self.stale_time,
             },
         )
         .expect("the kernel can be talked to")
@@ -546,3 +551,46 @@ async fn routes_can_carry_their_own_gateway() {
     fib.stop().await;
     task.await.unwrap().unwrap();
 }
+
+/// When the daemon starts, its table is empty until its peers have sent
+/// theirs. The routes an earlier run left are kept for a while, so that
+/// the machine goes on forwarding; then those nobody announced again go.
+#[tokio::test(flavor = "multi_thread")]
+#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
+async fn routes_of_an_earlier_run_are_kept_while_peers_come_back() {
+    let mut machine = Machine::new(7);
+    machine.stale_time = Duration::from_secs(3);
+    let (table, interface) = (machine.table(), machine.interface());
+    let add = |route: &str| ip(&format!("route add {route} dev {interface} table {table}"));
+    add("203.0.113.0/24 via 10.7.0.2 proto bgp metric 20");
+    add("198.18.0.0/15 via 10.7.0.2 proto bgp metric 20");
+    let left = [
+        machine.shown("203.0.113.0/24", machine.neighbor(2)),
+        machine.shown("198.18.0.0/15", machine.neighbor(2)),
+    ];
+
+    // The daemon starts with nothing in its table: both stay.
+    let (fib, task) = machine.start();
+    status_becomes(&fib, "in step, keeping two", |status| {
+        status.synced && status.installed == 2 && status.kept == 2
+    });
+    assert_eq!(machine.kernel_routes(), left.iter().cloned().collect());
+
+    // A peer comes back with one of them.
+    let peer = machine.peer(2);
+    let via = machine.via(machine.neighbor(2).into(), 0);
+    machine
+        .rib
+        .announce(peer, &via, &[prefix("203.0.113.0/24")]);
+    status_becomes(&fib, "one confirmed", |status| status.kept == 1);
+    assert_eq!(machine.kernel_routes(), left.iter().cloned().collect());
+
+    // Time is up: the one nobody announced goes.
+    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
+    status_becomes(&fib, "none kept", |status| {
+        status.installed == 1 && status.kept == 0
+    });
+
+    fib.stop().await;
+    task.await.unwrap().unwrap();
+}

crates/packet/src/errors.rs

@@ -170,6 +170,10 @@ pub enum UpdateMsgSubcode {
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Error, Serialize, FromRepr)]
 #[repr(u8)]
 pub enum FsmSubcode {
+    /// RFC 6608: what speakers from before it send, and FRR still does.
+    #[error("unspecified")]
+    Unspecified = 0,
+
     #[error("Received an unexpected message in OpenSent state")]
     UnexpectedOpenSent = 1,
 

crates/packet/src/message.rs

@@ -1954,7 +1954,7 @@ mod tests {
         use crate::errors::BgpError;
 
         let ctx = default_v6_context();
-        for (code, subcode) in [(6, 0), (5, 0), (2, 5), (3, 7), (4, 9), (99, 1)] {
+        for (code, subcode) in [(6, 0), (5, 9), (2, 5), (3, 7), (4, 9), (99, 1)] {
             let decoded = Message::decode(&ctx, &framed(21, 3, &[code, subcode])).unwrap();
             let Message::Notification(notification) = &decoded else {
                 panic!("expected a NOTIFICATION, got {decoded:?}");
@@ -1962,6 +1962,14 @@ mod tests {
             assert_eq!(notification.error, BgpError::Other { code, subcode });
             assert_eq!(notification.error.code_subcode(), (code, subcode));
         }
+        // A state machine error without a subcode has a name (RFC 6608);
+        // FRR sends it.
+        let decoded = Message::decode(&ctx, &framed(21, 3, &[5, 0])).unwrap();
+        assert!(matches!(
+            &decoded,
+            Message::Notification(n)
+                if n.error == BgpError::FsmError(crate::errors::FsmSubcode::Unspecified)
+        ));
         // Too short to hold a code and subcode.
         assert!(Message::decode(&ctx, &framed(20, 3, &[6])).is_err());
     }

crates/server/proto/config.proto

@@ -79,6 +79,14 @@ message Kernel {
   // route` then shows routes with "nhid". True has every route carry its
   // own gateway, as on older kernels.
   bool without_nexthop_objects = 3;
+
+  // How long, after the daemon starts, routes of its own that an earlier
+  // run left in the kernel are kept though no peer has announced them
+  // (again). Defaults to 120. The daemon's table is empty until its peers
+  // have sent theirs; removing the old routes at once would stop the
+  // machine forwarding for as long as that takes. After this time, what
+  // was not announced again is removed.
+  uint32 stale_seconds = 4;
 }
 
 // Where to get the validated ROA payloads that say which AS may originate

crates/server/src/config.rs

@@ -184,6 +184,10 @@ impl Config {
                     table: or(kernel.table, defaults.table),
                     metric: or(kernel.metric, defaults.metric),
                     nexthop_objects: !kernel.without_nexthop_objects,
+                    stale_time: match kernel.stale_seconds {
+                        0 => defaults.stale_time,
+                        seconds => Duration::from_secs(seconds.into()),
+                    },
                 }
             }),
             rpki,
@@ -746,11 +750,13 @@ mod tests {
                 table: 254,
                 metric: 20,
                 nexthop_objects: true,
+                stale_time: Duration::from_secs(120),
             })
         );
         assert_eq!(
             parse(&format!(
-                "{BASE}kernel {{ table: 1000 metric: 150 without_nexthop_objects: true }}"
+                "{BASE}kernel {{ table: 1000 metric: 150 without_nexthop_objects: true \
+                 stale_seconds: 30 }}"
             ))
             .unwrap()
             .kernel,
@@ -758,6 +764,7 @@ mod tests {
                 table: 1000,
                 metric: 150,
                 nexthop_objects: false,
+                stale_time: Duration::from_secs(30),
             })
         );
     }

crates/web/src/api.rs

@@ -867,6 +867,9 @@ pub struct KernelView {
     pub synced: bool,
     /// How many routes of ours the kernel has.
     pub installed: usize,
+    /// How many of them are kept from before the daemon started though no
+    /// peer has announced them again; they go when `stale_seconds` is up.
+    pub kept: usize,
     /// Prefixes whose best route is not in the kernel because its next hop
     /// cannot be used, by why.
     pub unresolved: BTreeMap<&'static str, usize>,
@@ -899,6 +902,7 @@ pub async fn kernel(State(state): State<AppState>) -> Json<KernelView> {
         metric: Some(config.metric),
         synced: status.synced,
         installed: status.installed,
+        kept: status.kept,
         unresolved: status.unresolved.into_iter().collect(),
         refused: status.refused,
         errors: status.errors,

crates/web/static/statusz.html

@@ -318,6 +318,7 @@ from <select id="lookup-peer"><option value="">any peer</option></select>
       (i.addresses.length ? ` <span class="mono dim">${i.addresses.map(esc).join(' ')}</span>` : '')).join(' &middot; ');
     setHTML(b, `<span class="${k.synced ? 'ok' : 'warn'}">${k.synced ? 'In step' : 'Catching up'}</span> with the routing table: ` +
       `<b>${k.installed}</b> routes installed <span class="dim">(table ${k.table}, metric ${k.metric}, proto bgp)</span>` +
+      (k.kept ? `; <span class="warn">${k.kept} kept from before the restart</span>, not announced again yet` : '') +
       (unresolved ? `; <span class="warn">not installed:</span> ${unresolved}` : '') +
       (k.refused ? `; <span class="bad">${k.refused} refused by the kernel</span>` : '') + '.' +
       (k.last_error ? `<br><span class="bad">Last error</span> (${k.errors} in all): ${esc(k.last_error)}` : '') +