Install best routes in the Linux kernel over netlink

New crate bgp-fib. It follows the routing table's best routes and keeps
the kernel's routing table the same, and learns the machine's interfaces
and addresses from the kernel, which is what says whether a next hop can
be reached.

- netlink: the rtnetlink messages used (routes, addresses, links, errors),
  as bytes and back. No I/O; property tested and fuzzed.
- interfaces: what is directly reachable, and from which interface.
- plan: what to ask the kernel for, given the best routes and what the
  kernel has. No I/O; tested against a model kernel.
- kernel (Linux only): the sockets and the task. Requests go in batches;
  routes of ours found at startup are kept if still right and removed if
  not; a change of interfaces has every route worked out again; a clean
  stop removes our routes.

Only routes marked proto bgp with the configured table and metric are
ever touched. A best route is installed if its next hop is a neighbor;
next hops further away are not resolved through other routes yet, and
such prefixes are counted and shown instead.

Configured with `kernel { table metric }`; off by default. The daemon
needs CAP_NET_ADMIN and stops if the kernel refuses it. On shutdown it
closes sessions and then removes its routes.

/api/kernel and the status page show installed, unresolved and refused
counts, the last error and the interfaces; five bgp_kernel_* metrics.

Testing: scripts/kernel-tests.sh runs, in network namespaces of their
own and with only the capabilities the daemon needs, the installer
against `ip route`, the daemon with a peer, and two daemons that route
for each other with pings across over IPv4 and IPv6.
scripts/linux-vm.sh runs that, or anything else, in a Lima VM from
macOS. CI runs the kernel tests and fuzzes the netlink decoder.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

crates/integration-tests/Cargo.toml

@@ -9,6 +9,7 @@ version.workspace = true
 publish = false
 
 [dependencies]
+bgp-fib.workspace = true
 bgp-packet.workspace = true
 bgp-primitives.workspace = true
 bgp-rib.workspace = true

crates/integration-tests/src/test_peer.rs

@@ -79,7 +79,7 @@ impl TestPeer {
             match self.recv().await? {
                 Message::Update(u) => return Ok(u),
                 Message::KeepAlive => {}
-                other => bail!("Expected UPDATE, got {:?}", other),
+                other => return Err(eyre::eyre!("Expected UPDATE, got {:?}", other)),
             }
         }
     }
@@ -96,7 +96,7 @@ impl TestPeer {
                 self.ctx.four_octet_asn = four_byte && self.offered_four_octet_asn;
                 Ok(o)
             }
-            other => bail!("Expected OPEN, got {:?}", other),
+            other => Err(eyre::eyre!("Expected OPEN, got {:?}", other)),
         }
     }
 
@@ -189,7 +189,12 @@ impl TestPeer {
         // Expect server KEEPALIVE
         match self.recv().await? {
             Message::KeepAlive => {}
-            other => bail!("Expected KEEPALIVE from server, got {:?}", other),
+            other => {
+                return Err(eyre::eyre!(
+                    "Expected KEEPALIVE from server, got {:?}",
+                    other
+                ));
+            }
         }
         // Send our KEEPALIVE
         self.send(&Message::KeepAlive).await?;

crates/integration-tests/src/test_server.rs

@@ -1,6 +1,7 @@
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::sync::Arc;
 
+use bgp_fib::Fib;
 use bgp_packet::message::{NlriNextHop, PathAttribute};
 use bgp_primitives::ip_prefix::IpPrefix;
 use bgp_rib::LocalRib;
@@ -32,6 +33,8 @@ pub struct TestServer {
     pub refusals: RefusalLog,
     /// The data for route origin validation, if the server validates.
     pub rpki: Option<Rpki>,
+    /// What keeps the best routes in the kernel, if the server does.
+    pub kernel: Option<Fib>,
     pub local_asn: u32,
 }
 
@@ -63,6 +66,7 @@ impl TestServer {
             originate: vec![],
             http_endpoint: None,
             rpki: None,
+            kernel: None,
         };
         customize(&mut config);
 
@@ -77,6 +81,7 @@ impl TestServer {
         let registry = bound.registry.clone();
         let refusals = bound.refusals.clone();
         let rpki = bound.rpki.clone();
+        let kernel = bound.kernel.clone();
 
         let handle = tokio::spawn(bound.run());
         Ok(Self {
@@ -87,6 +92,7 @@ impl TestServer {
             registry,
             refusals,
             rpki,
+            kernel,
             local_asn,
         })
     }
@@ -206,6 +212,7 @@ impl TestServer {
             rib: Arc::clone(&self.rib),
             refusals: self.refusals.clone(),
             rpki: self.rpki.clone(),
+            kernel: self.kernel.clone(),
         };
         let handle = tokio::spawn(bgp_web::serve_on(state, listener));
         Ok((base_url, HttpGuard { handle }))

crates/integration-tests/tests/kernel.rs

@@ -0,0 +1,169 @@
+//! The whole daemon against a real kernel: a peer announces routes over a
+//! session, and `ip route` shows them.
+//!
+//! This changes the machine's interfaces and routes, so it is not run by
+//! `cargo test`; `scripts/kernel-tests.sh` runs it in a network namespace
+//! of its own.
+#![cfg(target_os = "linux")]
+
+use std::collections::BTreeSet;
+use std::net::Ipv4Addr;
+use std::process::Command;
+use std::time::{Duration, Instant};
+
+use bgp_fib::FibConfig;
+use bgp_integration_tests::test_peer::TestPeer;
+use bgp_integration_tests::test_server::TestServer;
+use bgp_packet::message::{
+    AsPathAttribute, Message, NextHopPathAttribute, OriginPathAttribute, PathAttribute,
+    UpdateMessage,
+};
+use bgp_primitives::ip_prefix::IpPrefix;
+use eyre::Result;
+use serde_json::Value;
+
+const TIMEOUT: Duration = Duration::from_secs(10);
+const TABLE: u32 = 1100;
+const INTERFACE: &str = "ferrix100";
+
+#[track_caller]
+fn ip(arguments: &str) {
+    let output = Command::new("ip")
+        .args(arguments.split_whitespace())
+        .output()
+        .expect("the ip command is there");
+    assert!(
+        output.status.success(),
+        "ip {arguments}: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+}
+
+fn kernel_routes() -> BTreeSet<String> {
+    let output = Command::new("ip")
+        .args(["route", "show", "table", &TABLE.to_string()])
+        .output()
+        .unwrap();
+    String::from_utf8_lossy(&output.stdout)
+        .lines()
+        .map(|line| line.split_whitespace().collect::<Vec<_>>().join(" "))
+        .filter(|line| !line.is_empty())
+        .collect()
+}
+
+#[track_caller]
+fn kernel_becomes(expected: &[&str]) {
+    let expected: BTreeSet<String> = expected.iter().map(|route| route.to_string()).collect();
+    let start = Instant::now();
+    while kernel_routes() != expected {
+        assert!(
+            start.elapsed() < TIMEOUT,
+            "the kernel has {:#?}; expected {expected:#?}",
+            kernel_routes()
+        );
+        std::thread::sleep(Duration::from_millis(20));
+    }
+}
+
+/// Removes the test's interface, and its routes with it, when dropped.
+struct Interface;
+
+impl Drop for Interface {
+    fn drop(&mut self) {
+        let _ = Command::new("ip").args(["link", "del", INTERFACE]).output();
+        let _ = Command::new("ip")
+            .args(["route", "flush", "table", &TABLE.to_string()])
+            .output();
+    }
+}
+
+async fn announce(peer: &mut TestPeer, prefixes: &[&str], next_hop: Ipv4Addr) -> Result<()> {
+    peer.send(&Message::Update(UpdateMessage {
+        withdrawn_routes: vec![],
+        path_attributes: vec![
+            PathAttribute::Origin(OriginPathAttribute::IGP),
+            AsPathAttribute::from_asns(vec![65001]),
+            PathAttribute::NextHop(NextHopPathAttribute(next_hop)),
+        ],
+        announced_routes: prefixes
+            .iter()
+            .map(|p| IpPrefix::try_from(*p).unwrap().into())
+            .collect(),
+    }))
+    .await
+}
+
+/// A peer's routes go from the session to the kernel, and out of it again
+/// when they are withdrawn and when the session ends.
+#[tokio::test(flavor = "multi_thread")]
+#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
+async fn test_routes_from_a_session_reach_the_kernel() -> Result<()> {
+    ip(&format!("link add {INTERFACE} type dummy"));
+    let _interface = Interface;
+    ip(&format!("addr add 10.100.0.1/24 dev {INTERFACE}"));
+    ip(&format!("link set {INTERFACE} up"));
+
+    let server = TestServer::start_with(65000, |config| {
+        config.kernel = Some(FibConfig {
+            table: TABLE,
+            metric: 20,
+        });
+    })
+    .await?;
+    let (base, _http) = server.serve_http().await?;
+    let mut peer = TestPeer::connect(server.addr).await?;
+    peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 2), 90)
+        .await?;
+
+    // The peer speaks from the loopback and names a router on the test's
+    // interface as next hop, as a route server would.
+    let neighbor = Ipv4Addr::new(10, 100, 0, 2);
+    announce(&mut peer, &["203.0.113.0/24", "198.18.0.0/15"], neighbor).await?;
+    // One by way of a router that is not a neighbor.
+    announce(&mut peer, &["192.88.99.0/24"], Ipv4Addr::new(172, 31, 0, 1)).await?;
+    kernel_becomes(&[
+        "203.0.113.0/24 via 10.100.0.2 dev ferrix100 proto bgp metric 20",
+        "198.18.0.0/15 via 10.100.0.2 dev ferrix100 proto bgp metric 20",
+    ]);
+
+    let kernel: Value = reqwest::get(format!("{base}/api/kernel"))
+        .await?
+        .json()
+        .await?;
+    assert_eq!(kernel["enabled"], true);
+    assert_eq!(kernel["table"], TABLE);
+    assert_eq!(kernel["installed"], 2);
+    assert_eq!(
+        kernel["unresolved"],
+        serde_json::json!({"next hop not directly connected": 1})
+    );
+    let shown = kernel["interfaces"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|interface| interface["name"] == INTERFACE)
+        .expect("the test's interface is listed");
+    assert_eq!(shown["up"], true);
+    // Its own address, beside the link-local one the kernel gave it.
+    assert!(
+        shown["addresses"]
+            .as_array()
+            .unwrap()
+            .contains(&serde_json::json!("10.100.0.1/24")),
+        "{shown}"
+    );
+
+    // One is withdrawn.
+    peer.send(&Message::Update(UpdateMessage {
+        withdrawn_routes: vec![IpPrefix::try_from("198.18.0.0/15").unwrap().into()],
+        path_attributes: vec![],
+        announced_routes: vec![],
+    }))
+    .await?;
+    kernel_becomes(&["203.0.113.0/24 via 10.100.0.2 dev ferrix100 proto bgp metric 20"]);
+
+    // The session ends: packets must not go on following its routes.
+    drop(peer);
+    kernel_becomes(&[]);
+    Ok(())
+}