Files
ferrix/crates/fib/tests/kernel.rs
T
Rayhaan JaufeerallyandClaude Fable 5.1 1de75fc610 Install a full table in the kernel in about a second
A million IPv4 routes took 1.6 s to install (release build); they take
1.1 s now, and 0.5 s to remove instead of 1.15 s. Most of what is left is
the kernel's own work on each route.

- Nexthop objects (Linux 5.3 and later): a gateway is put in the kernel
  once and routes name it, so the kernel checks it once and not for every
  route. The daemon makes its own objects, uses nobody else's, is told
  when one of its own is removed and puts it back, and removes them when
  it stops, which takes all the routes that name them along at once.
  `without_nexthop_objects` turns this off; a kernel without them is
  noticed and served as before.
- Writing to the kernel is a task of its own, fed through a short queue,
  so the next batch is worked out while the kernel is busy with the last.
  "In step" is said when the kernel has done the batch that completes the
  table, not when it was handed over.
- What a next hop resolves to is kept, by next hop: it was looked up among
  the machine's addresses for every route.
- A quicker hash for the plan's maps, and no lookups in the ones that are
  empty.

Fixes a hole found on the way: when an interface went down and came back
within the second it takes us to react, the kernel had dropped our routes
and we did not put them back, since nothing about them had changed for
us. After such a change the kernel is now asked what it actually has.

scripts/in-namespace.sh runs one command in a scratch namespace;
FERRIX_ROUTES sets the size of the timing test.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-02 17:13:44 +02:00

549 lines
20 KiB
Rust

//! The kernel's routes against a real kernel: what `ip route` shows after
//! the routing table changes.
//!
//! These change the machine's interfaces and routes, so they are not run
//! by `cargo test`. `scripts/kernel-tests.sh` runs them in a network
//! namespace of their own, with the capability they need
//! (CAP_NET_ADMIN) and nothing more.
#![cfg(target_os = "linux")]
use std::collections::BTreeSet;
use std::net::{IpAddr, Ipv4Addr};
use std::process::Command;
use std::sync::Arc;
use std::time::{Duration, Instant};
use bgp_fib::{Fib, FibConfig, FibError};
use bgp_packet::message::{
AsPathAttribute, MultiExitDiscPathAttribute, NextHopPathAttribute, NlriNextHop,
OriginPathAttribute, PathAttribute,
};
use bgp_primitives::ip_prefix::IpPrefix;
use bgp_rib::table::{AttrSet, PeerIndex, PeerInfo, RibTable};
use tokio::task::JoinHandle;
const TIMEOUT: Duration = Duration::from_secs(10);
fn prefix(s: &str) -> IpPrefix {
IpPrefix::try_from(s).unwrap()
}
/// Runs `ip` and returns what it printed. Panics if it fails.
#[track_caller]
fn ip(arguments: &str) -> String {
let output = Command::new("ip")
.args(arguments.split_whitespace())
.output()
.expect("the ip command is there");
assert!(
output.status.success(),
"ip {arguments}: {}",
String::from_utf8_lossy(&output.stderr)
);
String::from_utf8_lossy(&output.stdout).into_owned()
}
/// What a test has of the machine: an interface of its own with
/// 10.`n`.0.1/24 and 2001:db8:`n`::1/64 on it, and a routing table of its
/// own. Tests run side by side in one network namespace, and `n` keeps
/// them apart.
struct Machine {
n: u8,
rib: Arc<RibTable>,
/// Whether routes name nexthop objects, which is how it is unless a
/// test says otherwise.
nexthop_objects: bool,
}
impl Machine {
fn new(n: u8) -> Self {
let machine = Self {
n,
rib: Arc::new(RibTable::new(4)),
nexthop_objects: true,
};
ip(&format!("link add {} type dummy", machine.interface()));
machine.add_addresses();
ip(&format!("link set {} up", machine.interface()));
machine
}
fn interface(&self) -> String {
format!("ferrix{}", self.n)
}
fn add_addresses(&self) {
let (n, interface) = (self.n, self.interface());
ip(&format!("addr add 10.{n}.0.1/24 dev {interface}"));
ip(&format!(
"-6 addr add 2001:db8:{n}::1/64 dev {interface} nodad"
));
}
fn add_addresses_v4(&self) {
ip(&format!(
"addr add 10.{}.0.1/24 dev {}",
self.n,
self.interface()
));
}
fn table(&self) -> u32 {
1000 + u32::from(self.n)
}
/// A neighbor on the test's interface.
fn neighbor(&self, host: u8) -> Ipv4Addr {
Ipv4Addr::new(10, self.n, 0, host)
}
fn start(&self) -> (Fib, JoinHandle<Result<(), FibError>>) {
Fib::spawn(
Arc::clone(&self.rib),
FibConfig {
table: self.table(),
metric: 20,
nexthop_objects: self.nexthop_objects,
},
)
.expect("the kernel can be talked to")
}
fn peer(&self, host: u8) -> PeerIndex {
self.rib
.add_peer(PeerInfo {
asn: 65000 + u32::from(host),
router_id: self.neighbor(host),
address: self.neighbor(host).into(),
ibgp: false,
})
.unwrap()
}
/// Attributes with the next hop `next_hop` and, to say which of two
/// routes is the better, `med`.
fn via(&self, next_hop: IpAddr, med: u32) -> Arc<AttrSet> {
let mut attributes = vec![
PathAttribute::Origin(OriginPathAttribute::IGP),
AsPathAttribute::from_asns(vec![65001]),
PathAttribute::MultiExitDisc(MultiExitDiscPathAttribute(med)),
];
let mp_next_hop = match next_hop {
IpAddr::V4(v4) => {
attributes.push(PathAttribute::NextHop(NextHopPathAttribute(v4)));
None
}
IpAddr::V6(v6) => Some(NlriNextHop::Ipv6(v6)),
};
self.rib
.attrs()
.intern(&attributes, mp_next_hop.as_ref())
.unwrap()
}
/// The routes in the test's table, as `ip route` prints them.
fn kernel_routes(&self) -> BTreeSet<String> {
let table = self.table();
let mut routes = BTreeSet::new();
for family in ["-4", "-6"] {
// A table nothing was ever put in does not exist yet.
let output = Command::new("ip")
.args([family, "route", "show", "table", &table.to_string()])
.output()
.unwrap();
routes.extend(
String::from_utf8_lossy(&output.stdout)
.lines()
.map(|line| line.split_whitespace().collect::<Vec<_>>().join(" "))
// IPv6 routes are shown with "pref medium".
.map(|line| line.replace(" pref medium", ""))
// A route that names a nexthop object is shown with
// its number, which is the kernel's to choose.
.map(|line| match line.split_once(" nhid ") {
Some((prefix, rest)) => {
let rest = rest.split_once(' ').map_or("", |(_, rest)| rest);
format!("{prefix} {rest}")
}
None => line,
})
.filter(|line| !line.is_empty()),
);
}
routes
}
/// Waits until the kernel's table holds exactly `expected`.
#[track_caller]
fn kernel_becomes(&self, expected: &[String]) {
let expected: BTreeSet<String> = expected.iter().cloned().collect();
let start = Instant::now();
while self.kernel_routes() != expected {
assert!(
start.elapsed() < TIMEOUT,
"the kernel has {:#?}; expected {expected:#?}",
self.kernel_routes()
);
std::thread::sleep(Duration::from_millis(20));
}
}
/// How `ip route` shows a route of ours.
fn shown(&self, prefix: &str, gateway: impl std::fmt::Display) -> String {
format!(
"{prefix} via {gateway} dev {} proto bgp metric 20",
self.interface()
)
}
}
impl Drop for Machine {
fn drop(&mut self) {
let _ = Command::new("ip")
.args(["link", "del", &self.interface()])
.output();
for family in ["-4", "-6"] {
let _ = Command::new("ip")
.args([family, "route", "flush", "table", &self.table().to_string()])
.output();
}
}
}
/// Waits for `condition` to hold of the status.
#[track_caller]
fn status_becomes(fib: &Fib, what: &str, condition: impl Fn(&bgp_fib::Status) -> bool) {
let start = Instant::now();
while !condition(&fib.status()) {
assert!(
start.elapsed() < TIMEOUT,
"{what}: the status is {:#?}",
fib.status()
);
std::thread::sleep(Duration::from_millis(20));
}
}
/// The best route of each prefix is in the kernel, and follows the table:
/// when the best route changes, when it goes, and when there is none whose
/// next hop can be reached.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn the_kernel_has_the_best_route_of_each_prefix() {
let machine = Machine::new(1);
let (near, far) = (machine.peer(2), machine.peer(3));
let via_near = machine.via(machine.neighbor(2).into(), 50);
// What was there before the daemon started is installed too.
machine
.rib
.announce(near, &via_near, &[prefix("203.0.113.0/24")]);
let (fib, task) = machine.start();
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
status_becomes(&fib, "in step", |status| {
status.synced && status.installed == 1
});
// More routes, IPv6 among them.
let v6_neighbor: IpAddr = "2001:db8:1::2".parse().unwrap();
machine.rib.announce(
near,
&machine.via(v6_neighbor, 50),
&[prefix("2001:db8:100::/48")],
);
machine
.rib
.announce(near, &via_near, &[prefix("198.18.0.0/15")]);
machine.kernel_becomes(&[
machine.shown("203.0.113.0/24", machine.neighbor(2)),
machine.shown("198.18.0.0/15", machine.neighbor(2)),
machine.shown("2001:db8:100::/48", v6_neighbor),
]);
// A better route from another peer takes the first one's place.
let via_far = machine.via(machine.neighbor(3).into(), 10);
machine
.rib
.announce(far, &via_far, &[prefix("203.0.113.0/24")]);
machine.kernel_becomes(&[
machine.shown("203.0.113.0/24", machine.neighbor(3)),
machine.shown("198.18.0.0/15", machine.neighbor(2)),
machine.shown("2001:db8:100::/48", v6_neighbor),
]);
// It goes: the first is the best again.
machine.rib.withdraw(far, &[prefix("203.0.113.0/24")]);
machine.kernel_becomes(&[
machine.shown("203.0.113.0/24", machine.neighbor(2)),
machine.shown("198.18.0.0/15", machine.neighbor(2)),
machine.shown("2001:db8:100::/48", v6_neighbor),
]);
// A best route whose next hop is not a neighbor is not installed, and
// the route it replaced does not stay.
let unreachable = machine.via("172.31.9.9".parse().unwrap(), 5);
machine
.rib
.announce(far, &unreachable, &[prefix("198.18.0.0/15")]);
machine.kernel_becomes(&[
machine.shown("203.0.113.0/24", machine.neighbor(2)),
machine.shown("2001:db8:100::/48", v6_neighbor),
]);
status_becomes(&fib, "one unresolved", |status| {
status.installed == 2 && status.unresolved == [("next hop not directly connected", 1)]
});
// A session ends: its routes go.
machine.rib.remove_peer(near);
machine.kernel_becomes(&[]);
status_becomes(&fib, "nothing installed", |status| status.installed == 0);
fib.stop().await;
task.await.unwrap().unwrap();
}
/// Routes of ours left by a run that ended badly are removed at the next
/// start unless they are still wanted. Routes that are not ours are left
/// alone, whatever they are for.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_left_behind_are_cleared_and_others_are_not_touched() {
let machine = Machine::new(2);
let (table, interface) = (machine.table(), machine.interface());
let add = |route: &str| ip(&format!("route add {route} dev {interface} table {table}"));
// Ours, from before: one that is still right, one that has another
// next hop now, one that nobody announces any more.
add("203.0.113.0/24 via 10.2.0.2 proto bgp metric 20");
add("198.18.0.0/15 via 10.2.0.9 proto bgp metric 20");
add("192.88.99.0/24 via 10.2.0.2 proto bgp metric 20");
// Not ours: someone's static route, and a BGP route with another
// metric, which is how another daemon's would differ.
add("192.0.2.0/24 via 10.2.0.7 proto static metric 20");
add("192.88.99.0/24 via 10.2.0.7 proto bgp metric 30");
let others = [
format!("192.0.2.0/24 via 10.2.0.7 dev {interface} proto static metric 20"),
format!("192.88.99.0/24 via 10.2.0.7 dev {interface} proto bgp metric 30"),
];
let peer = machine.peer(2);
let via = machine.via(machine.neighbor(2).into(), 0);
machine.rib.announce(
peer,
&via,
&[prefix("203.0.113.0/24"), prefix("198.18.0.0/15")],
);
let (fib, task) = machine.start();
machine.kernel_becomes(&[
machine.shown("203.0.113.0/24", machine.neighbor(2)),
machine.shown("198.18.0.0/15", machine.neighbor(2)),
others[0].clone(),
others[1].clone(),
]);
// A run that ends in good order takes its routes with it, and only
// its own.
fib.stop().await;
task.await.unwrap().unwrap();
machine.kernel_becomes(&others);
}
/// What a next hop resolves to depends on the machine's interfaces. When
/// they change, the routes are worked out again.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_follow_the_machines_interfaces() {
let machine = Machine::new(3);
let interface = machine.interface();
// The interface has no addresses yet: nothing is a neighbor.
ip(&format!("addr flush dev {interface}"));
let peer = machine.peer(2);
let via = machine.via(machine.neighbor(2).into(), 0);
machine
.rib
.announce(peer, &via, &[prefix("203.0.113.0/24")]);
let (fib, task) = machine.start();
status_becomes(&fib, "unresolved", |status| {
status.synced && status.unresolved == [("next hop not directly connected", 1)]
});
assert!(machine.kernel_routes().is_empty());
// It gets its address: the next hop is a neighbor now.
machine.add_addresses();
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
status_becomes(&fib, "the interface is shown", |status| {
status.unresolved.is_empty()
&& status.interfaces.iter().any(|shown| {
shown.name == interface
&& shown.up
&& shown
.addresses
.contains(&(IpAddr::from(machine.neighbor(1)), 24))
})
});
// It loses it. The kernel drops the route by itself; what matters is
// that we know it has.
ip(&format!("addr del 10.3.0.1/24 dev {interface}"));
machine.kernel_becomes(&[]);
status_becomes(&fib, "unresolved again", |status| {
status.installed == 0 && status.unresolved.len() == 1
});
// The interface goes down and comes straight back, faster than we
// work the routes out again. The kernel dropped the route when it went
// down, though nothing about the route has changed for us.
machine.add_addresses_v4();
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
ip(&format!("link set {interface} down"));
ip(&format!("link set {interface} up"));
assert!(machine.kernel_routes().is_empty(), "the kernel dropped it");
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
ip(&format!("addr del 10.3.0.1/24 dev {interface}"));
machine.kernel_becomes(&[]);
// And gets it back, with the interface having been down meanwhile.
ip(&format!("link set {interface} down"));
ip(&format!("addr add 10.3.0.1/24 dev {interface}"));
ip(&format!("link set {interface} up"));
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
fib.stop().await;
task.await.unwrap().unwrap();
machine.kernel_becomes(&[]);
}
/// A table's worth of routes goes in, and comes out, in reasonable time.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn many_routes_are_installed_and_removed() {
// FERRIX_ROUTES=1000000 for the size of the Internet's IPv4 table.
let routes: usize = std::env::var("FERRIX_ROUTES")
.ok()
.and_then(|routes| routes.parse().ok())
.unwrap_or(200_000);
let machine = Machine::new(4);
let peer = machine.peer(2);
let via = machine.via(machine.neighbor(2).into(), 0);
let prefixes: Vec<IpPrefix> = (0..routes as u32)
.map(|n| {
let [_, a, b, c] = n.to_be_bytes();
IpPrefix::new(
bgp_primitives::constants::IpVersion::Ipv4,
[100 + a, b, c, 0],
24,
)
.unwrap()
})
.collect();
machine.rib.announce(peer, &via, &prefixes);
let started = Instant::now();
let (fib, task) = machine.start();
status_becomes(&fib, "all installed", |status| {
status.synced && status.installed == routes
});
let installed_in = started.elapsed();
assert_eq!(machine.kernel_routes().len(), routes);
assert_eq!(fib.status().errors, 0, "{:?}", fib.status().last_error);
let started = Instant::now();
fib.stop().await;
task.await.unwrap().unwrap();
let removed_in = started.elapsed();
assert!(machine.kernel_routes().is_empty());
println!(
"{routes} routes: installed in {installed_in:?} ({:.0}/s), removed in {removed_in:?}",
routes as f64 / installed_in.as_secs_f64()
);
}
/// Routes name a nexthop object for their gateway where the kernel has
/// such objects. The object is ours: made when first needed, put back if
/// someone removes it, and removed when we stop. Other objects are not
/// touched.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_name_a_nexthop_object_that_is_ours_to_keep() {
let machine = Machine::new(5);
let interface = machine.interface();
// Someone else's object, to the same gateway even.
ip(&format!(
"nexthop add id 50005 via 10.5.0.2 dev {interface} protocol bgp"
));
let nexthops = || -> Vec<String> {
ip("nexthop show")
.lines()
.filter(|line| line.contains(&format!("dev {interface} ")))
.map(|line| line.trim().to_string())
.collect()
};
let peer = machine.peer(2);
let via = machine.via(machine.neighbor(2).into(), 0);
machine.rib.announce(
peer,
&via,
&[prefix("203.0.113.0/24"), prefix("198.18.0.0/15")],
);
let (fib, task) = machine.start();
let both = [
machine.shown("203.0.113.0/24", machine.neighbor(2)),
machine.shown("198.18.0.0/15", machine.neighbor(2)),
];
machine.kernel_becomes(&both);
// One object of ours beside the other, and both routes name it.
let ours: Vec<String> = nexthops()
.into_iter()
.filter(|line| !line.starts_with("id 50005 "))
.collect();
assert_eq!(ours.len(), 1, "{:?}", nexthops());
assert!(ours[0].contains("via 10.5.0.2") && ours[0].contains("proto bgp"));
let id = ours[0].split_whitespace().nth(1).unwrap().to_string();
let raw = ip(&format!("route show table {}", machine.table()));
assert_eq!(raw.matches(&format!("nhid {id} ")).count(), 2, "{raw}");
// Someone removes our object, and the kernel our routes with it. We
// are told, and put both back.
ip(&format!("nexthop del id {id}"));
machine.kernel_becomes(&both);
// When we stop, our object goes with our routes; the other stays.
fib.stop().await;
task.await.unwrap().unwrap();
machine.kernel_becomes(&[]);
assert_eq!(nexthops().len(), 1, "{:?}", nexthops());
assert!(nexthops()[0].starts_with("id 50005 "));
ip("nexthop del id 50005");
}
/// On a kernel without nexthop objects, or where the configuration says
/// so, every route carries its own gateway.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_can_carry_their_own_gateway() {
let mut machine = Machine::new(6);
machine.nexthop_objects = false;
let peer = machine.peer(2);
let via = machine.via(machine.neighbor(2).into(), 0);
machine
.rib
.announce(peer, &via, &[prefix("203.0.113.0/24")]);
let (fib, task) = machine.start();
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
let raw = ip(&format!("route show table {}", machine.table()));
assert!(!raw.contains("nhid"), "{raw}");
assert!(
!ip("nexthop show").contains(&format!("dev {} ", machine.interface())),
"no nexthop object was made"
);
// The next hop changes, and the route goes.
let other = machine.via(machine.neighbor(3).into(), 0);
machine
.rib
.announce(peer, &other, &[prefix("203.0.113.0/24")]);
machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(3))]);
machine.rib.withdraw(peer, &[prefix("203.0.113.0/24")]);
machine.kernel_becomes(&[]);
fib.stop().await;
task.await.unwrap().unwrap();
}