ferrix/crates/fib/tests/kernel.rs

//! The kernel's routes against a real kernel: what `ip route` shows after
//! the routing table changes.
//!
//! These change the machine's interfaces and routes, so they are not run
//! by `cargo test`. `scripts/kernel-tests.sh` runs them in a network
//! namespace of their own, with the capability they need
//! (CAP_NET_ADMIN) and nothing more.
#![cfg(target_os = "linux")]

use std::collections::BTreeSet;
use std::net::{IpAddr, Ipv4Addr};
use std::process::Command;
use std::sync::Arc;
use std::time::{Duration, Instant};

use bgp_fib::{Fib, FibConfig, FibError};
use bgp_packet::message::{
    AsPathAttribute, MultiExitDiscPathAttribute, NextHopPathAttribute, NlriNextHop,
    OriginPathAttribute, PathAttribute,
};
use bgp_primitives::ip_prefix::IpPrefix;
use bgp_rib::table::{AttrSet, PeerIndex, PeerInfo, RibTable};
use tokio::task::JoinHandle;

const TIMEOUT: Duration = Duration::from_secs(10);

fn prefix(s: &str) -> IpPrefix {
    IpPrefix::try_from(s).unwrap()
}

/// Runs `ip` and returns what it printed. Panics if it fails.
#[track_caller]
fn ip(arguments: &str) -> String {
    let output = Command::new("ip")
        .args(arguments.split_whitespace())
        .output()
        .expect("the ip command is there");
    assert!(
        output.status.success(),
        "ip {arguments}: {}",
        String::from_utf8_lossy(&output.stderr)
    );
    String::from_utf8_lossy(&output.stdout).into_owned()
}

/// What a test has of the machine: an interface of its own with
/// 10.`n`.0.1/24 and 2001:db8:`n`::1/64 on it, and a routing table of its
/// own. Tests run side by side in one network namespace, and `n` keeps
/// them apart.
struct Machine {
    n: u8,
    rib: Arc<RibTable>,
    /// Whether routes name nexthop objects, which is how it is unless a
    /// test says otherwise.
    nexthop_objects: bool,
    /// How long routes of an earlier run are kept: not at all, unless a
    /// test says otherwise.
    stale_time: Duration,
}

impl Machine {
    fn new(n: u8) -> Self {
        let machine = Self {
            n,
            rib: Arc::new(RibTable::new(4)),
            nexthop_objects: true,
            stale_time: Duration::ZERO,
        };
        ip(&format!("link add {} type dummy", machine.interface()));
        machine.add_addresses();
        ip(&format!("link set {} up", machine.interface()));
        machine
    }

    fn interface(&self) -> String {
        format!("ferrix{}", self.n)
    }

    fn add_addresses(&self) {
        let (n, interface) = (self.n, self.interface());
        ip(&format!("addr add 10.{n}.0.1/24 dev {interface}"));
        ip(&format!(
            "-6 addr add 2001:db8:{n}::1/64 dev {interface} nodad"
        ));
    }

    fn add_addresses_v4(&self) {
        ip(&format!(
            "addr add 10.{}.0.1/24 dev {}",
            self.n,
            self.interface()
        ));
    }

    fn table(&self) -> u32 {
        1000 + u32::from(self.n)
    }

    /// A neighbor on the test's interface.
    fn neighbor(&self, host: u8) -> Ipv4Addr {
        Ipv4Addr::new(10, self.n, 0, host)
    }

    fn start(&self) -> (Fib, JoinHandle<Result<(), FibError>>) {
        Fib::spawn(
            Arc::clone(&self.rib),
            FibConfig {
                table: self.table(),
                metric: 20,
                nexthop_objects: self.nexthop_objects,
                stale_time: self.stale_time,
            },
        )
        .expect("the kernel can be talked to")
    }

    fn peer(&self, host: u8) -> PeerIndex {
        self.rib
            .add_peer(PeerInfo {
                asn: 65000 + u32::from(host),
                router_id: self.neighbor(host),
                address: self.neighbor(host).into(),
                ibgp: false,
            })
            .unwrap()
    }

    /// Attributes with the next hop `next_hop` and, to say which of two
    /// routes is the better, `med`.
    fn via(&self, next_hop: IpAddr, med: u32) -> Arc<AttrSet> {
        let mut attributes = vec![
            PathAttribute::Origin(OriginPathAttribute::IGP),
            AsPathAttribute::from_asns(vec![65001]),
            PathAttribute::MultiExitDisc(MultiExitDiscPathAttribute(med)),
        ];
        let mp_next_hop = match next_hop {
            IpAddr::V4(v4) => {
                attributes.push(PathAttribute::NextHop(NextHopPathAttribute(v4)));
                None
            }
            IpAddr::V6(v6) => Some(NlriNextHop::Ipv6(v6)),
        };
        self.rib
            .attrs()
            .intern(&attributes, mp_next_hop.as_ref())
            .unwrap()
    }

    /// The routes in the test's table, as `ip route` prints them.
    fn kernel_routes(&self) -> BTreeSet<String> {
        let table = self.table();
        let mut routes = BTreeSet::new();
        for family in ["-4", "-6"] {
            // A table nothing was ever put in does not exist yet.
            let output = Command::new("ip")
                .args([family, "route", "show", "table", &table.to_string()])
                .output()
                .unwrap();
            routes.extend(
                String::from_utf8_lossy(&output.stdout)
                    .lines()
                    .map(|line| line.split_whitespace().collect::<Vec<_>>().join(" "))
                    // IPv6 routes are shown with "pref medium".
                    .map(|line| line.replace(" pref medium", ""))
                    // A route that names a nexthop object is shown with
                    // its number, which is the kernel's to choose.
                    .map(|line| match line.split_once(" nhid ") {
                        Some((prefix, rest)) => {
                            let rest = rest.split_once(' ').map_or("", |(_, rest)| rest);
                            format!("{prefix} {rest}")
                        }
                        None => line,
                    })
                    .filter(|line| !line.is_empty()),
            );
        }
        routes
    }

    /// Waits until the kernel's table holds exactly `expected`.
    #[track_caller]
    fn kernel_becomes(&self, expected: &[String]) {
        let expected: BTreeSet<String> = expected.iter().cloned().collect();
        let start = Instant::now();
        while self.kernel_routes() != expected {
            assert!(
                start.elapsed() < TIMEOUT,
                "the kernel has {:#?}; expected {expected:#?}",
                self.kernel_routes()
            );
            std::thread::sleep(Duration::from_millis(20));
        }
    }

    /// How `ip route` shows a route of ours.
    fn shown(&self, prefix: &str, gateway: impl std::fmt::Display) -> String {
        format!(
            "{prefix} via {gateway} dev {} proto bgp metric 20",
            self.interface()
        )
    }
}

impl Drop for Machine {
    fn drop(&mut self) {
        let _ = Command::new("ip")
            .args(["link", "del", &self.interface()])
            .output();
        for family in ["-4", "-6"] {
            let _ = Command::new("ip")
                .args([family, "route", "flush", "table", &self.table().to_string()])
                .output();
        }
    }
}

/// Waits for `condition` to hold of the status.
#[track_caller]
fn status_becomes(fib: &Fib, what: &str, condition: impl Fn(&bgp_fib::Status) -> bool) {
    let start = Instant::now();
    while !condition(&fib.status()) {
        assert!(
            start.elapsed() < TIMEOUT,
            "{what}: the status is {:#?}",
            fib.status()
        );
        std::thread::sleep(Duration::from_millis(20));
    }
}

/// The best route of each prefix is in the kernel, and follows the table:
/// when the best route changes, when it goes, and when there is none whose
/// next hop can be reached.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn the_kernel_has_the_best_route_of_each_prefix() {
    let machine = Machine::new(1);
    let (near, far) = (machine.peer(2), machine.peer(3));
    let via_near = machine.via(machine.neighbor(2).into(), 50);
    // What was there before the daemon started is installed too.
    machine
        .rib
        .announce(near, &via_near, &[prefix("203.0.113.0/24")]);
    let (fib, task) = machine.start();
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    status_becomes(&fib, "in step", |status| {
        status.synced && status.installed == 1
    });

    // More routes, IPv6 among them.
    let v6_neighbor: IpAddr = "2001:db8:1::2".parse().unwrap();
    machine.rib.announce(
        near,
        &machine.via(v6_neighbor, 50),
        &[prefix("2001:db8:100::/48")],
    );
    machine
        .rib
        .announce(near, &via_near, &[prefix("198.18.0.0/15")]);
    machine.kernel_becomes(&[
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
        machine.shown("2001:db8:100::/48", v6_neighbor),
    ]);

    // A better route from another peer takes the first one's place.
    let via_far = machine.via(machine.neighbor(3).into(), 10);
    machine
        .rib
        .announce(far, &via_far, &[prefix("203.0.113.0/24")]);
    machine.kernel_becomes(&[
        machine.shown("203.0.113.0/24", machine.neighbor(3)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
        machine.shown("2001:db8:100::/48", v6_neighbor),
    ]);
    // It goes: the first is the best again.
    machine.rib.withdraw(far, &[prefix("203.0.113.0/24")]);
    machine.kernel_becomes(&[
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
        machine.shown("2001:db8:100::/48", v6_neighbor),
    ]);

    // A best route whose next hop is not a neighbor is not installed, and
    // the route it replaced does not stay.
    let unreachable = machine.via("172.31.9.9".parse().unwrap(), 5);
    machine
        .rib
        .announce(far, &unreachable, &[prefix("198.18.0.0/15")]);
    machine.kernel_becomes(&[
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("2001:db8:100::/48", v6_neighbor),
    ]);
    status_becomes(&fib, "one unresolved", |status| {
        status.installed == 2 && status.unresolved == [("next hop not directly connected", 1)]
    });

    // A session ends: its routes go.
    machine.rib.remove_peer(near);
    machine.kernel_becomes(&[]);
    status_becomes(&fib, "nothing installed", |status| status.installed == 0);

    fib.stop().await;
    task.await.unwrap().unwrap();
}

/// Routes of ours left by a run that ended badly are removed at the next
/// start unless they are still wanted. Routes that are not ours are left
/// alone, whatever they are for.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_left_behind_are_cleared_and_others_are_not_touched() {
    let machine = Machine::new(2);
    let (table, interface) = (machine.table(), machine.interface());
    let add = |route: &str| ip(&format!("route add {route} dev {interface} table {table}"));
    // Ours, from before: one that is still right, one that has another
    // next hop now, one that nobody announces any more.
    add("203.0.113.0/24 via 10.2.0.2 proto bgp metric 20");
    add("198.18.0.0/15 via 10.2.0.9 proto bgp metric 20");
    add("192.88.99.0/24 via 10.2.0.2 proto bgp metric 20");
    // Not ours: someone's static route, and a BGP route with another
    // metric, which is how another daemon's would differ.
    add("192.0.2.0/24 via 10.2.0.7 proto static metric 20");
    add("192.88.99.0/24 via 10.2.0.7 proto bgp metric 30");
    let others = [
        format!("192.0.2.0/24 via 10.2.0.7 dev {interface} proto static metric 20"),
        format!("192.88.99.0/24 via 10.2.0.7 dev {interface} proto bgp metric 30"),
    ];

    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    machine.rib.announce(
        peer,
        &via,
        &[prefix("203.0.113.0/24"), prefix("198.18.0.0/15")],
    );
    let (fib, task) = machine.start();
    machine.kernel_becomes(&[
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
        others[0].clone(),
        others[1].clone(),
    ]);

    // A run that ends in good order takes its routes with it, and only
    // its own.
    fib.stop().await;
    task.await.unwrap().unwrap();
    machine.kernel_becomes(&others);
}

/// What a next hop resolves to depends on the machine's interfaces. When
/// they change, the routes are worked out again.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_follow_the_machines_interfaces() {
    let machine = Machine::new(3);
    let interface = machine.interface();
    // The interface has no addresses yet: nothing is a neighbor.
    ip(&format!("addr flush dev {interface}"));
    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    machine
        .rib
        .announce(peer, &via, &[prefix("203.0.113.0/24")]);
    let (fib, task) = machine.start();
    status_becomes(&fib, "unresolved", |status| {
        status.synced && status.unresolved == [("next hop not directly connected", 1)]
    });
    assert!(machine.kernel_routes().is_empty());

    // It gets its address: the next hop is a neighbor now.
    machine.add_addresses();
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    status_becomes(&fib, "the interface is shown", |status| {
        status.unresolved.is_empty()
            && status.interfaces.iter().any(|shown| {
                shown.name == interface
                    && shown.up
                    && shown
                        .addresses
                        .contains(&(IpAddr::from(machine.neighbor(1)), 24))
            })
    });

    // It loses it. The kernel drops the route by itself; what matters is
    // that we know it has.
    ip(&format!("addr del 10.3.0.1/24 dev {interface}"));
    machine.kernel_becomes(&[]);
    status_becomes(&fib, "unresolved again", |status| {
        status.installed == 0 && status.unresolved.len() == 1
    });

    // The interface goes down and comes straight back, faster than we
    // work the routes out again. The kernel dropped the route when it went
    // down, though nothing about the route has changed for us.
    machine.add_addresses_v4();
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    ip(&format!("link set {interface} down"));
    ip(&format!("link set {interface} up"));
    assert!(machine.kernel_routes().is_empty(), "the kernel dropped it");
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    ip(&format!("addr del 10.3.0.1/24 dev {interface}"));
    machine.kernel_becomes(&[]);

    // And gets it back, with the interface having been down meanwhile.
    ip(&format!("link set {interface} down"));
    ip(&format!("addr add 10.3.0.1/24 dev {interface}"));
    ip(&format!("link set {interface} up"));
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);

    fib.stop().await;
    task.await.unwrap().unwrap();
    machine.kernel_becomes(&[]);
}

/// A table's worth of routes goes in, and comes out, in reasonable time.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn many_routes_are_installed_and_removed() {
    // FERRIX_ROUTES=1000000 for the size of the Internet's IPv4 table.
    let routes: usize = std::env::var("FERRIX_ROUTES")
        .ok()
        .and_then(|routes| routes.parse().ok())
        .unwrap_or(200_000);
    let machine = Machine::new(4);
    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    let prefixes: Vec<IpPrefix> = (0..routes as u32)
        .map(|n| {
            let [_, a, b, c] = n.to_be_bytes();
            IpPrefix::new(
                bgp_primitives::constants::IpVersion::Ipv4,
                [100 + a, b, c, 0],
                24,
            )
            .unwrap()
        })
        .collect();
    machine.rib.announce(peer, &via, &prefixes);

    let started = Instant::now();
    let (fib, task) = machine.start();
    status_becomes(&fib, "all installed", |status| {
        status.synced && status.installed == routes
    });
    let installed_in = started.elapsed();
    assert_eq!(machine.kernel_routes().len(), routes);
    assert_eq!(fib.status().errors, 0, "{:?}", fib.status().last_error);

    let started = Instant::now();
    fib.stop().await;
    task.await.unwrap().unwrap();
    let removed_in = started.elapsed();
    assert!(machine.kernel_routes().is_empty());
    println!(
        "{routes} routes: installed in {installed_in:?} ({:.0}/s), removed in {removed_in:?}",
        routes as f64 / installed_in.as_secs_f64()
    );
}

/// Routes name a nexthop object for their gateway where the kernel has
/// such objects. The object is ours: made when first needed, put back if
/// someone removes it, and removed when we stop. Other objects are not
/// touched.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_name_a_nexthop_object_that_is_ours_to_keep() {
    let machine = Machine::new(5);
    let interface = machine.interface();
    // Someone else's object, to the same gateway even.
    ip(&format!(
        "nexthop add id 50005 via 10.5.0.2 dev {interface} protocol bgp"
    ));
    let nexthops = || -> Vec<String> {
        ip("nexthop show")
            .lines()
            .filter(|line| line.contains(&format!("dev {interface} ")))
            .map(|line| line.trim().to_string())
            .collect()
    };
    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    machine.rib.announce(
        peer,
        &via,
        &[prefix("203.0.113.0/24"), prefix("198.18.0.0/15")],
    );
    let (fib, task) = machine.start();
    let both = [
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
    ];
    machine.kernel_becomes(&both);

    // One object of ours beside the other, and both routes name it.
    let ours: Vec<String> = nexthops()
        .into_iter()
        .filter(|line| !line.starts_with("id 50005 "))
        .collect();
    assert_eq!(ours.len(), 1, "{:?}", nexthops());
    assert!(ours[0].contains("via 10.5.0.2") && ours[0].contains("proto bgp"));
    let id = ours[0].split_whitespace().nth(1).unwrap().to_string();
    let raw = ip(&format!("route show table {}", machine.table()));
    assert_eq!(raw.matches(&format!("nhid {id} ")).count(), 2, "{raw}");

    // Someone removes our object, and the kernel our routes with it. We
    // are told, and put both back.
    ip(&format!("nexthop del id {id}"));
    machine.kernel_becomes(&both);

    // When we stop, our object goes with our routes; the other stays.
    fib.stop().await;
    task.await.unwrap().unwrap();
    machine.kernel_becomes(&[]);
    assert_eq!(nexthops().len(), 1, "{:?}", nexthops());
    assert!(nexthops()[0].starts_with("id 50005 "));
    ip("nexthop del id 50005");
}

/// On a kernel without nexthop objects, or where the configuration says
/// so, every route carries its own gateway.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_can_carry_their_own_gateway() {
    let mut machine = Machine::new(6);
    machine.nexthop_objects = false;
    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    machine
        .rib
        .announce(peer, &via, &[prefix("203.0.113.0/24")]);
    let (fib, task) = machine.start();
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    let raw = ip(&format!("route show table {}", machine.table()));
    assert!(!raw.contains("nhid"), "{raw}");
    assert!(
        !ip("nexthop show").contains(&format!("dev {} ", machine.interface())),
        "no nexthop object was made"
    );

    // The next hop changes, and the route goes.
    let other = machine.via(machine.neighbor(3).into(), 0);
    machine
        .rib
        .announce(peer, &other, &[prefix("203.0.113.0/24")]);
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(3))]);
    machine.rib.withdraw(peer, &[prefix("203.0.113.0/24")]);
    machine.kernel_becomes(&[]);

    fib.stop().await;
    task.await.unwrap().unwrap();
}

/// When the daemon starts, its table is empty until its peers have sent
/// theirs. The routes an earlier run left are kept for a while, so that
/// the machine goes on forwarding; then those nobody announced again go.
#[tokio::test(flavor = "multi_thread")]
#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"]
async fn routes_of_an_earlier_run_are_kept_while_peers_come_back() {
    let mut machine = Machine::new(7);
    machine.stale_time = Duration::from_secs(3);
    let (table, interface) = (machine.table(), machine.interface());
    let add = |route: &str| ip(&format!("route add {route} dev {interface} table {table}"));
    add("203.0.113.0/24 via 10.7.0.2 proto bgp metric 20");
    add("198.18.0.0/15 via 10.7.0.2 proto bgp metric 20");
    let left = [
        machine.shown("203.0.113.0/24", machine.neighbor(2)),
        machine.shown("198.18.0.0/15", machine.neighbor(2)),
    ];

    // The daemon starts with nothing in its table: both stay.
    let (fib, task) = machine.start();
    status_becomes(&fib, "in step, keeping two", |status| {
        status.synced && status.installed == 2 && status.kept == 2
    });
    assert_eq!(machine.kernel_routes(), left.iter().cloned().collect());

    // A peer comes back with one of them.
    let peer = machine.peer(2);
    let via = machine.via(machine.neighbor(2).into(), 0);
    machine
        .rib
        .announce(peer, &via, &[prefix("203.0.113.0/24")]);
    status_becomes(&fib, "one confirmed", |status| status.kept == 1);
    assert_eq!(machine.kernel_routes(), left.iter().cloned().collect());

    // Time is up: the one nobody announced goes.
    machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]);
    status_becomes(&fib, "none kept", |status| {
        status.installed == 1 && status.kept == 0
    });

    fib.stop().await;
    task.await.unwrap().unwrap();
}