From 321af51e4bdd6c39aa09558f57efd1c0e4f910db Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Fri, 2 Oct 2026 17:44:49 +0200 Subject: [PATCH] Add an interoperability lab with FRR, BIRD and Routinator scripts/interop/lab.sh runs the daemon against FRR, BIRD and Routinator, each in a network namespace on one exchange LAN, and checks what every side ends up with: sessions and routes over IPv4 and IPv6 with their attributes, four-octet AS numbers, ADD-PATH, RFC 9234 roles and Only to Customer, RTR version 2 and invalid routes, route refresh, session resets, killed peers, a cut wire, the daemon stopped and killed, a table of a million routes and a restart under it, role mismatch and the prefix limit. All of it passes against FRR 10.5.1, BIRD 2.18 and Routinator 0.15.2. scripts/interop/real-rpki.sh checks the RTR client against a Routinator with the real RPKI: 713,258 IPv4 and 187,479 IPv6 VRPs and 3,379 ASPAs, the same numbers on both sides. What the lab found, fixed here: - A daemon restarted with its routes in the kernel removed them all at once, its table being empty until the peers had sent theirs again, and then put them back: no forwarding meanwhile. Routes of an earlier run are now kept for `stale_seconds` (120 by default) and only those nobody announced again are removed. A restart under a million routes takes 1.1 s to be in step, with nothing removed. - FRR ends a session it loses to a collision with a state machine error without a subcode, which was shown as "error code 5 subcode 0". CI gets a job that runs the lab. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 13 + bin/bgp/README.md | 31 +- crates/fib/src/kernel.rs | 40 +- crates/fib/src/lib.rs | 15 + crates/fib/src/plan.rs | 105 ++++- crates/fib/tests/kernel.rs | 48 +++ crates/packet/src/errors.rs | 4 + crates/packet/src/message.rs | 10 +- crates/server/proto/config.proto | 8 + crates/server/src/config.rs | 9 +- crates/web/src/api.rs | 4 + crates/web/static/statusz.html | 1 + scripts/interop/install.sh | 9 + scripts/interop/lab.sh | 673 +++++++++++++++++++++++++++++++ scripts/interop/real-rpki.sh | 70 ++++ scripts/linux-vm.sh | 1 + 16 files changed, 1024 insertions(+), 17 deletions(-) create mode 100755 scripts/interop/install.sh create mode 100755 scripts/interop/lab.sh create mode 100755 scripts/interop/real-rpki.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc13271..aa62695 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,19 @@ jobs: - name: Kernel tests run: scripts/kernel-tests.sh + # The daemon against FRR, BIRD and Routinator; see bin/bgp/README.md. + interop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Install FRR, BIRD and Routinator + run: scripts/interop/install.sh + - name: Build + run: cargo build --release --locked -p bgp + - name: Interoperability lab + run: scripts/interop/lab.sh + # A short run of each fuzz target: enough to notice a parser change that # breaks on input the seeds are close to. Long runs are done by hand; see # fuzz/README.md. diff --git a/bin/bgp/README.md b/bin/bgp/README.md index e2b7077..26561a4 100644 --- a/bin/bgp/README.md +++ b/bin/bgp/README.md @@ -205,7 +205,9 @@ kernel { run two daemons that install BGP routes into the same table with the same metric. - **Starting and stopping.** At startup, routes of ours left by an earlier - run are kept if they are still right and removed if not. On SIGINT or + run are kept, so that the machine goes on forwarding while the peers send + their tables again; those that no peer has announced again after + `stale_seconds` (120 unless set) are removed. On SIGINT or SIGTERM the daemon closes its sessions and then removes its routes. After a crash they stay until the next start. - **Interfaces.** The daemon learns the machine's interfaces and addresses @@ -251,6 +253,33 @@ and two daemons that route for each other with pings across. From macOS, `scripts/linux-vm.sh` runs it (or any command, such as `cargo test`) in a Lima virtual machine that it creates on first use. +## Interoperability + +`scripts/interop/lab.sh` runs the daemon against FRR, BIRD and Routinator, +each in a network namespace of its own on one exchange LAN, and checks what +every side ends up with. It needs Linux (`scripts/linux-vm.sh +scripts/interop/lab.sh` from macOS; `scripts/interop/install.sh` installs +the others on Ubuntu). Last run against FRR 10.5.1, BIRD 2.18 and +Routinator 0.15.2. What it covers: + +| Phase | What is checked | +|-------|-----------------| +| `basics` | Sessions over IPv4 and IPv6; routes both ways with path, next hop, communities and large communities intact; four-octet AS numbers; ADD-PATH from BIRD; pings across by the installed routes | +| `roles` | RFC 9234 roles agree with FRR (peer) and BIRD (provider); Only to Customer is set where it should be, in both directions | +| `rpki` | RTR version 2 with Routinator; an invalid route from FRR is refused and held, and taken after Routinator restarts with data that allows it | +| `refresh` | Routes added and withdrawn by FRR; FRR's ROUTE-REFRESH answered; BIRD re-sending | +| `flaps` | Five session resets; FRR and BIRD killed and restarted; a cut wire noticed by the hold timer | +| `restart` | The daemon stopped in good order (kernel emptied, peers told) and killed (routes kept, stale one cleared by the next start) | +| `scale` | A table from BIRD (`SCALE_ROUTES`, a million for the Internet's size) into the table and the kernel; the daemon killed and restarted under it; the table withdrawn | +| `limits` | A peer whose role does not fit gets no session; a peer over its prefix limit gets Cease / Maximum Number of Prefixes and is held down | + +`scripts/interop/real-rpki.sh` checks the RTR client against a Routinator +that has the real RPKI, ASPAs included: the daemon has to end up with as +many VRPs and ASPAs as the validator says it has. + +Not covered: routers other than FRR and BIRD; IPv6 routes over an IPv4 +session (the daemon does not send them); long runs. + ## Finding a route The status page has a look-up box over the received routes, and diff --git a/crates/fib/src/kernel.rs b/crates/fib/src/kernel.rs index 7bcb0a3..f36df44 100644 --- a/crates/fib/src/kernel.rs +++ b/crates/fib/src/kernel.rs @@ -354,6 +354,10 @@ struct Task { synced: bool, /// When the routes were last worked out again for a refusal. mended_at: Option, + /// Whether routes of an earlier run are still being kept; counting + /// them means looking at every route, and is not done once there are + /// none to find. + keeping: bool, } impl Task { @@ -448,7 +452,11 @@ impl Task { } /// Finds the routes of ours that the kernel has from before. - async fn adopt_routes(&mut self, nexthops: &HashMap) -> Result<(), FibError> { + async fn adopt_routes( + &mut self, + nexthops: &HashMap, + keep: bool, + ) -> Result<(), FibError> { for route in self.kernel_routes().await? { if !self.is_ours(&route) { continue; @@ -464,7 +472,7 @@ impl Task { } _ => None, }; - self.plan.adopt(route.prefix, target); + self.plan.adopt(route.prefix, target, keep); } Ok(()) } @@ -745,7 +753,7 @@ impl Task { self.synced = false; self.plan = Plan::default(); let nexthops = self.kernel_nexthops().await?; - self.adopt_routes(&nexthops).await?; + self.adopt_routes(&nexthops, false).await?; self.use_nexthops(&nexthops); Ok(()) } @@ -793,7 +801,7 @@ impl Task { // What is left, if anything, is routes from an earlier run // that carry their own gateway. self.plan = Plan::default(); - self.adopt_routes(&HashMap::new()).await?; + self.adopt_routes(&HashMap::new(), false).await?; } let ops = self.plan.clear(); self.send(ops, false).await?; @@ -834,6 +842,7 @@ impl Task { self.fib.update(|status| { status.synced = self.synced; status.installed = self.plan.installed(); + status.kept = if self.keeping { self.plan.kept() } else { 0 }; status.refused = self.plan.refused(); status.unresolved = self .plan @@ -879,18 +888,26 @@ pub(crate) async fn run( marks: 0, synced: false, mended_at: None, + keeping: false, }; // The socket for news was bound before this: what changes while the // interfaces are being learned is told there. task.learn_interfaces().await?; let nexthops = task.kernel_nexthops().await?; - task.adopt_routes(&nexthops).await?; + // Kept for a while, if there is to be any keeping: our table is empty + // until the peers have sent theirs again. + let keep = !config.stale_time.is_zero(); + task.adopt_routes(&nexthops, keep).await?; task.use_nexthops(&nexthops); + let mut release_at = None; if task.plan.installed() > 0 { info!( routes = task.plan.installed(), + kept_for = ?config.stale_time, "Found routes of ours in the kernel from an earlier run" ); + release_at = keep.then(|| Instant::now() + config.stale_time); + task.keeping = keep; } task.publish(true); info!( @@ -963,6 +980,19 @@ pub(crate) async fn run( task.relearn(&mut heard).await?; reader = task.rib.reader(); } + () = async { sleep_until(release_at.expect("checked by the guard")).await }, + if release_at.is_some() => + { + release_at = None; + task.keeping = false; + let stale = task.plan.release(); + info!( + removed = stale.len(), + "Routes of an earlier run that were not announced again are removed" + ); + task.send(stale, false).await?; + task.publish(false); + } ended = &mut task.writer => { return Err(match ended { Ok(Err(e)) => e, diff --git a/crates/fib/src/lib.rs b/crates/fib/src/lib.rs index b41c51d..68dfca9 100644 --- a/crates/fib/src/lib.rs +++ b/crates/fib/src/lib.rs @@ -39,6 +39,7 @@ pub mod plan; use std::net::IpAddr; use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; use bgp_rib::table::RibTable; use tokio::sync::watch; @@ -49,6 +50,10 @@ use tokio::task::JoinHandle; /// against static and connected routes as an operator expects. pub const DEFAULT_METRIC: u32 = 20; +/// How long routes left by an earlier run are kept unless configured +/// otherwise: time for sessions to come up and tables to be sent. +pub const DEFAULT_STALE_TIME: Duration = Duration::from_secs(120); + /// Where our routes go in the kernel. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FibConfig { @@ -60,6 +65,12 @@ pub struct FibConfig { /// 5.3 and later), which makes installing many routes a good deal /// faster. Without, every route carries its own gateway. pub nexthop_objects: bool, + /// How long routes of ours that an earlier run left in the kernel are + /// kept after a start without the routing table having them. The + /// table is empty until the peers have sent theirs again; removing + /// everything at once would stop the machine forwarding meanwhile. + /// After this long, what has not been announced again goes. + pub stale_time: Duration, } impl Default for FibConfig { @@ -68,6 +79,7 @@ impl Default for FibConfig { table: netlink::TABLE_MAIN, metric: DEFAULT_METRIC, nexthop_objects: true, + stale_time: DEFAULT_STALE_TIME, } } } @@ -108,6 +120,9 @@ pub struct Status { pub synced: bool, /// How many routes of ours the kernel has. pub installed: usize, + /// How many of them are kept from an earlier run though the routing + /// table does not have them (yet); see [`FibConfig::stale_time`]. + pub kept: usize, /// Prefixes whose best route is not in the kernel because its next /// hop is not usable, by why. pub unresolved: Vec<(&'static str, usize)>, diff --git a/crates/fib/src/plan.rs b/crates/fib/src/plan.rs index b713568..77ddb86 100644 --- a/crates/fib/src/plan.rs +++ b/crates/fib/src/plan.rs @@ -142,6 +142,9 @@ struct Entry { target: Option, /// Whether the prefix was heard of since the last reset. confirmed: bool, + /// Whether the route is one an earlier run left that is being kept + /// for now though not confirmed: see [`Plan::adopt`]. + kept: bool, } /// The routes of ours in the kernel, kept in step with the best routes. @@ -209,18 +212,49 @@ impl Plan { self.installed.keys() } - /// Notes a route of ours that the kernel already has, from before this - /// run. It stays only if the routing table turns out to have the same. - pub fn adopt(&mut self, prefix: IpPrefix, target: Option) { + /// Notes a route of ours that the kernel already has. It stays only if + /// the routing table turns out to have the same. + /// + /// With `keep` it is not removed for being absent from a whole table, + /// only by [`Plan::release`]. That is for the routes an earlier run + /// left: when the daemon starts, its table is empty until its peers + /// have sent theirs again, and to remove everything meanwhile would + /// stop the machine forwarding for as long as that takes. + pub fn adopt(&mut self, prefix: IpPrefix, target: Option, keep: bool) { self.installed.insert( prefix, Entry { target, confirmed: false, + kept: keep, }, ); } + /// How many routes are being kept from an earlier run without the + /// routing table having confirmed them. + pub fn kept(&self) -> usize { + self.installed + .values() + .filter(|entry| entry.kept && !entry.confirmed) + .count() + } + + /// The time for keeping unconfirmed routes of an earlier run is over: + /// those the routing table still does not have go. + pub fn release(&mut self) -> Vec { + let mut stale = Vec::new(); + self.installed.retain(|prefix, entry| { + let goes = entry.kept && !entry.confirmed; + entry.kept = false; + if goes { + stale.push(Op::Remove { prefix: *prefix }); + } + !goes + }); + stale + } + /// The whole table follows: nothing is known to be wanted until it is /// heard of again. pub fn reset(&mut self) { @@ -246,6 +280,7 @@ impl Plan { let entry = self.installed.entry(prefix).or_insert(Entry { target: None, confirmed: true, + kept: false, }); entry.confirmed = true; if entry.target == Some(target) { @@ -279,7 +314,7 @@ impl Plan { let stale: Vec = self .installed .iter() - .filter(|(_, entry)| !entry.confirmed) + .filter(|(_, entry)| !entry.confirmed && !entry.kept) .map(|(prefix, _)| *prefix) .collect(); stale @@ -493,10 +528,10 @@ mod tests { ); let a = target("192.0.2.2", 2); // Left in the kernel by an earlier run. - plan.adopt(kept, Some(a)); - plan.adopt(changed, Some(a)); - plan.adopt(stale, Some(a)); - plan.adopt(odd, None); + plan.adopt(kept, Some(a), false); + plan.adopt(changed, Some(a), false); + plan.adopt(stale, Some(a), false); + plan.adopt(odd, None, false); assert_eq!(plan.installed(), 4); plan.reset(); @@ -520,6 +555,58 @@ mod tests { assert_eq!(plan.synced(), []); } + /// When the daemon starts its table is empty, and stays so until its + /// peers have sent theirs. What an earlier run left in the kernel is + /// kept meanwhile. + #[test] + fn routes_of_an_earlier_run_are_kept_until_released() { + let mut plan = Plan::default(); + let (still_right, moved, gone) = ( + prefix("203.0.113.0/24"), + prefix("198.18.0.0/15"), + prefix("192.88.99.0/24"), + ); + let (a, b) = (target("192.0.2.2", 2), target("192.0.2.3", 2)); + for p in [still_right, moved, gone] { + plan.adopt(p, Some(a), true); + } + // The first whole table is the empty one: nothing goes. + plan.reset(); + assert_eq!(plan.synced(), []); + assert_eq!((plan.installed(), plan.kept()), (3, 3)); + + // The peers come back. + assert_eq!(plan.best(still_right, Ok(a)), None); + assert_eq!( + plan.best(moved, Ok(b)), + Some(Op::Install { + prefix: moved, + target: b + }) + ); + assert_eq!(plan.kept(), 1); + // Time is up: what nobody announced again goes, and only that. + assert_eq!(plan.release(), [Op::Remove { prefix: gone }]); + assert_eq!((plan.installed(), plan.kept()), (2, 0)); + assert_eq!(plan.release(), []); + + // From then on a whole table decides as always. + plan.reset(); + plan.best(moved, Ok(b)); + assert_eq!( + plan.synced(), + [Op::Remove { + prefix: still_right + }] + ); + + // A route that is withdrawn while kept goes at once. + let mut plan = Plan::default(); + plan.adopt(gone, Some(a), true); + plan.best(gone, Ok(a)); + assert_eq!(plan.withdrawn(gone), Some(Op::Remove { prefix: gone })); + } + #[test] fn a_route_the_kernel_refuses_is_not_counted_as_there() { let mut plan = Plan::default(); @@ -578,7 +665,7 @@ mod tests { let mut plan = Plan::default(); for (prefix, gateway) in left_behind { kernel.insert(p(prefix), t(gateway)); - plan.adopt(p(prefix), Some(t(gateway))); + plan.adopt(p(prefix), Some(t(gateway)), false); } // The best routes: the prefix's next hop, if it resolves. let mut table: HashMap> = HashMap::default(); diff --git a/crates/fib/tests/kernel.rs b/crates/fib/tests/kernel.rs index 5ff7609..204cf08 100644 --- a/crates/fib/tests/kernel.rs +++ b/crates/fib/tests/kernel.rs @@ -53,6 +53,9 @@ struct Machine { /// Whether routes name nexthop objects, which is how it is unless a /// test says otherwise. nexthop_objects: bool, + /// How long routes of an earlier run are kept: not at all, unless a + /// test says otherwise. + stale_time: Duration, } impl Machine { @@ -61,6 +64,7 @@ impl Machine { n, rib: Arc::new(RibTable::new(4)), nexthop_objects: true, + stale_time: Duration::ZERO, }; ip(&format!("link add {} type dummy", machine.interface())); machine.add_addresses(); @@ -104,6 +108,7 @@ impl Machine { table: self.table(), metric: 20, nexthop_objects: self.nexthop_objects, + stale_time: self.stale_time, }, ) .expect("the kernel can be talked to") @@ -546,3 +551,46 @@ async fn routes_can_carry_their_own_gateway() { fib.stop().await; task.await.unwrap().unwrap(); } + +/// When the daemon starts, its table is empty until its peers have sent +/// theirs. The routes an earlier run left are kept for a while, so that +/// the machine goes on forwarding; then those nobody announced again go. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"] +async fn routes_of_an_earlier_run_are_kept_while_peers_come_back() { + let mut machine = Machine::new(7); + machine.stale_time = Duration::from_secs(3); + let (table, interface) = (machine.table(), machine.interface()); + let add = |route: &str| ip(&format!("route add {route} dev {interface} table {table}")); + add("203.0.113.0/24 via 10.7.0.2 proto bgp metric 20"); + add("198.18.0.0/15 via 10.7.0.2 proto bgp metric 20"); + let left = [ + machine.shown("203.0.113.0/24", machine.neighbor(2)), + machine.shown("198.18.0.0/15", machine.neighbor(2)), + ]; + + // The daemon starts with nothing in its table: both stay. + let (fib, task) = machine.start(); + status_becomes(&fib, "in step, keeping two", |status| { + status.synced && status.installed == 2 && status.kept == 2 + }); + assert_eq!(machine.kernel_routes(), left.iter().cloned().collect()); + + // A peer comes back with one of them. + let peer = machine.peer(2); + let via = machine.via(machine.neighbor(2).into(), 0); + machine + .rib + .announce(peer, &via, &[prefix("203.0.113.0/24")]); + status_becomes(&fib, "one confirmed", |status| status.kept == 1); + assert_eq!(machine.kernel_routes(), left.iter().cloned().collect()); + + // Time is up: the one nobody announced goes. + machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]); + status_becomes(&fib, "none kept", |status| { + status.installed == 1 && status.kept == 0 + }); + + fib.stop().await; + task.await.unwrap().unwrap(); +} diff --git a/crates/packet/src/errors.rs b/crates/packet/src/errors.rs index 50cc7e5..0c132b1 100644 --- a/crates/packet/src/errors.rs +++ b/crates/packet/src/errors.rs @@ -170,6 +170,10 @@ pub enum UpdateMsgSubcode { #[derive(Debug, Clone, Copy, PartialEq, Eq, Error, Serialize, FromRepr)] #[repr(u8)] pub enum FsmSubcode { + /// RFC 6608: what speakers from before it send, and FRR still does. + #[error("unspecified")] + Unspecified = 0, + #[error("Received an unexpected message in OpenSent state")] UnexpectedOpenSent = 1, diff --git a/crates/packet/src/message.rs b/crates/packet/src/message.rs index d64f1c9..f4c1028 100644 --- a/crates/packet/src/message.rs +++ b/crates/packet/src/message.rs @@ -1954,7 +1954,7 @@ mod tests { use crate::errors::BgpError; let ctx = default_v6_context(); - for (code, subcode) in [(6, 0), (5, 0), (2, 5), (3, 7), (4, 9), (99, 1)] { + for (code, subcode) in [(6, 0), (5, 9), (2, 5), (3, 7), (4, 9), (99, 1)] { let decoded = Message::decode(&ctx, &framed(21, 3, &[code, subcode])).unwrap(); let Message::Notification(notification) = &decoded else { panic!("expected a NOTIFICATION, got {decoded:?}"); @@ -1962,6 +1962,14 @@ mod tests { assert_eq!(notification.error, BgpError::Other { code, subcode }); assert_eq!(notification.error.code_subcode(), (code, subcode)); } + // A state machine error without a subcode has a name (RFC 6608); + // FRR sends it. + let decoded = Message::decode(&ctx, &framed(21, 3, &[5, 0])).unwrap(); + assert!(matches!( + &decoded, + Message::Notification(n) + if n.error == BgpError::FsmError(crate::errors::FsmSubcode::Unspecified) + )); // Too short to hold a code and subcode. assert!(Message::decode(&ctx, &framed(20, 3, &[6])).is_err()); } diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index 791f50e..53af4b7 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -79,6 +79,14 @@ message Kernel { // route` then shows routes with "nhid". True has every route carry its // own gateway, as on older kernels. bool without_nexthop_objects = 3; + + // How long, after the daemon starts, routes of its own that an earlier + // run left in the kernel are kept though no peer has announced them + // (again). Defaults to 120. The daemon's table is empty until its peers + // have sent theirs; removing the old routes at once would stop the + // machine forwarding for as long as that takes. After this time, what + // was not announced again is removed. + uint32 stale_seconds = 4; } // Where to get the validated ROA payloads that say which AS may originate diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 24070fc..9625341 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -184,6 +184,10 @@ impl Config { table: or(kernel.table, defaults.table), metric: or(kernel.metric, defaults.metric), nexthop_objects: !kernel.without_nexthop_objects, + stale_time: match kernel.stale_seconds { + 0 => defaults.stale_time, + seconds => Duration::from_secs(seconds.into()), + }, } }), rpki, @@ -746,11 +750,13 @@ mod tests { table: 254, metric: 20, nexthop_objects: true, + stale_time: Duration::from_secs(120), }) ); assert_eq!( parse(&format!( - "{BASE}kernel {{ table: 1000 metric: 150 without_nexthop_objects: true }}" + "{BASE}kernel {{ table: 1000 metric: 150 without_nexthop_objects: true \ + stale_seconds: 30 }}" )) .unwrap() .kernel, @@ -758,6 +764,7 @@ mod tests { table: 1000, metric: 150, nexthop_objects: false, + stale_time: Duration::from_secs(30), }) ); } diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 000301b..3dd27d4 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -867,6 +867,9 @@ pub struct KernelView { pub synced: bool, /// How many routes of ours the kernel has. pub installed: usize, + /// How many of them are kept from before the daemon started though no + /// peer has announced them again; they go when `stale_seconds` is up. + pub kept: usize, /// Prefixes whose best route is not in the kernel because its next hop /// cannot be used, by why. pub unresolved: BTreeMap<&'static str, usize>, @@ -899,6 +902,7 @@ pub async fn kernel(State(state): State) -> Json { metric: Some(config.metric), synced: status.synced, installed: status.installed, + kept: status.kept, unresolved: status.unresolved.into_iter().collect(), refused: status.refused, errors: status.errors, diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index 4888d6b..6f59dcf 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -318,6 +318,7 @@ from (i.addresses.length ? ` ${i.addresses.map(esc).join(' ')}` : '')).join(' · '); setHTML(b, `${k.synced ? 'In step' : 'Catching up'} with the routing table: ` + `${k.installed} routes installed (table ${k.table}, metric ${k.metric}, proto bgp)` + + (k.kept ? `; ${k.kept} kept from before the restart, not announced again yet` : '') + (unresolved ? `; not installed: ${unresolved}` : '') + (k.refused ? `; ${k.refused} refused by the kernel` : '') + '.' + (k.last_error ? `
Last error (${k.errors} in all): ${esc(k.last_error)}` : '') + diff --git a/scripts/interop/install.sh b/scripts/interop/install.sh new file mode 100755 index 0000000..43581f6 --- /dev/null +++ b/scripts/interop/install.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Installs what scripts/interop/lab.sh needs, on Ubuntu or Debian: FRR and +# BIRD from the distribution, Routinator from crates.io (a few minutes to +# build). +set -eu +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y frr bird2 jq curl +# They are run by the lab, in namespaces, not as services of this machine. +sudo systemctl disable --now frr bird 2>/dev/null || true +command -v routinator >/dev/null || cargo install --locked routinator diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh new file mode 100755 index 0000000..c799bb0 --- /dev/null +++ b/scripts/interop/lab.sh @@ -0,0 +1,673 @@ +#!/bin/bash +# An interoperability lab: this daemon against FRR, BIRD and Routinator, +# each in a network namespace of its own on one exchange LAN. +# +# 10.9.0.0/24, 2001:db8:9::/64 +# ┌───────────────────┬──────────────────────┐ +# dut (.1) AS 64601 frr (.2) AS 64602 bird (.3) AS 4200000003 +# lan 10.1.0.1/24 lan 10.2.0.1/24 lan 10.3.0.1/24 +# 2001:db8:1::1/48 2001:db8:2::1/48 2001:db8:3::1/48 +# + Routinator on 127.0.0.1:3323 +# +# "dut" is the daemon under test. FRR is its peer, BIRD its provider; both +# say so in their OPEN (RFC 9234). Routinator serves validated ROA payloads +# made up for the lab from a local exceptions file, so nothing is fetched +# from the Internet. +# +# scripts/interop/lab.sh run every phase, then take the lab down +# scripts/interop/lab.sh up bring the lab up and leave it up +# scripts/interop/lab.sh down take it down +# PHASES="basics scale" scripts/interop/lab.sh only these phases +# SCALE_ROUTES=1000000 ... the size of the table BIRD sends +# +# Needs Linux, sudo, and: frr, bird2, jq, routinator, a built `bgp` +# (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS; +# scripts/interop/install.sh installs what is missing on Ubuntu. +set -euo pipefail + +cd "$(dirname "$0")/../.." +bgp="${CARGO_TARGET_DIR:-target}/release/bgp" +work=/tmp/ferrix-lab +pathspace=ferrixlab +all_phases="basics roles rpki refresh flaps restart scale limits" +phases="${PHASES:-$all_phases}" +scale_routes="${SCALE_ROUTES:-200000}" + +# --- plumbing --------------------------------------------------------------- + +in_ns() { local name=$1; shift; sudo ip netns exec "lab-$name" "$@"; } +api() { in_ns dut curl -sf "http://127.0.0.1:8080$1"; } +vty() { in_ns frr vtysh -N "$pathspace" "$@" 2>/dev/null; } +bird_ctl() { sudo birdc -s "$work/bird.ctl" "$@"; } +say() { printf '%s\n' "$*"; } +ok() { say " ok: $*"; } +fail() { say "FAILED: $*" >&2; exit 1; } + +# wait_for "what" command...: until the command succeeds. +wait_for() { + local what=$1; shift + local tries=${WAIT_TRIES:-150} + for _ in $(seq "$tries"); do + if "$@" >/dev/null 2>&1; then return 0; fi + sleep 0.2 + done + say "timed out waiting for: $what" >&2 + return 1 +} + +down() { + for pidfile in "$work"/dut.pid "$work"/routinator.pid; do + [ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true + done + [ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true + for daemon in bgpd zebra; do + local pidfile="/var/run/frr/$pathspace/$daemon.pid" + sudo test -f "$pidfile" && sudo kill "$(sudo cat "$pidfile")" 2>/dev/null || true + done + sleep 0.3 + for name in dut frr bird ix; do sudo ip netns del "lab-$name" 2>/dev/null || true; done + sudo rm -rf "/etc/frr/$pathspace" "/var/run/frr/$pathspace" +} + +# --- the network ------------------------------------------------------------ + +network_up() { + sudo modprobe dummy 2>/dev/null || true + sudo ip netns add lab-ix + in_ns ix ip link add ix type bridge + in_ns ix ip link set ix up + local n=0 + for name in dut frr bird; do + n=$((n + 1)) + sudo ip netns add "lab-$name" + sudo ip link add ix0 netns "lab-$name" type veth peer name "to-$name" netns lab-ix + in_ns ix ip link set "to-$name" master ix up + in_ns "$name" ip link set lo up + in_ns "$name" ip addr add "10.9.0.$n/24" dev ix0 + in_ns "$name" ip -6 addr add "2001:db8:9::$n/64" dev ix0 nodad + in_ns "$name" ip link set ix0 up + in_ns "$name" ip link add lan type dummy + in_ns "$name" ip addr add "10.$n.0.1/24" dev lan + in_ns "$name" ip -6 addr add "2001:db8:$n::1/48" dev lan nodad + in_ns "$name" ip link set lan up + done +} + +# --- Routinator --------------------------------------------------------------- + +# routinator_up [asn]: with `asn` the AS that may originate 10.66.0.0/24. +routinator_up() { + local owner=${1:-64999} + mkdir -p "$work/rpki/repository" "$work/rpki/tals" + # 10.2.0.0/24 is FRR's to originate. 10.66.0.0/24 is someone else's, + # and FRR announces it all the same. 10.3.0.0/16 is BIRD's, as far as + # /24. + cat > "$work/rpki/slurm.json" < /dev/null 2>&1 & + echo $! | sudo tee "$work/routinator.pid" > /dev/null +} + +# --- the daemon under test ------------------------------------------------------ + +# dut_config [frr-role] [frr-max-prefixes] +dut_config() { + local frr_role=${1:-PEER} frr_limit=${2:-2000000} + cat > "$work/dut.textpb" <> "$work/dut.log" 2>&1 & + echo $! > "$work/dut.pid" + # Not the shell's to report on when it is killed, which the tests do. + disown + wait_for "the daemon's status interface" api /api/status +} + +# dut_down [signal]: stops the daemon and waits until it has gone. +dut_down() { + local signal=${1:-TERM} + # The pid recorded is sudo's; the daemon is the one in the namespace + # that is not Routinator. + local pid + for pid in $(sudo ip netns pids lab-dut); do + if [ "$(cat "/proc/$pid/comm" 2>/dev/null)" = bgp ]; then + sudo kill "-$signal" "$pid" + while [ -e "/proc/$pid" ]; do sleep 0.05; done + fi + done +} + +# --- FRR ------------------------------------------------------------------------ + +frr_up() { + sudo mkdir -p "/etc/frr/$pathspace" "/var/run/frr/$pathspace" + sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null < "$work/$daemon.out" 2>&1 \ + || fail "$daemon did not start: $(cat "$work/$daemon.out")" + # zebra makes the directory its sockets are in root's. + sudo chown -R frr:frr "/var/run/frr/$pathspace" + done +} + +frr_down() { + local signal=${1:-TERM} daemon + for daemon in bgpd zebra; do + local pidfile="/var/run/frr/$pathspace/$daemon.pid" + sudo test -f "$pidfile" && sudo kill "-$signal" "$(sudo cat "$pidfile")" 2>/dev/null || true + done + sleep 0.5 +} + +# --- BIRD ----------------------------------------------------------------------- + +# bird_config [extra-static-routes-file] +bird_config() { + local extra=${1:-} + cat > "$work/bird.conf" < 0' >/dev/null; } +dut_kernel() { # dut_kernel family "route as ip shows it, without nhid" + # All of it is read: with a table's worth of routes, a grep that + # stopped at the first match would leave `ip` writing to nobody. + in_ns dut ip "$1" route show proto bgp | sed 's/ */ /g; s/ nhid [0-9]*//; s/ pref medium//; s/ *$//' | grep -xF "$2" >/dev/null +} +dut_kernel_lacks() { ! in_ns dut ip "$1" route show proto bgp | grep "^$2 " >/dev/null; } +frr_has() { # frr_has family prefix: FRR has a route for it by way of the daemon + vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | any(.aspath.string == "64601")' +} +frr_lacks() { ! vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | length > 0' >/dev/null; } +bird_has() { bird_ctl show route for "$1" all | grep -q "BGP.as_path: 64601"; } + +lab_up() { + down + sudo rm -rf "$work"; mkdir -p "$work"; chmod 777 "$work" + [ -x "$bgp" ] || fail "$bgp not found: cargo build --release -p bgp" + network_up + routinator_up + frr_up + bird_config; bird_up + dut_config; dut_up +} + +converged() { + wait_for "four sessions" established 4 + wait_for "FRR's routes at the daemon" dut_has 10.2.0.0/24 frr4 + wait_for "BIRD's routes at the daemon" dut_has 10.3.0.0/24 bird4 + wait_for "FRR's IPv6 route" dut_has 2001:db8:2::/48 frr6 + wait_for "BIRD's IPv6 route" dut_has 2001:db8:3::/48 bird6 + wait_for "routes in the kernel" dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20" + wait_for "routes in the kernel" dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" + wait_for "routes in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20" + wait_for "routes in the kernel" dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20" + wait_for "the daemon's route at FRR" frr_has ipv4 10.1.0.0/24 + wait_for "the daemon's IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48 + wait_for "the daemon's route at BIRD" bird_has 10.1.0.0/24 + wait_for "the daemon's IPv6 route at BIRD" bird_has 2001:db8:1::/48 +} + +# --- the phases --------------------------------------------------------------------- + +frr_neighbor() { # frr_neighbor address jq-filter + vty -c "show bgp neighbors $1 json" | jq -e --arg n "$1" ".[\$n] | $2" +} +frr_configure() { # each argument a line of configuration + local args=(-c "configure terminal") + for line in "$@"; do args+=(-c "$line"); done + vty "${args[@]}" +} +peer_is() { # peer_is name jq-filter: of the daemon's view of a peer + api /api/peers | jq -e --arg n "$1" ".[] | select(.name == \$n) | $2" +} + +# Sessions come up, routes go both ways with their attributes intact, and +# packets follow them. +phase_basics() { + # What FRR has of ours: the path, the next hop, the communities. + local route + route=$(vty -c "show bgp ipv4 unicast 10.1.0.0/24 json") + jq -e '.paths[0] | .aspath.string == "64601" and .nexthops[0].ip == "10.9.0.1" + and .community.string == "64601:100" and .origin == "IGP"' <<<"$route" >/dev/null \ + || fail "FRR's copy of 10.1.0.0/24 is not as sent: $route" + route=$(vty -c "show bgp ipv6 unicast 2001:db8:1::/48 json") + jq -e '.paths[0] | .aspath.string == "64601" and (.nexthops | any(.ip == "2001:db8:9::1")) + and .largeCommunity.string == "64601:1:2"' <<<"$route" >/dev/null \ + || fail "FRR's copy of 2001:db8:1::/48 is not as sent: $route" + ok "FRR has our routes, with path, next hop and communities" + + route=$(bird_ctl show route for 10.1.0.0/24 all) + grep -q "BGP.next_hop: 10.9.0.1" <<<"$route" && grep -q "BGP.community: (64601,100)" <<<"$route" \ + || fail "BIRD's copy of 10.1.0.0/24 is not as sent: $route" + route=$(bird_ctl show route for 2001:db8:1::/48 all) + grep -q "BGP.next_hop: 2001:db8:9::1" <<<"$route" && grep -q "BGP.large_community: (64601, 1, 2)" <<<"$route" \ + || fail "BIRD's copy of 2001:db8:1::/48 is not as sent: $route" + ok "BIRD has our routes, with path, next hop and communities" + + # What we have of theirs. BIRD's AS number needs four octets, and it + # sends path identifiers (ADD-PATH). + api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0] + | .summary.as_path == [4200000003] and .summary.next_hop == "10.9.0.3" and .path_id != 0' >/dev/null \ + || fail "our copy of BIRD's 10.3.0.0/24 is not as sent" + api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0] + | .summary.as_path == [64602] and .summary.next_hop == "10.9.0.2" and .best' >/dev/null \ + || fail "our copy of FRR's 10.2.0.0/24 is not as sent" + peer_is bird4 '.four_octet_asn and (.add_path_receive == ["IPv4 unicast"])' >/dev/null \ + || fail "the session with BIRD did not agree on four-octet AS numbers and ADD-PATH" + ok "we have their routes; four-octet AS numbers and ADD-PATH with BIRD" + + # Packets, there and back by the routes BGP put in three kernels. + in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no ping to FRR's lan" + in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no ping to BIRD's lan" + in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan" + in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan" + ok "packets get through to both, over IPv4 and IPv6" +} + +# RFC 9234: each end says what it is to the other, and routes are marked +# Only to Customer where they should be. +phase_roles() { + api /api/peers | jq -e 'all(.[]; .role_confirmed)' >/dev/null \ + || fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')" + frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \ + || fail "FRR does not see us as its peer" + bird_ctl show protocols all dut4 | grep -A12 "Neighbor capabilities" | grep -q "Role: customer" \ + || fail "BIRD does not see us as its customer" + ok "roles agree with FRR (peer) and BIRD (provider)" + + # To a peer our routes are marked; to a provider they are not. + vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].otc == 64601' >/dev/null \ + || fail "FRR's copy of our route is not marked Only to Customer by us" + if bird_ctl show route for 10.1.0.0/24 all | grep -qi "otc\|only_to_customer"; then + fail "our route to our provider is marked Only to Customer" + fi + # What comes from a peer or a provider is marked by them, or by us. + api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 64602)' >/dev/null \ + || fail "FRR's route is not marked Only to Customer" + api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 4200000003)' >/dev/null \ + || fail "BIRD's route is not marked Only to Customer" + ok "Only to Customer is where it should be, in both directions" +} + +# Routinator's data over RTR: an invalid route is refused, and taken when +# the data comes to allow it. +phase_rpki() { + wait_for "the daemon to have Routinator's data" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/rpki | jq -e '.state == \"synchronised\" and .vrps_ipv4 == 3 and .vrps_ipv6 == 1'" + ok "synchronised with Routinator over RTR version $(api /api/rpki | jq .protocol_version): 3 IPv4 and 1 IPv6 VRPs" + + # FRR announces 10.66.0.0/24, which is AS 64999's to announce. + wait_for "the invalid route to be refused" dut_lacks 10.66.0.0/24 + wait_for "the invalid route to leave the kernel" dut_kernel_lacks -4 10.66.0.0/24 + api "/api/routes/rejected?peer=frr4" | jq -e '.[0].by_reason["RPKI invalid"] == 1' >/dev/null \ + || fail "the refusal is not on record: $(api /api/routes/rejected)" + peer_is frr4 '.held_routes == 1' >/dev/null || fail "the invalid route is not held" + api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \ + || fail "FRR's own route is not valid" + api "/api/routes/received?prefix=10.3.7.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \ + || fail "BIRD's more specific route is not valid" + ok "FRR's invalid route is refused and held; the others are valid" + + # The data changes, by way of a Routinator that is restarted: the + # route is authorised now. + sudo kill "$(cat "$work/routinator.pid")"; sleep 0.5 + routinator_up 64602 + wait_for "the held route to be taken" dut_has 10.66.0.0/24 frr4 + wait_for "the route in the kernel" dut_kernel -4 "10.66.0.0/24 via 10.9.0.2 dev ix0 metric 20" + peer_is frr4 '.held_routes == 0' >/dev/null || fail "the route is still held" + ok "after Routinator restarts with new data, the held route is taken" +} + +# Changes after the first exchange: new routes, withdrawals, and being +# asked to send everything again. +phase_refresh() { + frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.9.0/24" + wait_for "FRR's new route" dut_has 10.2.9.0/24 frr4 + wait_for "it in the kernel" dut_kernel -4 "10.2.9.0/24 via 10.9.0.2 dev ix0 metric 20" + frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.9.0/24" + wait_for "FRR's withdrawal" dut_lacks 10.2.9.0/24 + wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.9.0/24 + ok "a route FRR adds and withdraws comes and goes, in the table and the kernel" + + # FRR asks for our routes again (ROUTE-REFRESH), and gets them. + local before + before=$(frr_neighbor 10.9.0.1 '.messageStats.updatesRecv') + vty -c "clear bgp ipv4 unicast 10.9.0.1 soft in" + wait_for "our routes to be sent again" \ + bash -c "[ \"\$(sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq '.[\"10.9.0.1\"].messageStats.updatesRecv')\" -gt $before ]" + frr_neighbor 10.9.0.1 '.bgpState == "Established"' >/dev/null || fail "the session did not survive a route refresh" + wait_for "our route still at FRR" frr_has ipv4 10.1.0.0/24 + ok "FRR's route refresh is answered with our routes" + + # BIRD sends its routes again unasked; nothing changes. + bird_ctl reload out dut4 >/dev/null + sleep 0.5 + converged + ok "BIRD re-sending its routes changes nothing" +} + +# Sessions that end in every way they can, and come back. +phase_flaps() { + for round in 1 2 3 4 5; do + vty -c "clear bgp *" + wait_for "FRR's routes to go (round $round)" dut_lacks 10.2.0.0/24 + converged + done + ok "five resets by FRR: routes go and come back each time" + + bird_ctl restart dut4 >/dev/null; bird_ctl restart dut6 >/dev/null + converged + ok "BIRD restarting its sessions" + + # FRR dies without a word. + frr_down KILL + wait_for "FRR's routes to go" dut_lacks 10.2.0.0/24 + wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24 + dut_has 10.3.0.0/24 bird4 >/dev/null || fail "BIRD's routes went with FRR's" + frr_up + converged + ok "FRR killed and started again" + + # The wire to FRR is cut: nothing says so but the silence. The hold + # timer (nine seconds here) has to notice. + in_ns ix ip link set to-frr down + WAIT_TRIES=100 wait_for "the hold timer to end the session" dut_lacks 10.2.0.0/24 + wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24 + in_ns ix ip link set to-frr up + converged + ok "a cut wire is noticed by the hold timer, and mended" + + sudo kill -KILL "$(sudo cat "$work/bird.pid")" + wait_for "BIRD's routes to go" dut_lacks 10.3.0.0/24 + sudo rm -f "$work/bird.ctl" + bird_up + converged + ok "BIRD killed and started again" +} + +# The daemon itself stopping, in good order and not. +phase_restart() { + dut_down TERM + [ -z "$(in_ns dut ip route show proto bgp)" ] && [ -z "$(in_ns dut ip -6 route show proto bgp)" ] \ + || fail "routes left in the kernel after a clean stop: $(in_ns dut ip route show proto bgp)" + wait_for "FRR to drop our route" frr_lacks ipv4 10.1.0.0/24 + frr_neighbor 10.9.0.1 '.lastNotificationReason // .lastResetDueTo | test("Cease|Notification"; "i")' >/dev/null \ + || fail "FRR was not told we were going: $(frr_neighbor 10.9.0.1 '{lastResetDueTo, lastNotificationReason}')" + dut_up + converged + ok "stopped in good order: kernel emptied, peers told; and back" + + # Killed: its routes stay in the kernel. While it is dead BIRD stops + # announcing one of them; the daemon that starts next has to clear + # that one out and keep the rest. + dut_down KILL + dut_kernel -4 "10.3.7.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "the kernel's routes went with the killed daemon" + sed -i '/route 10.3.7.0\/24 blackhole;/d' "$work/bird.conf" + bird_ctl configure >/dev/null + dut_up + converged + wait_for "the stale route to be cleared" dut_kernel_lacks -4 10.3.7.0/24 + ok "killed: routes stay in the kernel, and the next start clears the stale one" +} + +# A full table's worth of routes from BIRD, into the table and the kernel. +phase_scale() { + # Out of 100.0.0.0/6 and up: as many /24s as asked for. + awk -v n="$scale_routes" 'BEGIN { for (i = 0; i < n; i++) + printf "route %d.%d.%d.0/24 blackhole;\n", 100 + int(i / 65536), int(i / 256) % 256, i % 256 }' \ + > "$work/bird-table.conf" + bird_config "$work/bird-table.conf" + local base start + base=$(api /api/status | jq .received_prefixes) + start=$(date +%s.%N) + bird_ctl configure >/dev/null + WAIT_TRIES=1500 wait_for "the table to arrive" \ + bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -ge $((base + scale_routes)) ]" + local arrived + arrived=$(date +%s.%N) + # By the daemon's own count first: listing a million routes with `ip` + # takes longer than installing them. + WAIT_TRIES=1500 wait_for "the table in the kernel" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.installed >= $scale_routes'" + local installed + installed=$(date +%s.%N) + local listed + listed=$(in_ns dut ip -4 route show proto bgp | wc -l) + [ "$listed" -ge "$scale_routes" ] || fail "the kernel lists $listed routes, not $scale_routes" + api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')" + ok "$scale_routes routes from BIRD: in the table after $(echo "$arrived - $start" | bc)s, in the kernel after $(echo "$installed - $start" | bc)s" + peer_is frr4 '.state == "Established"' >/dev/null || fail "the session with FRR did not survive the table" + + # The daemon is killed with the table in the kernel, and started + # again: it has to find its routes there, take the table from BIRD + # anew, and end up with the same, none refused and none left over. + dut_down KILL + start=$(date +%s.%N) + dut_up + WAIT_TRIES=1500 wait_for "the table in the kernel again" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.synced and .installed >= $scale_routes'" + installed=$(date +%s.%N) + converged + local listed_again + listed_again=$(in_ns dut ip -4 route show proto bgp | wc -l) + [ "$listed_again" -eq "$listed" ] || fail "the kernel lists $listed_again routes after a restart; it listed $listed before" + # They were kept, not taken out and put back: next to nothing was + # removed on the way. + local removed + removed=$(api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }') + [ "${removed:-0}" -lt 100 ] || fail "$removed routes were removed from the kernel on the way" + api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')" + ok "killed and restarted under the table: in step again after $(echo "$installed - $start" | bc)s, the kernel's routes as before" + + bird_config + start=$(date +%s.%N) + bird_ctl configure >/dev/null + WAIT_TRIES=1500 wait_for "the table to be withdrawn" \ + bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -le $((base + 1)) ]" + WAIT_TRIES=1500 wait_for "the table out of the kernel" \ + bash -c "[ \"\$(sudo ip netns exec lab-dut ip -4 route show proto bgp | wc -l)\" -le $((base + 1)) ]" + ok "withdrawn again after $(echo "$(date +%s.%N) - $start" | bc)s" + converged +} + +# What the daemon refuses: a peer that is not what it is configured as, +# and one that sends more than it may. +phase_limits() { + # FRR says it is our peer; here it is configured as our customer. + dut_down TERM + dut_config CUSTOMER + dut_up + wait_for "the sessions that should come up" established 3 + wait_for "the role mismatch to be on record" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | .state != \"Established\" and (.last_error // \"\" | test(\"role mismatch\"; \"i\"))'" + sleep 1 + peer_is frr4 '.state != "Established"' >/dev/null || fail "a session came up across a role mismatch" + frr_neighbor 10.9.0.1 '.bgpState != "Established"' >/dev/null || fail "FRR thinks the session is up" + ok "a peer whose role does not fit ours gets no session: $(peer_is frr4 '.last_error')" + + # FRR sends at least two routes that pass; it may have one. + frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.8.0/24" + dut_down TERM + dut_config PEER 1 + dut_up + wait_for "the limit to end the session" \ + bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq -e '.[\"10.9.0.1\"] | (.lastNotificationReason // \"\") | test(\"Maximum Number of Prefixes\"; \"i\")'" + wait_for "the peer to be held down" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | (.last_error // \"\") | test(\"held down|prefix\"; \"i\")'" + dut_lacks 10.2.0.0/24 || fail "routes of a peer over its limit are still there" + ok "a peer over its prefix limit is told so (Cease / Maximum Number of Prefixes) and held down" + + frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.8.0/24" + dut_down TERM + dut_config + dut_up + converged + ok "back to normal" +} + +# --- main ----------------------------------------------------------------------- + +case "${1:-test}" in + down) down; exit 0 ;; + up) lab_up; converged; say "the lab is up; work files in $work"; exit 0 ;; + test) ;; + *) fail "usage: $0 [test|up|down]" ;; +esac + +on_exit() { + status=$? + if [ "$status" -ne 0 ]; then + say "--- the daemon's last words:" >&2 + tail -n 25 "$work/dut.log" >&2 || true + fi + [ -n "${KEEP:-}" ] || down + exit "$status" +} +trap on_exit EXIT +lab_up +say "lab: converging" +converged +for phase in $phases; do + say "phase: $phase" + "phase_$phase" +done +say "all phases passed: $phases" diff --git a/scripts/interop/real-rpki.sh b/scripts/interop/real-rpki.sh new file mode 100755 index 0000000..6983abc --- /dev/null +++ b/scripts/interop/real-rpki.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# The RTR client against a validator that has the real RPKI: what the +# daemon ends up with has to be what the validator says it has. +# +# The lab (lab.sh) gives Routinator made-up data, which says nothing about +# ASPAs (Routinator does not take those from a local file) nor about data +# of the Internet's size. This does, against a Routinator you run +# yourself, which takes a quarter of an hour and some gigabytes the first +# time: +# +# routinator --enable-aspa server --rtr 127.0.0.1:3324 --http 127.0.0.1:8324 +# scripts/interop/real-rpki.sh # those are the defaults +# scripts/interop/real-rpki.sh 127.0.0.1 3324 http://127.0.0.1:8324 +# +# The daemon is run with no peers and nothing but the validator +# configured, on a port of its own; nothing else is touched. +set -euo pipefail + +cd "$(dirname "$0")/../.." +address=${1:-127.0.0.1} +port=${2:-3324} +validator=${3:-http://127.0.0.1:8324} +bgp="${CARGO_TARGET_DIR:-target}/release/bgp" +[ -x "$bgp" ] || { echo "$bgp not found: cargo build --release -p bgp" >&2; exit 1; } +work=$(mktemp -d) +trap 'kill "$pid" 2>/dev/null || true; rm -rf "$work"' EXIT + +cat > "$work/config.textpb" < "$work/log" 2>&1 & +pid=$! + +for _ in $(seq 600); do + if curl -sf http://127.0.0.1:18080/api/rpki | jq -e '.state == "synchronised"' >/dev/null 2>&1; then + break + fi + sleep 0.2 +done +ours=$(curl -sf http://127.0.0.1:18080/api/rpki) +jq -e '.state == "synchronised"' <<<"$ours" >/dev/null || { + echo "FAILED: not synchronised: $ours" >&2 + tail -n 5 "$work/log" >&2 + exit 1 +} + +# What the validator says it has: distinct VRPs, and ASes with an ASPA. +theirs=$(curl -sf "$validator/json") +their_v4=$(jq '[.roas[] | select(.prefix | contains(":") | not) | {asn, prefix, maxLength}] | unique | length' <<<"$theirs") +their_v6=$(jq '[.roas[] | select(.prefix | contains(":")) | {asn, prefix, maxLength}] | unique | length' <<<"$theirs") +their_aspas=$(jq '[(.aspas // [])[] | .customer] | unique | length' <<<"$theirs") +our_v4=$(jq .vrps_ipv4 <<<"$ours") +our_v6=$(jq .vrps_ipv6 <<<"$ours") +our_aspas=$(jq .aspas <<<"$ours") + +echo "RTR version $(jq .protocol_version <<<"$ours"), serial $(jq .serial <<<"$ours")" +echo "IPv4 VRPs: validator $their_v4, daemon $our_v4" +echo "IPv6 VRPs: validator $their_v6, daemon $our_v6" +echo "ASPAs: validator $their_aspas, daemon $our_aspas" +# The validator may have moved on a serial between the two looks; a +# handful apart is that, more is not. +close() { [ $(( $1 > $2 ? $1 - $2 : $2 - $1 )) -le $(( $1 / 200 + 5 )) ]; } +close "$their_v4" "$our_v4" && close "$their_v6" "$our_v6" && close "$their_aspas" "$our_aspas" \ + || { echo "FAILED: the daemon does not have what the validator has" >&2; exit 1; } +[ "$our_v4" -gt 0 ] || { echo "FAILED: no data at all" >&2; exit 1; } +echo "ok: the daemon has what the validator has" diff --git a/scripts/linux-vm.sh b/scripts/linux-vm.sh index b50eecc..0ca5822 100755 --- a/scripts/linux-vm.sh +++ b/scripts/linux-vm.sh @@ -5,6 +5,7 @@ # # scripts/linux-vm.sh the kernel tests # scripts/linux-vm.sh cargo test the whole test suite, on Linux +# scripts/linux-vm.sh scripts/interop/lab.sh the lab with FRR and BIRD # scripts/linux-vm.sh bash a shell # # The machine is made on first use with Lima (https://lima-vm.io; on macOS