use std::collections::{BTreeMap, HashMap}; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use bgp_packet::message::{ AsPathAttribute, LocalPrefPathAttribute, MultiExitDiscPathAttribute, NextHopPathAttribute, OriginPathAttribute, PathAttribute, }; use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; use proptest::prelude::*; use super::*; pub(super) fn prefix(s: &str) -> IpPrefix { IpPrefix::try_from(s).unwrap() } /// What a test says about a route; everything selection looks at. #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct Route { pub(super) local_pref: Option, pub(super) path: Vec, pub(super) origin: u8, pub(super) med: Option, } impl Route { pub(super) fn via(path: &[u32]) -> Self { Self { local_pref: None, path: path.to_vec(), origin: 0, med: None, } } pub(super) fn attributes(&self) -> Vec { let mut attributes = vec![ PathAttribute::Origin(match self.origin { 0 => OriginPathAttribute::IGP, 1 => OriginPathAttribute::EGP, _ => OriginPathAttribute::INCOMPLETE, }), AsPathAttribute::from_asns(self.path.clone()), PathAttribute::NextHop(NextHopPathAttribute(Ipv4Addr::new(192, 0, 2, 1))), ]; attributes.extend( self.med .map(|med| PathAttribute::MultiExitDisc(MultiExitDiscPathAttribute(med))), ); attributes.extend( self.local_pref .map(|lp| PathAttribute::LocalPref(LocalPrefPathAttribute(lp))), ); attributes } } pub(super) fn external(number: u8) -> PeerInfo { PeerInfo { asn: 65000 + u32::from(number), router_id: Ipv4Addr::new(10, 0, 0, number), address: IpAddr::from(Ipv4Addr::new(192, 0, 2, number)), ibgp: false, } } pub(super) fn internal(number: u8) -> PeerInfo { PeerInfo { asn: 64999, ibgp: true, ..external(number) } } /// A table and the peers added to it, for tests that speak of peers by the /// order they were added in. pub(super) struct Fixture { pub(super) table: Arc, pub(super) peers: Vec, } impl Fixture { pub(super) fn new(peers: &[PeerInfo]) -> Self { // More than one shard, so that a test's prefixes are spread. Self::on(RibTable::new(4), peers) } pub(super) fn on(table: RibTable, peers: &[PeerInfo]) -> Self { let table = Arc::new(table); let peers = peers.iter().map(|p| table.add_peer(*p).unwrap()).collect(); Self { table, peers } } pub(super) fn announce(&self, peer: usize, prefix: &IpPrefix, route: &Route) -> Outcome { let attrs = self .table .attrs() .intern(&route.attributes(), None) .unwrap(); self.table.announce(self.peers[peer], &attrs, &[*prefix]) } pub(super) fn withdraw(&self, peer: usize, prefix: &IpPrefix) -> Outcome { self.table.withdraw(self.peers[peer], &[*prefix]) } /// As `announce`, from a peer that sends several routes for a prefix /// and calls this one `path_id`. pub(super) fn announce_path( &self, peer: usize, prefix: &IpPrefix, path_id: u32, route: &Route, ) -> Outcome { let attrs = self .table .attrs() .intern(&route.attributes(), None) .unwrap(); let nlri = Nlri { prefix: *prefix, path_id, }; self.table.announce(self.peers[peer], &attrs, &[nlri]) } pub(super) fn withdraw_path(&self, peer: usize, prefix: &IpPrefix, path_id: u32) -> Outcome { let nlri = Nlri { prefix: *prefix, path_id, }; self.table.withdraw(self.peers[peer], &[nlri]) } /// The best route for `prefix`: which peer (by order added) it is from /// and that peer's identifier for it. pub(super) fn best_path(&self, prefix: &IpPrefix) -> Option<(usize, u32)> { let best = self.table.routes(prefix).into_iter().find(|r| r.best)?; let peer = self.peers.iter().position(|p| *p == best.peer)?; Some((peer, best.path_id)) } /// The routes held for `prefix`, as (peer, path identifier), in order. pub(super) fn held(&self, prefix: &IpPrefix) -> Vec<(usize, u32)> { let mut held: Vec<(usize, u32)> = self .table .routes(prefix) .iter() .map(|r| { let peer = self.peers.iter().position(|p| *p == r.peer).unwrap(); (peer, r.path_id) }) .collect(); held.sort_unstable(); held } /// Which peer (by order added) the best route for `prefix` is from. fn best(&self, prefix: &IpPrefix) -> Option { let (peer, _) = self.table.best(prefix)?; self.peers.iter().position(|p| *p == peer) } } #[test] fn each_step_of_selection_decides_when_the_ones_before_tie() { let p = prefix("10.0.0.0/8"); // Peer 0 has the lower identifier, so it wins whenever nothing else // decides; each case makes peer 1's route better in one step. let cases: [(&str, [PeerInfo; 2], Route, Route); 6] = [ ( "higher LOCAL_PREF, despite a longer path", [internal(1), internal(2)], Route { local_pref: Some(100), ..Route::via(&[1]) }, Route { local_pref: Some(200), ..Route::via(&[1, 2, 3]) }, ), ( "shorter AS path", [external(1), external(2)], Route::via(&[1, 2, 3]), Route::via(&[4, 5]), ), ( "lower ORIGIN", [external(1), external(2)], Route { origin: 2, ..Route::via(&[1, 2]) }, Route { origin: 0, ..Route::via(&[3, 4]) }, ), ( "lower MED from the same neighbouring AS", [external(1), external(2)], Route { med: Some(20), ..Route::via(&[7, 2]) }, Route { med: Some(10), ..Route::via(&[7, 3]) }, ), ( "no MED counts as the lowest", [external(1), external(2)], Route { med: Some(1), ..Route::via(&[7, 2]) }, Route::via(&[7, 3]), ), ( "external over internal", [internal(1), external(2)], Route::via(&[7, 2]), Route::via(&[8, 3]), ), ]; for (case, peers, first, second) in cases { // In both orders of arrival: the choice must not depend on it. for order in [[0, 1], [1, 0]] { let fixture = Fixture::new(&peers); for peer in order { fixture.announce(peer, &p, [&first, &second][peer]); } assert_eq!(fixture.best(&p), Some(1), "{case}, arriving {order:?}"); } } // And when nothing decides: the lowest identifier. let fixture = Fixture::new(&[external(2), external(1)]); fixture.announce(0, &p, &Route::via(&[1, 2])); fixture.announce(1, &p, &Route::via(&[3, 4])); assert_eq!(fixture.best(&p), Some(1)); } /// MEDs are compared only within a neighbouring AS. Taken pair by pair /// that is not an order: A beats B, B beats C, C beats A. Whatever the /// order of arrival, the answer must be the same. #[test] fn med_is_compared_only_within_a_neighbouring_as_and_never_goes_in_circles() { let p = prefix("10.0.0.0/8"); let routes = [ // From AS 1, MED 10, and the highest identifier. Route { med: Some(10), ..Route::via(&[1, 9]) }, // From AS 2: its MED is not comparable with the others'. Route { med: Some(50), ..Route::via(&[2, 9]) }, // From AS 1, MED 20: loses to the first on MED. Route { med: Some(20), ..Route::via(&[1, 8]) }, ]; // Identifiers: the third route's peer lowest, then the second's, then // the first's. Without MED the third would win; it is knocked out by // the first, and of the two left the second has the lower identifier. let peers = [external(3), external(2), external(1)]; for order in [ [0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0], ] { let fixture = Fixture::new(&peers); for peer in order { fixture.announce(peer, &p, &routes[peer]); } assert_eq!(fixture.best(&p), Some(1), "arriving {order:?}"); } // Take away the route that knocked the third out, and the third wins. let fixture = Fixture::new(&peers); for (peer, route) in routes.iter().enumerate() { fixture.announce(peer, &p, route); } let outcome = fixture.withdraw(0, &p); assert_eq!(outcome.best_changed, 1); assert_eq!(fixture.best(&p), Some(2)); } /// A route that is not the best can still decide which is: by its MED it /// keeps another route of its neighbouring AS out. When it gets worse or /// goes, that other route is back in the running. The shortcuts must not /// skip this. #[test] fn a_route_that_was_keeping_another_out_matters_when_it_changes() { let p = prefix("10.0.0.0/8"); // Peer 0: from AS 1 with the lowest identifier, but MED 1. // Peer 1: from AS 2. Best while peer 0's route is kept out. // Peer 2: from AS 1 with MED 0, which keeps peer 0's route out. let peers = [external(1), external(2), external(3)]; let held_out = Route { med: Some(1), ..Route::via(&[1, 9]) }; let best = Route::via(&[2, 9]); let holding_out = Route { med: Some(0), ..Route::via(&[1, 8]) }; let set_up = || { let fixture = Fixture::new(&peers); fixture.announce(0, &p, &held_out); fixture.announce(1, &p, &best); fixture.announce(2, &p, &holding_out); assert_eq!(fixture.best(&p), Some(1)); fixture }; // It is replaced by a route with a longer path. let fixture = set_up(); let outcome = fixture.announce(2, &p, &Route::via(&[1, 8, 7])); assert_eq!(fixture.best(&p), Some(0)); assert_eq!(outcome.best_changed, 1); // It is withdrawn. let fixture = set_up(); let outcome = fixture.withdraw(2, &p); assert_eq!(fixture.best(&p), Some(0)); assert_eq!(outcome.best_changed, 1); // Its peer goes away. let fixture = set_up(); let outcome = fixture.table.remove_peer(fixture.peers[2]); assert_eq!(fixture.best(&p), Some(0)); assert_eq!(outcome.best_changed, 1); // Its MED gets worse than the one it was keeping out. let fixture = set_up(); fixture.announce( 2, &p, &Route { med: Some(5), ..Route::via(&[1, 8]) }, ); assert_eq!(fixture.best(&p), Some(0)); } #[test] fn what_a_change_did_is_reported() { let fixture = Fixture::new(&[external(1), external(2)]); let p = prefix("10.0.0.0/8"); let long = Route::via(&[1, 2, 3]); let short = Route::via(&[4]); let first = fixture.announce(0, &p, &long); assert_eq!((first.added, first.best_changed), (1, 1)); let worse = fixture.announce(1, &p, &Route::via(&[1, 2, 3, 4])); assert_eq!((worse.added, worse.best_changed), (1, 0)); // The same route again changes nothing. let again = fixture.announce(0, &p, &long); assert_eq!((again.replaced, again.best_changed), (1, 0)); // The best route replaced by a better one is still a change of best. let better = fixture.announce(0, &p, &short); assert_eq!((better.replaced, better.best_changed), (1, 1)); assert_eq!((fixture.table.len(), fixture.table.prefix_count()), (2, 1)); assert_eq!(fixture.withdraw(1, &p).best_changed, 0); let last = fixture.withdraw(0, &p); assert_eq!((last.removed, last.best_changed), (1, 1)); assert_eq!(fixture.withdraw(0, &p), Outcome::default(), "nothing there"); assert!(fixture.table.is_empty()); assert_eq!(fixture.best(&p), None); } #[test] fn removing_a_peer_takes_its_routes_and_frees_its_number() { let fixture = Fixture::new(&[external(1), external(2)]); let prefixes: Vec = (0..200u32) .map(|i| { IpPrefix::new( bgp_primitives::constants::IpVersion::Ipv4, (i << 8).to_be_bytes(), 24, ) .unwrap() }) .collect(); let attrs = |route: &Route| { fixture .table .attrs() .intern(&route.attributes(), None) .unwrap() }; // Peer 0 is best everywhere; peer 1 has only the first hundred. fixture .table .announce(fixture.peers[0], &attrs(&Route::via(&[1])), &prefixes); fixture.table.announce( fixture.peers[1], &attrs(&Route::via(&[2, 3])), &prefixes[..100], ); assert_eq!(fixture.table.len(), 300); let outcome = fixture.table.remove_peer(fixture.peers[0]); assert_eq!(outcome.removed, 200); assert_eq!(outcome.best_changed, 200); assert_eq!( (fixture.table.len(), fixture.table.prefix_count()), (100, 100) ); assert_eq!(fixture.best(&prefixes[0]), Some(1)); assert_eq!(fixture.best(&prefixes[150]), None); // Its routes are gone, so its attribute set can be dropped. assert_eq!(fixture.table.attrs().sweep(), 1); // And its number is given to the next peer. assert_eq!(fixture.table.add_peer(external(3)), Ok(fixture.peers[0])); } #[test] fn readers_see_every_route_and_which_is_best() { let fixture = Fixture::new(&[external(1), external(2)]); let (a, b) = (prefix("10.0.0.0/8"), prefix("2001:db8::/32")); fixture.announce(0, &a, &Route::via(&[1, 2])); fixture.announce(1, &a, &Route::via(&[3])); fixture.announce(0, &b, &Route::via(&[1, 2])); let mut best = Vec::new(); fixture .table .for_each_best(|prefix, peer, _| best.push((*prefix, peer))); best.sort_by_key(|(prefix, _)| prefix.to_string()); assert_eq!(best, [(a, fixture.peers[1]), (b, fixture.peers[0])]); let mut all = 0; let mut marked_best = 0; fixture.table.for_each_route(|route| { all += 1; marked_best += usize::from(route.best); }); assert_eq!((all, marked_best), (3, 2)); let routes = fixture.table.routes(&a); assert_eq!(routes.len(), 2); assert!( routes .iter() .any(|route| route.peer == fixture.peers[1] && route.best) ); // Routes with the same attributes share one set. let shared = &fixture.table.routes(&b)[0].attrs; assert!(routes.iter().any(|route| Arc::ptr_eq(&route.attrs, shared))); } /// With ADD-PATH a peer sends several routes for one prefix and tells them /// apart by a path identifier. Each is a route of its own: kept, replaced /// and withdrawn by prefix and identifier together. #[test] fn a_peer_can_have_several_routes_for_a_prefix() { let fixture = Fixture::new(&[external(1), external(2)]); let p = prefix("10.0.0.0/8"); fixture.announce_path(0, &p, 1, &Route::via(&[1, 2, 3])); let second = fixture.announce_path(0, &p, 2, &Route::via(&[1, 4])); assert_eq!( (second.added, second.replaced, second.best_changed), (1, 0, 1) ); fixture.announce(1, &p, &Route::via(&[2, 5, 6, 7])); assert_eq!(fixture.held(&p), [(0, 1), (0, 2), (1, 0)]); assert_eq!(fixture.best_path(&p), Some((0, 2)), "the shorter path"); assert_eq!((fixture.table.len(), fixture.table.prefix_count()), (3, 1)); // Announcing a path again replaces that path and no other. let replaced = fixture.announce_path(0, &p, 1, &Route::via(&[1])); assert_eq!((replaced.added, replaced.replaced), (0, 1)); assert_eq!(fixture.held(&p), [(0, 1), (0, 2), (1, 0)]); assert_eq!(fixture.best_path(&p), Some((0, 1))); // Withdrawing one leaves the peer's other. assert_eq!(fixture.withdraw_path(0, &p, 1).removed, 1); assert_eq!(fixture.held(&p), [(0, 2), (1, 0)]); assert_eq!(fixture.best_path(&p), Some((0, 2))); // A path that is not there is not withdrawn: not the same prefix // under another identifier, and not the peer's plain route. assert_eq!(fixture.withdraw_path(0, &p, 9), Outcome::default()); assert_eq!(fixture.withdraw(0, &p), Outcome::default()); // When the session ends, all of the peer's routes go. fixture.announce_path(0, &p, 3, &Route::via(&[1, 8, 9])); let outcome = fixture.table.remove_peer(fixture.peers[0]); assert_eq!((outcome.removed, outcome.best_changed), (2, 1)); assert_eq!(fixture.held(&p), [(1, 0)]); } /// Two routes from one peer that nothing else tells apart: the lower path /// identifier is chosen, whichever arrived first. #[test] fn equal_routes_from_one_peer_are_chosen_by_path_identifier() { let p = prefix("10.0.0.0/8"); for order in [[5, 3], [3, 5]] { let fixture = Fixture::new(&[external(1)]); for path_id in order { fixture.announce_path(0, &p, path_id, &Route::via(&[1, path_id])); } assert_eq!(fixture.best_path(&p), Some((0, 3)), "arriving {order:?}"); } } // --------------------------------------------------------------------------- // The table against a model // --------------------------------------------------------------------------- /// Route selection written the plain way: all routes of a prefix, step by /// step, with no shortcuts. The table takes shortcuts and must agree. /// Routes are known by peer and path identifier. fn chosen_plainly( routes: &BTreeMap<(usize, u32), Route>, peers: &[PeerInfo], ) -> Option<(usize, u32)> { let local_pref = |peer: usize, route: &Route| { if peers[peer].ibgp { route.local_pref.unwrap_or(DEFAULT_LOCAL_PREF) } else { DEFAULT_LOCAL_PREF } }; let mut left: Vec<((usize, u32), &Route)> = routes.iter().map(|(k, route)| (*k, route)).collect(); let most = left.iter().map(|(k, r)| local_pref(k.0, r)).max()?; left.retain(|(k, r)| local_pref(k.0, r) == most); let shortest = left.iter().map(|(_, r)| r.path.len()).min()?; left.retain(|(_, r)| r.path.len() == shortest); let lowest = left.iter().map(|(_, r)| r.origin).min()?; left.retain(|(_, r)| r.origin == lowest); let neighbor = |route: &Route| route.path.first().copied().unwrap_or(0); let med = |route: &Route| route.med.unwrap_or(0); let mut lowest_med: HashMap = HashMap::new(); for (_, route) in &left { let entry = lowest_med.entry(neighbor(route)).or_insert(u32::MAX); *entry = (*entry).min(med(route)); } left.retain(|(_, r)| med(r) == lowest_med[&neighbor(r)]); left.into_iter() .map(|(key, _)| key) .min_by_key(|&(peer, path_id)| { ( peers[peer].ibgp, peers[peer].router_id, peers[peer].address, peer, path_id, ) }) } #[derive(Debug, Clone)] enum Op { Announce { peer: usize, prefix: usize, path_id: u32, route: Route, }, Withdraw { peer: usize, prefix: usize, path_id: u32, }, /// The session goes down and comes straight back. Flap { peer: usize }, } const PEERS: usize = 6; // Few prefixes, so that each gathers routes from most peers. const PREFIXES: usize = 2; // Most routes have the identifier zero, as without ADD-PATH; some peers // send a second and third route. const PATH_IDS: u32 = 3; pub(super) fn route() -> impl Strategy { // Few values of everything, so that ties at every step are common: // ties are where the shortcuts could be wrong. ( proptest::option::of(100u32..102), proptest::collection::vec(1u32..3, 1..3), 0u8..2, proptest::option::of(0u32..3), ) .prop_map(|(local_pref, path, origin, med)| Route { local_pref, path, origin, med, }) } fn path_id() -> impl Strategy { prop_oneof![3 => Just(0), 2 => 1..PATH_IDS] } fn op() -> impl Strategy { prop_oneof![ 6 => (0..PEERS, 0..PREFIXES, path_id(), route()).prop_map( |(peer, prefix, path_id, route)| Op::Announce { peer, prefix, path_id, route } ), 3 => (0..PEERS, 0..PREFIXES, path_id()) .prop_map(|(peer, prefix, path_id)| Op::Withdraw { peer, prefix, path_id }), 1 => (0..PEERS).prop_map(|peer| Op::Flap { peer }), ] } proptest! { #![proptest_config(ProptestConfig::with_cases(2000))] /// After any sequence of changes the table holds exactly the routes it /// should, has chosen for every prefix what plain selection chooses, /// and has reported a change of best route exactly when there was one. #[test] fn the_table_agrees_with_plain_selection(ops in proptest::collection::vec(op(), 1..80)) { // Internal and external peers, and identifiers that are not in the // order the peers are added in. let infos: Vec = (0..PEERS) .map(|i| { let number = (PEERS - i) as u8; if i % 3 == 0 { internal(number) } else { external(number) } }) .collect(); let mut fixture = Fixture::new(&infos); let prefixes: Vec = (0..PREFIXES) .map(|i| prefix(&format!("10.{i}.0.0/16"))) .collect(); let mut model: Vec> = vec![BTreeMap::new(); PREFIXES]; for op in ops { let before: Vec> = model .iter() .map(|routes| chosen_plainly(routes, &infos).map(|key| (key, routes[&key].clone()))) .collect(); let outcome = match &op { Op::Announce { peer, prefix, path_id, route } => { model[*prefix].insert((*peer, *path_id), route.clone()); fixture.announce_path(*peer, &prefixes[*prefix], *path_id, route) } Op::Withdraw { peer, prefix, path_id } => { model[*prefix].remove(&(*peer, *path_id)); fixture.withdraw_path(*peer, &prefixes[*prefix], *path_id) } Op::Flap { peer } => { for routes in &mut model { routes.retain(|(from, _), _| from != peer); } let outcome = fixture.table.remove_peer(fixture.peers[*peer]); fixture.peers[*peer] = fixture.table.add_peer(infos[*peer]).unwrap(); outcome } }; let mut best_changes = 0; for (i, routes) in model.iter().enumerate() { let expected = chosen_plainly(routes, &infos); prop_assert_eq!(fixture.best_path(&prefixes[i]), expected, "after {:?}", &op); let now = expected.map(|key| (key, routes[&key].clone())); best_changes += usize::from(now != before[i]); prop_assert_eq!( fixture.held(&prefixes[i]), routes.keys().copied().collect::>() ); } prop_assert_eq!(outcome.best_changed, best_changes, "after {:?}", &op); prop_assert_eq!( fixture.table.len(), model.iter().map(BTreeMap::len).sum::() ); } } } /// Shards and lock stripes are picked by bits of the hash that the hash /// table inside does not use, so that keys which share a shard still /// differ where that table looks. #[test] fn keys_of_one_shard_differ_in_the_bits_their_table_uses() { let hasher = RandomState::default(); let shards = 64; let mut counts = vec![0usize; shards]; let mut tags = vec![std::collections::HashSet::new(); shards]; for n in 0..64_000u32 { let hash = hasher.hash_one(n); let shard = spread(hash, shards); counts[shard] += 1; // The seven bits hashbrown keeps as a tag. tags[shard].insert(hash >> 57); } for shard in 0..shards { assert!( (700..1300).contains(&counts[shard]), "shard {shard} has {} of 64000 keys", counts[shard] ); assert_eq!(tags[shard].len(), 128, "shard {shard}"); } assert_eq!(spread(u64::MAX, 3), 2); assert_eq!(spread(0, 3), 0); } /// A peer that sends several routes for a prefix can send more than fit a /// sixteen-bit index. The best route was kept as one, and wrapped. #[test] fn a_row_can_hold_more_routes_than_a_sixteen_bit_index_counts() { let store = AttrStore::new(); let worse = store .intern(&Route::via(&[1, 2]).attributes(), None) .unwrap(); let better = store.intern(&Route::via(&[1]).attributes(), None).unwrap(); let info = external(1); let peers = [Some(info)]; let peer = PeerIndex(0); let path = |attrs: &Arc| Path::new(select::preference(attrs.key(), &info), peer, attrs); let seqs = journal::Journals::new(journal::Limits::default()).appeared(prefix("10.0.0.0/8")); let mut row = Row::new(prefix("10.0.0.0/8"), path(&worse), 1, seqs); let mut scratch = select::Scratch::default(); let count = usize::from(u16::MAX) + 10; for path_id in 2..=count as u32 { row.push(path(&worse), path_id); } // Equal routes from one peer: the lowest identifier is the best. assert!(!select::after_add(&mut row, &peers, &mut scratch)); assert_eq!(row.best(), 0); row.push(path(&better), count as u32 + 1); assert!(select::after_add(&mut row, &peers, &mut scratch)); assert_eq!(row.best(), count); assert_eq!(row.route(row.best()).path_id, count as u32 + 1); // With the better one gone, the choice among all the rest is the // lowest identifier again. assert!(select::after_remove(&mut row, count, &peers, &mut scratch)); assert_eq!(row.route(row.best()).path_id, 1); }