diff --git a/README.md b/README.md index b1a9740..5b76717 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ What it does: - TCP-AO, TCP MD5, and GTSM - Long-Lived Graceful Restart, BFD, and configuration changes without a restart -- Recursive resolution of next hops that are not directly connected - Address families other than IPv4 and IPv6 unicast ## Building diff --git a/crates/fib/src/kernel.rs b/crates/fib/src/kernel.rs index 7e414a6..8e54767 100644 --- a/crates/fib/src/kernel.rs +++ b/crates/fib/src/kernel.rs @@ -29,6 +29,8 @@ use crate::netlink::{ use crate::plan::{ HashMap as FastHashMap, Neighbor, Op, Plan, Target, Unresolved, resolve_next_hop, }; +use crate::routes::KernelRoutes; +use crate::topology::Topology; use crate::{Fib, FibConfig, FibError, Stop}; /// How many best routes are taken from the routing table at a time. @@ -70,6 +72,11 @@ const RECEIVE_BUFFER: usize = 8 * 1024 * 1024; /// not at all. const ANSWER_TIMEOUT: Duration = Duration::from_secs(10); +/// How much room the socket that hints at route changes has: enough for +/// a few hundred messages, and no more, because what it hears is never +/// read. See `Sockets::hints`. +const HINT_BUFFER: usize = 64 * 1024; + /// The errors of the kernel that have a meaning here. mod errno { pub const EPERM: i32 = 1; @@ -128,20 +135,21 @@ struct Socket { impl Socket { /// A socket that is sent the notifications of `groups`. fn open(groups: u32) -> io::Result { + Self::open_with_room(groups, RECEIVE_BUFFER) + } + + /// Opens a socket with `room` octets of receive buffer, as far as the + /// kernel allows. + fn open_with_room(groups: u32, room: usize) -> io::Result { let mut socket = TokioSocket::new(NETLINK_ROUTE)?; socket.socket_mut().bind(&SocketAddr::new(0, groups))?; // Room for the answers to a whole write, and answers that do not // repeat the request. Both are for the better and neither is // needed: what is lost for want of room is noticed and made good. let raw = socket.socket_ref(); - if nix::sys::socket::setsockopt( - raw, - nix::sys::socket::sockopt::RcvBufForce, - &RECEIVE_BUFFER, - ) - .is_err() + if nix::sys::socket::setsockopt(raw, nix::sys::socket::sockopt::RcvBufForce, &room).is_err() { - let _ = raw.set_rx_buf_sz(RECEIVE_BUFFER); + let _ = raw.set_rx_buf_sz(room); } let _ = raw.set_cap_ack(true); Ok(Self { @@ -220,7 +228,7 @@ impl Socket { 0 => Ok(()), errno => Err(FibError::Dump { what: match what { - Dump::Routes(_) => "its routes", + Dump::Routes(_) | Dump::RoutesIn { .. } => "its routes", Dump::Addresses => "its addresses", Dump::Links => "its interfaces", Dump::Nexthops => "its nexthop objects", @@ -307,6 +315,17 @@ pub(crate) struct Sockets { /// For what the kernel tells unasked: changes to the interfaces, /// their addresses, and nexthop objects. news: Socket, + /// For asking which routes the resolution table has, and only that: + /// with strict checking on, the kernel answers with that table alone + /// rather than with every table it has, which may be a million routes + /// of ours. `None` on a kernel from before it could (Linux 4.20). + resolution: Option, + /// Told of every change to any route, as a hint that the resolution + /// table may have changed: what it says is never read, only that it + /// said something. The kernel tells of our own routes going in too, a + /// million at a time, so this socket has little room and is expected + /// to overflow; an overflow is as good a hint as a message. + hints: Socket, } impl Sockets { @@ -318,10 +337,18 @@ impl Sockets { .socket .socket_ref() .add_membership(netlink::GROUP_NEXTHOP); + let resolution = Socket::open(0)?; + let strict = resolution + .socket + .socket_ref() + .set_netlink_get_strict_chk(true) + .is_ok(); Ok(Self { control: Socket::open(0)?, routes: Socket::open(0)?, news, + resolution: strict.then_some(resolution), + hints: Socket::open_with_room(group::IPV4_ROUTE | group::IPV6_ROUTE, HINT_BUFFER)?, }) } } @@ -427,17 +454,22 @@ struct Task { fib: Fib, config: FibConfig, control: Socket, + /// See `Sockets::resolution`. + resolution: Option, /// Where changes to routes go to be written; `None` once they have /// all been handed over, at the end. queue: Option>, writer: JoinHandle>, - interfaces: Interfaces, + /// The kernel's view of the machine, as far as next hops are + /// concerned: interfaces, the routes that say the way to a next hop, + /// and nexthop objects. + topology: Topology, /// What each next hop seen so far resolves to, by the next hop, the /// IP version of the routes, and for a link-local one the peer. /// There are as many next hops as neighbors, and a million routes - /// share them; looking through the interfaces for each route would - /// take as long as there are addresses. Emptied when the interfaces - /// change. + /// share them; looking through the interfaces and routes for each + /// route would take as long as there are of those. Emptied when the + /// topology changes. resolved: FastHashMap>, plan: Plan, /// The kernel's nexthop objects that our routes name, by where they @@ -470,7 +502,9 @@ impl Task { && route.metric == self.config.metric } - /// Learns the machine's interfaces and addresses anew. + /// Learns the machine's interfaces and addresses anew. The routes + /// and nexthop objects are learned where the kernel is asked for them + /// anyway: `kernel_routes` and `kernel_nexthops`. async fn learn_interfaces(&mut self) -> Result<(), FibError> { let mut interfaces = Interfaces::default(); for what in [Dump::Links, Dump::Addresses] { @@ -480,18 +514,18 @@ impl Task { }) .await?; } - self.interfaces = interfaces; + self.topology.interfaces = interfaces; self.resolved.clear(); Ok(()) } - /// Asks which nexthop objects of BGP's the kernel has, and returns - /// those that lead somewhere, by number. + /// Asks which nexthop objects the kernel has. Every one goes into + /// the topology, where a route of the resolution table may name it; + /// those of BGP's that lead somewhere are returned by number, for + /// finding our routes among the kernel's. async fn kernel_nexthops(&mut self) -> Result, FibError> { let mut by_id = HashMap::new(); - if self.nexthops.is_none() { - return Ok(by_id); - } + let mut all = FastHashMap::default(); let asked = self .control .dump(Dump::Nexthops, |message| { @@ -499,10 +533,14 @@ impl Task { present: true, nexthop, } = message - && nexthop.protocol == PROTOCOL_BGP - && let (Some(gateway), Some(interface)) = (nexthop.gateway, nexthop.interface) { - by_id.insert(nexthop.id, Target { gateway, interface }); + all.insert(nexthop.id, nexthop); + if nexthop.protocol == PROTOCOL_BGP + && let (Some(gateway), Some(interface)) = + (nexthop.gateway, nexthop.interface) + { + by_id.insert(nexthop.id, Target { gateway, interface }); + } } }) .await; @@ -510,14 +548,18 @@ impl Task { Ok(()) => {} // A kernel from before it had them (Linux 5.3). Err(FibError::Dump { errno, .. }) => { - info!( - errno, - "The kernel has no nexthop objects; routes will carry their own gateway" - ); + if self.nexthops.is_some() { + info!( + errno, + "The kernel has no nexthop objects; routes will carry their own gateway" + ); + } self.nexthops = None; } Err(e) => return Err(e), } + self.topology.nexthops = all; + self.resolved.clear(); Ok(by_id) } @@ -553,6 +595,36 @@ impl Task { Ok(found) } + /// Asks the kernel for the routes of the resolution table anew, and + /// returns whether they differ from those known. Asked of the kernel + /// with strict checking where it has that, which gives the one table; + /// otherwise every table is asked for and all but the one is thrown + /// away. + async fn learn_routes(&mut self) -> Result { + let table = self.topology.routes.table(); + let mut routes = KernelRoutes::new(table); + for version in [IpVersion::Ipv4, IpVersion::Ipv6] { + let each = |message: Message| { + routes.apply(&message); + }; + match &mut self.resolution { + Some(socket) => socket.dump(Dump::RoutesIn { version, table }, each).await?, + None => self.control.dump(Dump::Routes(version), each).await?, + } + } + if routes == self.topology.routes { + return Ok(false); + } + debug!( + table, + routes = routes.len(), + "The routes that say the way to next hops changed" + ); + self.topology.routes = routes; + self.resolved.clear(); + Ok(true) + } + /// Finds the routes of ours that the kernel has from before. async fn adopt_routes( &mut self, @@ -720,7 +792,7 @@ impl Task { Op::Install { prefix, target } => format!( "install {prefix} via {} on {}", target.gateway, - self.interfaces.link(target.interface).map_or_else( + self.topology.interfaces.link(target.interface).map_or_else( || format!("interface {}", target.interface), |link| link.name.clone() ) @@ -800,10 +872,10 @@ impl Task { address: peer.address, interface: peer.interface, }); - let interfaces = &self.interfaces; + let topology = &self.topology; // Asked of every route, and not remembered with the // next hop: it is about the prefix. - let target = if interfaces.has_network_around(&prefix) { + let target = if topology.interfaces.has_network_around(&prefix) { Err(Unresolved::OwnNetwork) } else { *self @@ -811,7 +883,7 @@ impl Task { .entry((next_hop, prefix.address_family(), peer)) .or_insert_with(|| { resolve_next_hop( - interfaces, + topology, prefix.address_family(), next_hop.as_ref(), peer, @@ -886,7 +958,11 @@ impl Task { while !rest.is_empty() { let (_, message, used) = decode(rest)?; rest = &rest[used..]; - if self.interfaces.apply(&message) { + // A nexthop object of BGP's is one of ours, or another + // daemon's: never the way to a next hop, and made by the + // hundred as a table goes in. Only the others matter here. + let of_bgp = matches!(&message, Message::Nexthop { nexthop, .. } if nexthop.protocol == PROTOCOL_BGP); + if self.topology.apply(&message) && !of_bgp { self.resolved.clear(); changed = true; } @@ -917,6 +993,7 @@ impl Task { self.flush(heard).await?; self.synced = false; self.plan = Plan::default(); + self.learn_routes().await?; let nexthops = self.kernel_nexthops().await?; // While routes of an earlier run are being kept for the peers to // announce again, they are kept through this too. @@ -991,8 +1068,8 @@ impl Task { UNRESOLVED.set(self.plan.unresolved() as i64); REFUSED.set(self.plan.refused() as i64); let interfaces = interfaces_changed.then(|| { - self.fib.publish_interfaces(&self.interfaces); - self.interfaces.status() + self.fib.publish_topology(&self.topology); + self.topology.interfaces.status() }); self.fib.update(|status| { status.synced = self.synced; @@ -1039,6 +1116,8 @@ pub(crate) async fn run( control, routes, mut news, + resolution, + mut hints, } = sockets; let (queue, batches) = mpsc::channel(QUEUE); let (reports, mut heard) = mpsc::unbounded_channel(); @@ -1048,9 +1127,10 @@ pub(crate) async fn run( rib, fib, control, + resolution, queue: Some(queue), writer: tokio::spawn(write(routes, batches, reports)), - interfaces: Interfaces::default(), + topology: Topology::new(config.resolution_table, config.resolve_via_default), resolved: FastHashMap::default(), plan: Plan::default(), nexthops: config.nexthop_objects.then(FastHashMap::default), @@ -1064,6 +1144,7 @@ pub(crate) async fn run( // The socket for news was bound before this: what changes while the // interfaces are being learned is told there. task.learn_interfaces().await?; + task.learn_routes().await?; let nexthops = task.kernel_nexthops().await?; // Kept for a while, if there is to be any keeping: our table is empty // until the peers have sent theirs again. @@ -1090,7 +1171,10 @@ pub(crate) async fn run( table = config.table, metric = config.metric, nexthop_objects = task.nexthops.is_some(), - interfaces = task.interfaces.links().count(), + interfaces = task.topology.interfaces.links().count(), + resolution_table = config.resolution_table, + routes_to_next_hops = task.topology.routes.len(), + filtered_dumps = task.resolution.is_some(), "Installing best routes in the kernel" ); @@ -1098,17 +1182,44 @@ pub(crate) async fn run( // kernel in step; starting a new one does it over. let mut reader = task.rib.reader(); let mut again = Again::default(); + // When the routes that say the way to next hops are next to be asked + // for again, if they are. + let mut routes_again = Again::default(); loop { tokio::select! { + hint = hints.read() => { + match hint { + // Something about some route changed, or so much did + // that the socket overflowed: either way, ask. + Ok(_) => routes_again.soon(), + Err(e) if e.raw_os_error() == Some(errno::ENOBUFS) => routes_again.soon(), + Err(e) => return Err(e.into()), + } + } + () = at(routes_again.due()), if routes_again.due().is_some() => { + routes_again = Again::default(); + if task.learn_routes().await? { + // The way to a next hop may be another now: every + // route is worked out again, which asks the kernel + // only for what comes out different. + info!( + routes = task.topology.routes.len(), + "The kernel's routes changed; working out next hops again" + ); + reader = task.rib.reader(); + task.publish(true); + } + } () = reader.changed() => { task.follow(&mut reader).await?; task.publish(false); } Some(report) = heard.recv() => { - if task.hear(report).await? { + let changed = task.hear(report).await?; + if changed { again.soon(); } - task.publish(false); + task.publish(changed); } read = news.read() => { if task.on_news(read).await? { @@ -1123,6 +1234,7 @@ pub(crate) async fn run( info!("Working the kernel's routes out again"); task.relearn(&mut heard).await?; reader = task.rib.reader(); + task.publish(true); } _ = async { tokio::select! { diff --git a/crates/fib/src/lib.rs b/crates/fib/src/lib.rs index eddcbb7..64c2d22 100644 --- a/crates/fib/src/lib.rs +++ b/crates/fib/src/lib.rs @@ -9,7 +9,9 @@ //! //! - [`netlink`] is the kernel's messages, as bytes and back. //! - [`interfaces`] is the machine's interfaces and what is reachable on -//! them. +//! them; [`routes`] the kernel's own routes, which say the way to what +//! is not; [`topology`] the two together with the kernel's nexthop +//! objects, as one view. //! - [`plan`] works out what to ask the kernel for. //! - The rest, which only Linux has, asks it. //! @@ -25,17 +27,23 @@ //! //! # What is installed //! -//! A best route whose next hop is a neighbor: an address in the network -//! of one of the machine's interfaces. A route whose next hop is further -//! away than that, as with routes from a peer in our own AS that did not -//! set itself as next hop, would need the way to the next hop looked up in -//! turn, which is not done: the prefix then has no route in the kernel. +//! A best route whose next hop can be reached: a neighbor, which is an +//! address in the network of one of the machine's interfaces, or an +//! address the kernel's own routes know the way to. In the second case +//! the route goes into the kernel with the gateway and interface of the +//! route that knows the way, since the kernel does not look a gateway up +//! in turn: a route whose next hop is a router of our own AS, reached over +//! a tunnel by a static route, is installed as a route into the tunnel. +//! Which of the kernel's routes count for this, and which never do, is the +//! business of [`routes`]. pub mod interfaces; #[cfg(target_os = "linux")] mod kernel; pub mod netlink; pub mod plan; +pub mod routes; +pub mod topology; use std::net::IpAddr; use std::sync::{Arc, Mutex, PoisonError}; @@ -71,6 +79,12 @@ pub struct FibConfig { /// everything at once would stop the machine forwarding meanwhile. /// After this long, what has not been announced again goes. pub stale_time: Duration, + /// The kernel's table whose routes say the way to a next hop that is + /// not a neighbor: normally `main`, with the connected and static + /// routes. BGP's own routes in it never count; see [`routes`]. + pub resolution_table: u32, + /// Whether the default route counts as the way to a next hop. + pub resolve_via_default: bool, } impl Default for FibConfig { @@ -80,6 +94,8 @@ impl Default for FibConfig { metric: DEFAULT_METRIC, nexthop_objects: true, stale_time: DEFAULT_STALE_TIME, + resolution_table: netlink::TABLE_MAIN, + resolve_via_default: false, } } } @@ -135,9 +151,10 @@ pub struct Status { pub interfaces: Vec, } -/// The machine's interfaces as the kernel last described them; `None` -/// until it has. Whoever holds one is told when they change. -pub type InterfacesWatch = watch::Receiver>>; +/// The machine's interfaces, routes and nexthop objects as the kernel +/// last described them, as far as next hops are concerned; `None` until +/// it has. Whoever holds one is told when they change. +pub type TopologyWatch = watch::Receiver>>; /// How the task that keeps the kernel's routes is to end. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -151,7 +168,7 @@ enum Stop { struct Shared { config: FibConfig, status: Mutex, - interfaces: watch::Sender>>, + topology: watch::Sender>>, stop: watch::Sender>, /// How many routes of an earlier run were found in the kernel at the /// start, once that has been looked at. @@ -185,7 +202,7 @@ impl Fib { shared: Arc::new(Shared { config, status: Mutex::default(), - interfaces: watch::channel(None).0, + topology: watch::channel(None).0, stop, adopted: watch::channel(None).0, release: watch::channel(false).0, @@ -200,17 +217,18 @@ impl Fib { self.shared.config } - /// The machine's interfaces, for those who want to know whether a - /// next hop can be reached before the kernel is asked to use it. - pub fn interfaces(&self) -> InterfacesWatch { - self.shared.interfaces.subscribe() + /// The kernel's view of the machine, for those who want to know + /// whether a next hop can be reached before the kernel is asked to use + /// it. + pub fn topology(&self) -> TopologyWatch { + self.shared.topology.subscribe() } #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - fn publish_interfaces(&self, interfaces: &interfaces::Interfaces) { + fn publish_topology(&self, topology: &topology::Topology) { self.shared - .interfaces - .send_replace(Some(Arc::new(interfaces.clone()))); + .topology + .send_replace(Some(Arc::new(topology.clone()))); } pub fn status(&self) -> Status { diff --git a/crates/fib/src/netlink.rs b/crates/fib/src/netlink.rs index c30abf4..8da2e41 100644 --- a/crates/fib/src/netlink.rs +++ b/crates/fib/src/netlink.rs @@ -62,7 +62,9 @@ mod flag { pub mod group { pub const LINK: u32 = 0x1; pub const IPV4_ADDRESS: u32 = 0x10; + pub const IPV4_ROUTE: u32 = 0x40; pub const IPV6_ADDRESS: u32 = 0x100; + pub const IPV6_ROUTE: u32 = 0x400; } /// The number of the group that is told of changes to nexthop objects @@ -467,7 +469,15 @@ pub enum Change { /// What to ask for all of. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Dump { + /// Every route of the IP version, of every table. Routes(IpVersion), + /// The routes of one table. The kernel honours the table only for a + /// socket with strict checking on (`NETLINK_GET_STRICT_CHK`, Linux + /// 4.20 and later); otherwise it answers as for [`Dump::Routes`]. + RoutesIn { + version: IpVersion, + table: u32, + }, Addresses, Links, Nexthops, @@ -619,8 +629,24 @@ pub fn nexthop_removal(out: &mut BytesMut, sequence: u32, id: u32) { /// Writes the request for all of `what` to the end of `out`. The answer is /// a message for each, and then [`Message::Done`]. pub fn dump_request(out: &mut BytesMut, sequence: u32, what: Dump) { + if let Dump::RoutesIn { version, table } = what { + let start = begin(out, message_type::GET_ROUTE, flag::DUMP, sequence); + // rtmsg: family, destination length, source length, TOS, table, + // protocol, scope, type, and four octets of flags; the table in + // its octet if it fits, else as an attribute. + out.put_u8(family_of(version)); + out.put_bytes(0, 3); + out.put_u8(u8::try_from(table).unwrap_or(TABLE_IN_ATTRIBUTE)); + out.put_bytes(0, 7); + if u8::try_from(table).is_err() { + put_attribute(out, route_attribute::TABLE, &table.to_ne_bytes()); + } + finish(out, start); + return; + } let (message_type, family, fixed_length) = match what { Dump::Routes(version) => (message_type::GET_ROUTE, family_of(version), 12), + Dump::RoutesIn { .. } => unreachable!("dealt with above"), Dump::Addresses => (message_type::GET_ADDRESS, 0, 8), Dump::Links => (message_type::GET_LINK, 0, 16), Dump::Nexthops => (message_type::GET_NEXTHOP, 0, 8), @@ -883,6 +909,42 @@ mod tests { } } + /// A dump of one table names the table in its octet, or, for a number + /// the octet cannot hold, in an attribute. + #[test] + fn a_dump_of_one_table_names_the_table() { + let mut wire = BytesMut::new(); + dump_request( + &mut wire, + 3, + Dump::RoutesIn { + version: IpVersion::Ipv4, + table: TABLE_MAIN, + }, + ); + assert_eq!(wire.len(), 28); + assert_eq!(wire[16], 2, "AF_INET"); + assert_eq!(wire[20], TABLE_MAIN as u8); + + let mut wire = BytesMut::new(); + dump_request( + &mut wire, + 3, + Dump::RoutesIn { + version: IpVersion::Ipv6, + table: 1000, + }, + ); + // The fixed part, then an attribute of four octets with its + // four-octet header. + assert_eq!(wire.len(), 36); + assert_eq!(wire[16], 10, "AF_INET6"); + assert_eq!(wire[20], TABLE_IN_ATTRIBUTE); + assert_eq!(wire[28..30], 8u16.to_ne_bytes()); + assert_eq!(wire[30..32], route_attribute::TABLE.to_ne_bytes()); + assert_eq!(wire[32..36], 1000u32.to_ne_bytes()); + } + /// A message as the kernel would send it: header, fixed part, /// attributes. fn message(message_type: u16, sequence: u32, fixed: &[u8], attrs: &[(u16, &[u8])]) -> Vec { diff --git a/crates/fib/src/plan.rs b/crates/fib/src/plan.rs index 1a142fc..6002b54 100644 --- a/crates/fib/src/plan.rs +++ b/crates/fib/src/plan.rs @@ -9,7 +9,8 @@ use bgp_packet::message::NlriNextHop; use bgp_primitives::constants::IpVersion; use bgp_primitives::ip_prefix::IpPrefix; -use crate::interfaces::Interfaces; +use crate::netlink::KIND_UNICAST; +use crate::topology::Topology; // Asked for every route: a quick hash matters here. pub(crate) type HashMap = std::collections::HashMap; @@ -32,10 +33,17 @@ pub enum Unresolved { OtherFamily, /// The next hop is one of this machine's own addresses. Ourselves, - /// The next hop is not in the network of any interface that is up. - /// Finding the way to such a next hop through other routes is not - /// done. - NotConnected, + /// The next hop is not in the network of any interface that is up, + /// and none of the kernel's routes that count says the way to it + /// (see `crate::routes`). + NoRoute, + /// The only route to the next hop is the default route, which is not + /// taken as the way to a next hop unless asked for + /// (`Topology::via_default`). + OnlyDefault, + /// The route to the next hop is one that discards packets: a + /// blackhole, unreachable or prohibit route. + Discarded, /// The route is for a network this machine is on itself, or for part /// of one. The kernel reaches that directly; a route from a peer that /// said otherwise would send the neighbors' traffic, and the sessions @@ -44,11 +52,13 @@ pub enum Unresolved { } impl Unresolved { - pub const ALL: [Self; 5] = [ + pub const ALL: [Self; 7] = [ Self::Missing, Self::OtherFamily, Self::Ourselves, - Self::NotConnected, + Self::NoRoute, + Self::OnlyDefault, + Self::Discarded, Self::OwnNetwork, ]; @@ -57,7 +67,9 @@ impl Unresolved { Self::Missing => "no next hop", Self::OtherFamily => "IPv4 next hop for an IPv6 route", Self::Ourselves => "next hop is this machine", - Self::NotConnected => "next hop not directly connected", + Self::NoRoute => "no route to the next hop", + Self::OnlyDefault => "next hop reachable only by the default route", + Self::Discarded => "the route to the next hop discards packets", Self::OwnNetwork => "inside a network of this machine", } } @@ -95,19 +107,68 @@ impl From for Neighbor { /// Works out where the kernel is to send packets for a route to `prefix` /// with the next hop `next_hop`, learned from `peer`. /// -/// Only a next hop that is a neighbor on one of our interfaces is -/// resolved. A link-local next hop means something only on one link, and -/// that is taken to be the link the peer itself is on. +/// A next hop that is a neighbor on one of our interfaces is where the +/// packets go. One that is not is looked up in the kernel's own routes, +/// and the packets go where the route that covers it sends them: to its +/// gateway, or onto its interface. A link-local next hop means something +/// only on one link, and that is taken to be the link the peer itself is +/// on. pub fn resolve( - interfaces: &Interfaces, + topology: &Topology, prefix: &IpPrefix, next_hop: Option<&NlriNextHop>, peer: Option, ) -> Result { - if interfaces.has_network_around(prefix) { + if topology.interfaces.has_network_around(prefix) { return Err(Unresolved::OwnNetwork); } - resolve_next_hop(interfaces, prefix.address_family(), next_hop, peer) + resolve_next_hop(topology, prefix.address_family(), next_hop, peer) +} + +/// The way to `gateway`, which is not a neighbor, by the kernel's routes +/// that count: where the route that covers it sends packets. One step +/// only: the kernel sees to it that a route's own gateway is a neighbor. +fn through_routes( + topology: &Topology, + routes: IpVersion, + gateway: IpAddr, +) -> Result { + let route = topology.routes.lookup(gateway).ok_or(Unresolved::NoRoute)?; + if route.prefix.length() == 0 && !topology.via_default { + return Err(Unresolved::OnlyDefault); + } + if route.kind != KIND_UNICAST { + return Err(Unresolved::Discarded); + } + let (hop, interface) = match route.nexthop { + Some(id) => { + // A group of nexthops has no gateway or interface of its own; + // which of its members packets take is the kernel's to say, + // and nothing to give it as a route. + let nexthop = topology.nexthops.get(&id).ok_or(Unresolved::NoRoute)?; + ( + nexthop.gateway, + nexthop.interface.ok_or(Unresolved::NoRoute)?, + ) + } + None => (route.gateway, route.interface.ok_or(Unresolved::NoRoute)?), + }; + if !topology + .interfaces + .link(interface) + .is_some_and(|link| link.up) + { + return Err(Unresolved::NoRoute); + } + // A route without a gateway puts the destination on its interface: + // the next hop itself is the neighbor there. + let gateway = hop.unwrap_or(gateway); + // An IPv4 route may go by an IPv6 gateway (RFC 8950, and the kernel + // allows it); an IPv6 route by an IPv4 gateway is nothing. + if routes == IpVersion::Ipv6 && gateway.is_ipv4() { + return Err(Unresolved::NoRoute); + } + Ok(Target { gateway, interface }) } /// The part of [`resolve`] that is the same for every route of IP version @@ -115,11 +176,12 @@ pub fn resolve( /// those who ask it of many such routes, and ask of each only whether it /// is for one of the machine's own networks. pub fn resolve_next_hop( - interfaces: &Interfaces, + topology: &Topology, routes: IpVersion, next_hop: Option<&NlriNextHop>, peer: Option, ) -> Result { + let interfaces = &topology.interfaces; let next_hop = next_hop.ok_or(Unresolved::Missing)?; let version = match next_hop { NlriNextHop::Ipv4(_) => IpVersion::Ipv4, @@ -135,17 +197,17 @@ pub fn resolve_next_hop( if interfaces.is_local(gateway) { return Err(Unresolved::Ourselves); } - interfaces - .connected(gateway) - .map(|connected| Target { + match interfaces.connected(gateway) { + Some(connected) => Ok(Target { gateway, interface: connected.interface, - }) - .ok_or(Unresolved::NotConnected) + }), + None => through_routes(topology, routes, gateway), + } }; // The interface the peer is a neighbor on, for a link-local gateway. let on_peers_link = |gateway: Ipv6Addr| { - let interface = match peer.ok_or(Unresolved::NotConnected)? { + let interface = match peer.ok_or(Unresolved::NoRoute)? { // The session is on a link-local address itself, and so says // which link: every interface has fe80::/64, and looking the // peer up there would find any of them. @@ -157,7 +219,7 @@ pub fn resolve_next_hop( .connected(address) .map(|connected| connected.interface), } - .ok_or(Unresolved::NotConnected)?; + .ok_or(Unresolved::NoRoute)?; if interfaces .addresses_of(interface) .any(|ours| ours.local == IpAddr::V6(gateway)) @@ -182,8 +244,11 @@ pub fn resolve_next_hop( { on_peers_link(link_local) } + // Where there is no way to the global one, the link-local one is + // tried; a way that is there and is bad (ourselves, a blackhole) + // is an answer as it stands. NlriNextHop::Ipv6WithLl { global, link_local } => match direct(global.into()) { - Err(Unresolved::NotConnected) => on_peers_link(link_local), + Err(Unresolved::NoRoute | Unresolved::OnlyDefault) => on_peers_link(link_local), resolved => resolved, }, } @@ -435,7 +500,7 @@ mod tests { use proptest::prelude::*; use super::*; - use crate::netlink::{Address, Link, Message}; + use crate::netlink::{Address, Link, Message, Nexthop, PROTOCOL_BGP, Route, TABLE_MAIN}; fn prefix(s: &str) -> IpPrefix { IpPrefix::try_from(s).unwrap() @@ -448,14 +513,14 @@ mod tests { } } - fn interfaces() -> Interfaces { - let mut interfaces = Interfaces::default(); + fn interfaces() -> Topology { + let mut topology = Topology::new(TABLE_MAIN, false); for (index, local, network) in [ (2, "192.0.2.1", "192.0.2.0/24"), (2, "2001:db8::1", "2001:db8::/64"), (3, "198.51.100.1", "198.51.100.0/24"), ] { - interfaces.apply(&Message::Link { + topology.apply(&Message::Link { present: true, link: Link { index, @@ -463,7 +528,7 @@ mod tests { up: true, }, }); - interfaces.apply(&Message::Address { + topology.apply(&Message::Address { present: true, address: Address { interface: index, @@ -472,7 +537,28 @@ mod tests { }, }); } - interfaces + topology + } + + /// A route of the kernel's main table, as a static route is. + fn kernel_route(prefix_text: &str, gateway: Option<&str>, interface: Option) -> Route { + Route { + prefix: prefix(prefix_text), + table: TABLE_MAIN, + protocol: 4, + metric: 0, + gateway: gateway.map(|g| g.parse().unwrap()), + interface, + nexthop: None, + kind: KIND_UNICAST, + } + } + + fn with_route(topology: &mut Topology, route: Route) { + assert!(topology.apply(&Message::Route { + present: true, + route, + })); } fn neighbor(address: &str) -> Neighbor { @@ -502,7 +588,7 @@ mod tests { Ok(target("2001:db8::2", 2)) ); for (hop, why) in [ - (Some(v4_hop("10.9.9.9")), Unresolved::NotConnected), + (Some(v4_hop("10.9.9.9")), Unresolved::NoRoute), (Some(v4_hop("192.0.2.1")), Unresolved::Ourselves), (None, Unresolved::Missing), ] { @@ -588,7 +674,7 @@ mod tests { let far = Some(neighbor("2001:db8:ffff::9")); assert_eq!( resolve(&interfaces, &v6, Some(&NlriNextHop::Ipv6(link_local)), far), - Err(Unresolved::NotConnected) + Err(Unresolved::NoRoute) ); assert_eq!( resolve(&interfaces, &v6, Some(&both("2001:db8::1")), peer), @@ -645,7 +731,7 @@ mod tests { // An interface that is not there, or is down. assert_eq!( resolve(&interfaces, &v6, Some(&next_hop), on(9)), - Err(Unresolved::NotConnected) + Err(Unresolved::NoRoute) ); } // Our own address on that link, which on another is a neighbor's. @@ -666,6 +752,158 @@ mod tests { ); } + /// A next hop that is no neighbor is reached the way the kernel's + /// own routes say, and the route goes into the kernel with that way's + /// gateway and interface. + #[test] + fn a_far_next_hop_resolves_by_the_kernels_routes() { + let mut topology = interfaces(); + let v4 = prefix("203.0.113.0/24"); + let v6 = prefix("2001:db8:100::/48"); + let peer = Some(neighbor("192.0.2.2")); + let v4_hop = |s: &str| NlriNextHop::Ipv4(s.parse().unwrap()); + let v6_hop = |s: &str| NlriNextHop::Ipv6(s.parse().unwrap()); + let far = v4_hop("10.9.9.9"); + + // Nothing says the way. + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Err(Unresolved::NoRoute) + ); + // The default route does not count unless asked for. + with_route( + &mut topology, + kernel_route("0.0.0.0/0", Some("192.0.2.254"), Some(2)), + ); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Err(Unresolved::OnlyDefault) + ); + topology.via_default = true; + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Ok(target("192.0.2.254", 2)) + ); + topology.via_default = false; + // A static route with a gateway: the gateway is where packets go. + with_route( + &mut topology, + kernel_route("10.9.0.0/16", Some("198.51.100.9"), Some(3)), + ); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Ok(target("198.51.100.9", 3)) + ); + // A more specific route onto an interface, as a tunnel's is: the + // next hop itself is the neighbor there. + topology.apply(&Message::Link { + present: true, + link: Link { + index: 7, + name: "wg0".into(), + up: true, + }, + }); + with_route(&mut topology, kernel_route("10.9.9.0/24", None, Some(7))); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Ok(target("10.9.9.9", 7)) + ); + // The tunnel goes down: the kernel keeps such routes, we do not + // use them. + topology.apply(&Message::Link { + present: true, + link: Link { + index: 7, + name: "wg0".into(), + up: false, + }, + }); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Err(Unresolved::NoRoute) + ); + // A route that names a nexthop object goes where the object says. + with_route( + &mut topology, + Route { + nexthop: Some(77), + ..kernel_route("10.9.9.9/32", None, None) + }, + ); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Err(Unresolved::NoRoute), + "an object the kernel has not told of" + ); + topology.apply(&Message::Nexthop { + present: true, + nexthop: Nexthop { + id: 77, + protocol: 4, + gateway: Some("192.0.2.77".parse().unwrap()), + interface: Some(2), + }, + }); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Ok(target("192.0.2.77", 2)) + ); + // A route that discards packets is a way to nowhere. + with_route( + &mut topology, + Route { + kind: 6, // blackhole + ..kernel_route("10.9.9.9/32", None, None) + }, + ); + assert_eq!( + resolve(&topology, &v4, Some(&far), peer), + Err(Unresolved::Discarded) + ); + // Routes of BGP's never say the way, be the prefix ever so long. + assert!(!topology.apply(&Message::Route { + present: true, + route: Route { + protocol: PROTOCOL_BGP, + metric: 20, + ..kernel_route("10.9.9.9/32", Some("198.51.100.9"), Some(3)) + }, + })); + // An IPv6 next hop by an IPv4 gateway is no way for an IPv6 route, + // but is one for an IPv4 route (RFC 8950 the other way about is + // what the kernel allows). + with_route( + &mut topology, + kernel_route("2001:db8:9::/48", Some("198.51.100.9"), Some(3)), + ); + assert_eq!( + resolve(&topology, &v6, Some(&v6_hop("2001:db8:9::9")), peer), + Err(Unresolved::NoRoute) + ); + with_route( + &mut topology, + kernel_route("2001:db8:9::/48", Some("2001:db8::9"), Some(2)), + ); + assert_eq!( + resolve(&topology, &v6, Some(&v6_hop("2001:db8:9::9")), peer), + Ok(target("2001:db8::9", 2)) + ); + assert_eq!( + resolve(&topology, &v4, Some(&v6_hop("2001:db8:9::9")), peer), + Ok(target("2001:db8::9", 2)) + ); + // A neighbor is a neighbor, whatever the routes say. + with_route( + &mut topology, + kernel_route("192.0.2.2/32", Some("198.51.100.9"), Some(3)), + ); + assert_eq!( + resolve(&topology, &v4, Some(&v4_hop("192.0.2.2")), peer), + Ok(target("192.0.2.2", 2)) + ); + } + #[test] fn the_kernel_is_asked_only_for_what_changes() { let mut plan = Plan::default(); @@ -692,13 +930,13 @@ mod tests { assert_eq!(plan.installed(), 1); // A best route that cannot be installed takes the old one out. assert_eq!( - plan.best(p, Err(Unresolved::NotConnected)), + plan.best(p, Err(Unresolved::NoRoute)), Some(Op::Remove { prefix: p }) ); assert_eq!((plan.installed(), plan.unresolved()), (0, 1)); assert_eq!( plan.unresolved_by_reason().collect::>(), - [(Unresolved::NotConnected, 1)] + [(Unresolved::NoRoute, 1)] ); assert_eq!(plan.withdrawn(p), None); assert_eq!(plan.unresolved_by_reason().count(), 0); @@ -866,7 +1104,7 @@ mod tests { Some(Op::Remove { prefix }) => { kernel.remove(&prefix); } None => {} }; - let resolved = |gateway: Option| gateway.map(t).ok_or(Unresolved::NotConnected); + let resolved = |gateway: Option| gateway.map(t).ok_or(Unresolved::NoRoute); // A new reader is first given the whole table. let steps = std::iter::once(Step::Resync).chain(steps); for step in steps { diff --git a/crates/fib/src/routes.rs b/crates/fib/src/routes.rs new file mode 100644 index 0000000..6d125b2 --- /dev/null +++ b/crates/fib/src/routes.rs @@ -0,0 +1,266 @@ +//! The kernel's own routes, as far as next hops are concerned: the way to +//! a next hop that is not a neighbor on any of our interfaces. +//! +//! Only the routes of one table count, the resolution table, which is +//! normally `main`: the connected and static routes, and an IGP's if there +//! is one. Routes put there by BGP, ours or another daemon's, never count: +//! a next hop that is reached only by way of a BGP route is reached by way +//! of something that may itself depend on a next hop, and a table that +//! leans on itself like that is one that falls over in a way nobody can +//! predict. The default route counts only if asked for: on a router, the +//! default points at a transit provider, and a next hop that is "reached" +//! only by sending the packets to the Internet at large is not reached. + +use std::net::IpAddr; + +use bgp_primitives::constants::IpVersion; +use bgp_primitives::ip_prefix::IpPrefix; + +use crate::netlink::{Message, PROTOCOL_BGP, Route}; +use crate::plan::HashMap; + +/// The routes of the resolution table that may say the way to a next hop. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KernelRoutes { + table: u32, + /// By prefix, the routes there are for it: usually one, but the kernel + /// keeps one for each metric. + routes: HashMap>, + /// How many prefixes there are of each length, by IP version: the + /// lengths a lookup has to try. + lengths: [[u32; 129]; 2], +} + +/// `address` as the prefix that is only it. +pub(crate) fn host(address: IpAddr) -> IpPrefix { + match address { + IpAddr::V4(v4) => IpPrefix::new(IpVersion::Ipv4, v4.octets(), 32), + IpAddr::V6(v6) => IpPrefix::new(IpVersion::Ipv6, v6.octets(), 128), + } + .expect("an address is a prefix of its own length") +} + +fn version_index(version: IpVersion) -> usize { + match version { + IpVersion::Ipv4 => 0, + IpVersion::Ipv6 => 1, + } +} + +impl KernelRoutes { + pub fn new(table: u32) -> Self { + Self { + table, + routes: HashMap::default(), + lengths: [[0; 129]; 2], + } + } + + /// The table the routes are of. + pub fn table(&self) -> u32 { + self.table + } + + /// Whether `route` is one that may say the way to a next hop. + fn counts(&self, route: &Route) -> bool { + route.table == self.table && route.protocol != PROTOCOL_BGP + } + + /// Takes in what a message from the kernel says about routes. Returns + /// whether anything is different for it. + pub fn apply(&mut self, message: &Message) -> bool { + let Message::Route { present, route } = message else { + return false; + }; + if !self.counts(route) { + return false; + } + let version = version_index(route.prefix.address_family()); + let length = usize::from(route.prefix.length()); + if *present { + let routes = self.routes.entry(route.prefix).or_default(); + // A route for a prefix with a metric there is one for already + // takes that one's place: the kernel keeps one per metric. + match routes.iter_mut().find(|known| known.metric == route.metric) { + Some(known) if known == route => false, + Some(known) => { + *known = *route; + true + } + None => { + if routes.is_empty() { + self.lengths[version][length] += 1; + } + // In order of metric, so that two tables with the + // same routes compare equal however they were told. + let at = routes.partition_point(|known| known.metric < route.metric); + routes.insert(at, *route); + true + } + } + } else { + let Some(routes) = self.routes.get_mut(&route.prefix) else { + return false; + }; + let before = routes.len(); + routes.retain(|known| known.metric != route.metric); + let (after, none_left) = (routes.len(), routes.is_empty()); + if none_left { + self.routes.remove(&route.prefix); + self.lengths[version][length] -= 1; + } + before != after + } + } + + /// How many prefixes have a route. + pub fn len(&self) -> usize { + self.routes.len() + } + + pub fn is_empty(&self) -> bool { + self.routes.is_empty() + } + + /// The route the kernel would send packets for `address` by, from + /// this table: the one for the longest prefix that covers it, and of + /// several for that prefix the one with the lowest metric. The + /// default route too, if there is one and nothing longer covers the + /// address. + pub fn lookup(&self, address: IpAddr) -> Option<&Route> { + let whole = host(address); + let lengths = &self.lengths[version_index(whole.address_family())]; + (0..=whole.length()) + .rev() + .filter(|length| lengths[usize::from(*length)] > 0) + .find_map(|length| self.routes.get(&whole.shortened(length))) + .and_then(|routes| routes.iter().min_by_key(|route| route.metric)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::netlink::{KIND_UNICAST, TABLE_MAIN}; + + fn prefix(s: &str) -> IpPrefix { + IpPrefix::try_from(s).unwrap() + } + + fn route(prefix_text: &str, gateway: Option<&str>, interface: u32, metric: u32) -> Route { + Route { + prefix: prefix(prefix_text), + table: TABLE_MAIN, + protocol: 4, // static + metric, + gateway: gateway.map(|g| g.parse().unwrap()), + interface: Some(interface), + nexthop: None, + kind: KIND_UNICAST, + } + } + + fn present(route: Route) -> Message { + Message::Route { + present: true, + route, + } + } + + fn gone(route: Route) -> Message { + Message::Route { + present: false, + route, + } + } + + fn lookup(routes: &KernelRoutes, address: &str) -> Option<(String, Option, u32)> { + routes.lookup(address.parse().unwrap()).map(|route| { + ( + route.prefix.to_string(), + route.gateway.map(|g| g.to_string()), + route.metric, + ) + }) + } + + #[test] + fn the_longest_prefix_with_the_lowest_metric_is_the_way() { + let mut routes = KernelRoutes::new(TABLE_MAIN); + for r in [ + route("0.0.0.0/0", Some("192.0.2.1"), 2, 100), + route("10.0.0.0/8", Some("192.0.2.9"), 2, 0), + route("10.1.0.0/16", None, 7, 0), + route("10.1.0.0/16", Some("198.51.100.1"), 3, 50), + route("2001:db8::/32", Some("2001:db8::1"), 2, 1024), + ] { + assert!(routes.apply(&present(r))); + } + assert_eq!(routes.len(), 4); + assert_eq!( + lookup(&routes, "10.2.3.4"), + Some(("10.0.0.0/8".into(), Some("192.0.2.9".into()), 0)) + ); + // Two routes for the prefix: the lower metric wins. + assert_eq!( + lookup(&routes, "10.1.2.3"), + Some(("10.1.0.0/16".into(), None, 0)) + ); + assert_eq!( + lookup(&routes, "203.0.113.1"), + Some(("0.0.0.0/0".into(), Some("192.0.2.1".into()), 100)) + ); + assert_eq!( + lookup(&routes, "2001:db8:1::1"), + Some(("2001:db8::/32".into(), Some("2001:db8::1".into()), 1024)) + ); + assert_eq!(lookup(&routes, "2001:db9::1"), None); + + // Told again what is known: nothing changes. + assert!(!routes.apply(&present(route("10.0.0.0/8", Some("192.0.2.9"), 2, 0)))); + // The same prefix and metric with another gateway: replaced. + assert!(routes.apply(&present(route("10.0.0.0/8", Some("192.0.2.8"), 2, 0)))); + assert_eq!( + lookup(&routes, "10.2.3.4"), + Some(("10.0.0.0/8".into(), Some("192.0.2.8".into()), 0)) + ); + // The better of the two for 10.1/16 goes: the other is the way. + assert!(routes.apply(&gone(route("10.1.0.0/16", None, 7, 0)))); + assert_eq!( + lookup(&routes, "10.1.2.3"), + Some(("10.1.0.0/16".into(), Some("198.51.100.1".into()), 50)) + ); + assert!(routes.apply(&gone(route("10.1.0.0/16", Some("198.51.100.1"), 3, 50)))); + assert_eq!( + lookup(&routes, "10.1.2.3"), + Some(("10.0.0.0/8".into(), Some("192.0.2.8".into()), 0)) + ); + // Gone already: nothing changes. + assert!(!routes.apply(&gone(route("10.1.0.0/16", None, 7, 0)))); + assert_eq!(routes.len(), 3); + } + + #[test] + fn routes_of_other_tables_and_of_bgp_do_not_count() { + let mut routes = KernelRoutes::new(TABLE_MAIN); + let other_table = Route { + table: 200, + ..route("10.0.0.0/8", Some("192.0.2.9"), 2, 0) + }; + let bgp = Route { + protocol: PROTOCOL_BGP, + ..route("10.0.0.0/8", Some("192.0.2.9"), 2, 20) + }; + assert!(!routes.apply(&present(other_table))); + assert!(!routes.apply(&present(bgp))); + assert!(routes.is_empty()); + assert_eq!(lookup(&routes, "10.2.3.4"), None); + // What is not about routes changes nothing either. + assert!(!routes.apply(&Message::Done)); + + let mut table_200 = KernelRoutes::new(200); + assert!(table_200.apply(&present(other_table))); + assert_eq!(table_200.table(), 200); + assert_eq!(table_200.len(), 1); + } +} diff --git a/crates/fib/src/topology.rs b/crates/fib/src/topology.rs new file mode 100644 index 0000000..a8e47b0 --- /dev/null +++ b/crates/fib/src/topology.rs @@ -0,0 +1,123 @@ +//! What the kernel knows that says whether, and how, a next hop can be +//! reached: the machine's interfaces and addresses, the routes of the +//! resolution table, and the kernel's nexthop objects that those routes +//! may name. One value, replaced as a whole when any of it changes, so +//! that everyone judging next hops judges them by one moment's view. + +use crate::interfaces::Interfaces; +use crate::netlink::{Message, Nexthop}; +use crate::plan::HashMap; +use crate::routes::KernelRoutes; + +/// The kernel's view of the machine, as far as next hops are concerned. +#[derive(Debug, Clone)] +pub struct Topology { + pub interfaces: Interfaces, + /// The routes a next hop that is not a neighbor is reached by. + pub routes: KernelRoutes, + /// Every nexthop object the kernel has, by number, whoever made it: + /// a route of the resolution table may name one instead of carrying + /// its own gateway. + pub nexthops: HashMap, + /// Whether the default route counts as the way to a next hop. + pub via_default: bool, +} + +impl Topology { + /// Nothing known yet, with next hops to be resolved by the routes of + /// `resolution_table`. + pub fn new(resolution_table: u32, via_default: bool) -> Self { + Self { + interfaces: Interfaces::default(), + routes: KernelRoutes::new(resolution_table), + nexthops: HashMap::default(), + via_default, + } + } + + /// Takes in what a message from the kernel says. Returns whether + /// anything is different for it: then what any next hop resolves to + /// may be different too. + pub fn apply(&mut self, message: &Message) -> bool { + match message { + Message::Nexthop { + present: true, + nexthop, + } => self.nexthops.insert(nexthop.id, *nexthop) != Some(*nexthop), + Message::Nexthop { + present: false, + nexthop, + } => self.nexthops.remove(&nexthop.id).is_some(), + Message::Route { .. } => self.routes.apply(message), + _ => self.interfaces.apply(message), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::netlink::{KIND_UNICAST, Link, PROTOCOL_BGP, Route, TABLE_MAIN}; + use bgp_primitives::ip_prefix::IpPrefix; + + #[test] + fn each_kind_of_news_goes_where_it_belongs() { + let mut topology = Topology::new(TABLE_MAIN, false); + let nexthop = Nexthop { + id: 7, + protocol: 4, + gateway: Some("192.0.2.1".parse().unwrap()), + interface: Some(2), + }; + let present = |nexthop| Message::Nexthop { + present: true, + nexthop, + }; + assert!(topology.apply(&present(nexthop))); + assert!(!topology.apply(&present(nexthop))); + assert!(topology.apply(&present(Nexthop { + interface: Some(3), + ..nexthop + }))); + assert_eq!(topology.nexthops[&7].interface, Some(3)); + assert!(topology.apply(&Message::Nexthop { + present: false, + nexthop, + })); + assert!(topology.nexthops.is_empty()); + + let route = Route { + prefix: IpPrefix::try_from("10.0.0.0/8").unwrap(), + table: TABLE_MAIN, + protocol: 4, + metric: 0, + gateway: Some("192.0.2.1".parse().unwrap()), + interface: Some(2), + nexthop: None, + kind: KIND_UNICAST, + }; + assert!(topology.apply(&Message::Route { + present: true, + route, + })); + assert_eq!(topology.routes.len(), 1); + // BGP's routes are not the way to anything. + assert!(!topology.apply(&Message::Route { + present: true, + route: Route { + protocol: PROTOCOL_BGP, + ..route + }, + })); + assert!(topology.apply(&Message::Link { + present: true, + link: Link { + index: 2, + name: "eth0".into(), + up: true, + }, + })); + assert!(topology.interfaces.link(2).is_some()); + assert!(!topology.apply(&Message::Done)); + } +} diff --git a/crates/fib/tests/kernel.rs b/crates/fib/tests/kernel.rs index 620e628..7660f2b 100644 --- a/crates/fib/tests/kernel.rs +++ b/crates/fib/tests/kernel.rs @@ -109,6 +109,7 @@ impl Machine { metric: 20, nexthop_objects: self.nexthop_objects, stale_time: self.stale_time, + ..FibConfig::default() }, ) .expect("the kernel can be talked to") @@ -294,7 +295,7 @@ async fn the_kernel_has_the_best_route_of_each_prefix() { machine.shown("2001:db8:100::/48", v6_neighbor), ]); status_becomes(&fib, "one unresolved", |status| { - status.installed == 2 && status.unresolved == [("next hop not directly connected", 1)] + status.installed == 2 && status.unresolved == [("no route to the next hop", 1)] }); // A session ends: its routes go. @@ -367,7 +368,7 @@ async fn routes_follow_the_machines_interfaces() { .announce(peer, &via, &[prefix("203.0.113.0/24")]); let (fib, task) = machine.start(); status_becomes(&fib, "unresolved", |status| { - status.synced && status.unresolved == [("next hop not directly connected", 1)] + status.synced && status.unresolved == [("no route to the next hop", 1)] }); assert!(machine.kernel_routes().is_empty()); @@ -416,6 +417,88 @@ async fn routes_follow_the_machines_interfaces() { machine.kernel_becomes(&[]); } +/// A next hop that is no neighbor goes the way the kernel's own routes +/// say: a static route's gateway, a tunnel's interface, a nexthop object. +/// When those routes change, so do ours; a route that discards packets is +/// the way to nowhere. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"] +async fn a_far_next_hop_goes_the_way_the_kernels_routes_say() { + let machine = Machine::new(10); + let interface = machine.interface(); + let peer = machine.peer(2); + // Next hops that are not in any network of ours. + let far = machine.via("172.31.10.9".parse().unwrap(), 0); + let farther = machine.via("172.31.11.9".parse().unwrap(), 0); + machine + .rib + .announce(peer, &far, &[prefix("203.0.113.0/24")]); + machine + .rib + .announce(peer, &farther, &[prefix("198.18.0.0/15")]); + let (fib, task) = machine.start(); + status_becomes(&fib, "no way to either", |status| { + status.synced && status.unresolved == [("no route to the next hop", 2)] + }); + assert!(machine.kernel_routes().is_empty()); + + // A static route with a gateway says the way to one: the route goes + // in with that gateway. + ip(&format!( + "route add 172.31.10.0/24 via 10.10.0.9 dev {interface}" + )); + machine.kernel_becomes(&[machine.shown("203.0.113.0/24", "10.10.0.9")]); + // A route onto the interface, as a tunnel's is, says the way to the + // other: the next hop itself is the neighbor there. + ip(&format!("route add 172.31.11.0/24 dev {interface}")); + machine.kernel_becomes(&[ + machine.shown("203.0.113.0/24", "10.10.0.9"), + machine.shown("198.18.0.0/15", "172.31.11.9"), + ]); + status_becomes(&fib, "both resolved", |status| { + status.unresolved.is_empty() && status.installed == 2 + }); + + // The way changes: so does the route. Then it goes, and so does the + // route. + ip(&format!( + "route replace 172.31.10.0/24 via 10.10.0.8 dev {interface}" + )); + machine.kernel_becomes(&[ + machine.shown("203.0.113.0/24", "10.10.0.8"), + machine.shown("198.18.0.0/15", "172.31.11.9"), + ]); + ip("route del 172.31.10.0/24"); + machine.kernel_becomes(&[machine.shown("198.18.0.0/15", "172.31.11.9")]); + status_becomes(&fib, "one unresolved again", |status| { + status.unresolved == [("no route to the next hop", 1)] + }); + + // A route that names a nexthop object goes where the object says. + ip(&format!( + "nexthop add id 1010 via 10.10.0.7 dev {interface}" + )); + ip("route add 172.31.10.0/24 nhid 1010"); + machine.kernel_becomes(&[ + machine.shown("203.0.113.0/24", "10.10.0.7"), + machine.shown("198.18.0.0/15", "172.31.11.9"), + ]); + + // The tunnel's route becomes a blackhole: the way to nowhere. + ip("route replace blackhole 172.31.11.0/24"); + machine.kernel_becomes(&[machine.shown("203.0.113.0/24", "10.10.0.7")]); + status_becomes(&fib, "discarded", |status| { + status.unresolved == [("the route to the next hop discards packets", 1)] + }); + + ip("route del 172.31.10.0/24"); + ip("route del blackhole 172.31.11.0/24"); + ip("nexthop del id 1010"); + fib.stop().await; + task.await.unwrap().unwrap(); + machine.kernel_becomes(&[]); +} + /// A table's worth of routes goes in, and comes out, in reasonable time. #[tokio::test(flavor = "multi_thread")] #[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"] diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index e74ea34..9f7bc51 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -220,6 +220,22 @@ message Kernel { // was not announced again is removed. // Zero removes them at once. optional uint32 stale_seconds = 4; + + // The kernel's table whose routes say how a next hop that is not a + // neighbour is reached: its connected and static routes, and an IGP's + // if there is one. Defaults to 254, the main table. A route whose next + // hop is covered by such a route is installed with that route's gateway + // and interface, the kernel not looking gateways up in turn. Routes put + // in the table by BGP, this daemon's or another's, never count: a next + // hop reached only by way of a BGP route rests on what may itself rest + // on a next hop. + uint32 resolution_table = 5; + + // Whether the default route in that table counts as the way to a next + // hop. Off by default: on a router the default route points at a + // transit provider, and a next hop "reached" only by sending the + // packets to the Internet at large is not reached. + bool resolve_via_default_route = 6; } // Where to get the validated ROA payloads that say which AS may originate diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 8fa6c9f..aa3b8aa 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -408,6 +408,11 @@ fn parse_kernel(kernel: &proto::Kernel) -> FibConfig { }, metric: kernel.metric.unwrap_or(defaults.metric), nexthop_objects: !kernel.without_nexthop_objects, + resolution_table: match kernel.resolution_table { + 0 => defaults.resolution_table, + table => table, + }, + resolve_via_default: kernel.resolve_via_default_route, stale_time: kernel.stale_seconds.map_or(defaults.stale_time, |seconds| { Duration::from_secs(seconds.into()) }), @@ -1290,6 +1295,8 @@ mod tests { metric: 20, nexthop_objects: true, stale_time: Duration::from_secs(120), + resolution_table: 254, + resolve_via_default: false, }) ); assert_eq!( @@ -1304,6 +1311,8 @@ mod tests { metric: 150, nexthop_objects: false, stale_time: Duration::from_secs(30), + resolution_table: 254, + resolve_via_default: false, }) ); } diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index 7c26bfc..4564f47 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -13,7 +13,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use bgp_fib::InterfacesWatch; +use bgp_fib::TopologyWatch; use bgp_packet::errors::{BgpError, DecodeError, MsgHeaderSubcode}; use bgp_packet::message::RouteRefreshMessage; use bgp_packet::message::{ @@ -87,8 +87,9 @@ pub struct ServerContext { pub rejections: RejectionLog, /// The data for route origin validation, where that is configured. pub rpki: Option, - /// The machine's interfaces, where routes are installed in the kernel. - pub interfaces: Option, + /// The kernel's view of the machine, where routes are installed in + /// the kernel: what says whether a next hop can be reached. + pub topology: Option, /// The routes kept of peers that are restarting, and how far the peers /// are with sending theirs since the start (graceful restart). pub restarts: Arc, @@ -1078,9 +1079,9 @@ impl Session { .filter(|_| self.server.rpki.is_some()); let reach = self .server - .interfaces + .topology .clone() - .map(|interfaces| Reach::new(interfaces, neighbor_at(self.peer_addr))); + .map(|topology| Reach::new(topology, neighbor_at(self.peer_addr))); Validator::new(self.server.rpki.clone().zip(subject), reach).map(|validator| Validation { validator, changes: handle.validate(subject), @@ -1605,7 +1606,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), - interfaces: None, + topology: None, rejections: RejectionLog::default(), rpki: None, restarts: Restarts::new(rib, true, [], 120), @@ -1656,7 +1657,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), - interfaces: None, + topology: None, rejections: RejectionLog::default(), rpki: None, restarts: Restarts::new(rib, true, [], 120), diff --git a/crates/server/src/reach.rs b/crates/server/src/reach.rs index 59dc7a5..bd3747a 100644 --- a/crates/server/src/reach.rs +++ b/crates/server/src/reach.rs @@ -9,10 +9,11 @@ //! aside like a route the RPKI finds invalid, and looked at again when //! the interfaces change (see `crate::routes::Held`). //! -//! Only next hops that are neighbors on one of the machine's interfaces -//! count as reachable: the way to any other is not looked up. Routes from -//! a peer in our own AS that did not set itself as next hop are therefore -//! held, where they used to be chosen and then not installed. +//! A next hop that is a neighbor on one of the machine's interfaces can +//! be reached, and so can one that the kernel's own routes know the way +//! to; see `bgp_fib::routes` for which routes count. Routes from a peer +//! in our own AS that did not set itself as next hop are taken if a static +//! route or an IGP's says how to reach it, and held otherwise. use std::cell::RefCell; use std::collections::HashMap; @@ -20,9 +21,9 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; -use bgp_fib::InterfacesWatch; -use bgp_fib::interfaces::Interfaces; +use bgp_fib::TopologyWatch; use bgp_fib::plan::{Neighbor, resolve, resolve_next_hop}; +use bgp_fib::topology::Topology; use bgp_packet::message::NlriNextHop; use bgp_primitives::constants::IpVersion; use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; @@ -38,10 +39,10 @@ use crate::session::SessionRegistry; /// judged it by the interfaces as they were. const SETTLE: Duration = Duration::from_millis(500); -/// The interfaces to judge one peer's next hops by. +/// The kernel's view of the machine to judge one peer's next hops by. #[derive(Clone, Copy)] pub(crate) struct NextHops<'a> { - pub(crate) interfaces: &'a Interfaces, + pub(crate) topology: &'a Topology, /// The peer: where a link-local next hop is. pub(crate) neighbor: Neighbor, } @@ -50,7 +51,7 @@ impl NextHops<'_> { /// Whether the kernel can be given a route for `prefix` by way of /// `next_hop`. pub(crate) fn reaches(&self, prefix: &IpPrefix, next_hop: Option<&NlriNextHop>) -> bool { - resolve(self.interfaces, prefix, next_hop, Some(self.neighbor)).is_ok() + resolve(self.topology, prefix, next_hop, Some(self.neighbor)).is_ok() } /// The two halves of [`NextHops::reaches`], for an UPDATE's worth of @@ -58,52 +59,52 @@ impl NextHops<'_> { /// is asked once, and whether a prefix is one the machine reaches by /// itself, which is asked of each. pub(crate) fn next_hop_is_near(&self, routes: IpVersion, next_hop: &NlriNextHop) -> bool { - resolve_next_hop(self.interfaces, routes, Some(next_hop), Some(self.neighbor)).is_ok() + resolve_next_hop(self.topology, routes, Some(next_hop), Some(self.neighbor)).is_ok() } pub(crate) fn is_own_network(&self, prefix: &IpPrefix) -> bool { - self.interfaces.has_network_around(prefix) + self.topology.interfaces.has_network_around(prefix) } } -/// A session's view of the interfaces: those in use, replaced when the -/// kernel says they have changed. Asking whether they have costs the load -/// of one number, which is done for every UPDATE. +/// A session's view of the topology: the one in use, replaced when the +/// kernel says it has changed. Asking whether it has costs the load of +/// one number, which is done for every UPDATE. pub(crate) struct Reach { - watch: InterfacesWatch, + watch: TopologyWatch, neighbor: Neighbor, - interfaces: Option>, + topology: Option>, } impl Reach { - pub(crate) fn new(watch: InterfacesWatch, neighbor: Neighbor) -> Self { - let interfaces = watch.borrow().clone(); + pub(crate) fn new(watch: TopologyWatch, neighbor: Neighbor) -> Self { + let topology = watch.borrow().clone(); Self { watch, neighbor, - interfaces, + topology, } } /// What to judge next hops by now; nothing until the kernel has said - /// what interfaces there are. + /// what interfaces and routes there are. pub(crate) fn next_hops(&mut self) -> Option> { if self.watch.has_changed().unwrap_or(false) { - self.interfaces = self.watch.borrow_and_update().clone(); + self.topology = self.watch.borrow_and_update().clone(); } - self.interfaces.as_deref().map(|interfaces| NextHops { - interfaces, + self.topology.as_deref().map(|topology| NextHops { + topology, neighbor: self.neighbor, }) } } -/// Starts the task that, whenever the machine's interfaces change, finds -/// the routes in the table whose next hop can no longer be reached and has -/// their sessions take them out, and has every session look again at the -/// routes it holds. +/// Starts the task that, whenever the machine's interfaces or routes +/// change, finds the routes in the table whose next hop can no longer be +/// reached and has their sessions take them out, and has every session +/// look again at the routes it holds. pub(crate) fn spawn_revalidator( - mut watch: InterfacesWatch, + mut watch: TopologyWatch, rib: Arc, registry: SessionRegistry, ) -> JoinHandle<()> { @@ -112,7 +113,7 @@ pub(crate) fn spawn_revalidator( // which the server notices for itself. while watch.changed().await.is_ok() { tokio::time::sleep(SETTLE).await; - let Some(interfaces) = watch.borrow_and_update().clone() else { + let Some(topology) = watch.borrow_and_update().clone() else { continue; }; let sessions = registry.list(); @@ -127,10 +128,9 @@ pub(crate) fn spawn_revalidator( } let table = Arc::clone(&rib); // Looks at every route: not for a thread that sessions run on. - let found = tokio::task::spawn_blocking(move || { - now_unreachable(&table, &interfaces, &neighbors) - }) - .await; + let found = + tokio::task::spawn_blocking(move || now_unreachable(&table, &topology, &neighbors)) + .await; let mut suspects = match found { Ok(suspects) => suspects, Err(e) => { @@ -141,7 +141,7 @@ pub(crate) fn spawn_revalidator( info!( now_unreachable = suspects.values().map(Vec::len).sum::(), sessions = sessions.len(), - "The machine's interfaces changed; looking at next hops again" + "The machine's interfaces or routes changed; looking at next hops again" ); // Every session, also those with nothing found: each has its // held routes to look at again. @@ -162,11 +162,11 @@ pub(crate) fn neighbor_at(address: SocketAddr) -> Neighbor { } /// The routes in the table that the kernel could not be given, as the -/// interfaces are now, by peer. `neighbors` says, for each peer by its +/// machine is now, by peer. `neighbors` says, for each peer by its /// number in the table, where it is. fn now_unreachable( rib: &RibTable, - interfaces: &Interfaces, + topology: &Topology, neighbors: &[Option], ) -> HashMap> { // There are as many next hops as neighbors, and a table's worth of @@ -177,19 +177,13 @@ fn now_unreachable( let Some(Some(neighbor)) = neighbors.get(route.peer.as_usize()) else { return false; }; - if interfaces.has_network_around(&route.prefix) { + if topology.interfaces.has_network_around(&route.prefix) { return true; } let next_hop = route.attrs.next_hop(); let key = (next_hop, route.prefix.address_family(), route.peer); !*resolved.borrow_mut().entry(key).or_insert_with(|| { - resolve( - interfaces, - &route.prefix, - next_hop.as_ref(), - Some(*neighbor), - ) - .is_ok() + resolve(topology, &route.prefix, next_hop.as_ref(), Some(*neighbor)).is_ok() }) }); let mut by_peer: HashMap> = HashMap::new(); @@ -206,7 +200,7 @@ fn now_unreachable( mod tests { use std::net::{IpAddr, Ipv4Addr}; - use bgp_fib::netlink::{Address, Link, Message}; + use bgp_fib::netlink::{Address, Link, Message, TABLE_MAIN}; use bgp_packet::message::{ AsPathAttribute, NextHopPathAttribute, OriginPathAttribute, PathAttribute, UpdateMessage, }; @@ -224,8 +218,8 @@ mod tests { /// A machine with eth2 (192.0.2.1/24), and eth3 (198.51.100.1/24) if /// `both`. - fn interfaces(both: bool) -> Interfaces { - let mut interfaces = Interfaces::default(); + fn interfaces(both: bool) -> Topology { + let mut interfaces = Topology::new(TABLE_MAIN, false); let all = [ (2, "192.0.2.1", "192.0.2.0/24"), (3, "198.51.100.1", "198.51.100.0/24"), @@ -298,7 +292,7 @@ mod tests { fn with_import( &self, peer: &mut Peer, - interfaces: &Interfaces, + interfaces: &Topology, work: impl FnOnce(&RibTable, &Import<'_>, &mut Held) -> T, ) -> T { let address = IpAddr::V4(peer.address); @@ -317,7 +311,7 @@ mod tests { policy: &self.policy, checks: Checks { next_hops: Some(NextHops { - interfaces, + topology: interfaces, neighbor: address.into(), }), ..Checks::NONE @@ -329,7 +323,7 @@ mod tests { /// The peer announces `route` with a path of `hops` AS numbers and /// itself as next hop. - fn announce(&self, peer: &mut Peer, interfaces: &Interfaces, route: &str, hops: usize) { + fn announce(&self, peer: &mut Peer, interfaces: &Topology, route: &str, hops: usize) { let update = UpdateMessage { withdrawn_routes: vec![], path_attributes: vec![ @@ -344,7 +338,7 @@ mod tests { }); } - fn revalidate(&self, peer: &mut Peer, interfaces: &Interfaces, suspects: &[Nlri]) { + fn revalidate(&self, peer: &mut Peer, interfaces: &Topology, suspects: &[Nlri]) { self.with_import(peer, interfaces, |table, import, held| { routes::revalidate(table, import, held, suspects) }); diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 2ad8656..4b7f208 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -192,7 +192,7 @@ impl BgpServer { hold_downs: HoldDowns::default(), rejections: self.rejections.clone(), rpki: self.rpki.clone(), - interfaces: self.kernel.as_ref().map(Fib::interfaces), + topology: self.kernel.as_ref().map(Fib::topology), restarts: Arc::clone(&self.restarts), }); // What an earlier run left in the kernel and no peer has announced @@ -277,10 +277,10 @@ impl BgpServer { spawn_revalidator(rpki, Arc::clone(&self.rib), self.registry.clone()), )); } - if let Some(interfaces) = context.interfaces.clone() { + if let Some(topology) = context.topology.clone() { tasks.push(named( "next hop revalidator", - reach::spawn_revalidator(interfaces, Arc::clone(&self.rib), self.registry.clone()), + reach::spawn_revalidator(topology, Arc::clone(&self.rib), self.registry.clone()), )); } for peer in self.config.peers.iter().filter(|peer| peer.passive) { diff --git a/docs/content/docs/development/architecture.md b/docs/content/docs/development/architecture.md index 8bb9893..1907cc5 100644 --- a/docs/content/docs/development/architecture.md +++ b/docs/content/docs/development/architecture.md @@ -163,9 +163,12 @@ UPDATE message, and waits when the session's output queue is full. ### Kernel Route Installation The `fib` crate reads best path changes and maintains the kernel routing -table over rtnetlink, with one route for each prefix through the next hop -of its best path. It also receives the router's interfaces and addresses -from the kernel, which determine whether a next hop is resolvable. +table over rtnetlink, with one route for each prefix through the resolved +next hop of its best path. It also receives the router's interfaces, +addresses and the routes of the resolution table from the kernel, which +determine whether and how a next hop is resolvable; the import policy +consults the same view, so that a route with an unresolvable next hop is +held rather than selected. ### HTTP Interface diff --git a/docs/content/docs/getting-started/introduction.md b/docs/content/docs/getting-started/introduction.md index 22c2565..76e4279 100644 --- a/docs/content/docs/getting-started/introduction.md +++ b/docs/content/docs/getting-started/introduction.md @@ -67,7 +67,6 @@ Ferrix 0.2.5 does not support the following features: Generalized TTL Security Mechanism (GTSM). - Long-Lived Graceful Restart, Bidirectional Forwarding Detection (BFD), and configuration changes without a restart. -- Recursive resolution of next hops that are not directly connected. For the implementation status of each RFC, see [Supported Standards](@/docs/reference/rfcs.md). diff --git a/docs/content/docs/guides/kernel.md b/docs/content/docs/guides/kernel.md index 23a3c70..a2f84d6 100644 --- a/docs/content/docs/guides/kernel.md +++ b/docs/content/docs/guides/kernel.md @@ -33,6 +33,8 @@ kernel { | `metric` | Metric of the installed routes. The kernel prefers the route with the lowest metric. | 20 | | `without_nexthop_objects` | Installs each route with its own gateway instead of a nexthop object | Disabled | | `stale_seconds` | Time for which routes from a previous run are retained at startup | 120 seconds | +| `resolution_table` | Kernel routing table whose routes resolve next hops that are not directly connected | 254 (`main`) | +| `resolve_via_default_route` | Permits the default route to resolve a next hop | Disabled | Ferrix requires the `CAP_NET_ADMIN` capability to modify the kernel routing table. If the kernel rejects a request for lack of privileges, the daemon @@ -44,15 +46,55 @@ exits. See [Granting Privileges](@/docs/getting-started/installation.md#granting ## Next Hop Resolution -Ferrix installs a route only if its next hop is directly connected: the -next hop must be an address in a subnet of an interface that is up. For an -IPv6 route with a link-local next hop, the outgoing interface is the -interface on which the neighbor's session is established. +Ferrix resolves the next hop of each active route to a gateway and an +outgoing interface before it installs the route: -Ferrix does not resolve next hops recursively through other routes. -Therefore, a route whose next hop is not directly connected, such as a -route from an IBGP neighbor that does not set itself as the next hop, is -not installed. +- A next hop that is directly connected, that is, an address in a subnet + of an interface that is up, is the gateway, and the interface is that + interface. For an IPv6 route with a link-local next hop, the outgoing + interface is the interface on which the neighbor's session is + established. +- A next hop that is not directly connected is resolved through the + routes of the resolution table (`resolution_table`, by default the + `main` table). Ferrix selects the route with the longest prefix that + contains the next hop, and of several routes for that prefix the one + with the lowest metric. If the route has a gateway, the gateway and the + route's interface are used. If the route has no gateway, as a route to a + point-to-point interface or a tunnel does not, the next hop itself is + the gateway and the route's interface is used. A route that names a + kernel nexthop object is resolved through that object. + +The Linux kernel does not resolve gateways recursively, so Ferrix installs +the route with the resolved gateway and interface rather than with the +BGP next hop. For example, a route from an IBGP neighbor with the next hop +`192.0.2.9`, when the kernel has the static route +`192.0.2.0/24 via 10.0.0.2 dev wg0`, is installed as a route +`via 10.0.0.2 dev wg0`. When the static route changes or is removed, +Ferrix resolves the next hop again and updates or removes the installed +route. + +The following routes do not resolve a next hop: + +- Routes with protocol `bgp`, whether installed by Ferrix or by another + BGP daemon. A next hop that is reachable only through a BGP route would + depend on another next hop, and such dependencies between routes are not + predictable. +- The default route, unless `resolve_via_default_route` is `true`. On a + router, the default route typically points to a transit provider, and a + next hop that is reachable only through it is not a usable next hop. +- Blackhole, unreachable and prohibit routes. A next hop covered by such a + route is unresolvable. +- Routes whose interface is down. + +Ferrix learns the routes of the resolution table from the kernel at +startup and is notified of changes. Changes to the routes take effect +within a few seconds. + +> [!NOTE] +> Routes from an IBGP neighbor that does not set itself as the next hop, +> such as a route reflector that preserves the next hop, require a route to +> the next hop in the resolution table: a static route, a route to the +> tunnel on which the next hop is reached, or a route from an IGP. ### Exclusion of Unresolvable Routes diff --git a/docs/content/docs/guides/passing-routes-on.md b/docs/content/docs/guides/passing-routes-on.md index 3973bc2..6aa3237 100644 --- a/docs/content/docs/guides/passing-routes-on.md +++ b/docs/content/docs/guides/passing-routes-on.md @@ -367,10 +367,12 @@ not affected. Ferrix advertises a reflected route with the local address of the session as next hop, as it does every route. A standard route reflector does not -change the next hop. Ferrix does, because it does not resolve next hops -that are not directly connected: a client that is not connected to the -router that learned the route can then still use it, and sends the -traffic through the route reflector. +change the next hop. Ferrix does, so that a client needs no route to the +router that learned the route: the client sends the traffic through the +route reflector, which forwards it. Preserving the next hop on reflection +is not yet configurable; clients that have a route to the originating +router, for example over a tunnel, would then forward to it directly. See +[Next Hop Resolution](@/docs/guides/kernel.md#next-hop-resolution). ### Loop Prevention diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md index a33caa7..f5a849b 100644 --- a/docs/content/docs/reference/configuration.md +++ b/docs/content/docs/reference/configuration.md @@ -214,6 +214,8 @@ Statements for kernel route installation. See | `metric` uint32, optional | 20 | Specifies the metric of installed routes. | | `without_nexthop_objects` bool | `false` | Installs each route with its own gateway instead of a nexthop object. | | `stale_seconds` uint32, optional | 120 | Specifies the time, in seconds, for which routes installed by a previous run are retained at startup. A value of 0 removes them immediately. | +| `resolution_table` uint32 | 254 (`main`) | Specifies the kernel routing table whose routes resolve next hops that are not directly connected. Routes with protocol `bgp` in this table are not used. See [Next Hop Resolution](@/docs/guides/kernel.md#next-hop-resolution). | +| `resolve_via_default_route` bool | `false` | Permits the default route of the resolution table to resolve a next hop. | ## GracefulRestart diff --git a/docs/content/docs/reference/rfcs.md b/docs/content/docs/reference/rfcs.md index 3e66219..744e14e 100644 --- a/docs/content/docs/reference/rfcs.md +++ b/docs/content/docs/reference/rfcs.md @@ -38,9 +38,9 @@ The base protocol specification. | §8.1.1 | Optional `CollisionDetectEstablishedState` (`collision_detect_established_state`) | ✅ | | §6.2 | Bad BGP Identifier check (neighbor uses the local identifier, or 0.0.0.0) | ✅ | | §9 | Decision process / best-path selection | ✅ | -| §9.1.2 | Best routes installed for forwarding (Linux kernel, over netlink), where the next hop is directly connected | ✅ | +| §9.1.2 | Best routes installed for forwarding (Linux kernel, over netlink) | ✅ | | §9.1.2.1 | Routes with an unresolvable next hop excluded from selection | ✅ where routes are installed in the kernel: such a route is held out of the table until its next hop can be reached | -| §9.1.2.1 | Next hops resolved through other routes | ❌ (only next hops that are neighbors count as reachable) | +| §9.1.2.1 | Next hops resolved through other routes | ✅ (through the kernel's connected, static and IGP routes; not through BGP routes) | | §9 | Adj-RIB-In (per-peer table of received routes) | ✅ | | §9 | Locally originated routes, advertised to all established peers | ✅ | | §9 | Advertisement of the best path for each prefix to neighbors (`export.learned_routes`), with the local AS prepended and the local address as next hop; MED and LOCAL_PREF not sent to EBGP neighbors; LOCAL_PREF sent to IBGP neighbors (§5.1.4, §5.1.5, §9.1.3) | ✅ | @@ -116,7 +116,7 @@ The base protocol specification. | ORIGINATOR_ID and CLUSTER_LIST added on reflection, not sent to EBGP neighbors | ✅ | | Routes with the local router ID as ORIGINATOR_ID, or the local cluster ID in CLUSTER_LIST, rejected | ✅ | | Configurable cluster ID | ✅ | -| Next hop unchanged on reflection | ❌ The route reflector is the next hop: next hops that are not directly connected are not resolved | +| Next hop unchanged on reflection | ❌ The route reflector sets itself as the next hop | | ORIGINATOR_ID and CLUSTER_LIST length as best path tie breakers (§9) | ❌ | ### RFC 8538 — Notification Message Support for BGP Graceful Restart