diff --git a/bin/bgp/src/main.rs b/bin/bgp/src/main.rs index 5f453c5..0a78cb2 100644 --- a/bin/bgp/src/main.rs +++ b/bin/bgp/src/main.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; +use std::process::ExitCode; use std::sync::Arc; use bgp_server::config::Config; use bgp_server::proto::config::BgpConfig; -use bgp_server::server::{BgpServer, shut_down}; +use bgp_server::server::{BgpServer, leave_for_restart, shut_down}; use bgp_web::{AppState, RouterInfo}; use clap::Parser; use eyre::{Context, Result}; @@ -20,6 +21,11 @@ mod logging; #[global_allocator] static ALLOCATOR: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +/// The exit status after leaving to be started again (SIGUSR1): +/// EX_TEMPFAIL, on which the systemd unit starts the daemon again +/// (`RestartForceExitStatus=75`). +const RESTART_EXIT_STATUS: u8 = 75; + /// A BGP speaker. #[derive(Parser, Debug)] #[command(version)] @@ -37,7 +43,7 @@ struct ServerArgs { } #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> Result { let args = ServerArgs::parse(); args.logging.install()?; log_panics(); @@ -55,7 +61,7 @@ async fn main() -> Result<()> { config.peers.len(), config.originate.len() ); - return Ok(()); + return Ok(ExitCode::SUCCESS); } let server = BgpServer::bind(config).await?; @@ -81,6 +87,7 @@ async fn main() -> Result<()> { refusals: server.refusals.clone(), rpki: server.rpki.clone(), kernel: server.kernel.clone(), + restarts: Arc::clone(&server.restarts), }; tokio::spawn(async move { if let Err(e) = bgp_web::serve_on(state, listener).await { @@ -94,13 +101,28 @@ async fn main() -> Result<()> { // makes them slower to take us back. // And take our routes out of the kernel: with us gone nothing would // keep them true. + // Unless we are to be back at once (SIGUSR1, for an upgrade): then the + // routes stay, and peers that agreed to graceful restart keep ours. let (registry, kernel) = (server.registry.clone(), server.kernel.clone()); tokio::select! { - result = server.run() => Ok(result?), - signal = shutdown_signal() => { - info!(signal, "Shutting down, closing BGP sessions"); - shut_down(®istry, kernel.as_ref()).await; - Ok(()) + result = server.run() => { + result?; + Ok(ExitCode::SUCCESS) + } + signal = signalled() => match signal { + Signal::Stop(name) => { + info!(signal = name, "Shutting down, closing BGP sessions"); + shut_down(®istry, kernel.as_ref()).await; + Ok(ExitCode::SUCCESS) + } + Signal::Restart => { + info!( + signal = "SIGUSR1", + "Leaving to be started again: routes stay in the kernel, and peers keep ours" + ); + leave_for_restart(®istry, kernel.as_ref()).await; + Ok(ExitCode::from(RESTART_EXIT_STATUS)) + } } } } @@ -127,20 +149,30 @@ fn log_panics() { })); } -/// Resolves with the name of the first termination signal received. -async fn shutdown_signal() -> &'static str { +/// What a signal asks of the daemon. +enum Signal { + /// To stop: SIGINT or SIGTERM, by name. + Stop(&'static str), + /// To leave, to be started again at once: SIGUSR1. + Restart, +} + +/// Resolves with the first signal received that ends the daemon. +async fn signalled() -> Signal { #[cfg(unix)] { use tokio::signal::unix::{SignalKind, signal}; let mut term = signal(SignalKind::terminate()).expect("install SIGTERM handler"); + let mut restart = signal(SignalKind::user_defined1()).expect("install SIGUSR1 handler"); tokio::select! { - _ = tokio::signal::ctrl_c() => "SIGINT", - _ = term.recv() => "SIGTERM", + _ = tokio::signal::ctrl_c() => Signal::Stop("SIGINT"), + _ = term.recv() => Signal::Stop("SIGTERM"), + _ = restart.recv() => Signal::Restart, } } #[cfg(not(unix))] { let _ = tokio::signal::ctrl_c().await; - "ctrl-c" + Signal::Stop("ctrl-c") } } diff --git a/crates/fib/src/kernel.rs b/crates/fib/src/kernel.rs index 977ca10..4278cb1 100644 --- a/crates/fib/src/kernel.rs +++ b/crates/fib/src/kernel.rs @@ -31,7 +31,7 @@ use crate::netlink::{ /// "Input/output error": for what should not happen. const EIO: i32 = 5; use crate::plan::{Neighbor, Op, Plan, Target, Unresolved, resolve}; -use crate::{Fib, FibConfig, FibError, InterfaceStatus}; +use crate::{Fib, FibConfig, FibError, InterfaceStatus, Stop}; /// How many best routes are taken from the routing table at a time. const BATCH: usize = 1024; @@ -932,6 +932,18 @@ impl Task { Ok(()) } + /// Ends with our routes left in the kernel, for the next run to find: + /// what was on its way there is written, and nothing is taken out. + async fn leave(&mut self) -> Result<(), FibError> { + info!( + routes = self.plan.installed(), + "Leaving our routes in the kernel for the next run" + ); + self.queue = None; + (&mut self.writer).await.map_err(io::Error::other)??; + Ok(()) + } + /// Takes everything of ours out of the kernel. async fn withdraw( &mut self, @@ -1022,7 +1034,7 @@ pub(crate) async fn run( rib: Arc, fib: Fib, sockets: Sockets, - mut stopping: watch::Receiver, + mut stopping: watch::Receiver>, ) -> Result<(), FibError> { // There from the first scrape, not from the first time they count. LazyLock::force(&ROUTES); @@ -1075,6 +1087,11 @@ pub(crate) async fn run( release_at = keep.then(|| Instant::now() + config.stale_time); task.keeping = keep; } + task.fib + .shared + .adopted + .send_replace(Some(task.plan.installed())); + let mut release = task.fib.shared.release.subscribe(); task.publish(true); info!( table = config.table, @@ -1114,7 +1131,13 @@ pub(crate) async fn run( task.relearn(&mut heard).await?; reader = task.rib.reader(); } - () = at(release_at), if release_at.is_some() => { + _ = async { + tokio::select! { + () = at(release_at) => {} + // Sooner, if the peers are known to have sent everything. + _ = release.wait_for(|now| *now) => {} + } + }, if release_at.is_some() => { release_at = None; task.keeping = false; let stale = task.plan.release(); @@ -1133,7 +1156,11 @@ pub(crate) async fn run( }); } _ = stopping.changed() => { - task.withdraw(&mut heard).await?; + if *stopping.borrow() == Some(Stop::Keep) { + task.leave().await?; + } else { + task.withdraw(&mut heard).await?; + } task.publish(false); return Ok(()); } diff --git a/crates/fib/src/lib.rs b/crates/fib/src/lib.rs index c28d957..a0cb2e4 100644 --- a/crates/fib/src/lib.rs +++ b/crates/fib/src/lib.rs @@ -139,11 +139,25 @@ pub struct Status { /// until it has. Whoever holds one is told when they change. pub type InterfacesWatch = watch::Receiver>>; +/// How the task that keeps the kernel's routes is to end. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Stop { + /// Taking our routes out of the kernel. + Withdraw, + /// Leaving them there, for the next run to find. + Keep, +} + struct Shared { config: FibConfig, status: Mutex, interfaces: watch::Sender>>, - stop: watch::Sender, + stop: watch::Sender>, + /// How many routes of an earlier run were found in the kernel at the + /// start, once that has been looked at. + adopted: watch::Sender>, + /// Set to have what is kept of an earlier run go now. + release: watch::Sender, /// Whether the task has ended. ended: watch::Sender, } @@ -165,13 +179,15 @@ impl Fib { rib: Arc, config: FibConfig, ) -> Result<(Self, JoinHandle>), FibError> { - let (stop, stopping) = watch::channel(false); + let (stop, stopping) = watch::channel(None); let fib = Self { shared: Arc::new(Shared { config, status: Mutex::default(), interfaces: watch::channel(None).0, stop, + adopted: watch::channel(None).0, + release: watch::channel(false).0, ended: watch::channel(false).0, }), }; @@ -208,13 +224,52 @@ impl Fib { /// until it has. Returns at once if it has ended already, for whatever /// reason; the task's own result says how. pub async fn stop(&self) { + self.end(Stop::Withdraw).await; + } + + /// Has the task stop following the routing table and end, leaving our + /// routes in the kernel for the next run to find and keep for a while + /// (see [`FibConfig::stale_time`]), and waits until it has. For a + /// restart that is to go unnoticed by the packets being forwarded. + pub async fn leave(&self) { + self.end(Stop::Keep).await; + } + + async fn end(&self, how: Stop) { let mut ended = self.shared.ended.subscribe(); - self.shared.stop.send_replace(true); + self.shared.stop.send_if_modified(|stop| { + let first = stop.is_none(); + if first { + *stop = Some(how); + } + first + }); // Fails only if the handle that holds the sender is gone, and this // is one. let _ = ended.wait_for(|ended| *ended).await; } + /// How many routes of an earlier run the kernel had at the start, once + /// the task has looked; zero if it ended before it did. + pub async fn adopted(&self) -> usize { + let mut adopted = self.shared.adopted.subscribe(); + let mut ended = self.shared.ended.subscribe(); + tokio::select! { + found = adopted.wait_for(Option::is_some) => { + found.ok().and_then(|found| *found).unwrap_or(0) + } + _ = ended.wait_for(|ended| *ended) => 0, + } + } + + /// Has the routes of an earlier run that the routing table does not + /// have go now, rather than when [`FibConfig::stale_time`] is up: the + /// peers have all sent their routes, and what is not among them is no + /// longer anybody's. + pub fn release_kept(&self) { + self.shared.release.send_replace(true); + } + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] fn update(&self, change: impl FnOnce(&mut Status)) { change( @@ -231,7 +286,7 @@ impl Fib { fn start( rib: Arc, fib: Fib, - stopping: watch::Receiver, + stopping: watch::Receiver>, ) -> Result>, FibError> { // Opened here, so that a machine that cannot is known of at once. let sockets = kernel::Sockets::open()?; @@ -246,7 +301,7 @@ fn start( fn start( _rib: Arc, _fib: Fib, - _stopping: watch::Receiver, + _stopping: watch::Receiver>, ) -> Result>, FibError> { Err(FibError::Unsupported) } diff --git a/crates/fib/tests/kernel.rs b/crates/fib/tests/kernel.rs index 49611d7..b76f201 100644 --- a/crates/fib/tests/kernel.rs +++ b/crates/fib/tests/kernel.rs @@ -649,6 +649,55 @@ async fn routes_of_an_earlier_run_are_kept_while_peers_come_back() { task.await.unwrap().unwrap(); } +/// A restart that is to go unnoticed (graceful restart): the routes are +/// left in the kernel, by way of their nexthop objects; the next run +/// finds them, says how many, and lets those nobody announced again go +/// as soon as it is told the peers have all sent theirs. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "changes the machine's routes; run scripts/kernel-tests.sh"] +async fn routes_stay_in_the_kernel_over_a_restart() { + let mut machine = Machine::new(12); + machine.stale_time = Duration::from_secs(60); + let (fib, task) = machine.start(); + assert_eq!(fib.adopted().await, 0); + let peer = machine.peer(2); + let via = machine.via(machine.neighbor(2).into(), 0); + machine.rib.announce( + peer, + &via, + &[prefix("203.0.113.0/24"), prefix("198.18.0.0/15")], + ); + let both = [ + machine.shown("198.18.0.0/15", machine.neighbor(2)), + machine.shown("203.0.113.0/24", machine.neighbor(2)), + ]; + machine.kernel_becomes(&both); + + fib.leave().await; + task.await.unwrap().unwrap(); + assert_eq!(machine.kernel_routes(), both.iter().cloned().collect()); + + // The next run, with a table that is empty until the peers are back. + machine.rib = Arc::new(RibTable::new(4)); + let (fib, task) = machine.start(); + assert_eq!(fib.adopted().await, 2); + status_becomes(&fib, "keeping both", |status| status.kept == 2); + let peer = machine.peer(2); + let via = machine.via(machine.neighbor(2).into(), 0); + machine + .rib + .announce(peer, &via, &[prefix("203.0.113.0/24")]); + status_becomes(&fib, "one confirmed", |status| status.kept == 1); + + // Every peer has sent its routes: what was not among them goes now, + // and not a minute later. + fib.release_kept(); + machine.kernel_becomes(&[machine.shown("203.0.113.0/24", machine.neighbor(2))]); + + fib.stop().await; + task.await.unwrap().unwrap(); +} + /// An IPv4 route whose next hop is an IPv6 neighbor: routes over a link /// that has only IPv6 addresses (RFC 8950). The kernel takes such a route, /// by way of a nexthop object or with its gateway spelled out. diff --git a/crates/integration-tests/src/test_server.rs b/crates/integration-tests/src/test_server.rs index 4fc8dce..364e7cd 100644 --- a/crates/integration-tests/src/test_server.rs +++ b/crates/integration-tests/src/test_server.rs @@ -10,6 +10,7 @@ use bgp_rpki::Rpki; use bgp_server::config::Config; use bgp_server::error::ServerError; use bgp_server::refusals::RefusalLog; +use bgp_server::restart::Restarts; use bgp_server::server::BgpServer; use bgp_server::session::SessionRegistry; use bgp_web::{AppState, RouterInfo}; @@ -35,6 +36,8 @@ pub struct TestServer { pub rpki: Option, /// What keeps the best routes in the kernel, if the server does. pub kernel: Option, + /// The routes kept of restarting peers (graceful restart). + pub restarts: Arc, pub local_asn: u32, } @@ -69,6 +72,7 @@ impl TestServer { kernel: None, export: Default::default(), control: None, + graceful_restart: Default::default(), }; customize(&mut config); @@ -84,6 +88,7 @@ impl TestServer { let refusals = bound.refusals.clone(); let rpki = bound.rpki.clone(); let kernel = bound.kernel.clone(); + let restarts = Arc::clone(&bound.restarts); let handle = tokio::spawn(bound.run()); Ok(Self { @@ -95,6 +100,7 @@ impl TestServer { refusals, rpki, kernel, + restarts, local_asn, }) } @@ -215,6 +221,7 @@ impl TestServer { refusals: self.refusals.clone(), rpki: self.rpki.clone(), kernel: self.kernel.clone(), + restarts: Arc::clone(&self.restarts), }; let handle = tokio::spawn(bgp_web::serve_on(state, listener)); Ok((base_url, HttpGuard { handle })) diff --git a/crates/integration-tests/tests/add_path.rs b/crates/integration-tests/tests/add_path.rs index 01fb781..1daa6af 100644 --- a/crates/integration-tests/tests/add_path.rs +++ b/crates/integration-tests/tests/add_path.rs @@ -40,6 +40,7 @@ async fn server_offering( .collect(); TestServer::start_with(LOCAL_ASN, |c| { c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "route-server-client".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/admission.rs b/crates/integration-tests/tests/admission.rs index 7ba8a8f..26fb3d9 100644 --- a/crates/integration-tests/tests/admission.rs +++ b/crates/integration-tests/tests/admission.rs @@ -18,6 +18,7 @@ const TIMEOUT: Duration = Duration::from_secs(5); fn peer_at(endpoint: SocketAddr) -> PeerConfig { PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/aspa.rs b/crates/integration-tests/tests/aspa.rs index 04df306..9853e5c 100644 --- a/crates/integration-tests/tests/aspa.rs +++ b/crates/integration-tests/tests/aspa.rs @@ -57,6 +57,7 @@ async fn server_with( ..RpkiConfig::default() }); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/collision.rs b/crates/integration-tests/tests/collision.rs index dbf50ef..7be5673 100644 --- a/crates/integration-tests/tests/collision.rs +++ b/crates/integration-tests/tests/collision.rs @@ -36,6 +36,7 @@ async fn server_dialing(listener: &TcpListener) -> Result { TestServer::start_with(LOCAL_ASN, |c| { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs index b976a26..68b439a 100644 --- a/crates/integration-tests/tests/debugging.rs +++ b/crates/integration-tests/tests/debugging.rs @@ -167,6 +167,7 @@ async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { let server = TestServer::start_with(1299, |c| { c.accept_unconfigured_peers = false; c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "refused-demo".into(), endpoint: endpoint.into(), remote_asn: 3333, diff --git a/crates/integration-tests/tests/dialer.rs b/crates/integration-tests/tests/dialer.rs index fc5c6c3..83f1492 100644 --- a/crates/integration-tests/tests/dialer.rs +++ b/crates/integration-tests/tests/dialer.rs @@ -27,6 +27,7 @@ async fn server_dialing(listener: &TcpListener, remote_asn: u32) -> Result Result<()> { let server = TestServer::start_with(LOCAL_ASN, |c| { c.connect_retry = Duration::from_secs(60); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, @@ -237,6 +239,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { let _server = TestServer::start_with(LOCAL_ASN, |c| { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, @@ -258,6 +261,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { let server = TestServer::start_with(LOCAL_ASN, |c| { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/graceful_restart.rs b/crates/integration-tests/tests/graceful_restart.rs new file mode 100644 index 0000000..c52daee --- /dev/null +++ b/crates/integration-tests/tests/graceful_restart.rs @@ -0,0 +1,359 @@ +//! Graceful restart (RFC 4724, RFC 8538): a peer that restarts keeps its +//! routes with us while it does, and we keep ours with it while we do. + +use std::net::Ipv4Addr; +use std::time::Duration; + +use bgp_integration_tests::test_peer::TestPeer; +use bgp_integration_tests::test_server::TestServer; +use bgp_packet::errors::{BgpError, CeaseSubcode}; +use bgp_packet::message::{ + AsPathAttribute, Message, NextHopPathAttribute, NotificationMessage, OriginPathAttribute, + PathAttribute, UpdateMessage, +}; +use bgp_packet::open::{Capability, GracefulRestartEntry}; +use bgp_primitives::constants::{Afi, AfiSafi, Safi}; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_server::restart::Leave; +use eyre::{Result, bail}; + +const TIMEOUT: Duration = Duration::from_secs(5); +const PEER_ASN: u32 = 65001; +const PEER_ID: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 2); +const PREFIXES: [&str; 3] = ["10.1.0.0/16", "10.2.0.0/16", "10.3.0.0/16"]; + +/// The graceful restart capability of a peer that speaks IPv4 and IPv6. +fn restart(restarted: bool, notification: bool, forwarding: bool, time: u16) -> Capability { + Capability::GracefulRestart { + restart_state: restarted, + restart_time: time, + notification, + entries: [Afi::IPV4, Afi::IPV6] + .into_iter() + .map(|afi| GracefulRestartEntry { + afi, + safi: Safi::UNICAST, + preserve_forwarding: forwarding, + }) + .collect(), + } +} + +/// What a dual-stack peer offers, with `more` besides. +fn capabilities(more: impl IntoIterator) -> Vec { + let mut capabilities = vec![ + Capability::FourByteAsn { asn: PEER_ASN }, + Capability::MultiProtocol { + afi: Afi::IPV4, + safi: Safi::UNICAST, + }, + Capability::MultiProtocol { + afi: Afi::IPV6, + safi: Safi::UNICAST, + }, + Capability::RouteRefresh, + ]; + capabilities.extend(more); + capabilities +} + +fn announcement(prefixes: &[&str]) -> Message { + Message::Update(UpdateMessage { + withdrawn_routes: vec![], + path_attributes: vec![ + PathAttribute::Origin(OriginPathAttribute::IGP), + AsPathAttribute::from_asns(vec![PEER_ASN]), + PathAttribute::NextHop(NextHopPathAttribute(Ipv4Addr::new(192, 0, 2, 1))), + ], + announced_routes: prefixes + .iter() + .map(|p| IpPrefix::try_from(*p).unwrap().into()) + .collect(), + }) +} + +fn end_of_rib(family: AfiSafi) -> Message { + Message::Update(UpdateMessage::end_of_rib(family)) +} + +/// Waits until `done` holds, or fails with `what`. +async fn until(what: &str, mut done: impl FnMut() -> bool) -> Result<()> { + let waited = tokio::time::timeout(TIMEOUT, async { + while !done() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + if waited.is_err() { + bail!("timed out waiting until {what}"); + } + Ok(()) +} + +/// The prefixes in the server's table, and how many of its routes are +/// stale. +fn table(server: &TestServer) -> (Vec, usize) { + let mut prefixes = Vec::new(); + let mut stale = 0; + server.rib.for_each_route(|route| { + prefixes.push(route.prefix.to_string()); + stale += usize::from(route.stale); + }); + prefixes.sort(); + (prefixes, stale) +} + +/// A peer that has announced `PREFIXES` with graceful restart as `offer` +/// says, and is Established. +async fn peer_with_routes(server: &TestServer, offer: Capability) -> Result { + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake_with(PEER_ASN, PEER_ID, 90, capabilities([offer])) + .await?; + peer.send(&announcement(&PREFIXES)).await?; + until("the routes are in", || server.rib.len() == 3).await?; + Ok(peer) +} + +/// The peer's connection fails; its routes are kept, stale, and used as +/// before. It comes back, sends some of them again, and its End-of-RIB: +/// the others go. +#[tokio::test] +async fn test_a_restarting_peer_keeps_its_routes_until_it_has_sent_them_again() -> Result<()> { + let server = TestServer::start(65000).await?; + let peer = peer_with_routes(&server, restart(false, true, true, 90)).await?; + drop(peer); + + until("the session is down", || server.registry.list().is_empty()).await?; + until("the routes are stale", || table(&server).1 == 3).await?; + let kept = server.restarts.kept(); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].1.routes, 3); + // Still chosen: the RFC has stale routes used like any others. + assert!(server.received_routes().iter().all(|route| route.best)); + // And shown, as the peer's, stale. + let (base, _http) = server.serve_http().await?; + let received: serde_json::Value = reqwest::get(format!("{base}/api/routes/received")) + .await? + .json() + .await?; + let routes = received["routes"].as_array().unwrap(); + assert_eq!(routes.len(), 3, "{received}"); + for route in routes { + assert_eq!(route["stale"], true, "{route}"); + assert_eq!(route["peer_asn"], PEER_ASN, "{route}"); + } + + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake_with( + PEER_ASN, + PEER_ID, + 90, + capabilities([restart(true, true, true, 90)]), + ) + .await?; + until("the routes are taken over", || { + server.registry.list().len() == 1 && server.restarts.kept().is_empty() + }) + .await?; + peer.send(&announcement(&PREFIXES[..2])).await?; + until("two are fresh", || table(&server).1 == 1).await?; + // The peer has them all, as far as its limit goes. + assert_eq!(server.registry.list()[0].counters.routes(), 3); + + peer.send(&end_of_rib(AfiSafi::IPV4_UNICAST)).await?; + until("the one not sent again is gone", || { + table(&server) == (vec!["10.1.0.0/16".into(), "10.2.0.0/16".into()], 0) + }) + .await?; + until("it is counted out", || { + server + .registry + .list() + .first() + .is_some_and(|session| session.counters.routes() == 2) + }) + .await?; + Ok(()) +} + +/// What ends a session decides whether the peer is restarting: a Hard +/// Reset never is, and a NOTIFICATION is only where both sides said so +/// (RFC 8538). A peer that does not offer graceful restart never is. +#[tokio::test] +async fn test_only_a_restart_keeps_the_routes() -> Result<()> { + let cease = |subcode| NotificationMessage { + error: BgpError::Cease(subcode), + data: vec![], + }; + let hard_reset = NotificationMessage { + error: BgpError::Cease(CeaseSubcode::HardReset), + data: vec![6, 2], + }; + for (offer, goodbye, kept) in [ + ( + Some(restart(false, true, true, 90)), + Some(cease(CeaseSubcode::AdminReset)), + true, + ), + ( + Some(restart(false, true, true, 90)), + Some(hard_reset.clone()), + false, + ), + ( + Some(restart(false, false, true, 90)), + Some(cease(CeaseSubcode::AdminReset)), + false, + ), + (Some(restart(false, false, true, 90)), None, true), + (Some(restart(false, true, true, 0)), None, false), + (None, None, false), + ] { + let server = TestServer::start(65000).await?; + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake_with(PEER_ASN, PEER_ID, 90, capabilities(offer)) + .await?; + peer.send(&announcement(&PREFIXES)).await?; + until("the routes are in", || server.rib.len() == 3).await?; + if let Some(goodbye) = &goodbye { + peer.send(&Message::Notification(goodbye.clone())).await?; + } + drop(peer); + until("the session is down", || server.registry.list().is_empty()).await?; + if kept { + until("the routes are stale", || table(&server).1 == 3).await?; + } else { + until("the routes are gone", || server.rib.is_empty()).await?; + assert!(server.restarts.kept().is_empty()); + } + } + Ok(()) +} + +/// A peer that does not come back within its restart time has its routes +/// removed then. +#[tokio::test] +async fn test_the_routes_go_when_the_peer_does_not_come_back_in_time() -> Result<()> { + let server = TestServer::start(65000).await?; + let peer = peer_with_routes(&server, restart(false, true, true, 1)).await?; + drop(peer); + until("the routes are stale", || table(&server).1 == 3).await?; + until("the routes are gone", || server.rib.is_empty()).await?; + assert!(server.restarts.kept().is_empty()); + Ok(()) +} + +/// A peer that comes back without having kept its forwarding (F bit), or +/// without graceful restart at all, has its stale routes removed at once. +#[tokio::test] +async fn test_routes_whose_forwarding_was_lost_go_when_the_peer_is_back() -> Result<()> { + for back_with in [Some(restart(true, true, false, 90)), None] { + let server = TestServer::start(65000).await?; + let peer = peer_with_routes(&server, restart(false, true, true, 90)).await?; + drop(peer); + until("the routes are stale", || table(&server).1 == 3).await?; + + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake_with(PEER_ASN, PEER_ID, 90, capabilities(back_with.clone())) + .await?; + until("the stale routes are gone", || server.rib.is_empty()).await?; + until("they are counted out", || { + server + .registry + .list() + .first() + .is_some_and(|session| session.counters.routes() == 0) + }) + .await?; + drop(peer); + } + Ok(()) +} + +/// Our OPEN offers graceful restart, as just restarted, and a peer that +/// agreed is sent End-of-RIB markers once it has our routes. +#[tokio::test] +async fn test_we_offer_graceful_restart_and_say_when_our_routes_are_all_sent() -> Result<()> { + let server = TestServer::start(65000).await?; + let mut peer = TestPeer::connect(server.addr).await?; + let open = peer + .handshake_with( + PEER_ASN, + PEER_ID, + 90, + capabilities([restart(false, true, true, 90)]), + ) + .await?; + let offered = open.capabilities().find_map(|capability| match capability { + Capability::GracefulRestart { + restart_state, + restart_time, + notification, + entries, + } => Some(( + *restart_state, + *restart_time, + *notification, + entries.clone(), + )), + _ => None, + }); + let Some((restarted, time, notification, entries)) = offered else { + bail!("no graceful restart in {open:?}"); + }; + assert!(restarted, "started just now"); + assert_eq!(time, 120); + assert!(notification); + // Without kernel routes, our forwarding is not ours to lose. + assert!(entries.iter().all(|entry| entry.preserve_forwarding)); + + // Nothing is originated: the markers are all there is. + let mut markers = Vec::new(); + while markers.len() < 2 { + let update = tokio::time::timeout(TIMEOUT, peer.recv_update()).await??; + match update.end_of_rib_family() { + Some(family) => markers.push(family), + None => bail!("expected End-of-RIB, got {update:?}"), + } + } + markers.sort(); + assert_eq!(markers, [AfiSafi::IPV4_UNICAST, AfiSafi::IPV6_UNICAST]); + Ok(()) +} + +/// Leaving to restart closes the connection without a NOTIFICATION where +/// graceful restart was agreed, so that the peer keeps our routes; leaving +/// for good is a Hard Reset there, and a Cease elsewhere. +#[tokio::test] +async fn test_leaving_tells_each_peer_as_graceful_restart_has_it() -> Result<()> { + let hard_reset = BgpError::Cease(CeaseSubcode::HardReset); + let shutdown = BgpError::Cease(CeaseSubcode::AdminShutdown); + for (leave, offer, told) in [ + (Leave::Restart, Some(restart(false, true, true, 90)), None), + (Leave::Restart, None, Some(shutdown.clone())), + ( + Leave::Stop, + Some(restart(false, true, true, 90)), + Some(hard_reset), + ), + (Leave::Stop, None, Some(shutdown)), + ] { + let server = TestServer::start(65000).await?; + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake_with(PEER_ASN, PEER_ID, 90, capabilities(offer.clone())) + .await?; + until("the session is up", || server.registry.list().len() == 1).await?; + server.registry.leave(leave, Duration::from_secs(2)).await; + let mut notification = None; + // Whatever came before (End-of-RIB markers) is passed over. + while let Ok(Ok(message)) = tokio::time::timeout(TIMEOUT, peer.recv()).await { + if let Message::Notification(n) = message { + notification = Some(n.error); + break; + } + } + assert_eq!(notification, told, "{leave:?} with {offer:?}"); + assert!(peer.is_closed().await); + } + Ok(()) +} diff --git a/crates/integration-tests/tests/import_policy.rs b/crates/integration-tests/tests/import_policy.rs index b04e990..d909db1 100644 --- a/crates/integration-tests/tests/import_policy.rs +++ b/crates/integration-tests/tests/import_policy.rs @@ -41,6 +41,7 @@ async fn server_with(import: ImportPolicy) -> Result<(TestServer, TcpListener)> c.accept_unconfigured_peers = false; c.connect_retry = Duration::from_millis(50); c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "r1".into(), endpoint: endpoint.into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/negotiation.rs b/crates/integration-tests/tests/negotiation.rs index bf9eefc..c8b65d3 100644 --- a/crates/integration-tests/tests/negotiation.rs +++ b/crates/integration-tests/tests/negotiation.rs @@ -183,6 +183,7 @@ async fn test_no_family_in_common_refuses_the_session() -> Result<()> { let server = TestServer::start_with(LOCAL_ASN, |c| { // Inbound connections from this address inherit the peer's settings. c.peers = vec![PeerConfig { + graceful_restart: Default::default(), name: "v6-only".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, diff --git a/crates/integration-tests/tests/roles.rs b/crates/integration-tests/tests/roles.rs index de8a028..9a5e2da 100644 --- a/crates/integration-tests/tests/roles.rs +++ b/crates/integration-tests/tests/roles.rs @@ -48,6 +48,7 @@ async fn server_with(role: Option, require_role: bool) -> Result Self { + let path_attributes = if family == AfiSafi::IPV4_UNICAST { + vec![] + } else { + vec![PathAttribute::MpUnreachNlri(MpUnreachNlriPathAttribute { + family, + prefixes: vec![], + })] + }; + Self { + withdrawn_routes: vec![], + path_attributes, + announced_routes: vec![], + } + } + + /// The address family whose End-of-RIB marker this is, if it is one. + pub fn end_of_rib_family(&self) -> Option { + if !self.withdrawn_routes.is_empty() || !self.announced_routes.is_empty() { + return None; + } + match self.path_attributes.as_slice() { + [] => Some(AfiSafi::IPV4_UNICAST), + [PathAttribute::MpUnreachNlri(unreach)] if unreach.prefixes.is_empty() => { + Some(unreach.family) + } + _ => None, + } + } + /// Writes the body of the UPDATE (what follows the 19-octet header). pub fn to_wire(&self, ctx: &ParserContext, out: &mut BytesMut) -> Result<(), EncodeError> { // The prefixes in the body are IPv4 unicast (RFC 4271 §4.3). @@ -1246,6 +1280,50 @@ mod tests { ); } + /// End-of-RIB markers (RFC 4724 §2) go over the wire and are known + /// for what they are; an UPDATE with anything else in it is not one. + #[test] + fn end_of_rib_markers_are_written_and_recognised() -> Result<()> { + let ctx = default_v6_context(); + for family in [AfiSafi::IPV4_UNICAST, AfiSafi::IPV6_UNICAST] { + let marker = Message::Update(UpdateMessage::end_of_rib(family)); + let mut wire = BytesMut::new(); + marker.to_wire(&ctx, &mut wire)?; + let Message::Update(decoded) = + Message::decode(&ctx, &wire).map_err(|e| eyre!("{e:?}"))? + else { + return Err(eyre!("not an UPDATE")); + }; + assert_eq!(decoded.end_of_rib_family(), Some(family)); + } + // The IPv4 one is 23 octets: a header and two empty lengths. + let mut wire = BytesMut::new(); + Message::Update(UpdateMessage::end_of_rib(AfiSafi::IPV4_UNICAST)) + .to_wire(&ctx, &mut wire)?; + assert_eq!(wire.len(), 23); + + let withdrawal = UpdateMessage { + withdrawn_routes: vec![Nlri::from( + IpPrefix::try_from("192.0.2.0/24").map_err(|e| eyre!("{e:?}"))?, + )], + path_attributes: vec![], + announced_routes: vec![], + }; + assert_eq!(withdrawal.end_of_rib_family(), None); + let ipv6_withdrawal = UpdateMessage { + withdrawn_routes: vec![], + path_attributes: vec![PathAttribute::MpUnreachNlri(MpUnreachNlriPathAttribute { + family: AfiSafi::IPV6_UNICAST, + prefixes: vec![Nlri::from( + IpPrefix::try_from("2001:db8::/32").map_err(|e| eyre!("{e:?}"))?, + )], + })], + announced_routes: vec![], + }; + assert_eq!(ipv6_withdrawal.end_of_rib_family(), None); + Ok(()) + } + #[test] fn decode_ignores_bytes_after_the_message() { let ctx = default_v6_context(); diff --git a/crates/packet/src/open.rs b/crates/packet/src/open.rs index b6773f2..327e695 100644 --- a/crates/packet/src/open.rs +++ b/crates/packet/src/open.rs @@ -280,6 +280,9 @@ pub enum Capability { /// The estimated time in seconds for the session to be /// re-established after a restart. restart_time: u16, + /// The speaker applies graceful restart to sessions that end with + /// a NOTIFICATION too, other than a Hard Reset (RFC 8538). + notification: bool, /// The address families whose routes are kept over a restart. entries: Vec, }, @@ -450,6 +453,7 @@ impl Capability { rest, Self::GracefulRestart { restart_state: flags_restart_time & (1 << 15) != 0, + notification: flags_restart_time & (1 << 14) != 0, restart_time: flags_restart_time & 0xfff, entries, }, @@ -567,6 +571,7 @@ impl Capability { Capability::GracefulRestart { restart_state, restart_time, + notification, entries, } => { if *restart_time > 0xfff { @@ -576,7 +581,9 @@ impl Capability { max: 0xfff, }); } - out.put_u16(u16::from(*restart_state) << 15 | restart_time); + out.put_u16( + u16::from(*restart_state) << 15 | u16::from(*notification) << 14 | restart_time, + ); for entry in entries { out.put_u16(entry.afi.0); out.put_u8(entry.safi.0); @@ -867,6 +874,7 @@ mod tests { Capability::GracefulRestart { restart_state: false, restart_time: 300, + notification: false, entries: vec![GracefulRestartEntry { afi: Afi::IPV4, safi: Safi::UNICAST, @@ -875,6 +883,23 @@ mod tests { } ); + // Restarted (R), with Notification support (N, RFC 8538), and the + // forwarding of IPv6 routes kept (F). + test_capability_roundtrip!( + test_graceful_restart_with_flags, + &[0x40, 0x06, 0xc0, 0x78, 0x00, 0x02, 0x01, 0x80], + Capability::GracefulRestart { + restart_state: true, + restart_time: 120, + notification: true, + entries: vec![GracefulRestartEntry { + afi: Afi::IPV6, + safi: Safi::UNICAST, + preserve_forwarding: true, + }] + } + ); + test_capability_roundtrip!( test_extended_nexthop, &[ @@ -930,6 +955,7 @@ mod tests { OpenOption::Capabilities(vec![Capability::GracefulRestart { restart_state: false, restart_time: 300, + notification: false, entries: vec![GracefulRestartEntry { afi: Afi::IPV4, safi: Safi::UNICAST, @@ -1028,6 +1054,7 @@ mod tests { Capability::GracefulRestart { restart_state: false, restart_time: 120, + notification: false, entries: vec![GracefulRestartEntry { afi: evpn.0, safi: evpn.1, diff --git a/crates/packet/tests/properties.rs b/crates/packet/tests/properties.rs index ba0b9ba..89c0b4f 100644 --- a/crates/packet/tests/properties.rs +++ b/crates/packet/tests/properties.rs @@ -396,13 +396,15 @@ fn capability() -> impl Strategy { role: BgpRole::from(role), }), ( + any::(), any::(), 0u16..4096, vec((afi(), safi(), any::()), 0..4) ) - .prop_map(|(restart_state, restart_time, entries)| { + .prop_map(|(restart_state, notification, restart_time, entries)| { Capability::GracefulRestart { restart_state, + notification, restart_time, entries: entries .into_iter() diff --git a/crates/rib/src/table/mod.rs b/crates/rib/src/table/mod.rs index cd3d1da..544e9c5 100644 --- a/crates/rib/src/table/mod.rs +++ b/crates/rib/src/table/mod.rs @@ -36,6 +36,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::sync::atomic::{AtomicU64, AtomicUsize}; use std::sync::{Arc, Mutex, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use bgp_primitives::constants::IpVersion; use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; use foldhash::fast::RandomState; use hashbrown::HashTable; @@ -85,6 +86,9 @@ pub struct Outcome { pub removed: usize, /// Prefixes whose best route is now another one, or none. pub best_changed: usize, + /// Stale routes that are no longer: replaced or withdrawn by their + /// peer (graceful restart; see [`RibTable::mark_stale`]). + pub stale_cleared: usize, } impl std::ops::AddAssign for Outcome { @@ -93,20 +97,31 @@ impl std::ops::AddAssign for Outcome { self.replaced += other.replaced; self.removed += other.removed; self.best_changed += other.best_changed; + self.stale_cleared += other.stale_cleared; } } /// One route of a prefix: sixteen bytes. struct Path { - /// Two things in one word. The high 48 bits are the first steps of + /// Three things in one word. The high 48 bits are the first steps of /// route selection as one number, the larger the better; see /// [`select::preference`]. The low 16 are the peer the route is from. + /// And one bit that the number never uses marks the route stale; see + /// [`STALE`]. word: u64, attrs: Arc, } +/// The bit of a [`Path`]'s word that marks a route stale: kept from a +/// session that ended, until the peer announces it again or it is swept +/// away (graceful restart, RFC 4724 §4.2). It is the lowest bit of the +/// preference, which is always zero ([`select::preference`]), and is left +/// out of it: a stale route is chosen like any other, as the RFC has it. +const STALE: u64 = 1 << 16; + impl Path { fn new(preference: u64, peer: PeerIndex, attrs: &Arc) -> Self { + debug_assert_eq!(preference & 1, 0, "the bit for stale routes is used"); Self { word: (preference << 16) | u64::from(peer.0), attrs: Arc::clone(attrs), @@ -114,7 +129,11 @@ impl Path { } fn preference(&self) -> u64 { - self.word >> 16 + (self.word & !STALE) >> 16 + } + + fn is_stale(&self) -> bool { + self.word & STALE != 0 } fn peer(&self) -> PeerIndex { @@ -170,11 +189,6 @@ impl Row { .position(|(index, path)| path.peer() == peer && self.path_id(index) == path_id) } - /// Where a route from `peer` is, whatever its identifier. - fn position_of_peer(&self, peer: PeerIndex) -> Option { - self.paths.iter().position(|path| path.peer() == peer) - } - fn push(&mut self, path: Path, path_id: u32) { if path_id != 0 && self.path_ids.is_empty() { self.path_ids = vec![0; self.paths.len()]; @@ -211,6 +225,7 @@ impl Row { path_id: self.path_id(index), attrs: &path.attrs, best: index == self.best(), + stale: path.is_stale(), } } @@ -248,6 +263,9 @@ pub struct RouteRef<'a> { pub attrs: &'a Arc, /// Whether it is the route chosen for its prefix. pub best: bool, + /// Whether it is kept from a session of the peer's that ended, and has + /// not been announced again since (graceful restart). + pub stale: bool, } impl RouteRef<'_> { @@ -259,6 +277,7 @@ impl RouteRef<'_> { path_id: self.path_id, attrs: Arc::clone(self.attrs), best: self.best, + stale: self.stale, } } } @@ -271,6 +290,7 @@ pub struct Route { pub path_id: u32, pub attrs: Arc, pub best: bool, + pub stale: bool, } struct Shard { @@ -430,6 +450,100 @@ impl RibTable { outcome } + /// Gives a peer number that is in use to a new session of the same + /// peer, whose identifier or address may have changed: what route + /// selection knows of the peer is replaced. Returns whether the number + /// was in use. + /// + /// This is how a peer that restarts finds the routes kept from its last + /// session (graceful restart): they are still under its number. + pub fn rebind_peer(&self, peer: PeerIndex, info: PeerInfo) -> bool { + let mut peers = self.peers.lock().unwrap_or_else(PoisonError::into_inner); + match peers.get_mut(peer.as_usize()) { + Some(slot @ Some(_)) => *slot = Some(info), + _ => return false, + } + for shard in &self.shards { + if let Some(slot) = write(shard).peers.get_mut(peer.as_usize()) { + *slot = Some(info); + } + } + true + } + + /// Marks every route of `peer` for prefixes of the IP versions in + /// `versions` stale, and returns how many there are. They stay in the + /// table and are chosen as before; announcing one again makes it fresh, + /// and [`RibTable::sweep_stale`] takes out those that are still stale. + /// + /// For graceful restart (RFC 4724 §4.2): the routes of a peer that is + /// restarting are kept while it does. Like [`RibTable::remove_peer`] + /// this looks at every prefix in the table. It changes no route, so + /// the readers are not told of anything. + pub fn mark_stale(&self, peer: PeerIndex, versions: &[IpVersion]) -> usize { + let mut marked = 0; + for shard in &self.shards { + let mut shard = write(shard); + for row in shard.rows.iter_mut() { + if !versions.contains(&row.prefix.address_family()) { + continue; + } + for path in &mut row.paths { + if path.peer() == peer { + path.word |= STALE; + marked += 1; + } + } + } + } + marked + } + + /// Takes out the routes of `peer` for prefixes of the IP versions in + /// `versions` that are still stale; see [`RibTable::mark_stale`]. The + /// peer keeps its number. + /// + /// Safe to run while the peer announces routes: one that is announced + /// again, before or during this, is fresh and stays. + pub fn sweep_stale(&self, peer: PeerIndex, versions: &[IpVersion]) -> Outcome { + let mut outcome = Outcome::default(); + let mut news = false; + let change = self.begin_change(); + for shard in &self.shards { + let mut shard = write(shard); + shard.journals.set_stamp(change.stamp); + let before = shard.journals.mark(); + outcome += shard.remove_where(|row, path| { + path.peer() == peer + && path.is_stale() + && versions.contains(&row.prefix.address_family()) + }); + shard.tidy_journals(); + news |= shard.journals.has_news_since(before); + } + drop(change); + if news { + self.wake_readers(); + } + outcome + } + + /// How many routes of `peer` are stale; see [`RibTable::mark_stale`]. + /// Looks at every prefix in the table. + pub fn count_stale(&self, peer: PeerIndex) -> usize { + self.shards + .iter() + .map(|shard| { + read(shard) + .rows + .iter() + .flat_map(|row| &row.paths) + .filter(|path| path.peer() == peer && path.is_stale()) + .count() + }) + .sum() + } + /// Announces `routes` from `peer`, all with the attributes `attrs`: /// what one UPDATE does. A route is known by its prefix and path /// identifier together ([`Nlri`]; a plain [`IpPrefix`] is the route @@ -757,6 +871,7 @@ impl Shard { let before = std::mem::replace(&mut row.paths[at], path); let outcome = Outcome { replaced: 1, + stale_cleared: usize::from(before.is_stale()), ..Outcome::default() }; if Arc::ptr_eq(&before.attrs, attrs) && before.preference() == preference { @@ -790,12 +905,14 @@ impl Shard { return Outcome::default(); }; self.routes -= 1; + let stale_cleared = usize::from(row.paths[at].is_stale()); if row.paths.len() == 1 { entry.remove(); self.journals.disappeared(prefix); return Outcome { removed: 1, best_changed: 1, + stale_cleared, ..Outcome::default() }; } @@ -804,11 +921,17 @@ impl Shard { Outcome { removed: 1, best_changed: best_changed.into(), + stale_cleared, ..Outcome::default() } } fn remove_peer(&mut self, peer: PeerIndex) -> Outcome { + self.remove_where(|_, path| path.peer() == peer) + } + + /// Takes out every route that `doomed` picks, given its row. + fn remove_where(&mut self, doomed: impl Fn(&Row, &Path) -> bool) -> Outcome { let mut outcome = Outcome::default(); let peers = &self.peers; let scratch = &mut self.scratch; @@ -818,7 +941,7 @@ impl Shard { let mut best_changed = false; // A peer that sends several routes for a prefix has several // here. - while let Some(at) = row.position_of_peer(peer) { + while let Some(at) = row.paths.iter().position(|path| doomed(row, path)) { removed += 1; if row.paths.len() == 1 { outcome.removed += removed; diff --git a/crates/rib/src/table/tests.rs b/crates/rib/src/table/tests.rs index cb069ad..acf959f 100644 --- a/crates/rib/src/table/tests.rs +++ b/crates/rib/src/table/tests.rs @@ -445,6 +445,129 @@ fn removing_a_peer_takes_its_routes_and_frees_its_number() { assert_eq!(fixture.table.add_peer(external(3)), Ok(fixture.peers[0])); } +/// Graceful restart (RFC 4724 §4.2): a restarting peer's routes are kept, +/// marked stale, and used as before; what it announces again is fresh, and +/// what it does not is swept away at the end. +#[test] +fn stale_routes_are_chosen_as_before_and_swept_unless_announced_again() { + let fixture = Fixture::new(&[external(1), external(2)]); + let v4 = [ + prefix("10.1.0.0/16"), + prefix("10.2.0.0/16"), + prefix("10.3.0.0/16"), + ]; + let v6 = prefix("2001:db8::/32"); + for p in v4.iter().chain([&v6]) { + fixture.announce(0, p, &Route::via(&[1])); + } + // Peer 1 has a worse route for the first prefix, and loses. + fixture.announce(1, &v4[0], &Route::via(&[2, 3])); + let mut reader = fixture.table.reader_from_now(); + + let peer = fixture.peers[0]; + let both = [IpVersion::Ipv4, IpVersion::Ipv6]; + assert_eq!(fixture.table.mark_stale(peer, &both), 4); + assert_eq!(fixture.table.count_stale(peer), 4); + // Nothing changed for anyone who follows the table, and the stale + // route is still the best. + let mut events = 0; + reader.poll(usize::MAX, |_| events += 1); + assert_eq!(events, 0); + assert_eq!(fixture.best(&v4[0]), Some(0)); + assert!(fixture.table.routes(&v4[0]).iter().any(|r| r.stale)); + + // The peer comes back and announces two of them again, one with the + // attributes it had: they are fresh. + assert_eq!( + fixture.announce(0, &v4[0], &Route::via(&[1])).stale_cleared, + 1 + ); + assert_eq!( + fixture + .announce(0, &v4[1], &Route::via(&[1, 4])) + .stale_cleared, + 1 + ); + // Again, and it was not stale any more. + assert_eq!( + fixture + .announce(0, &v4[1], &Route::via(&[1, 4])) + .stale_cleared, + 0 + ); + assert_eq!(fixture.table.count_stale(peer), 2); + + // Its End-of-RIB for IPv4: the IPv4 route it did not announce goes, + // the IPv6 one waits for the IPv6 marker. + let outcome = fixture.table.sweep_stale(peer, &[IpVersion::Ipv4]); + assert_eq!((outcome.removed, outcome.best_changed), (1, 1)); + assert_eq!(fixture.best(&v4[2]), None); + assert_eq!(fixture.best(&v4[0]), Some(0)); + assert_eq!(fixture.table.count_stale(peer), 1); + let outcome = fixture.table.sweep_stale(peer, &[IpVersion::Ipv6]); + assert_eq!(outcome.removed, 1); + assert_eq!(fixture.table.count_stale(peer), 0); + assert_eq!(fixture.table.len(), 3); + // Sweeping again finds nothing. + assert_eq!(fixture.table.sweep_stale(peer, &both), Outcome::default()); +} + +/// Stale routes are swept from one peer only, and only for the versions +/// asked: the other peer's routes, stale or not, stay. +#[test] +fn sweeping_takes_only_the_stale_routes_of_one_peer() { + let fixture = Fixture::new(&[external(1), external(2)]); + let p = prefix("10.0.0.0/8"); + fixture.announce(0, &p, &Route::via(&[1])); + fixture.announce(1, &p, &Route::via(&[2, 3])); + fixture + .table + .mark_stale(fixture.peers[1], &[IpVersion::Ipv4]); + assert_eq!( + fixture + .table + .sweep_stale(fixture.peers[0], &[IpVersion::Ipv4]), + Outcome::default() + ); + assert_eq!( + fixture + .table + .sweep_stale(fixture.peers[1], &[IpVersion::Ipv6]), + Outcome::default() + ); + assert_eq!(fixture.held(&p), vec![(0, 0), (1, 0)]); + let outcome = fixture + .table + .sweep_stale(fixture.peers[1], &[IpVersion::Ipv4]); + assert_eq!((outcome.removed, outcome.best_changed), (1, 0)); + assert_eq!(fixture.held(&p), vec![(0, 0)]); +} + +/// A new session of a restarting peer takes over its number, with what +/// selection knows of it brought up to date: here a new identifier, which +/// is what breaks the tie. +#[test] +fn a_peer_number_can_be_given_to_the_next_session_of_the_peer() { + let fixture = Fixture::new(&[external(1), external(2)]); + let p = prefix("10.0.0.0/8"); + fixture.announce(0, &p, &Route::via(&[1])); + fixture.announce(1, &p, &Route::via(&[2])); + assert_eq!(fixture.best(&p), Some(0)); + let renumbered = PeerInfo { + router_id: Ipv4Addr::new(10, 0, 0, 9), + ..external(1) + }; + assert!(fixture.table.rebind_peer(fixture.peers[0], renumbered)); + assert_eq!(fixture.table.peer(fixture.peers[0]), Some(renumbered)); + // Chosen again the next time the row changes. + fixture.announce(1, &p, &Route::via(&[5])); + assert_eq!(fixture.best(&p), Some(1)); + + // A number nobody has cannot be taken over. + fixture.table.remove_peer(fixture.peers[1]); + assert!(!fixture.table.rebind_peer(fixture.peers[1], external(3))); +} + #[test] fn readers_see_every_route_and_which_is_best() { let fixture = Fixture::new(&[external(1), external(2)]); diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index d066fe9..6d01a6e 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -79,6 +79,40 @@ message BgpConfig { // with AS n": mark routes with something else (large communities // L:2:n and up, say). bool control_communities = 13; + + // Graceful restart, for every peer that has no `graceful_restart` of its + // own. Left out, it is on, as the defaults below have it. + GracefulRestart graceful_restart = 14; +} + +// Graceful restart (RFC 4724): BGP restarting without the routes going. +// +// When the daemon is restarted in good order for an upgrade (on SIGUSR1), +// it leaves its routes in the kernel, and its peers keep the routes it +// announced to them, for as long as they were told a restart takes. When a +// peer restarts, its routes are kept and used as before, marked stale, +// until it has sent them again; those it does not send again are removed. +// Either way, nothing is withdrawn across the network for a restart. +// +// It is in use with a peer that offers it too. A session that ends with a +// Hard Reset (RFC 8538), or because of an error, has its routes removed at +// once, as without graceful restart. +message GracefulRestart { + // True does not offer graceful restart: a peer's routes are removed as + // soon as its session ends, and it removes ours as soon as it sees ours + // end. + optional bool disabled = 1; + + // How long, in seconds, peers are told a restart of this daemon takes: + // how long they keep our routes after our session with them ends. At + // most 4095. Defaults to 120. + optional uint32 restart_time_seconds = 2; + + // The longest time, in seconds, a restarting peer's routes are kept + // stale: while it is gone, whatever restart time it gave, and again once + // it is back, until it has sent all its routes. Zero keeps none. + // Defaults to 120. + optional uint32 stale_routes_seconds = 3; } // Installing routes in the kernel. Linux only; the daemon needs the @@ -242,6 +276,10 @@ message Peer { // `export` holds, and without that the routes we originate are // announced as they are. ExportPolicy export = 11; + + // Graceful restart with this peer. What is set here takes the place of + // what the top-level `graceful_restart` says. + GracefulRestart graceful_restart = 12; } // What is announced to a peer. diff --git a/crates/server/src/advertise.rs b/crates/server/src/advertise.rs index c2c6c84..e29908b 100644 --- a/crates/server/src/advertise.rs +++ b/crates/server/src/advertise.rs @@ -14,6 +14,13 @@ //! //! The advertiser itself only starts an exporter when a session comes up //! and stops it when the session goes. +//! +//! To a peer that agreed to graceful restart (RFC 4724) an exporter sends +//! an End-of-RIB marker for each address family once the peer has all our +//! routes: at once after the originated ones, or, where learned routes are +//! passed on too, once the routing table has been gone through and every +//! peer has sent its own routes since the start. A peer that kept our +//! routes over a restart of ours drops those it has not had again then. use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv6Addr}; @@ -33,11 +40,12 @@ use bgp_rib::{LocalRib, LocalRibEvent, LocalRoute}; use bytes::BytesMut; use tokio::sync::broadcast::Receiver; use tokio::sync::broadcast::error::RecvError; -use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, watch}; use tokio::task::JoinHandle; use tracing::{debug, info, warn}; use crate::export::{self, Egress, Outgoing, Source}; +use crate::family::FamilySet; use crate::marks::Control; use crate::metrics::ROUTES_EXPORTED; use crate::policy::PeerRole; @@ -51,8 +59,10 @@ pub fn spawn_advertiser( registry: SessionRegistry, local_asn: u32, control: Option, + converged: watch::Receiver, ) -> JoinHandle<()> { let advertiser = Advertiser { + converged, session_events: registry.subscribe(), local_rib, rib, @@ -91,6 +101,8 @@ struct Advertiser { resyncs: Arc, session_events: Receiver, exporters: HashMap, + /// Whether every peer has sent its routes since the start. + converged: watch::Receiver, } impl Advertiser { @@ -143,6 +155,11 @@ impl Advertiser { local_asn: self.local_asn, resend: Arc::clone(&resend), announced: HashSet::new(), + end_of_rib: session + .graceful_restart + .as_ref() + .map(|_| session.families.clone()), + converged: Some(self.converged.clone()).filter(|converged| !*converged.borrow()), }; let running = Running { session: session.clone(), @@ -188,6 +205,12 @@ struct Exporter { /// The prefixes announced to the peer and not withdrawn since: our /// Adj-RIB-Out for it (RFC 4271 §3.2), as far as it needs keeping. announced: HashSet, + /// The families to send End-of-RIB markers for, until they are sent; + /// `None` if graceful restart was not agreed. + end_of_rib: Option, + /// Word of every peer having sent its routes since the start, until it + /// comes. + converged: Option>, } /// How many changes of best route are taken from the table at a time. @@ -244,6 +267,8 @@ struct Learned { permit: Option, /// The prefixes announced to the peer and not withdrawn since. announced: HashSet, + /// Whether the peer has been told of the whole table once. + synced: bool, /// While the table is being gone through anew: what was announced /// before and has not been heard of again yet. stale: Option>, @@ -261,6 +286,7 @@ impl Learned { gathered: None, permit: None, announced: HashSet::new(), + synced: false, stale: None, dressed: HashMap::new(), } @@ -312,6 +338,18 @@ impl Learned { } } +/// Waits until every peer has sent its routes since the start; for ever if +/// that is not to be waited for. +async fn converged(converged: &mut Option>) { + match converged { + // If the sender is gone, so is the server: nothing to wait for. + Some(converged) => { + let _ = converged.wait_for(|done| *done).await; + } + None => std::future::pending().await, + } +} + /// Waits until there may be changes of best route to pass on; for ever if /// the peer is not told of them. async fn changed(learned: &mut Option) { @@ -340,6 +378,7 @@ impl Exporter { async fn export(&mut self) -> Result<(), Finished> { self.send_all().await?; + self.end_of_rib().await?; loop { tokio::select! { change = self.changes.recv() => match change { @@ -370,11 +409,39 @@ impl Exporter { // look at what each peer is told. tokio::time::sleep(COALESCE).await; self.follow().await?; + self.end_of_rib().await?; + } + () = converged(&mut self.converged), if self.converged.is_some() => { + self.converged = None; + self.end_of_rib().await?; } } } } + /// Sends the End-of-RIB markers, if they are still to be sent and the + /// peer has had all our routes: the originated ones, and the learned + /// ones as far as there are to be any. + async fn end_of_rib(&mut self) -> Result<(), Finished> { + let complete = match &self.learned { + None => true, + Some(learned) => learned.synced && self.converged.is_none(), + }; + if !complete { + return Ok(()); + } + let Some(families) = self.end_of_rib.take() else { + return Ok(()); + }; + debug!(peer = %self.session.peer, %families, "Sending End-of-RIB"); + for family in families.iter() { + self.session + .send(Message::Update(UpdateMessage::end_of_rib(family))) + .await?; + } + Ok(()) + } + /// The session as far as it decides what goes out on it and how. fn towards(&self) -> Towards { let session = &self.session; @@ -521,6 +588,9 @@ impl Exporter { } } } + if synced { + learned.synced = true; + } if synced && let Some(stale) = learned.stale.take() { for prefix in stale { withdrawn[version_index(prefix.address_family())].push(prefix.into()); @@ -1222,6 +1292,7 @@ mod tests { registry.clone(), 65000, None, + watch::channel(true).1, ); Self { registry, @@ -1246,6 +1317,7 @@ mod tests { add_path_send: FamilySet::default(), role_confirmed: false, extended_next_hop: false, + graceful_restart: None, }; let rib_peer = self .table diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 22b1783..4504079 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -18,6 +18,7 @@ use crate::family::FamilySet; use crate::marks::{Control, Marks}; use crate::policy::{ExportPolicy, ImportPolicy, PeerRole, PrefixSet}; use crate::proto::config::{self as proto, BgpConfig, OriginatedRoute, Peer}; +use crate::restart::GracefulRestartConfig; const DEFAULT_BGP_PORT: u16 = 179; const DEFAULT_HOLD_TIME_SECS: u16 = 90; @@ -58,6 +59,10 @@ pub struct Config { /// The communities by which routes say who is told of them, if they /// are honoured. pub control: Option, + /// Graceful restart for peers that have no setting of their own; and + /// the restart time this daemon gives, which decides how long it waits + /// after it starts for its peers to send their routes. + pub graceful_restart: GracefulRestartConfig, } #[derive(Debug, Clone)] @@ -84,6 +89,8 @@ pub struct PeerConfig { pub require_role: bool, /// What the peer is told, and how. pub export: Arc, + /// Graceful restart with the peer (RFC 4724). + pub graceful_restart: GracefulRestartConfig, } impl Config { @@ -123,6 +130,10 @@ impl Config { Some(export) => parse_export(export)?, None => ExportPolicy::default(), }); + let graceful_restart = match proto.graceful_restart.as_ref() { + Some(own) => GracefulRestartConfig::default().overridden_by(own)?, + None => GracefulRestartConfig::default(), + }; let mut peers: Vec = Vec::with_capacity(proto.peers.len()); for peer in proto.peers { let name = peer.name.clone(); @@ -130,7 +141,7 @@ impl Config { peer: name.clone(), problem, }; - let peer = parse_peer(peer, &export).map_err(rejected)?; + let peer = parse_peer(peer, &export, graceful_restart).map_err(rejected)?; // RFC 9234 is about what networks are to each other. if peer.role.is_some() && peer.remote_asn == proto.local_asn { return Err(rejected(PeerProblem::RoleInOwnAs)); @@ -210,6 +221,7 @@ impl Config { control: proto.control_communities.then_some(Control { local_asn: proto.local_asn, }), + graceful_restart, }) } } @@ -245,7 +257,11 @@ fn parse_export(export: &proto::ExportPolicy) -> Result) -> Result { +fn parse_peer( + peer: Peer, + export: &Arc, + graceful_restart: GracefulRestartConfig, +) -> Result { let ep = peer.endpoint.as_ref().ok_or(PeerProblem::MissingEndpoint)?; let endpoint = parse_port(ep.port) .and_then(|port| Ok(Endpoint::parse(&ep.address, port)?)) @@ -334,6 +350,10 @@ fn parse_peer(peer: Peer, export: &Arc) -> Result Arc::new(parse_export(own)?), None => Arc::clone(export), }, + graceful_restart: match peer.graceful_restart.as_ref() { + Some(own) => graceful_restart.overridden_by(own)?, + None => graceful_restart, + }, }) } diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index 244abf9..7db1e0b 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -21,6 +21,7 @@ use bgp_packet::message::{ }; use bgp_packet::open::AddPathSendRecv; use bgp_packet::update_decode::{UpdateError, merge_as4_path}; +use bgp_primitives::constants::{AfiSafi, IpVersion}; use bgp_primitives::ip_prefix::Nlri; use bgp_primitives::parser::ParserContext; use bgp_rib::table::{Outcome, PeerIndex, PeerInfo, RibTable}; @@ -30,7 +31,7 @@ use prometheus::IntGauge; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, watch}; use tokio::time::{Instant, Interval, MissedTickBehavior, Sleep, interval_at, sleep, timeout}; use tracing::{debug, error, info, warn}; @@ -48,6 +49,7 @@ use crate::panic::contained; use crate::policy::{self, ExportPolicy, HeldDown, HoldDowns, ImportPolicy, PeerRole}; use crate::reach::{Reach, neighbor_at}; use crate::refusals::{PeerRefusals, RefusalLog}; +use crate::restart::{self, Agreed, GracefulRestartConfig, Leave, Restarts}; use crate::routes::{self, Held}; use crate::rpki::{Checks, Subject, Validator}; use crate::session::{PeerId, SessionCounters, SessionGuard, SessionHandle, SessionRegistry}; @@ -86,6 +88,9 @@ pub struct ServerContext { pub rpki: Option, /// The machine's interfaces, where routes are installed in the kernel. pub interfaces: Option, + /// The routes kept of peers that are restarting, and how far the peers + /// are with sending theirs since the start (graceful restart). + pub restarts: Arc, } /// What is configured for the peer at the other end of a connection. @@ -109,13 +114,19 @@ pub struct PeerSettings { pub require_role: bool, /// What the peer is told, and how. pub export: Arc, + /// Graceful restart with the peer. + pub graceful_restart: GracefulRestartConfig, } impl PeerSettings { /// For a peer that connected to us and is not in the configuration. /// Such peers are taken only where that was asked for, on a closed /// test network, and there everything they announce is taken. - pub fn unconfigured(hold_time_secs: u16, export: Arc) -> Self { + pub fn unconfigured( + hold_time_secs: u16, + export: Arc, + graceful_restart: GracefulRestartConfig, + ) -> Self { Self { name: None, expected_asn: None, @@ -126,6 +137,7 @@ impl PeerSettings { role: None, require_role: false, export, + graceful_restart, } } } @@ -142,6 +154,7 @@ impl From<&PeerConfig> for PeerSettings { role: peer.role, require_role: peer.require_role, export: Arc::clone(&peer.export), + graceful_restart: peer.graceful_restart, } } } @@ -209,24 +222,63 @@ pub async fn run_session( } // Whatever ended the session, routes learned over it are gone - // (RFC 4271 §8.2.2). Do this before the registration drops so that the - // routes are withdrawn before the session is seen to be down. + // (RFC 4271 §8.2.2), unless the peer is restarting (RFC 4724 §4.2). + // Do this before the registration drops so that the routes are + // withdrawn, or marked stale, before the session is seen to be down. if let Some(established) = session.established.take() { + let leaving = *session.leave.borrow(); + let keep_for = established + .graceful_restart + .as_ref() + .filter(|agreed| leaving.is_none() && restart::ends_in_restart(&result, agreed)) + .map(|agreed| { + let theirs = Duration::from_secs(agreed.restart_time.into()); + ( + restart::versions(&agreed.families), + theirs.min(session.graceful_restart.stale_routes), + ) + }) + .filter(|(versions, time)| !versions.is_empty() && !time.is_zero()); let Established { peer_id, + rib_peer, routes, _registration, .. } = established; - let outcome = routes.remove().await; - // The peer has no routes now, and no session to count them for. + // The peer has no session to count its routes for. let _ = PEER_ROUTES.remove_label_values(&[session.refusals.label()]); - info!( - peer = %peer_id, - removed_routes = outcome.removed, - best_changed = outcome.best_changed, - "Session ended" - ); + if leaving == Some(Leave::Restart) { + // The daemon is about to go, and its routes with it; the + // kernel keeps its own. Taking them out first would only hold + // the restart up. + routes.forget(); + } else if let Some((versions, time)) = keep_for { + let kept = routes.keep(versions.clone()).await; + session.server.restarts.keep( + (peer_addr.into(), peer_id.asn), + peer_addr, + session.peer_name.clone(), + rib_peer, + versions, + kept, + time, + ); + info!( + peer = %peer_id, + kept_routes = kept, + kept_for = ?time, + "Session ended; the peer is restarting, and its routes are kept meanwhile" + ); + } else { + let outcome = routes.remove().await; + info!( + peer = %peer_id, + removed_routes = outcome.removed, + best_changed = outcome.best_changed, + "Session ended" + ); + } } session.close().await; SessionEnd { @@ -261,10 +313,32 @@ struct Established { /// The address families the peer can be asked to send again, if it can /// be asked at all. refreshable: Option, + /// The address families of the session. + families: FamilySet, + /// Graceful restart (RFC 4724), if both sides offered it. + graceful_restart: Option, + /// The families the peer has sent its End-of-RIB marker for. + end_of_rib: FamilySet, + /// The routes kept from the peer's last session that it has not sent + /// again yet, if there are any. + stale: Option, + /// Where the sweeps of stale routes say what they did. + swept: ( + mpsc::UnboundedSender, + mpsc::UnboundedReceiver, + ), /// Keeps the session listed in the registry. _registration: SessionGuard, } +/// The routes kept from a restarting peer's last session, while its new +/// one sends them again: those of the IP versions it has not sent an +/// End-of-RIB marker for yet, which are kept until `until` at most. +struct Stale { + versions: Vec, + until: Pin>, +} + impl Established { /// Whose routes are judged and by what rules, and where those that /// are invalid for now are held. @@ -296,6 +370,7 @@ impl Established { ) -> Result<(), SessionError> { let routes = self.counters.note(outcome.added, outcome.removed, rejected); self.counters.set_held(self.held.len()); + self.counters.clear_stale(outcome.stale_cleared); self.routes_metric.set(routes as i64); match import.max_prefixes { Some(limit) if routes > u64::from(limit) => { @@ -348,6 +423,40 @@ impl PeerRoutes { } } + /// Keeps the routes of the IP versions `versions` for the peer's next + /// session, marked stale, and removes the others. Returns how many + /// were kept. + async fn keep(mut self, versions: Vec) -> usize { + let (table, peer) = (Arc::clone(&self.table), self.peer); + let kept = tokio::task::spawn_blocking(move || { + let all = [IpVersion::Ipv4, IpVersion::Ipv6]; + let marked = table.mark_stale(peer, &all); + let others: Vec = all + .into_iter() + .filter(|version| !versions.contains(version)) + .collect(); + let removed = table.sweep_stale(peer, &others).removed; + marked - removed + }) + .await; + match kept { + Ok(kept) => { + self.removed = true; + kept + } + // Dropping this removes them all. + Err(e) => { + error!(err = %e, "Keeping a restarting peer's routes failed; removing them"); + 0 + } + } + } + + /// Leaves the routes where they are: the daemon is about to end. + fn forget(mut self) { + self.removed = true; + } + fn remove_now(table: &RibTable, peer: PeerIndex) -> Outcome { let outcome = table.remove_peer(peer); // Most of the peer's attribute sets were its alone. @@ -396,6 +505,10 @@ enum Wake { /// The validation data has changed; these routes of the peer's are /// now invalid. Revalidate(Vec), + /// Stale routes of the peer's were swept away, with this outcome. + Swept(Outcome), + /// The peer's stale routes have been kept as long as they may be. + StaleTimeout, } /// What a session says in the log only once in a while, because the peer @@ -456,6 +569,10 @@ struct Session { export: Arc, /// Where what is refused is noted. refusals: Arc, + /// Graceful restart with the peer, as configured. + graceful_restart: GracefulRestartConfig, + /// Says when the daemon is leaving, and how. + leave: watch::Receiver>, } impl Session { @@ -482,12 +599,14 @@ impl Session { // Over IPv6 our address is no next hop for an IPv4 route as // BGP first had it; this is how such routes go all the same. extended_next_hop: peer_addr.is_ipv6(), + graceful_restart: server.restarts.offer(&peer.graceful_restart), }); // The peer goes by its configured name, or by its address. let label = peer .name .clone() .unwrap_or_else(|| PeerAddress::from(peer_addr).to_string()); + let leave = server.registry.leaving(); let session = Session { refusals: server.refusals.of_peer(peer_addr.into(), &label), fsm, @@ -511,6 +630,8 @@ impl Session { role: peer.role, export: peer.export, said: Said::new(), + graceful_restart: peer.graceful_restart, + leave, }; Ok((session, first_actions)) } @@ -557,6 +678,11 @@ impl Session { actions = match self.next_event().await? { Wake::Event(event) => self.fsm.handle(event), Wake::Revalidate(suspects) => self.revalidate(&suspects), + Wake::Swept(outcome) => self.swept(&outcome), + Wake::StaleTimeout => { + self.sweep_all_stale(); + vec![] + } }; } } @@ -568,6 +694,7 @@ impl Session { frame = self.frames.next(&self.ctx) => match frame { Ok(mut decoded) => { self.report(&decoded.update_errors); + let repaired = !decoded.update_errors.is_empty(); // From here on a route has one path, whatever the // session brought it as. if !self.ctx.four_octet_asn @@ -578,6 +705,7 @@ impl Session { Event::Message(Received { message: decoded.message, attributes_wire: decoded.attributes_wire, + repaired, }) } Err(SessionError::Decode(e)) => Event::DecodeFailed(e), @@ -587,6 +715,7 @@ impl Session { () = ticked(&mut self.keepalive_timer) => Event::KeepaliveTimerExpired, wake = from_server(&mut self.established) => return Ok(wake), () = killed(&mut self.collision) => Event::CollisionLost, + leave = leaving(&mut self.leave) => Event::Leave(leave), }; Ok(Wake::Event(event)) } @@ -699,6 +828,7 @@ impl Session { queue = self.fsm.handle(Event::LocalFailure(e)).into(); } } + Action::EndOfRib(family) => self.end_of_rib(family), Action::RouteRefreshRequested => { if let Some(established) = &self.established { if let Some(unsaid) = self.said.refreshes.pass() { @@ -764,13 +894,23 @@ impl Session { addr: self.peer_addr, asn: negotiated.peer_asn, }; - let rib_peer = self.server.rib.add_peer(PeerInfo { + let info = PeerInfo { asn: negotiated.peer_asn, router_id: negotiated.peer_identifier, address: self.peer_addr.ip(), interface: PeerAddress::from(self.peer_addr).interface, ibgp: negotiated.ibgp, - })?; + }; + // A peer back from a restart takes over the routes kept of it. + let kept = self + .server + .restarts + .take((self.peer_addr.into(), negotiated.peer_asn)) + .filter(|kept| self.server.rib.rebind_peer(kept.rib_peer, info)); + let rib_peer = match &kept { + Some(kept) => kept.rib_peer, + None => self.server.rib.add_peer(info)?, + }; info!(peer_addr = %socket_text(self.peer_addr), peer_asn = negotiated.peer_asn, "Session established"); SESSIONS_ESTABLISHED_TOTAL.inc(); @@ -821,6 +961,11 @@ impl Session { refreshable: negotiated .route_refresh .then(|| negotiated.families.clone()), + families: negotiated.families.clone(), + graceful_restart: negotiated.graceful_restart.clone(), + end_of_rib: FamilySet::default(), + stale: None, + swept: mpsc::unbounded_channel(), peer_id, rib_peer, outbound, @@ -839,9 +984,130 @@ impl Session { routes_metric: PEER_ROUTES.with_label_values(&[self.refusals.label()]), _registration: self.server.registry.register(handle), }); + if let Some(kept) = kept { + self.take_over(kept, negotiated); + } Ok(()) } + /// Takes over the routes kept from the peer's last session, which are + /// stale until it sends them again (RFC 4724 §4.2). Those of the + /// families whose forwarding it did not keep over its restart, or all + /// of them if it no longer offers graceful restart, go at once; the + /// others go as it sends its End-of-RIB markers, or after as long as + /// stale routes are kept. + fn take_over(&mut self, kept: restart::Kept, negotiated: &Negotiated) { + let Some(established) = &mut self.established else { + return; + }; + let forwarded = negotiated + .graceful_restart + .as_ref() + .map(|agreed| restart::versions(&agreed.forwarding)) + .unwrap_or_default(); + let (stay, go): (Vec, Vec) = kept + .versions + .iter() + .partition(|version| forwarded.contains(version)); + // They are the peer's routes, as far as its limit goes. + let routes = established.counters.note(kept.routes, 0, 0); + established.counters.set_stale(kept.routes); + established.routes_metric.set(routes as i64); + info!( + peer = %established.peer_id, + kept_routes = kept.routes, + "The peer is back from a restart; its routes are kept until it sends them again" + ); + if !go.is_empty() { + sweep( + &self.server.rib, + established.rib_peer, + go, + &established.swept.0, + ); + } + if !stay.is_empty() { + established.stale = Some(Stale { + versions: stay, + until: Box::pin(sleep(self.graceful_restart.stale_routes)), + }); + } + } + + /// The peer has sent all its routes of `family`: the stale routes of + /// the family that it did not send again go (RFC 4724 §4.2). + fn end_of_rib(&mut self, family: AfiSafi) { + let Some(established) = &mut self.established else { + return; + }; + established.end_of_rib.insert(family); + debug!(peer = %established.peer_id, %family, "Received End-of-RIB"); + if let Some(stale) = &mut established.stale + && let Some(version) = restart::versions(&[family].into_iter().collect()).pop() + && let Some(at) = stale.versions.iter().position(|v| *v == version) + { + stale.versions.remove(at); + if stale.versions.is_empty() { + established.stale = None; + } + sweep( + &self.server.rib, + established.rib_peer, + vec![version], + &established.swept.0, + ); + } + if let Some(name) = &self.peer_name + && established + .families + .iter() + .all(|family| established.end_of_rib.contains(family)) + { + self.server.restarts.convergence().peer_done(name); + } + } + + /// The peer has had as long as it may to send its routes again: what + /// it has not sent goes. + fn sweep_all_stale(&mut self) { + let Some(established) = &mut self.established else { + return; + }; + if let Some(stale) = established.stale.take() { + warn!( + peer = %established.peer_id, + "The peer did not send all its routes again in time; removing those it did not" + ); + sweep( + &self.server.rib, + established.rib_peer, + stale.versions, + &established.swept.0, + ); + } + } + + /// Keeps count after stale routes of the peer's were swept away. + fn swept(&mut self, outcome: &Outcome) -> Vec { + let Some(established) = &mut self.established else { + return vec![]; + }; + if outcome.removed > 0 { + info!( + peer = %established.peer_id, + removed_routes = outcome.removed, + best_changed = outcome.best_changed, + "Removed stale routes the peer did not send again" + ); + } + // Every route swept away was stale. + established.counters.clear_stale(outcome.removed); + match established.account(outcome, 0, &self.import) { + Ok(()) => vec![], + Err(e) => self.fsm.handle(Event::LocalFailure(e)), + } + } + /// Puts the routes of a received UPDATE into the routing table. fn store( &mut self, @@ -949,12 +1215,40 @@ async fn ticked(timer: &mut Option) { } } +/// Sweeps the stale routes of `peer` of the IP versions `versions` away, +/// on a thread that can take its time, and says what that did on `done`. +fn sweep( + rib: &Arc, + peer: PeerIndex, + versions: Vec, + done: &mpsc::UnboundedSender, +) { + let (rib, done) = (Arc::clone(rib), done.clone()); + tokio::spawn(async move { + let swept = tokio::task::spawn_blocking(move || { + let outcome = rib.sweep_stale(peer, &versions); + rib.attrs().sweep(); + outcome + }) + .await; + match swept { + // The session may have ended meanwhile. + Ok(outcome) => { + let _ = done.send(outcome); + } + Err(e) => error!(err = %e, "Removing stale routes failed"), + } + }); +} + /// What the rest of the server has for an established session: a message /// to send to the peer, or word that the validation data has changed. async fn from_server(established: &mut Option) -> Wake { let Some(Established { outbound, validation, + stale, + swept, .. }) = established else { @@ -966,9 +1260,17 @@ async fn from_server(established: &mut Option) -> Wake { None => pending().await, } }; + let stale_timeout = async { + match stale { + Some(stale) => stale.until.as_mut().await, + None => pending().await, + } + }; tokio::select! { Some(msg) = outbound.recv() => Wake::Event(Event::Outbound(msg)), Some(suspects) = invalidated => Wake::Revalidate(suspects), + Some(outcome) = swept.1.recv() => Wake::Swept(outcome), + () = stale_timeout => Wake::StaleTimeout, // The registry holds the sending halves for as long as the session // is registered, so neither closes under us; if they somehow did, // there is simply nothing more to come. @@ -976,6 +1278,19 @@ async fn from_server(established: &mut Option) -> Wake { } } +/// Resolves with how the daemon is leaving, once it is. Cancel-safe. +async fn leaving(leave: &mut watch::Receiver>) -> Leave { + let leave = match leave.wait_for(Option::is_some).await { + Ok(leave) => *leave, + // The registry is gone, and so is everything else. + Err(_) => None, + }; + match leave { + Some(leave) => leave, + None => pending().await, + } +} + async fn killed(collision: &mut Option) { match collision { Some(guard) => guard.killed().await, @@ -1173,23 +1488,25 @@ mod tests { .await .unwrap(); let (stream, peer_addr) = listener.accept().await.unwrap(); + let rib = Arc::new(RibTable::new(4)); let server = Arc::new(ServerContext { router_id: Ipv4Addr::new(10, 0, 0, 1), local_asn: 65000, - rib: Arc::new(RibTable::new(4)), + rib: Arc::clone(&rib), registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), interfaces: None, refusals: RefusalLog::default(), rpki: None, + restarts: Restarts::new(rib, true, [], 120), }); let (mut session, _) = Session::open( stream, peer_addr, Direction::Inbound, server, - PeerSettings::unconfigured(90, Arc::default()), + PeerSettings::unconfigured(90, Arc::default(), GracefulRestartConfig::default()), ) .unwrap(); @@ -1222,23 +1539,25 @@ mod tests { .await .unwrap(); let (stream, peer_addr) = listener.accept().await.unwrap(); + let rib = Arc::new(RibTable::new(4)); let server = Arc::new(ServerContext { router_id: Ipv4Addr::new(10, 0, 0, 1), local_asn: 65000, - rib: Arc::new(RibTable::new(4)), + rib: Arc::clone(&rib), registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), interfaces: None, refusals: RefusalLog::default(), rpki: None, + restarts: Restarts::new(rib, true, [], 120), }); let (mut session, _) = Session::open( stream, peer_addr, Direction::Inbound, server, - PeerSettings::unconfigured(90, Arc::default()), + PeerSettings::unconfigured(90, Arc::default(), GracefulRestartConfig::default()), ) .unwrap(); session.send_hold_time = Duration::from_millis(100); diff --git a/crates/server/src/error.rs b/crates/server/src/error.rs index 2e024e3..b4d4f25 100644 --- a/crates/server/src/error.rs +++ b/crates/server/src/error.rs @@ -180,6 +180,8 @@ pub enum ConfigError { Export(#[from] ExportProblem), #[error("rpki: the port of cache {0:?} is required")] RpkiCachePort(String), + #[error("graceful_restart: {0}")] + GracefulRestart(#[from] GracefulRestartProblem), #[error("peer {peer:?}: {problem}")] Peer { peer: String, problem: PeerProblem }, #[error("originate {prefix:?}: {problem}")] @@ -222,6 +224,15 @@ pub enum PeerProblem { Import(#[from] ImportProblem), #[error("export: {0}")] Export(#[from] ExportProblem), + #[error("graceful_restart: {0}")] + GracefulRestart(#[from] GracefulRestartProblem), +} + +/// What is wrong with a `graceful_restart`. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum GracefulRestartProblem { + #[error("restart_time_seconds is {0}; it is at most 4095")] + RestartTime(u32), } /// What is wrong with a peer's `import` policy. diff --git a/crates/server/src/fsm.rs b/crates/server/src/fsm.rs index dc7613a..142f774 100644 --- a/crates/server/src/fsm.rs +++ b/crates/server/src/fsm.rs @@ -14,8 +14,8 @@ use std::net::Ipv4Addr; use std::time::Duration; -use bgp_packet::errors::{BgpError, DecodeError, OpenMsgSubcode}; -use bgp_packet::message::{AS_TRANS, BGP4_VERSION, Message, UpdateMessage}; +use bgp_packet::errors::{BgpError, CeaseSubcode, DecodeError, OpenMsgSubcode}; +use bgp_packet::message::{AS_TRANS, BGP4_VERSION, Message, NotificationMessage, UpdateMessage}; use bgp_packet::open::{ AddPathEntry, AddPathSendRecv, Capability, ExtendedNextHop, OpenMessage, OpenOption, }; @@ -25,6 +25,7 @@ use crate::error::SessionError; use crate::family::FamilySet; use crate::metrics::message_type_name; use crate::policy::PeerRole; +use crate::restart::{self, Agreed, Leave, Offer}; /// How long to wait for the peer's OPEN, and for its first KEEPALIVE when /// the negotiated hold time is zero. RFC 4271 §8.2.2 suggests four minutes. @@ -79,6 +80,8 @@ pub struct FsmConfig { /// Offer to exchange IPv4 routes with IPv6 next hops (RFC 8950): for a /// session over a link that has only IPv6 addresses. pub extended_next_hop: bool, + /// What graceful restart (RFC 4724) is offered as, if it is. + pub graceful_restart: Option, } /// What the two OPENs settled. @@ -110,6 +113,8 @@ pub struct Negotiated { /// Whether IPv4 routes go with IPv6 next hops on this session /// (RFC 8950): both sides offered it, and the session carries IPv4. pub extended_next_hop: bool, + /// Graceful restart (RFC 4724), if both sides offered it. + pub graceful_restart: Option, } /// A message from the peer, with what the decoder found out besides. @@ -120,6 +125,9 @@ pub struct Received { /// decoder kept them (`DecodedUpdate::attributes_wire`). They go with /// the UPDATE to whoever stores its routes. pub attributes_wire: Option>, + /// For an UPDATE: whether the decoder changed it because of errors in + /// it (RFC 7606). An UPDATE emptied that way is no End-of-RIB marker. + pub repaired: bool, } impl From for Received { @@ -127,6 +135,7 @@ impl From for Received { Self { message, attributes_wire: None, + repaired: false, } } } @@ -149,6 +158,8 @@ pub enum Event { /// table has no room for the peer's routes, say. The peer is told as /// the error has it. LocalFailure(SessionError), + /// The daemon is going, and the session with it. + Leave(Leave), } /// Something the driver must do, in the order given. @@ -170,6 +181,8 @@ pub enum Action { /// The peer sent this many routes of address families that were not /// negotiated. They have been left out of the update that follows. RoutesIgnored(usize), + /// The peer has sent all its routes of this family once (RFC 4724 §2). + EndOfRib(AfiSafi), /// Routes arrived; put them in the routing table. DeliverUpdate { update: UpdateMessage, @@ -252,6 +265,7 @@ impl Fsm { _ => vec![], }, Event::Outbound(msg) => self.on_outbound(msg), + Event::Leave(leave) => self.leave(leave), } } @@ -259,6 +273,7 @@ impl Fsm { let Received { message, attributes_wire, + repaired, } = received; match (&self.phase, message) { // A NOTIFICATION ends the session in any state, and is not @@ -284,6 +299,14 @@ impl Fsm { (Phase::Established(negotiated), Message::KeepAlive) => heard_from_peer(negotiated), (Phase::Established(negotiated), Message::Update(mut update)) => { let mut actions = heard_from_peer(negotiated); + // Before the families are restricted, which can leave an + // UPDATE that looks like one. + if let Some(family) = update.end_of_rib_family().filter(|_| !repaired) { + if negotiated.families.contains(family) { + actions.push(Action::EndOfRib(family)); + } + return actions; + } // A route of a family the session does not carry is ignored // rather than treated as an error: it harms nothing, and // resetting the session over it would. @@ -386,6 +409,12 @@ impl Fsm { } } + // RFC 4724 §3: in use if both offered it. + let graceful_restart = self + .config + .graceful_restart + .and_then(|_| Agreed::of(&open, &families)); + // RFC 8950 §4: in use only if both said so. let extended_next_hop = self.config.extended_next_hop && families.contains(AfiSafi::IPV4_UNICAST) @@ -408,6 +437,7 @@ impl Fsm { add_path_receive, add_path_send, role_confirmed, + graceful_restart, }; // With a hold time of zero the timer will not run once Established, // but a peer that never confirms must still be given up on. @@ -436,9 +466,25 @@ impl Fsm { /// Ends the session because of `error`, telling the peer with the /// NOTIFICATION that goes with it if there is one. + /// + /// With graceful restart agreed for NOTIFICATIONs too (RFC 8538), one + /// that goes for an error is a Hard Reset: the peer is to drop our + /// routes, and not keep them for a restart. Not for an expired hold + /// timer, which RFC 8538 §4 has the peer treat as a restart; nor for a + /// collision, where another connection takes over at once. fn fail(&mut self, error: SessionError) -> Vec { - let mut actions: Vec = error - .notification() + let for_good = !matches!( + error, + SessionError::HoldTimerExpired | SessionError::Collision + ); + let notification = error.notification().map(|notification| { + if for_good && self.notifications_restart() { + restart::hard_reset(notification) + } else { + notification + } + }); + let mut actions: Vec = notification .map(|notification| Action::Send(Message::Notification(notification))) .into_iter() .collect(); @@ -446,6 +492,44 @@ impl Fsm { actions } + /// Whether graceful restart applies to sessions that end with a + /// NOTIFICATION, other than a Hard Reset (RFC 8538). + fn notifications_restart(&self) -> bool { + match &self.phase { + Phase::OpenConfirm(negotiated) | Phase::Established(negotiated) => negotiated + .graceful_restart + .as_ref() + .is_some_and(|agreed| agreed.notification), + Phase::OpenSent | Phase::Closed => false, + } + } + + /// Ends the session because the daemon is going: for good, with a + /// NOTIFICATION that has the peer drop our routes now; or to come back + /// at once, by closing the connection without one where graceful + /// restart is agreed, which has the peer keep them (RFC 4724 §4.2). + fn leave(&mut self, leave: Leave) -> Vec { + let agreed = match &self.phase { + Phase::Established(negotiated) => negotiated.graceful_restart.is_some(), + _ => false, + }; + if leave == Leave::Restart && agreed { + return self.close(Ok(())); + } + let shutdown = NotificationMessage { + error: BgpError::Cease(CeaseSubcode::AdminShutdown), + data: vec![], + }; + let notification = if self.notifications_restart() { + restart::hard_reset(shutdown) + } else { + shutdown + }; + let mut actions = vec![Action::Send(Message::Notification(notification))]; + actions.extend(self.close(Ok(()))); + actions + } + fn close(&mut self, result: Result<(), SessionError>) -> Vec { self.phase = Phase::Closed; vec![Action::Close(result)] @@ -495,6 +579,9 @@ fn build_open(config: &FsmConfig) -> Message { entries: vec![IPV4_WITH_IPV6_NEXT_HOP], }); } + if let Some(offer) = &config.graceful_restart { + capabilities.push(restart::capability(offer, &config.families)); + } if let Some(send_recv) = config.add_path { capabilities.push(Capability::AddPath { entries: config @@ -563,6 +650,7 @@ mod tests { role: None, require_role: false, extended_next_hop: false, + graceful_restart: None, } } @@ -665,6 +753,7 @@ mod tests { add_path_send: FamilySet::default(), role_confirmed: false, extended_next_hop: false, + graceful_restart: None, }; assert!(matches!( &actions[..], @@ -1237,7 +1326,29 @@ mod tests { let actions = fsm.handle(received(Message::KeepAlive)); assert!(matches!(&actions[..], [Action::ResetHoldTimer(d)] if *d == hold)); + let withdrawal = UpdateMessage { + withdrawn_routes: vec![IpPrefix::try_from("192.0.2.0/24").unwrap().into()], + ..empty_update() + }; + let actions = fsm.handle(received(Message::Update(withdrawal))); + assert!(matches!( + &actions[..], + [Action::ResetHoldTimer(d), Action::DeliverUpdate { .. }] if *d == hold + )); + + // An UPDATE with nothing in it is the End-of-RIB marker for IPv4 + // (RFC 4724 §2), and has no routes to deliver. let actions = fsm.handle(received(Message::Update(empty_update()))); + assert!(matches!( + &actions[..], + [Action::ResetHoldTimer(d), Action::EndOfRib(AfiSafi::IPV4_UNICAST)] if *d == hold + )); + // Unless the decoder emptied it for errors in it (RFC 7606). + let actions = fsm.handle(Event::Message(Received { + message: Message::Update(empty_update()), + attributes_wire: None, + repaired: true, + })); assert!(matches!( &actions[..], [Action::ResetHoldTimer(d), Action::DeliverUpdate { .. }] if *d == hold @@ -1311,4 +1422,188 @@ mod tests { assert!(fsm.handle(event).is_empty()); } } + + // ---- graceful restart (RFC 4724, RFC 8538) ---------------------------- + + const OFFER: Offer = Offer { + restart_time: 120, + restarted: true, + forwarding_preserved: false, + }; + + /// A peer's capability: its restart time, N bit and the families with + /// their F bits. + fn peer_restart(notification: bool, entries: &[(AfiSafi, bool)]) -> Capability { + Capability::GracefulRestart { + restart_state: false, + restart_time: 90, + notification, + entries: entries + .iter() + .map( + |(family, forwarding)| bgp_packet::open::GracefulRestartEntry { + afi: family.afi, + safi: family.safi, + preserve_forwarding: *forwarding, + }, + ) + .collect(), + } + } + + /// A machine offering graceful restart, Established with a peer that + /// speaks both families and offers it as `restart` says, if at all. + fn restarting_fsm(restart: Option) -> Fsm { + let (mut fsm, _) = Fsm::new(FsmConfig { + graceful_restart: Some(OFFER), + ..config() + }); + let mut capabilities: Vec = SUPPORTED + .iter() + .map(|family| Capability::multiprotocol(*family)) + .collect(); + capabilities.extend(restart); + fsm.handle(received(Message::Open(OpenMessage { + options: vec![OpenOption::Capabilities(capabilities)], + ..peer_open(90) + }))); + fsm.handle(received(Message::KeepAlive)); + assert_eq!(fsm.state(), State::Established); + fsm + } + + #[test] + fn graceful_restart_is_offered_with_notifications_for_every_family() { + let (_, actions) = Fsm::new(FsmConfig { + graceful_restart: Some(OFFER), + ..config() + }); + let Action::Send(open) = &actions[0] else { + panic!("the OPEN comes first: {actions:?}"); + }; + let offered = capabilities(open) + .into_iter() + .find(|c| matches!(c, Capability::GracefulRestart { .. })) + .cloned(); + assert_eq!( + offered, + Some(Capability::GracefulRestart { + restart_state: true, + restart_time: 120, + notification: true, + entries: SUPPORTED + .iter() + .map(|family| bgp_packet::open::GracefulRestartEntry { + afi: family.afi, + safi: family.safi, + preserve_forwarding: false, + }) + .collect(), + }) + ); + // Not offered where it is not configured. + let (_, actions) = Fsm::new(config()); + let Action::Send(open) = &actions[0] else { + panic!("the OPEN comes first: {actions:?}"); + }; + assert!( + !capabilities(open) + .iter() + .any(|c| matches!(c, Capability::GracefulRestart { .. })) + ); + } + + #[test] + fn graceful_restart_is_agreed_for_the_families_the_peer_lists() { + let fsm = restarting_fsm(Some(peer_restart( + true, + &[ + (AfiSafi::IPV6_UNICAST, true), + (AfiSafi::new(Afi::L2VPN, Safi::EVPN), true), + ], + ))); + let agreed = fsm.negotiated().unwrap().graceful_restart.clone().unwrap(); + assert_eq!(agreed.restart_time, 90); + assert!(agreed.notification); + // A family the session does not carry is left out. + assert_eq!(agreed.families, only(AfiSafi::IPV6_UNICAST)); + assert_eq!(agreed.forwarding, only(AfiSafi::IPV6_UNICAST)); + + // Not agreed where only one side offers it. + let fsm = restarting_fsm(None); + assert_eq!(fsm.negotiated().unwrap().graceful_restart, None); + let mut fsm = fsm_in(State::OpenSent, 90); + fsm.handle(received(Message::Open(OpenMessage { + options: vec![OpenOption::Capabilities(vec![peer_restart(true, &[])])], + ..peer_open(90) + }))); + assert_eq!(fsm.negotiated().unwrap().graceful_restart, None); + } + + /// Leaving to restart is closing the connection without a word, which + /// a peer that agreed takes as a restart; without the agreement, and + /// when leaving for good, the peer is told, with a Hard Reset where it + /// would otherwise keep our routes. + #[test] + fn leaving_says_goodbye_as_graceful_restart_has_it() { + let shutdown = BgpError::Cease(CeaseSubcode::AdminShutdown); + let hard_reset = BgpError::Cease(CeaseSubcode::HardReset); + let both = &[(AfiSafi::IPV4_UNICAST, true), (AfiSafi::IPV6_UNICAST, true)]; + for (restart, leave, expected) in [ + (Some(peer_restart(true, both)), Leave::Restart, None), + (Some(peer_restart(false, both)), Leave::Restart, None), + (None, Leave::Restart, Some(shutdown.clone())), + ( + Some(peer_restart(true, both)), + Leave::Stop, + Some(hard_reset.clone()), + ), + ( + Some(peer_restart(false, both)), + Leave::Stop, + Some(shutdown.clone()), + ), + (None, Leave::Stop, Some(shutdown.clone())), + ] { + let mut fsm = restarting_fsm(restart.clone()); + let actions = fsm.handle(Event::Leave(leave)); + let (sent, result) = ending(&fsm, actions); + assert_eq!(sent, expected, "{leave:?} with {restart:?}"); + assert!(result.is_ok()); + } + // Before Established there is nothing to keep. + let mut fsm = fsm_in(State::OpenConfirm, 90); + let actions = fsm.handle(Event::Leave(Leave::Restart)); + assert_eq!(ending(&fsm, actions).0, Some(shutdown)); + } + + /// With NOTIFICATIONs covered (RFC 8538), one for an error of ours is a + /// Hard Reset, with the error in its data; an expired hold timer is + /// not. + #[test] + fn errors_end_the_session_with_a_hard_reset_where_notifications_restart() { + let both = &[(AfiSafi::IPV4_UNICAST, true), (AfiSafi::IPV6_UNICAST, true)]; + let mut fsm = restarting_fsm(Some(peer_restart(true, both))); + let actions = fsm.handle(Event::LocalFailure(SessionError::PrefixLimitExceeded { + limit: 10, + })); + let Action::Send(Message::Notification(sent)) = &actions[0] else { + panic!("a NOTIFICATION first: {actions:?}"); + }; + assert_eq!(sent.error, BgpError::Cease(CeaseSubcode::HardReset)); + assert_eq!(sent.data, [6, 1]); + + let mut fsm = restarting_fsm(Some(peer_restart(true, both))); + let actions = fsm.handle(Event::HoldTimerExpired); + assert_eq!(ending(&fsm, actions).0, Some(BgpError::HoldTimer)); + + let mut fsm = restarting_fsm(Some(peer_restart(false, both))); + let actions = fsm.handle(Event::LocalFailure(SessionError::PrefixLimitExceeded { + limit: 10, + })); + assert_eq!( + ending(&fsm, actions).0, + Some(BgpError::Cease(CeaseSubcode::MaxPrefixes)) + ); + } } diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index b89d9f3..bb58d69 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -14,6 +14,7 @@ mod panic; pub mod policy; mod reach; pub mod refusals; +pub mod restart; mod routes; pub mod rpki; pub mod server; diff --git a/crates/server/src/restart.rs b/crates/server/src/restart.rs new file mode 100644 index 0000000..9ba635f --- /dev/null +++ b/crates/server/src/restart.rs @@ -0,0 +1,447 @@ +//! Graceful restart (RFC 4724, with the additions of RFC 8538): a BGP +//! speaker that restarts without its routes, or its peers', being withdrawn +//! across the network. +//! +//! It has two sides, and the daemon takes both: +//! +//! - **When a peer restarts**, its routes are kept: marked stale in the +//! routing table and used as before, for at most the time the peer said +//! a restart takes. When it is back it sends its routes again, and then +//! an End-of-RIB marker for each address family; what it did not send +//! again is removed at that marker. This is what [`Restarts`] keeps +//! track of. +//! - **When this daemon restarts**, it leaves its routes in the kernel and +//! closes its sessions without a word, which peers that offered graceful +//! restart take as a restart. Started again, it says so in its OPENs, +//! and holds back its End-of-RIB markers for learned routes until its +//! peers have sent theirs, or for at most its restart time +//! ([`Convergence`]). + +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use bgp_packet::errors::{BgpError, CeaseSubcode}; +use bgp_packet::message::NotificationMessage; +use bgp_packet::open::{Capability, GracefulRestartEntry, OpenMessage}; +use bgp_primitives::constants::{AfiSafi, IpVersion}; +use bgp_rib::table::{PeerIndex, RibTable}; +use tokio::sync::watch; +use tracing::info; + +use crate::address::PeerAddress; +use crate::error::SessionError; +use crate::family::FamilySet; +use crate::sync::IgnorePoison; + +/// The restart time peers are told of unless configured otherwise. +pub const DEFAULT_RESTART_TIME: u16 = 120; + +/// How long a restarting peer's routes are kept stale at most, unless +/// configured otherwise. +pub const DEFAULT_STALE_ROUTES_TIME: Duration = Duration::from_secs(120); + +/// The longest restart time there is room for in the capability. +pub const MAX_RESTART_TIME: u16 = 0xfff; + +/// Graceful restart as configured for a peer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GracefulRestartConfig { + /// Whether it is offered to the peer, and done for it. + pub enabled: bool, + /// How long, in seconds, the peer is told a restart of ours takes. + pub restart_time: u16, + /// The longest time the peer's routes are kept stale, whatever it says. + pub stale_routes: Duration, +} + +impl Default for GracefulRestartConfig { + fn default() -> Self { + Self { + enabled: true, + restart_time: DEFAULT_RESTART_TIME, + stale_routes: DEFAULT_STALE_ROUTES_TIME, + } + } +} + +impl GracefulRestartConfig { + /// This with what `proto` sets put in its place. + pub(crate) fn overridden_by( + self, + proto: &crate::proto::config::GracefulRestart, + ) -> Result { + let restart_time = match proto.restart_time_seconds { + None => self.restart_time, + Some(seconds) => u16::try_from(seconds) + .ok() + .filter(|seconds| *seconds <= MAX_RESTART_TIME) + .ok_or(crate::error::GracefulRestartProblem::RestartTime(seconds))?, + }; + Ok(Self { + enabled: proto.disabled.map_or(self.enabled, |disabled| !disabled), + restart_time, + stale_routes: proto + .stale_routes_seconds + .map_or(self.stale_routes, |seconds| { + Duration::from_secs(seconds.into()) + }), + }) + } +} + +/// What our OPEN offers a peer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Offer { + /// How long, in seconds, a restart of ours takes. + pub restart_time: u16, + /// We have just started, and the peer is not to wait for our End-of-RIB + /// markers before it sends us its routes (the R bit). + pub restarted: bool, + /// The forwarding by the routes we announce was kept over our last + /// restart (the F bit, for every family of the session). + pub forwarding_preserved: bool, +} + +/// Graceful restart as both sides offered it on a session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Agreed { + /// How long, in seconds, the peer says a restart of its takes. + pub restart_time: u16, + /// The peer has just restarted (the R bit). + pub restarted: bool, + /// Both sides take a session that ends with a NOTIFICATION, other than + /// a Hard Reset, as one that restarts (RFC 8538). + pub notification: bool, + /// The families of the session that the peer listed: those whose + /// routes are kept over a restart. + pub families: FamilySet, + /// Of those, the ones whose forwarding the peer kept over its last + /// restart (the F bit). The routes of the others are not to be kept + /// once it is back. + pub forwarding: FamilySet, +} + +impl Agreed { + /// What the peer's OPEN says, if it offers graceful restart, for a + /// session that carries `families`. Only for a session on which we + /// offered it too. + pub(crate) fn of(open: &OpenMessage, families: &FamilySet) -> Option { + open.capabilities().find_map(|capability| match capability { + Capability::GracefulRestart { + restart_state, + restart_time, + notification, + entries, + } => { + let listed = |entry: &&GracefulRestartEntry| { + families.contains(AfiSafi::new(entry.afi, entry.safi)) + }; + Some(Self { + restart_time: *restart_time, + restarted: *restart_state, + // We always offer it: it is agreed if the peer does. + notification: *notification, + families: entries + .iter() + .filter(listed) + .map(|entry| AfiSafi::new(entry.afi, entry.safi)) + .collect(), + forwarding: entries + .iter() + .filter(listed) + .filter(|entry| entry.preserve_forwarding) + .map(|entry| AfiSafi::new(entry.afi, entry.safi)) + .collect(), + }) + } + _ => None, + }) + } +} + +/// The capability that makes `offer` for `families`. We always offer RFC +/// 8538 with it. +pub(crate) fn capability(offer: &Offer, families: &FamilySet) -> Capability { + Capability::GracefulRestart { + restart_state: offer.restarted, + restart_time: offer.restart_time, + notification: true, + entries: families + .iter() + .map(|family| GracefulRestartEntry { + afi: family.afi, + safi: family.safi, + preserve_forwarding: offer.forwarding_preserved, + }) + .collect(), + } +} + +/// How the daemon leaves its sessions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Leave { + /// For good: peers are to drop our routes now. + Stop, + /// To come back at once, as for an upgrade: peers that agreed to + /// graceful restart keep our routes meanwhile. + Restart, +} + +/// `notification` as a Hard Reset (RFC 8538 §3): the NOTIFICATION that +/// ends a session for good, with graceful restart agreed. The one it stands +/// for is its data. +pub(crate) fn hard_reset(notification: NotificationMessage) -> NotificationMessage { + if notification.error == BgpError::Cease(CeaseSubcode::HardReset) { + return notification; + } + let (code, subcode) = notification.error.code_subcode(); + let mut data = vec![code, subcode]; + data.extend_from_slice(¬ification.data); + NotificationMessage { + error: BgpError::Cease(CeaseSubcode::HardReset), + data, + } +} + +/// The IP versions of the unicast families in `families`: what a family is +/// in the routing table, which holds unicast routes only. +pub(crate) fn versions(families: &FamilySet) -> Vec { + let mut versions = Vec::new(); + if families.contains(AfiSafi::IPV4_UNICAST) { + versions.push(IpVersion::Ipv4); + } + if families.contains(AfiSafi::IPV6_UNICAST) { + versions.push(IpVersion::Ipv6); + } + versions +} + +/// Who a restarting peer is, to know it again when it is back: its address +/// and its AS. +pub type PeerKey = (PeerAddress, u32); + +/// The routes kept of a peer whose session ended, while it restarts. +#[derive(Debug, Clone)] +pub struct Kept { + /// Told apart from the routes kept of an earlier restart of the peer. + id: u64, + /// Where the peer's session was. + pub addr: SocketAddr, + /// The peer's number in the routing table, which its routes are under. + pub rib_peer: PeerIndex, + /// Its configured name, if it has one. + pub name: Option, + /// The IP versions of the routes kept. + pub versions: Vec, + /// How many routes were kept. + pub routes: usize, + /// When they are removed if the peer is not back. + pub until: SystemTime, +} + +/// What there is of graceful restart while the server runs: the routes +/// kept of peers that are restarting, and how far the peers are with +/// sending their routes after this daemon started. +pub struct Restarts { + rib: Arc, + started: Instant, + /// Whether the forwarding by our routes outlived the last run: always + /// where it is not ours to keep (no kernel routes), else if routes of + /// the last run were found in the kernel. + forwarding_preserved: bool, + kept: Mutex>, + ids: AtomicU64, + convergence: Convergence, +} + +impl Restarts { + /// For a server that started just now, whose configured peers are + /// `peers` and whose restart time is `restart_time`. + pub fn new( + rib: Arc, + forwarding_preserved: bool, + peers: impl IntoIterator, + restart_time: u16, + ) -> Arc { + Arc::new(Self { + rib, + started: Instant::now(), + forwarding_preserved, + kept: Mutex::default(), + ids: AtomicU64::new(0), + convergence: Convergence::new(peers, Duration::from_secs(restart_time.into())), + }) + } + + /// What to offer a peer configured with `config`, if anything. + /// + /// For as long as a restart takes after the start, peers are told that + /// we have restarted (RFC 4724 §3, the R bit): they are not to wait for + /// our End-of-RIB markers before sending us their routes. That much + /// later, the kernel has our routes whatever was found there at the + /// start. + pub fn offer(&self, config: &GracefulRestartConfig) -> Option { + if !config.enabled { + return None; + } + let since_start = self.started.elapsed(); + let restarting = since_start < Duration::from_secs(config.restart_time.into()); + Some(Offer { + restart_time: config.restart_time, + restarted: restarting, + forwarding_preserved: self.forwarding_preserved || !restarting, + }) + } + + /// Notes that the routes of `peer` under `rib_peer` have been kept + /// (`routes` of them, of the IP versions `versions`) until `for_at_most` + /// from now, and has them removed then unless the peer is back. + #[allow(clippy::too_many_arguments)] + pub(crate) fn keep( + self: &Arc, + peer: PeerKey, + addr: SocketAddr, + name: Option, + rib_peer: PeerIndex, + versions: Vec, + routes: usize, + for_at_most: Duration, + ) { + let id = self.ids.fetch_add(1, Ordering::Relaxed); + self.kept.locked().insert( + peer, + Kept { + id, + addr, + rib_peer, + name, + versions, + routes, + until: SystemTime::now() + for_at_most, + }, + ); + let restarts = Arc::clone(self); + tokio::spawn(async move { + tokio::time::sleep(for_at_most).await; + restarts.expire(peer, id).await; + }); + } + + /// Removes the routes kept of `peer` under `id`, if they are still + /// waiting for it: it did not come back in time. + async fn expire(&self, peer: PeerKey, id: u64) { + let kept = { + let mut all = self.kept.locked(); + match all.get(&peer) { + Some(kept) if kept.id == id => all.remove(&peer), + _ => None, + } + }; + let Some(kept) = kept else { + return; + }; + let rib = Arc::clone(&self.rib); + let outcome = tokio::task::spawn_blocking(move || { + let outcome = rib.remove_peer(kept.rib_peer); + rib.attrs().sweep(); + outcome + }) + .await + .unwrap_or_default(); + info!( + peer = %peer.0, + peer_asn = peer.1, + removed_routes = outcome.removed, + "The peer did not come back in time; removed the routes kept while it restarted" + ); + } + + /// The routes kept of `peer`, for its new session to take over. + pub(crate) fn take(&self, peer: PeerKey) -> Option { + self.kept.locked().remove(&peer) + } + + /// The routes kept of peers that are restarting, by peer. + pub fn kept(&self) -> Vec<(PeerKey, Kept)> { + self.kept + .locked() + .iter() + .map(|(peer, kept)| (*peer, kept.clone())) + .collect() + } + + /// See [`Convergence`]. + pub fn convergence(&self) -> &Convergence { + &self.convergence + } +} + +/// Whether the peers have all sent their routes since this daemon started: +/// an End-of-RIB marker for every family of their first session. Until +/// then, or until the restart time has passed, what is passed on of the +/// routes learned from them is not complete. The daemon's own End-of-RIB +/// markers for learned routes wait for it, and so does the removal of the +/// routes of an earlier run from the kernel. +pub struct Convergence { + waiting_for: Mutex>, + done: watch::Sender, +} + +impl Convergence { + fn new(peers: impl IntoIterator, at_most: Duration) -> Self { + let waiting_for: HashSet = peers.into_iter().collect(); + let done = watch::Sender::new(waiting_for.is_empty()); + if !waiting_for.is_empty() { + let timer = done.clone(); + // Within a runtime: where there is none (a test of something + // else), nobody waits. + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + tokio::time::sleep(at_most).await; + if !*timer.borrow() { + info!("Not every peer sent all its routes in time; going on without"); + } + timer.send_replace(true); + }); + } + } + Self { + waiting_for: Mutex::new(waiting_for), + done, + } + } + + /// Notes that the configured peer `name` has sent all its routes. + pub(crate) fn peer_done(&self, name: &str) { + let mut waiting_for = self.waiting_for.locked(); + if waiting_for.remove(name) && waiting_for.is_empty() && !*self.done.borrow() { + info!("Every peer has sent all its routes"); + self.done.send_replace(true); + } + } + + /// Whether it is so, and word when it becomes so. + pub fn watch(&self) -> watch::Receiver { + self.done.subscribe() + } +} + +/// Whether a session that ended as `result` ends in a restart of the peer, +/// whose routes are kept, given what was agreed (RFC 4724 §4.2, RFC 8538 +/// §4): when the connection failed, and, where NOTIFICATIONs are covered, +/// when the hold timer expired or the peer sent one that is not a Hard +/// Reset. Not when the session ended for an error of ours, nor when we +/// ended it. +pub(crate) fn ends_in_restart(result: &Result<(), SessionError>, agreed: &Agreed) -> bool { + match result { + Err(SessionError::Io(_)) => true, + Err(SessionError::HoldTimerExpired) => agreed.notification, + Err(SessionError::NotifiedByPeer(error)) => { + agreed.notification && *error != BgpError::Cease(CeaseSubcode::HardReset) + } + _ => false, + } +} diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 374df9e..c6a6b6e 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -30,6 +30,7 @@ use crate::panic; use crate::policy::HoldDowns; use crate::reach; use crate::refusals::RefusalLog; +use crate::restart::{Leave, Restarts}; use crate::rpki::spawn_revalidator; use crate::session::{ConfiguredPeer, SessionRegistry}; use crate::sync::IgnorePoison; @@ -93,6 +94,9 @@ pub struct BgpServer { /// What keeps the best routes in the kernel, where that is configured. pub kernel: Option, kernel_task: Option>>, + /// Graceful restart: the routes kept of restarting peers, and how far + /// the peers are with sending theirs since the start. + pub restarts: Arc, } impl BgpServer { @@ -134,7 +138,21 @@ impl BgpServer { None => None, } .unzip(); + // Whether the forwarding by our routes outlived the last run, which + // peers are told in our OPENs: so it has to be known before any. + // Where the kernel's routes are not ours, it is not ours to lose. + let forwarding_preserved = match &kernel { + Some(kernel) => kernel.adopted().await > 0, + None => true, + }; + let restarts = Restarts::new( + Arc::clone(&rib), + forwarding_preserved, + config.peers.iter().map(|peer| peer.name.clone()), + config.graceful_restart.restart_time, + ); Ok(BgpServer { + restarts, rpki, rpki_client, kernel, @@ -206,7 +224,18 @@ impl BgpServer { refusals: self.refusals.clone(), rpki: self.rpki.clone(), interfaces: self.kernel.as_ref().map(Fib::interfaces), + restarts: Arc::clone(&self.restarts), }); + // What an earlier run left in the kernel and no peer has announced + // again goes once every peer has sent its routes. + if let Some(kernel) = self.kernel.clone() { + let mut converged = self.restarts.convergence().watch(); + tokio::spawn(async move { + if converged.wait_for(|done| *done).await.is_ok() { + kernel.release_kept(); + } + }); + } // The tasks the server cannot do without. None of them ends while // the server runs. The advertiser comes first: it has to be @@ -220,6 +249,7 @@ impl BgpServer { self.registry.clone(), self.config.local_asn, self.config.control, + self.restarts.convergence().watch(), ), )); tasks.push(named("sweeper", spawn_sweeper(Arc::clone(&self.rib)))); @@ -323,6 +353,26 @@ pub async fn shut_down(registry: &SessionRegistry, kernel: Option<&Fib>) { } } +/// Leaves to come back at once, as for an upgrade (graceful restart, RFC +/// 4724): our routes stay in the kernel, so that the machine goes on +/// forwarding, and the sessions are closed in a way that has peers that +/// agreed to graceful restart keep our routes until we are back. The +/// kernel is let go of first: the routes of the sessions that close must +/// not be taken out of it. +pub async fn leave_for_restart(registry: &SessionRegistry, kernel: Option<&Fib>) { + if let Some(kernel) = kernel + && tokio::time::timeout(KERNEL_GRACE, kernel.leave()) + .await + .is_err() + { + warn!("Letting go of the kernel's routes took too long"); + } + let left = registry.leave(Leave::Restart, SHUTDOWN_GRACE).await; + if left > 0 { + warn!(sessions = left, "Some sessions did not close in time"); + } +} + /// Closes every session politely and waits a moment for them to go: each /// peer is told we are going (Cease / Administrative Shutdown), so that it /// stops using our routes now and not when its hold timer expires. @@ -367,9 +417,11 @@ fn admit( // for that peer. let peer = match config.peer_at(address) { Some(peer) => PeerSettings::from(peer), - None if config.accept_unconfigured_peers => { - PeerSettings::unconfigured(config.hold_time_secs, Arc::clone(&config.export)) - } + None if config.accept_unconfigured_peers => PeerSettings::unconfigured( + config.hold_time_secs, + Arc::clone(&config.export), + config.graceful_restart, + ), None => return Err(Refusal::NotAPeer), }; let slot = connections.take(address).ok_or(Refusal::TooMany)?; diff --git a/crates/server/src/session.rs b/crates/server/src/session.rs index 4dd12e7..af5edc6 100644 --- a/crates/server/src/session.rs +++ b/crates/server/src/session.rs @@ -11,13 +11,10 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime}; -use bgp_packet::errors::{BgpError, CeaseSubcode}; -use bgp_packet::message::NotificationMessage; - use bgp_packet::message::Message; use bgp_primitives::ip_prefix::Nlri; use bgp_rib::table::PeerIndex; -use tokio::sync::{broadcast, mpsc}; +use tokio::sync::{broadcast, mpsc, watch}; use crate::address::{Endpoint, PeerAddress, socket_text}; use crate::collision::Direction; @@ -25,6 +22,7 @@ use crate::family::FamilySet; use crate::fsm::Negotiated; use crate::metrics::{PEER_UP, SESSIONS_ESTABLISHED}; use crate::policy::{ExportPolicy, PeerRole}; +use crate::restart::{Agreed, Leave}; use crate::rpki::Subject; use crate::sync::IgnorePoisonRw; @@ -78,6 +76,9 @@ pub struct SessionCounters { /// Routes of the peer's that are held out of the table as invalid, to /// be taken if route origin validation comes to say otherwise. held: AtomicU64, + /// Routes of the peer's kept from its last session, which it has not + /// sent again yet (graceful restart). + stale: AtomicU64, } impl SessionCounters { @@ -97,6 +98,21 @@ impl SessionCounters { self.held.store(held as u64, Ordering::Relaxed); } + pub fn stale(&self) -> u64 { + self.stale.load(Ordering::Relaxed) + } + + pub(crate) fn set_stale(&self, stale: usize) { + self.stale.store(stale as u64, Ordering::Relaxed); + } + + /// Notes that `cleared` stale routes were sent again or went. + pub(crate) fn clear_stale(&self, cleared: usize) { + // Only the session writes this. + let left = self.stale().saturating_sub(cleared as u64); + self.stale.store(left, Ordering::Relaxed); + } + /// Notes what one UPDATE did, and returns how many routes the peer has /// in the table after it. pub(crate) fn note(&self, added: usize, removed: usize, rejected: usize) -> u64 { @@ -143,6 +159,8 @@ pub struct SessionHandle { /// The peer's number in the routing table: what its routes there are /// marked with. pub rib_peer: PeerIndex, + /// Graceful restart (RFC 4724), if both sides offered it. + pub graceful_restart: Option, /// How many routes the peer has, and how many were refused. pub counters: Arc, /// What the peer is told, and how. @@ -182,6 +200,7 @@ impl SessionHandle { add_path_receive: negotiated.add_path_receive.clone(), add_path_send: negotiated.add_path_send.clone(), rib_peer, + graceful_restart: negotiated.graceful_restart.clone(), counters: Arc::default(), export: Arc::default(), outbound, @@ -248,6 +267,8 @@ pub struct SessionRegistry { configured: Arc>, dial_states: Arc>>, shutting_down: Arc, + /// How the daemon is leaving, once it is: every connection hears it. + leaving: watch::Sender>, events: broadcast::Sender, } @@ -276,6 +297,7 @@ impl SessionRegistry { configured: Arc::new(configured), dial_states: Arc::new(RwLock::new(dial_states)), shutting_down: Arc::new(AtomicBool::new(false)), + leaving: watch::Sender::new(None), events, } } @@ -349,18 +371,17 @@ impl SessionRegistry { /// instead of discovering a dead TCP connection. Waits up to `grace` for /// the sessions to finish; returns how many were still open. pub async fn shutdown(&self, grace: Duration) -> usize { + self.leave(Leave::Stop, grace).await + } + + /// Closes every connection because the daemon is going, as `leave` + /// says: for good, or to restart (see [`Leave`]). Waits up to `grace` + /// for the sessions to finish; returns how many were still open. + pub async fn leave(&self, leave: Leave, grace: Duration) -> usize { self.shutting_down.store(true, Ordering::Relaxed); - for session in self.list() { - // Not waited for: a session whose queue is full is one whose - // peer is not reading, and it would hold up the goodbye to all - // the others. It ends when the process does. - let _ = session - .outbound - .try_send(Message::Notification(NotificationMessage { - error: BgpError::Cease(CeaseSubcode::AdminShutdown), - data: vec![], - })); - } + // Every connection watches this, whatever state it is in; one whose + // peer is not reading is not held up by the others. + self.leaving.send_replace(Some(leave)); let deadline = tokio::time::Instant::now() + grace; while !self.list().is_empty() && tokio::time::Instant::now() < deadline { tokio::time::sleep(Duration::from_millis(20)).await; @@ -368,6 +389,11 @@ impl SessionRegistry { self.list().len() } + /// How the daemon is leaving, once it is. + pub fn leaving(&self) -> watch::Receiver> { + self.leaving.subscribe() + } + pub fn subscribe(&self) -> broadcast::Receiver { self.events.subscribe() } diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 941687b..49b08c7 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -19,6 +19,7 @@ use bgp_rib::{LocalRoute, RibStopped}; use bgp_rpki::{CacheState, Update}; use bgp_server::address::{PeerAddress, socket_text}; use bgp_server::policy::PeerRole; +use bgp_server::restart::{Agreed, Restarts}; use bgp_server::rpki::PathCheck; use bgp_server::session::{ConfiguredPeer, DialState, PeerId, SessionHandle, SessionRegistry}; use serde::{Deserialize, Serialize}; @@ -92,6 +93,40 @@ pub struct PeerView { /// Why the last connection attempt or session ended, while in Backoff. pub last_error: Option, pub retry_at_unix: Option, + /// Graceful restart (RFC 4724), if both sides offered it. + pub graceful_restart: Option, + /// How many of the peer's routes are kept from its last session and + /// not sent again yet: while it restarts, and after, until it has sent + /// them all. + pub stale_routes: Option, + /// While the peer restarts: when its kept routes go if it is not back. + pub stale_until_unix: Option, +} + +/// Graceful restart as agreed with a peer. +#[derive(Debug, Clone, Serialize)] +pub struct GracefulRestartView { + /// How long the peer says a restart of its takes. + pub restart_time_secs: u16, + /// Whether sessions that end with a NOTIFICATION are covered too + /// (RFC 8538). + pub notification: bool, + /// The families whose routes are kept over a restart of the peer. + pub address_families: Vec, + /// Of those, the ones whose forwarding the peer kept over its last + /// restart. + pub forwarding_preserved: Vec, +} + +impl From<&Agreed> for GracefulRestartView { + fn from(agreed: &Agreed) -> Self { + Self { + restart_time_secs: agreed.restart_time, + notification: agreed.notification, + address_families: agreed.families.iter().map(|f| f.to_string()).collect(), + forwarding_preserved: agreed.forwarding.iter().map(|f| f.to_string()).collect(), + } + } } impl From<&SessionHandle> for PeerView { @@ -117,6 +152,9 @@ impl From<&SessionHandle> for PeerView { role_confirmed: s.role.map(|_| s.role_confirmed), last_error: None, retry_at_unix: None, + graceful_restart: s.graceful_restart.as_ref().map(GracefulRestartView::from), + stale_routes: Some(s.counters.stale()).filter(|stale| *stale > 0), + stale_until_unix: None, } } } @@ -151,13 +189,18 @@ impl PeerView { role_confirmed: None, last_error, retry_at_unix, + graceful_restart: None, + stale_routes: None, + stale_until_unix: None, } } } /// Configured peers first (with their live session when established), then -/// any unconfigured inbound sessions. -pub fn peer_views(registry: &SessionRegistry) -> Vec { +/// any unconfigured inbound sessions. A peer that is restarting has the +/// routes kept of it with it. +pub fn peer_views(registry: &SessionRegistry, restarts: &Restarts) -> Vec { + let kept = restarts.kept(); let sessions = registry.list(); let mut claimed = std::collections::HashSet::new(); let mut views: Vec = registry @@ -175,7 +218,17 @@ pub fn peer_views(registry: &SessionRegistry) -> Vec { ..PeerView::from(session) } } - None => PeerView::from_configured(peer, state), + None => { + let mut view = PeerView::from_configured(peer, state); + let restarting = kept.iter().find(|((address, asn), _)| { + *asn == peer.remote_asn && peer.endpoint.is_at(*address) + }); + if let Some((_, kept)) = restarting { + view.stale_routes = Some(kept.routes as u64); + view.stale_until_unix = Some(unix_secs(kept.until)); + } + view + } } }) .collect(); @@ -286,6 +339,9 @@ pub struct ReceivedRouteView { pub path_id: u32, /// Whether this is the route chosen for its prefix. pub best: bool, + /// Whether it is kept from a session of the peer's that ended, and not + /// sent again since (graceful restart). + pub stale: bool, /// What route origin validation says of the route: `valid` or /// `not found`. (An invalid route is not in the table, unless its /// peer's are taken regardless; then `invalid`.) Absent where @@ -312,12 +368,18 @@ pub(crate) struct PeerNames { impl PeerNames { pub(crate) fn of(state: &AppState) -> Self { + // A peer that is restarting has no session, and its routes are + // still in the table. + let restarting = state.restarts.kept().into_iter().map(|((_, asn), kept)| { + let peer = PeerId { + addr: kept.addr, + asn, + }; + (kept.rib_peer, (peer, kept.name, None)) + }); Self { - peers: state - .registry - .list() - .into_iter() - .map(|session| { + peers: restarting + .chain(state.registry.list().into_iter().map(|session| { // How the peer's paths are verified, or would be if // its invalid routes were not taken regardless. let check = session @@ -325,7 +387,7 @@ impl PeerNames { .filter(|_| !session.ibgp) .map(|role| PathCheck::new(role, session.peer.asn)); (session.rib_peer, (session.peer, session.name, check)) - }) + })) .collect(), rpki: state.rpki.as_ref().map(|rpki| rpki.current()), } @@ -356,6 +418,7 @@ impl PeerNames { prefix: route.prefix.to_string(), path_id: route.path_id, best: route.best, + stale: route.stale, rpki: self .rpki .as_ref() @@ -429,7 +492,7 @@ pub async fn config(State(state): State) -> Json { } pub async fn peers(State(state): State) -> Json> { - Json(peer_views(&state.registry)) + Json(peer_views(&state.registry, &state.restarts)) } pub async fn local_routes( diff --git a/crates/web/src/events.rs b/crates/web/src/events.rs index 0eca57d..4045cc8 100644 --- a/crates/web/src/events.rs +++ b/crates/web/src/events.rs @@ -133,6 +133,9 @@ fn best_routes(state: &AppState) -> impl Stream + use<> { path_id, attrs: Arc::clone(attrs), best: true, + // Not said in the journal; the event stream + // does not show it. + stale: false, })); } table::Event::Withdrawn { prefix } if !syncing => { diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index 258da61..70755a1 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -22,6 +22,7 @@ use bgp_rib::LocalRib; use bgp_rib::table::RibTable; use bgp_rpki::Rpki; use bgp_server::refusals::RefusalLog; +use bgp_server::restart::Restarts; use bgp_server::session::SessionRegistry; use tokio::net::TcpListener; use tracing::info; @@ -93,6 +94,8 @@ pub struct AppState { pub rpki: Option, /// What keeps the best routes in the kernel, where that is configured. pub kernel: Option, + /// Graceful restart: the routes kept of restarting peers. + pub restarts: Arc, } /// The status page, embedded so the binary is self-contained. diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index b418ec6..97c5193 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -193,14 +193,20 @@ from if (!state.peers.length) { setHTML(b, 'No peers configured and none connected.'); return; } setHTML(b, state.peers.map(p => { const up = p.state === 'Established'; - const detail = p.state === 'Backoff' ? `${esc(p.last_error || '')}; retry in ${until(p.retry_at_unix)}` : ''; + const restarting = !up && p.stale_until_unix + ? `restarting: ${p.stale_routes} routes kept for ${until(p.stale_until_unix)}` : ''; + const detail = [restarting, p.state === 'Backoff' ? `${esc(p.last_error || '')}; retry in ${until(p.retry_at_unix)}` : ''] + .filter(Boolean).join('; '); + const gr = p.graceful_restart; + const graceful = gr ? ` · graceful restart` : ''; + const stale = p.stale_routes ? ` (${p.stale_routes} stale)` : ''; return ` ${p.name ? esc(p.name) : '(inbound)'}${esc(p.peer_addr)}${asn(p.peer_asn)}${p.role ? ` ${esc(p.role)}${p.role_confirmed ? ' ✓' : ''}` : ''} ${esc(p.state)}${up ? (p.direction === 'outbound' ? 'us' : 'peer') : DASH} ${up ? esc(p.local_addr) : DASH}${up ? fmtTime(p.established_at_unix) : DASH} ${up ? age(p.established_at_unix) : DASH}${up ? (p.four_octet_asn ? 'yes' : 'no') : DASH} - ${up ? esc((p.address_families || []).join(', ')) + addPath(p) + (p.extended_next_hop ? ' · IPv4 with IPv6 next hops' : '') : DASH} - ${up ? p.routes : DASH} + ${up ? esc((p.address_families || []).join(', ')) + addPath(p) + (p.extended_next_hop ? ' · IPv4 with IPv6 next hops' : '') + graceful : DASH} + ${up ? p.routes + stale : DASH} ${up ? (p.rejected_routes ? `${p.rejected_routes}` : 0) + (p.held_routes ? ` · ${p.held_routes} held now` : '') : DASH}${detail}`; }).join('')); renderPeerChoices(); @@ -302,7 +308,7 @@ from ${s.next_hop ? esc(s.next_hop) : DASH}${s.origin ? esc(s.origin) : DASH}${s.med ?? DASH}${s.local_pref ?? DASH} ${commList(s.communities, true)}${commList(s.large_communities)} ${r.rpki ? `${esc(r.rpki)}` : DASH} - ${r.aspa ? `${esc(r.aspa)}` : DASH}${r.best ? 'best' : ''}`; }).join('')); + ${r.aspa ? `${esc(r.aspa)}` : DASH}${r.best ? 'best' : ''}${r.stale ? ` stale` : ''}`; }).join('')); } const RPKI_CLASS = { valid: 'ok', invalid: 'bad', 'not found': 'dim', unknown: 'dim' }; diff --git a/docs/content/docs/getting-started/introduction.md b/docs/content/docs/getting-started/introduction.md index 0985fe9..0ef708d 100644 --- a/docs/content/docs/getting-started/introduction.md +++ b/docs/content/docs/getting-started/introduction.md @@ -65,8 +65,8 @@ Ferrix 0.1.0 does not support the following features: ADD-PATH for advertised routes. - TCP Authentication Option (TCP-AO), TCP MD5 signatures, and the Generalized TTL Security Mechanism (GTSM). -- Graceful restart, Bidirectional Forwarding Detection (BFD), and - configuration changes without a restart. +- Long-Lived Graceful Restart, Bidirectional Forwarding Detection (BFD), + and configuration changes without a restart. - Recursive resolution of next hops that are not directly connected. For the implementation status of each RFC, see diff --git a/docs/content/docs/guides/examples.md b/docs/content/docs/guides/examples.md index c5c767e..4c08ddb 100644 --- a/docs/content/docs/guides/examples.md +++ b/docs/content/docs/guides/examples.md @@ -1,7 +1,7 @@ +++ title = "Configuration Examples" description = "Provides complete configurations for a single-homed edge network, a network with an Internet exchange connection, and a transit network." -weight = 7 +weight = 8 +++ This topic provides complete configurations for three common deployments. diff --git a/docs/content/docs/guides/graceful-restart.md b/docs/content/docs/guides/graceful-restart.md new file mode 100644 index 0000000..315b856 --- /dev/null +++ b/docs/content/docs/guides/graceful-restart.md @@ -0,0 +1,168 @@ ++++ +title = "Graceful Restart" +description = "Describes BGP graceful restart (RFC 4724, RFC 8538), which retains routes while a BGP speaker restarts, and how to restart Ferrix without withdrawing its routes." +weight = 7 ++++ + +Graceful restart is a BGP mechanism, specified in RFC 4724, that retains +the routes of a BGP speaker while the speaker restarts. The neighbors of +the restarting speaker continue to forward traffic by its routes, and do +not withdraw them from their own neighbors. Ferrix supports graceful +restart in both roles: as the restarting speaker, for example during a +software upgrade, and as the neighbor of a restarting speaker. + +## Understanding Graceful Restart + +Graceful restart is in use on a BGP session if both neighbors advertise the +Graceful Restart capability in their OPEN messages. The capability +contains the following information: + +- The restart time: the time within which the speaker expects to + reestablish the session after a restart. +- The address families for which the speaker retains routes. +- For each address family, whether the speaker preserved its forwarding + state during its last restart (the F bit). +- Whether the speaker has just restarted (the R bit). +- Whether the speaker applies graceful restart to sessions that end with a + NOTIFICATION message (the N bit, RFC 8538). + +When a session with a restarting neighbor ends, the router marks the +routes of the neighbor as stale. Stale routes remain in the routing table +and are selected and advertised in the same way as other routes. When the +neighbor reestablishes the session, it advertises its routes again, and +then sends an End-of-RIB marker for each address family. When the router +receives the End-of-RIB marker, it removes the stale routes of the address +family that the neighbor did not advertise again. + +RFC 8538 extends graceful restart to sessions that end with a NOTIFICATION +message. A Cease NOTIFICATION message with the subcode Hard Reset ends the +session without graceful restart: the receiving router removes the routes +immediately. + +## Restarting Ferrix + +Ferrix leaves for a restart when it receives the SIGUSR1 signal. Ferrix +then takes the following actions: + +1. It stops updating the kernel routing table, and leaves its routes in + the kernel. The router continues to forward traffic by these routes. +2. It closes the TCP connection of each session on which graceful restart + is in use, without a NOTIFICATION message. The neighbor retains the + routes of Ferrix for the restart time. +3. It sends a Cease NOTIFICATION message with the subcode Administrative + Shutdown to each other neighbor. +4. It exits with status 75. + +The systemd unit restarts the daemon when it exits with status 75. See +[Upgrading Ferrix](@/docs/operations/deployment.md#upgrading-ferrix). + +When the daemon starts again, it takes the following actions: + +1. It finds the routes of the previous run in the kernel and retains them. + See [Kernel Route Installation](@/docs/guides/kernel.md#startup-and-shutdown-behavior). +2. It advertises the Graceful Restart capability with the R bit set, for + the restart time after startup. It sets the F bit if it found routes of + the previous run in the kernel, or if kernel route installation is not + configured. +3. It advertises its originated routes, and sends an End-of-RIB marker for + each address family to each neighbor on which graceful restart is in use. + To a neighbor that receives learned routes (`learned_routes`), Ferrix + sends the End-of-RIB markers after every configured neighbor has sent + its own End-of-RIB markers, or after the restart time. +4. When every configured neighbor has sent its End-of-RIB markers, or after + `stale_seconds`, it removes the routes of the previous run that no + neighbor advertised again. + +When the daemon stops in response to SIGTERM, it ends each session for +good. On a session on which the N bit is in use, it sends a Cease +NOTIFICATION message with the subcode Hard Reset. + +## When a Neighbor Restarts + +Ferrix retains the routes of a neighbor whose session ends in one of the +following ways, if graceful restart is in use on the session: + +| Session ends because | Routes retained | +|---|---| +| The TCP connection fails or is closed | Yes | +| The hold timer expires | If the N bit is in use | +| The neighbor sends a NOTIFICATION message other than Hard Reset | If the N bit is in use | +| The neighbor sends a Cease NOTIFICATION message with the subcode Hard Reset | No | +| Ferrix detects an error, such as a malformed message or an exceeded prefix limit | No | + +Ferrix retains the routes of the address families that the neighbor listed +in its Graceful Restart capability. It removes the routes of other address +families immediately. + +Ferrix retains the routes for the restart time of the neighbor, or for +`stale_routes_seconds`, whichever is shorter. If the neighbor does not +reestablish the session within that time, Ferrix removes the routes. + +When the neighbor reestablishes the session, Ferrix takes the following +actions: + +- If the neighbor no longer advertises graceful restart, or does not set + the F bit for an address family, Ferrix removes the stale routes of the + address family immediately. +- Otherwise, Ferrix removes the stale routes of an address family when the + neighbor sends its End-of-RIB marker for the address family, or after + `stale_routes_seconds`. + +## Configuring Graceful Restart + +By default, graceful restart is enabled for every neighbor, with a restart +time of 120 seconds. The following example sets a shorter restart time for +all neighbors, and disables graceful restart for one neighbor: + +```textproto +graceful_restart { + restart_time_seconds: 60 + stale_routes_seconds: 60 +} + +peers { + name: "transit-a" + endpoint { address: "192.0.2.1" } + remote_asn: 64496 + graceful_restart { disabled: true } +} +``` + +The restart time determines for how long neighbors forward traffic to the +router after its session with them ends. Choose a restart time that covers +a restart of the daemon, including the time for systemd to start it again. + +For the statements, see +[GracefulRestart](@/docs/reference/configuration.md#gracefulrestart). + +## Verifying Graceful Restart + +The status page and the HTTP API display the following information: + +- For each neighbor on which graceful restart is in use, the restart time + of the neighbor and whether the N bit is in use. The `/api/peers` + endpoint returns this information in the `graceful_restart` field. +- For a neighbor that is restarting, the number of retained routes and the + time at which they are removed (`stale_routes` and `stale_until_unix`). +- For a neighbor that has reestablished its session, the number of routes + that are still stale (`stale_routes`). +- For each received route, whether it is stale (`stale` in + `/api/routes/received`). + +Ferrix logs the following events at the `info` level: a session that ends +with retained routes, a neighbor that reestablishes its session, and the +removal of stale routes. + +## Limitations + +- Ferrix does not support Long-Lived Graceful Restart (RFC 9494). +- Ferrix retains the routes of a neighbor whose session fails for any + reason, including a failure of the neighbor or of the link, for the + restart time of the neighbor. During that time, traffic is forwarded to + the neighbor. Ferrix does not support Bidirectional Forwarding Detection + (BFD), which would detect such failures. To remove the routes of a + neighbor immediately when its session fails, disable graceful restart + for the neighbor. +- Ferrix does not preserve its forwarding state when the router reboots, + or when the daemon is stopped with SIGTERM. In both cases the routes are + removed from the kernel. diff --git a/docs/content/docs/guides/kernel.md b/docs/content/docs/guides/kernel.md index f4568af..39f2530 100644 --- a/docs/content/docs/guides/kernel.md +++ b/docs/content/docs/guides/kernel.md @@ -113,10 +113,15 @@ Ferrix handles its kernel routes as follows: - **At startup**, Ferrix retains the routes installed by a previous run, so that the router continues to forward traffic while the neighbors - readvertise their routes. After `stale_seconds`, Ferrix removes the routes - that no neighbor has readvertised. + readvertise their routes. When every configured neighbor has sent an + End-of-RIB marker for each address family, or after `stale_seconds`, + whichever is first, Ferrix removes the routes that no neighbor has + readvertised. - **On SIGINT or SIGTERM**, Ferrix closes its BGP sessions and removes its routes from the kernel. +- **On SIGUSR1**, Ferrix leaves its routes in the kernel and exits, to be + started again (graceful restart). See + [Graceful Restart](@/docs/guides/graceful-restart.md). - **After a crash**, the routes remain in the kernel until the next start. ## Interface Changes diff --git a/docs/content/docs/operations/deployment.md b/docs/content/docs/operations/deployment.md index 111e3c5..bab1309 100644 --- a/docs/content/docs/operations/deployment.md +++ b/docs/content/docs/operations/deployment.md @@ -1,6 +1,6 @@ +++ title = "Deploying Ferrix" -description = "Describes how to run Ferrix as a systemd service, secure the hosts and interfaces around it, change its configuration, and restart it." +description = "Describes how to run Ferrix as a systemd service, secure the hosts and interfaces around it, change its configuration, and restart or upgrade it without withdrawing its routes." weight = 1 [extra] @@ -48,6 +48,10 @@ The `ExecStartPre` command validates the configuration before the daemon starts. If the configuration is invalid, the service does not start, and the journal contains the error. +When the daemon exits with status 75, after a graceful restart, systemd +starts it again (`RestartForceExitStatus=75`). See +[Upgrading Ferrix](#upgrading-ferrix). + The sandboxing options in the unit have been tested with kernel route installation enabled. The command `systemd-analyze security ferrix` rates the unit's exposure level as 2.1 (OK). @@ -134,31 +138,82 @@ configuration: sudo mv /etc/ferrix/server.textpb.new /etc/ferrix/server.textpb ``` -3. Restart the service. +3. Restart the daemon with a graceful restart. ```bash - sudo systemctl restart ferrix + sudo systemctl kill --signal=SIGUSR1 ferrix ``` +## Upgrading Ferrix + +A graceful restart (RFC 4724) restarts the daemon without withdrawing its +routes. The router continues to forward traffic during the restart, and +neighbors on which graceful restart is in use retain the routes of Ferrix. +See [Graceful Restart](@/docs/guides/graceful-restart.md). + +To upgrade Ferrix: + +1. Install the new executable. + + ```bash + sudo install -m 0755 bgp /usr/local/bin/bgp.new + sudo mv /usr/local/bin/bgp.new /usr/local/bin/bgp + ``` + + The `mv` command replaces the executable in one step, while the running + daemon continues to use the previous one. + +2. Validate the configuration with the new executable. + + ```bash + sudo bgp --check /etc/ferrix/server.textpb + ``` + +3. Send the SIGUSR1 signal to the daemon. + + ```bash + sudo systemctl kill --signal=SIGUSR1 ferrix + ``` + + The daemon leaves its routes in the kernel, closes its sessions, and + exits with status 75. systemd starts the new executable after + `RestartSec` (5 seconds). + +4. Verify that the sessions are reestablished. + + ```bash + journalctl -u ferrix -f + ``` + + On the status page, the neighbor table shows the restarted sessions, + and the kernel section shows the routes kept from the previous run + until every neighbor has sent its routes. + +The restart must complete within the restart time that Ferrix advertises +(`restart_time_seconds`, default: 120 seconds). Otherwise, the neighbors +remove the routes of Ferrix. + ## Restart Behavior -When the daemon stops in response to SIGTERM, it sends a Cease -NOTIFICATION message with the subcode Administrative Shutdown to every -neighbor, and removes its routes from the kernel. The neighbors withdraw -the routes that they received from Ferrix. Traffic uses alternative paths -until the sessions are reestablished and the routing tables are exchanged -again. For a full routing table, this takes several seconds. +Table 2 lists how the daemon stops, and the effect on its neighbors and on +forwarding. -If the daemon terminates unexpectedly, its routes remain in the kernel, and -the router continues to forward traffic. At the next start, Ferrix retains -these routes for `stale_seconds` (default: 120 seconds) while the neighbors -readvertise their routes, and then removes the routes that were not -readvertised. +| Event | Neighbors | Kernel routes | +|---|---|---| +| SIGUSR1 | Neighbors on which graceful restart is in use retain the routes of Ferrix for the restart time. Other neighbors receive a Cease NOTIFICATION message with the subcode Administrative Shutdown. | Retained | +| SIGTERM (`systemctl stop`, `systemctl restart`) | Every neighbor receives a Cease NOTIFICATION message, with the subcode Hard Reset where the neighbor supports RFC 8538, and withdraws the routes of Ferrix. | Removed | +| Unexpected termination | Neighbors on which graceful restart is in use retain the routes of Ferrix for the restart time. | Retained | + +At startup, Ferrix retains the routes that a previous run left in the +kernel until every neighbor has sent its routes, or for `stale_seconds` +(default: 120 seconds), and then removes the routes that no neighbor +advertised again. > [!WARNING] -> A restart interrupts forwarding through the router for the time that the -> sessions take to converge. Restart Ferrix in a maintenance window, or -> after traffic has been drained to another router. +> A stop with SIGTERM interrupts forwarding through the router until the +> sessions are reestablished and the routing tables are exchanged again. +> For a full routing table, this takes several seconds. To restart the +> daemon, use SIGUSR1. ## Running Ferrix in a Container diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md index 6415635..99d567f 100644 --- a/docs/content/docs/reference/configuration.md +++ b/docs/content/docs/reference/configuration.md @@ -55,6 +55,7 @@ Top-level statements. | `kernel` [Kernel](#kernel) | Disabled | Enables kernel route installation (Linux only). | | `export` [ExportPolicy](#exportpolicy) | Originated routes only | Specifies the export policy for neighbors that have no `export` statement. | | `control_communities` bool | `false` | Enables control communities, which restrict the neighbors to which a route is advertised. See [Control Communities](@/docs/guides/passing-routes-on.md#control-communities). | +| `graceful_restart` [GracefulRestart](#gracefulrestart) | Enabled | Specifies graceful restart for neighbors that have no `graceful_restart` statement. Its `restart_time_seconds` also limits how long Ferrix waits at startup for neighbors to send their routes. | ## Peer @@ -73,6 +74,7 @@ Statements for a BGP neighbor. | `role` [Role](#role) | None | Specifies the BGP role of the neighbor (RFC 9234). Not permitted for IBGP neighbors. | | `require_role` bool | `false` | Rejects the session if the neighbor does not advertise a BGP role. Requires `role`. | | `export` [ExportPolicy](#exportpolicy) | Top-level `export` | Specifies the export policy for the neighbor. Replaces the top-level `export` statement. | +| `graceful_restart` [GracefulRestart](#gracefulrestart) | Top-level `graceful_restart` | Specifies graceful restart for the neighbor. Each statement that is set replaces the corresponding top-level statement. | ## TcpEndpoint @@ -167,6 +169,17 @@ Statements for kernel route installation. See | `without_nexthop_objects` bool | `false` | Installs each route with its own gateway instead of a nexthop object. | | `stale_seconds` uint32, optional | 120 | Specifies the time, in seconds, for which routes installed by a previous run are retained at startup. A value of 0 removes them immediately. | +## GracefulRestart + +Statements for graceful restart (RFC 4724, RFC 8538). See +[Graceful Restart](@/docs/guides/graceful-restart.md). + +| Statement | Default | Description | +|---|---|---| +| `disabled` bool, optional | `false` | Disables graceful restart. Ferrix does not advertise the capability, and removes the routes of a neighbor as soon as its session ends. | +| `restart_time_seconds` uint32, optional | 120 | Specifies the restart time advertised to neighbors, in seconds: the time for which a neighbor retains the routes of Ferrix after the session ends. Range: 0 through 4095. | +| `stale_routes_seconds` uint32, optional | 120 | Specifies the maximum time, in seconds, for which Ferrix retains the routes of a restarting neighbor: while the neighbor is down, regardless of the restart time that it advertises, and again after it reconnects, until it has sent all its routes. A value of 0 retains no routes. | + ## Role | Value | Description | diff --git a/docs/content/docs/reference/http-api.md b/docs/content/docs/reference/http-api.md index 00b5482..49e2fcc 100644 --- a/docs/content/docs/reference/http-api.md +++ b/docs/content/docs/reference/http-api.md @@ -58,7 +58,15 @@ neighbor: "role": null, "role_confirmed": null, "last_error": null, - "retry_at_unix": null + "retry_at_unix": null, + "graceful_restart": { + "restart_time_secs": 120, + "notification": true, + "address_families": ["IPv4 unicast", "IPv6 unicast"], + "forwarding_preserved": ["IPv4 unicast", "IPv6 unicast"] + }, + "stale_routes": null, + "stale_until_unix": null } ``` @@ -71,6 +79,9 @@ neighbor: | `held_routes` | Number of held routes: routes rejected for a reason that can change without a new UPDATE, such as RPKI validity or next hop resolution. | | `role_confirmed` | Whether the neighbor advertised a BGP role that matches the configured role. | | `retry_at_unix` | For a neighbor without a session, the time of the next connection attempt. | +| `graceful_restart` | Graceful restart, if both neighbors advertised it: the restart time of the neighbor, whether the N bit is in use (RFC 8538), the address families whose routes are retained, and those for which the neighbor preserved its forwarding state. | +| `stale_routes` | Number of the neighbor's routes retained from its previous session and not yet advertised again, or `null` if there are none. | +| `stale_until_unix` | For a restarting neighbor without a session, the time at which its retained routes are removed. | ## Received Routes @@ -101,6 +112,7 @@ Prefixes are sorted by address, with IPv4 prefixes first. "prefix": "203.0.113.0/24", "path_id": 0, "best": true, + "stale": false, "rpki": null, "aspa": null, "summary": { @@ -123,6 +135,7 @@ Prefixes are sorted by address, with IPv4 prefixes first. |---|---| | `path_id` | ADD-PATH path identifier, or 0. | | `best` | Whether the route is the best path for the prefix. | +| `stale` | Whether the route is retained from a previous session of the neighbor and has not been advertised again (graceful restart). | | `rpki`, `aspa` | Route origin validation and ASPA verification state, if RPKI validation is configured. | | `attributes` | All path attributes, as received. | diff --git a/docs/content/docs/reference/rfcs.md b/docs/content/docs/reference/rfcs.md index 1cb5de1..9cf230f 100644 --- a/docs/content/docs/reference/rfcs.md +++ b/docs/content/docs/reference/rfcs.md @@ -101,8 +101,20 @@ The base protocol specification. | Feature | Status | |---------|--------| -| `GracefulRestart` capability wire format | 🟡 | -| Restart state and forwarding preserved across sessions | ❌ | +| `GracefulRestart` capability (parse, serialise) | ✅ | +| Receiving speaker: routes of a restarting neighbor retained as stale, and removed at its End-of-RIB or after the stale timer | ✅ | +| Restarting speaker: forwarding state preserved in the kernel over a restart (SIGUSR1); R and F bits | ✅ | +| End-of-RIB marker sent and received | ✅ | +| Route selection deferred until neighbors' End-of-RIB (for the removal of kept kernel routes and End-of-RIB markers for learned routes) | ✅ | + +### RFC 8538 — Notification Message Support for BGP Graceful Restart + + +| Feature | Status | +|---|---| +| N bit advertised and negotiated | ✅ | +| Routes retained after a NOTIFICATION other than Hard Reset, and after hold timer expiry | ✅ | +| Hard Reset sent for errors and for a stop, with the original NOTIFICATION as its data | ✅ | ### RFC 9494 — Long-Lived Graceful Restart for BGP diff --git a/docs/examples/ferrix.service b/docs/examples/ferrix.service index c0b4035..170246f 100644 --- a/docs/examples/ferrix.service +++ b/docs/examples/ferrix.service @@ -19,6 +19,10 @@ ExecStartPre=/usr/local/bin/bgp --check /etc/ferrix/server.textpb ExecStart=/usr/local/bin/bgp /etc/ferrix/server.textpb --level info Restart=on-failure RestartSec=5s +# On SIGUSR1 the daemon leaves its routes in place for a graceful restart +# and exits with status 75: start it again at once. +SuccessExitStatus=75 +RestartForceExitStatus=75 # Time to notify all neighbors and remove the kernel routes. TimeoutStopSec=30s diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh index badd2fe..81b15a3 100755 --- a/scripts/interop/lab.sh +++ b/scripts/interop/lab.sh @@ -29,7 +29,7 @@ cd "$(dirname "$0")/../.." bgp="${CARGO_TARGET_DIR:-target}/release/bgp" work=/tmp/ferrix-lab pathspace=ferrixlab -all_phases="basics roles rpki refresh flaps restart scale limits hijacks nexthops transit v6only linklocal" +all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit v6only linklocal" phases="${PHASES:-$all_phases}" scale_routes="${SCALE_ROUTES:-200000}" @@ -497,7 +497,7 @@ phase_roles() { || fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')" frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \ || fail "FRR does not see us as its peer" - bird_ctl show protocols all dut4 | grep -A12 "Neighbor capabilities" | grep -q "Role: customer" \ + bird_ctl show protocols all dut4 | grep -A40 "Neighbor capabilities" | grep -q "Role: customer" \ || fail "BIRD does not see us as its customer" ok "roles agree with FRR (peer) and BIRD (provider)" @@ -636,6 +636,65 @@ phase_restart() { ok "killed: routes stay in the kernel, and the next start clears the stale one" } +# Graceful restart (RFC 4724): the daemon restarts for an upgrade without +# its routes going anywhere, and keeps BIRD's while BIRD restarts. +frr_stale() { # frr_stale prefix: FRR keeps our route, as stale + vty -c "show bgp ipv4 unicast $1 json" | jq -e '.paths | any(.aspath.string == "64601" and .stale == true)' +} +frr_fresh() { frr_has ipv4 "$1" && ! frr_stale "$1"; } +dut_stale() { # dut_stale prefix peer-name: the daemon keeps the peer's route, as stale + api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer and .stale)' +} +kernel_removals() { + api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }' +} +phase_graceful() { + sed -i 's/graceful restart off;/graceful restart on;/' "$work/bird.conf" + bird_ctl configure >/dev/null + wait_for "BIRD to come back with graceful restart" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '[.[] | select(.name == \"bird4\" or .name == \"bird6\") | select(.graceful_restart != null)] | length == 2'" + converged + + # The daemon restarts: everything stays where it is meanwhile. + dut_down USR1 + grep -q "Leaving to be started again" "$work/dut.log" || fail "the daemon did not leave for a restart" + dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20" || fail "FRR's route left the kernel" + dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel" + dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20" || fail "BIRD's IPv6 route left the kernel" + wait_for "FRR to keep our route as stale" frr_stale 10.1.0.0/24 + bird_has 10.1.0.0/24 || fail "BIRD dropped our route" + bird_has 2001:db8:1::/48 || fail "BIRD dropped our IPv6 route" + in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "packets stopped while the daemon was away" + dut_up + converged + wait_for "FRR to take our route afresh" frr_fresh 10.1.0.0/24 + wait_for "nothing kept in the kernel any more" \ + bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.kept == 0'" + local removed + removed=$(kernel_removals) + [ "${removed:-0}" -eq 0 ] || fail "$removed routes were taken out of the kernel over the restart" + ok "restarted on SIGUSR1: kernel routes kept, FRR and BIRD kept ours, nothing withdrawn" + + # BIRD restarts: the daemon keeps its routes meanwhile, in the table + # and in the kernel, and takes them afresh when it is back. + bird_ctl graceful restart >/dev/null || true + while [ -e "/proc/$(sudo cat "$work/bird.pid" 2>/dev/null || echo 0)" ]; do sleep 0.05; done + wait_for "the daemon to keep BIRD's route as stale" dut_stale 10.3.0.0/24 bird4 + dut_stale 2001:db8:3::/48 bird6 >/dev/null || fail "BIRD's IPv6 route was not kept" + dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel" + peer_is bird4 '.stale_routes > 0' >/dev/null || fail "the daemon does not say it keeps BIRD's routes: $(peer_is bird4 .)" + sudo rm -f "$work/bird.ctl" + bird_up + converged + wait_for "BIRD's routes to be fresh" \ + bash -c "! sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?peer=bird4' | jq -e '.routes | any(.stale)' >/dev/null" + ok "BIRD restarted gracefully: its routes were kept, and taken afresh" + + sed -i 's/graceful restart on;/graceful restart off;/' "$work/bird.conf" + bird_ctl configure >/dev/null + converged +} + # A full table's worth of routes from BIRD, into the table and the kernel. phase_scale() { # Out of 100.0.0.0/6 and up: as many /24s as asked for.