diff --git a/crates/integration-tests/src/test_server.rs b/crates/integration-tests/src/test_server.rs index 93eda31..8fc834f 100644 --- a/crates/integration-tests/src/test_server.rs +++ b/crates/integration-tests/src/test_server.rs @@ -72,6 +72,7 @@ impl TestServer { rpki: None, kernel: None, export: Default::default(), + export_limits: None, control: None, graceful_restart: Default::default(), }; diff --git a/crates/integration-tests/tests/add_path.rs b/crates/integration-tests/tests/add_path.rs index b7f599d..75057b0 100644 --- a/crates/integration-tests/tests/add_path.rs +++ b/crates/integration-tests/tests/add_path.rs @@ -42,6 +42,7 @@ async fn server_offering( c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "route-server-client".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), diff --git a/crates/integration-tests/tests/admission.rs b/crates/integration-tests/tests/admission.rs index 8b74d4b..ae1cd0d 100644 --- a/crates/integration-tests/tests/admission.rs +++ b/crates/integration-tests/tests/admission.rs @@ -32,6 +32,7 @@ fn peer_at(endpoint: SocketAddr) -> PeerConfig { role: None, require_role: false, export: Default::default(), + export_limits: None, } } diff --git a/crates/integration-tests/tests/aspa.rs b/crates/integration-tests/tests/aspa.rs index 9817baa..075364e 100644 --- a/crates/integration-tests/tests/aspa.rs +++ b/crates/integration-tests/tests/aspa.rs @@ -59,6 +59,7 @@ async fn server_with( c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: endpoint.into(), diff --git a/crates/integration-tests/tests/collision.rs b/crates/integration-tests/tests/collision.rs index c1639c0..d6159ea 100644 --- a/crates/integration-tests/tests/collision.rs +++ b/crates/integration-tests/tests/collision.rs @@ -38,6 +38,7 @@ async fn server_dialing(listener: &TcpListener) -> Result { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: endpoint.into(), diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs index 3de9d83..29f0f24 100644 --- a/crates/integration-tests/tests/debugging.rs +++ b/crates/integration-tests/tests/debugging.rs @@ -169,6 +169,7 @@ async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "refused-demo".into(), endpoint: endpoint.into(), diff --git a/crates/integration-tests/tests/dialer.rs b/crates/integration-tests/tests/dialer.rs index 1c3f79e..200f77e 100644 --- a/crates/integration-tests/tests/dialer.rs +++ b/crates/integration-tests/tests/dialer.rs @@ -29,6 +29,7 @@ async fn server_dialing(listener: &TcpListener, remote_asn: u32) -> Result Result<()> { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), @@ -245,6 +247,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: endpoint.into(), @@ -269,6 +272,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: endpoint.into(), diff --git a/crates/integration-tests/tests/export_limits.rs b/crates/integration-tests/tests/export_limits.rs new file mode 100644 index 0000000..7917b00 --- /dev/null +++ b/crates/integration-tests/tests/export_limits.rs @@ -0,0 +1,187 @@ +//! Export limits, end to end: what a peer above or beside us is told is +//! held to the prefixes they list and to as many routes as they allow, +//! whatever else would send it more. + +use std::collections::BTreeSet; +use std::net::Ipv4Addr; +use std::sync::Arc; +use std::time::Duration; + +use bgp_integration_tests::default_policy; +use bgp_integration_tests::test_peer::TestPeer; +use bgp_integration_tests::test_server::TestServer; +use bgp_packet::errors::{BgpError, CeaseSubcode}; +use bgp_packet::message::Message; +use bgp_packet::open::{BgpRole, Capability}; +use bgp_primitives::constants::AfiSafi; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_rib::LocalRoute; +use bgp_server::config::PeerConfig; +use bgp_server::family::FamilySet; +use bgp_server::policy::{ExportLimits, PeerRole, PrefixSet}; +use eyre::{Result, bail}; +use tokio::net::TcpListener; + +const LOCAL_ASN: u32 = 1299; +const PEER_ASN: u32 = 3333; +const TIMEOUT: Duration = Duration::from_secs(5); + +fn prefix(s: &str) -> IpPrefix { + IpPrefix::try_from(s).unwrap() +} + +/// What a test holds on to: the server, and the listener its dialling +/// goes to. +struct Setup { + server: TestServer, + _listener: TcpListener, +} + +/// A server that originates `originated`, with one configured peer at the +/// address the test connects from, which is a peer of ours held to +/// `limits`. +async fn server_with(originated: &[&str], limits: ExportLimits) -> Result { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let endpoint = listener.local_addr()?; + let originate = originated + .iter() + .map(|p| LocalRoute::new(prefix(p), Default::default())) + .collect(); + let server = TestServer::start_with(LOCAL_ASN, |c| { + c.accept_unconfigured_peers = false; + c.originate = originate; + c.peers = vec![PeerConfig { + graceful_restart: Default::default(), + groups: Vec::new(), + export_limits: Some(Arc::new(limits)), + reflector_client: false, + name: "r1".into(), + endpoint: endpoint.into(), + remote_asn: PEER_ASN, + hold_time_secs: 90, + local_address: None, + families: FamilySet::supported(), + add_path: None, + import: default_policy(), + role: Some(PeerRole::Peer), + require_role: false, + export: Default::default(), + }]; + }) + .await?; + Ok(Setup { + server, + _listener: listener, + }) +} + +/// A session with the server, as its peer. +async fn established(server: &TestServer) -> Result { + let mut peer = TestPeer::connect(server.addr).await?; + let capabilities = vec![ + Capability::FourByteAsn { asn: PEER_ASN }, + Capability::multiprotocol(AfiSafi::IPV4_UNICAST), + Capability::multiprotocol(AfiSafi::IPV6_UNICAST), + Capability::BgpRole { + role: BgpRole::Peer, + }, + ]; + peer.handshake_with(PEER_ASN, Ipv4Addr::new(10, 0, 0, 2), 90, capabilities) + .await?; + Ok(peer) +} + +/// What the peer has been told once it has been quiet for a while: the +/// prefixes it holds, and the NOTIFICATION that ended the session, if one +/// did. +async fn told(peer: &mut TestPeer) -> (BTreeSet, Option<(BgpError, Vec)>) { + let mut routes = BTreeSet::new(); + while let Ok(Ok(message)) = tokio::time::timeout(Duration::from_millis(500), peer.recv()).await + { + match message { + Message::Update(update) => { + for route in &update.withdrawn_routes { + routes.remove(&route.prefix.to_string()); + } + for route in &update.announced_routes { + routes.insert(route.prefix.to_string()); + } + } + Message::Notification(notification) => { + return (routes, Some((notification.error, notification.data))); + } + _ => {} + } + } + (routes, None) +} + +fn allowing(prefixes: &[&str]) -> Option { + let mut set = PrefixSet::default(); + for p in prefixes { + let p = prefix(p); + set.allow(p, p.length()); + } + Some(set) +} + +/// A route of ours whose prefix is not in the peer's export limits is not +/// announced to it; the one that is, is. +#[tokio::test] +async fn test_a_route_outside_the_export_limits_is_not_announced() -> Result<()> { + let limits = ExportLimits { + prefixes: allowing(&["193.0.0.0/21"]), + max_prefixes: None, + }; + let Setup { server, .. } = server_with(&["193.0.0.0/21", "198.51.100.0/24"], limits).await?; + let mut peer = established(&server).await?; + let (routes, ended) = told(&mut peer).await; + assert_eq!(routes, BTreeSet::from(["193.0.0.0/21".to_string()])); + assert_eq!(ended, None); + Ok(()) +} + +/// One route more than the export limit allows is not announced: the +/// session ends with an Administrative Shutdown that says why (RFC 9003), +/// and the peer is kept down, so that it is told nothing rather than too +/// much. +#[tokio::test] +async fn test_going_over_the_export_limit_ends_the_session_and_keeps_the_peer_down() -> Result<()> { + let limits = ExportLimits { + prefixes: None, + max_prefixes: Some(2), + }; + let Setup { server, .. } = server_with(&["193.0.0.0/21", "193.0.8.0/21"], limits).await?; + let mut peer = established(&server).await?; + let (routes, ended) = told(&mut peer).await; + assert_eq!(routes.len(), 2); + assert_eq!(ended, None); + + server + .local_rib + .announce(LocalRoute::new(prefix("193.0.16.0/21"), Default::default())) + .await?; + let (more, ended) = told(&mut peer).await; + assert_eq!(more, BTreeSet::new(), "nothing more is announced"); + let Some((error, data)) = ended else { + bail!("the session was not ended"); + }; + assert_eq!(error, BgpError::Cease(CeaseSubcode::AdminShutdown)); + let text = String::from_utf8(data[1..].to_vec())?; + assert!( + text.starts_with("export limit of 2 routes reached"), + "{text}" + ); + assert!(peer.is_closed().await); + + // The peer is kept down: a new connection is closed unanswered. + let mut again = TestPeer::connect(server.addr).await?; + assert!( + matches!( + tokio::time::timeout(TIMEOUT, again.recv()).await, + Ok(Err(_)) + ), + "a held-down peer gets no session" + ); + Ok(()) +} diff --git a/crates/integration-tests/tests/import_policy.rs b/crates/integration-tests/tests/import_policy.rs index 4d8fd27..b1ffed1 100644 --- a/crates/integration-tests/tests/import_policy.rs +++ b/crates/integration-tests/tests/import_policy.rs @@ -44,6 +44,7 @@ async fn server_with(import: ImportPolicy) -> Result<(TestServer, TcpListener)> c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "r1".into(), endpoint: endpoint.into(), diff --git a/crates/integration-tests/tests/negotiation.rs b/crates/integration-tests/tests/negotiation.rs index b3128d8..9357bfa 100644 --- a/crates/integration-tests/tests/negotiation.rs +++ b/crates/integration-tests/tests/negotiation.rs @@ -185,6 +185,7 @@ async fn test_no_family_in_common_refuses_the_session() -> Result<()> { c.peers = vec![PeerConfig { graceful_restart: Default::default(), groups: Vec::new(), + export_limits: None, reflector_client: false, name: "v6-only".into(), endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), diff --git a/crates/integration-tests/tests/roles.rs b/crates/integration-tests/tests/roles.rs index 6a3c981..26c40e5 100644 --- a/crates/integration-tests/tests/roles.rs +++ b/crates/integration-tests/tests/roles.rs @@ -50,6 +50,7 @@ async fn server_with(role: Option, require_role: bool) -> Result + && !self.announced.contains(&prefix) + && limits.is_none_or(|limits| limits.allows(&prefix)) => { learned.dressed(&attrs, source, prefix.address_family(), &egress) } @@ -612,6 +620,16 @@ impl Exporter { withdrawn[version_index(prefix.address_family())].push(prefix.into()); } } + // What the peer would have once this goes out: none of it goes if + // that is more than it may. + let routes = self.announced.len() + + learned.announced.len() + + learned.stale.as_ref().map_or(0, HashSet::len); + if let Some(limits) = limits + && limits.exceeded_by(routes) + { + return Err(over_limit(session, limits, routes)); + } for (version, routes) in [IpVersion::Ipv4, IpVersion::Ipv6] .into_iter() .zip(withdrawn) @@ -673,6 +691,25 @@ impl Exporter { ); return Ok(()); } + if let Some(limits) = session.export_limits.as_deref() { + if !limits.allows(&route.prefix) { + debug!( + peer = %session.peer, + prefix = %route.prefix, + "Not announcing route: its prefix is not in the peer's export limits" + ); + return Ok(()); + } + let learned = self.learned.as_ref().map_or(0, |learned| { + learned.announced.len() + learned.stale.as_ref().map_or(0, HashSet::len) + }); + let routes = self.announced.len() + + usize::from(!self.announced.contains(&route.prefix)) + + learned; + if limits.exceeded_by(routes) { + return Err(over_limit(session, limits, routes)); + } + } let towards = self.towards(); let Some(mut update) = build_announce(route, &towards) else { debug!( @@ -728,6 +765,22 @@ impl Exporter { } } +/// Ends `session` because `routes` routes were to be announced on it, more +/// than `limits` allow; none of them has been. +fn over_limit(session: &SessionHandle, limits: &ExportLimits, routes: usize) -> Finished { + let limit = limits.max_prefixes.unwrap_or_default(); + error!( + peer = %session.peer, + name = session.name.as_deref().unwrap_or_default(), + routes, + limit, + "More routes were to be advertised to the peer than its export limit allows; advertising \ + none of them, ending the session and keeping the peer down" + ); + session.end(SessionError::ExportLimitExceeded { limit, routes }); + Finished +} + /// What about a session decides how a route is announced on it. #[derive(Debug, Clone, Copy)] pub struct Towards { @@ -1579,4 +1632,203 @@ mod tests { let mut second = fixture.session(1, 4); second.read_until_it_has(&routes).await; } + + // -- Export limits -- + + use crate::policy::{ExportLimits, ExportPolicy, PrefixSet}; + + /// A table with routes from a customer for `prefixes`, an advertiser + /// over it with `originated` routes of our own, and a session to a + /// peer of ours that is told of learned routes, within `limits`. + async fn limited( + prefixes: &[IpPrefix], + originated: &[IpPrefix], + limits: ExportLimits, + ) -> ( + mpsc::Receiver, + mpsc::UnboundedReceiver, + SessionGuard, + ) { + let registry = SessionRegistry::new(); + let rib = Arc::new(RibTable::new(4)); + let local_rib = LocalRib::spawn(); + for prefix in originated { + let route = LocalRoute::new(*prefix, LocalRouteAttrs::default()); + local_rib.announce(route).await.unwrap(); + } + spawn_advertiser( + local_rib, + Arc::clone(&rib), + registry.clone(), + 65000, + None, + Ipv4Addr::new(192, 0, 2, 1), + watch::channel(true).1, + ); + let peer = |n: u8, asn| PeerInfo { + asn, + router_id: Ipv4Addr::new(10, 0, 0, n), + address: IpAddr::V4(Ipv4Addr::new(192, 0, 2, n)), + interface: 0, + ibgp: false, + reflector_client: false, + }; + // A customer's routes carry no Only to Customer mark: as far as + // the roles go, they are for everyone. + let customer = rib.add_peer(peer(1, 64496)).unwrap(); + let came = [ + PathAttribute::Origin(OriginPathAttribute::IGP), + AsPathAttribute::from_asns(vec![64496]), + PathAttribute::NextHop(NextHopPathAttribute(Ipv4Addr::new(192, 0, 2, 1))), + ]; + let attrs = rib.attrs().intern(&came, None).unwrap(); + let nlri: Vec = prefixes.iter().copied().map(Nlri::from).collect(); + rib.announce(customer, &attrs, &nlri); + + let negotiated = Negotiated { + peer_asn: 64497, + ibgp: false, + peer_identifier: Ipv4Addr::new(10, 0, 0, 2), + hold_time_secs: 90, + four_octet_asn: true, + route_refresh: false, + families: FamilySet::supported(), + add_path_receive: FamilySet::default(), + add_path_send: FamilySet::default(), + role_confirmed: false, + extended_next_hop: false, + graceful_restart: None, + }; + let rib_peer = rib.add_peer(peer(2, 64497)).unwrap(); + let (tx, outbound) = mpsc::channel(1024); + let mut handle = SessionHandle::new( + PeerId { + addr: (Ipv4Addr::new(192, 0, 2, 2), 179).into(), + asn: 64497, + }, + None, + Direction::Inbound, + (Ipv4Addr::new(192, 0, 2, 100), 179).into(), + &negotiated, + rib_peer, + tx, + ); + handle.role = Some(PeerRole::Peer); + handle.export = Arc::new(ExportPolicy { + learned_routes: true, + ..ExportPolicy::default() + }); + handle.export_limits = Some(Arc::new(limits)); + let ending = handle.listen_for_end(); + (outbound, ending, registry.register(handle)) + } + + /// The routes announced on a session's queue until it has been quiet + /// for a while. + async fn announced(outbound: &mut mpsc::Receiver) -> HashSet { + let mut routes = HashSet::new(); + while let Ok(Some(message)) = + tokio::time::timeout(Duration::from_millis(500), outbound.recv()).await + { + if let Message::Update(update) = message { + for nlri in &update.withdrawn_routes { + routes.remove(&nlri.prefix); + } + routes.extend(update.announced_routes.iter().map(|nlri| nlri.prefix)); + if let Some(PathAttribute::MpReachNlri(reach)) = update + .path_attributes + .iter() + .find(|a| matches!(a, PathAttribute::MpReachNlri(_))) + { + routes.extend(reach.prefixes.iter().map(|nlri| nlri.prefix)); + } + } + } + routes + } + + fn allowing(prefixes: &[IpPrefix]) -> Option { + let mut set = PrefixSet::default(); + for prefix in prefixes { + set.allow(*prefix, prefix.length()); + } + Some(set) + } + + /// Only routes for the prefixes in the export limits go to a peer, + /// learned or our own, though nothing else keeps the others from it: + /// a customer's routes are for everyone, as far as the roles go. + #[tokio::test] + async fn routes_outside_the_export_limits_do_not_go_to_a_peer() { + let limits = ExportLimits { + prefixes: allowing(&[nth_prefix(2), nth_prefix(11)]), + max_prefixes: None, + }; + let (mut outbound, mut ending, _registration) = limited( + &[nth_prefix(1), nth_prefix(2), nth_prefix(3)], + &[nth_prefix(10), nth_prefix(11)], + limits, + ) + .await; + assert_eq!( + announced(&mut outbound).await, + HashSet::from([nth_prefix(2), nth_prefix(11)]) + ); + assert!(ending.try_recv().is_err(), "the session goes on"); + } + + /// When more routes would go to a peer than its export limit allows, + /// none of them does, and the session is ended: a leak becomes a + /// session that is down, and not a table spread across the Internet. + #[tokio::test] + async fn more_routes_than_the_export_limit_end_the_session_and_none_goes() { + let prefixes: Vec = (1..=5).map(nth_prefix).collect(); + let limits = ExportLimits { + prefixes: None, + max_prefixes: Some(3), + }; + let (mut outbound, mut ending, _registration) = limited(&prefixes, &[], limits).await; + let why = tokio::time::timeout(Duration::from_secs(10), ending.recv()) + .await + .expect("the session is asked to end") + .expect("by the exporter"); + assert!( + matches!( + why, + SessionError::ExportLimitExceeded { + limit: 3, + routes: 5 + } + ), + "{why:?}" + ); + assert_eq!(announced(&mut outbound).await, HashSet::new()); + } + + /// Our own routes count too, and none goes past the limit: those that + /// fit go, and the one that does not ends the session. + #[tokio::test] + async fn originated_routes_count_towards_the_export_limit() { + let originated: Vec = (10..13).map(nth_prefix).collect(); + let limits = ExportLimits { + prefixes: None, + max_prefixes: Some(2), + }; + let (mut outbound, mut ending, _registration) = limited(&[], &originated, limits).await; + let why = tokio::time::timeout(Duration::from_secs(10), ending.recv()) + .await + .expect("the session is asked to end") + .expect("by the exporter"); + assert!( + matches!( + why, + SessionError::ExportLimitExceeded { + limit: 2, + routes: 3 + } + ), + "{why:?}" + ); + assert_eq!(announced(&mut outbound).await.len(), 2); + } } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 8b50ed0..fe5caf8 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -16,7 +16,9 @@ use crate::address::{Endpoint, PeerAddress}; use crate::error::{ConfigError, ExportProblem, ImportProblem, OriginateProblem, PeerProblem}; use crate::family::FamilySet; use crate::marks::{Control, Marks, Patterns, Scrub}; -use crate::policy::{ExportPolicy, ImportPolicy, ImportRule, PeerRole, PrefixSet, Reflection}; +use crate::policy::{ + ExportLimits, ExportPolicy, ImportPolicy, ImportRule, PeerRole, PrefixSet, Reflection, +}; use crate::proto::config::{self as proto, BgpConfig, OriginatedRoute, Peer}; use crate::restart::GracefulRestartConfig; @@ -59,6 +61,9 @@ pub struct Config { pub rpki: Option, /// What a peer is told that has no `export` of its own. pub export: Arc, + /// Limits on what goes to a peer that has none of its own, if it is + /// one they are for ([`ExportLimits::apply_to`]). + pub export_limits: Option>, /// The communities by which routes say who is told of them, if they /// are honoured. pub control: Option, @@ -92,6 +97,9 @@ pub struct PeerConfig { pub require_role: bool, /// What the peer is told, and how. pub export: Arc, + /// Limits on what the peer is told, if it is one they are for + /// ([`ExportLimits::apply_to`]): its own, or the top-level ones. + pub export_limits: Option>, /// Graceful restart with the peer (RFC 4724). pub graceful_restart: GracefulRestartConfig, /// The groups the session is in, for control communities to name it @@ -147,6 +155,9 @@ impl Config { Some(export) => parse_export(export)?, None => ExportPolicy::default(), }); + let export_limits = parse_export_limits(proto.export_limits.as_ref()) + .map_err(ConfigError::ExportLimits)? + .map(Arc::new); let graceful_restart = match proto.graceful_restart.as_ref() { Some(own) => GracefulRestartConfig::default().overridden_by(own)?, None => GracefulRestartConfig::default(), @@ -158,7 +169,8 @@ impl Config { peer: name.clone(), problem, }; - let peer = parse_peer(peer, &export, graceful_restart).map_err(rejected)?; + let peer = + parse_peer(peer, &export, &export_limits, graceful_restart).map_err(rejected)?; // RFC 9234 is about what networks are to each other. if peer.role.is_some() && peer.remote_asn == proto.local_asn { return Err(rejected(PeerProblem::RoleInOwnAs)); @@ -179,6 +191,25 @@ impl Config { } peers.push(peer); } + // Where routes learned from one peer go on to others, which go + // where is decided by the peers' roles: a peer in another AS + // without one would have its routes go to everyone, and be told + // of everything. + let learned_go_on = + export.learned_routes || peers.iter().any(|peer| peer.export.learned_routes); + if learned_go_on + && let Some(peer) = peers + .iter() + .find(|peer| peer.remote_asn != proto.local_asn && peer.role.is_none()) + { + return Err(ConfigError::Peer { + peer: peer.name.clone(), + problem: PeerProblem::NoRole, + }); + } + if proto.accept_unconfigured_peers && export.learned_routes { + return Err(ConfigError::UnconfiguredPeersWithoutRole); + } let mut originate: Vec = Vec::with_capacity(proto.originate.len()); for entry in &proto.originate { @@ -257,6 +288,7 @@ impl Config { kernel: proto.kernel.as_ref().map(parse_kernel), rpki, export, + export_limits, control: proto.control_communities.then_some(Control { local_asn: proto.local_asn, }), @@ -296,9 +328,29 @@ fn parse_export(export: &proto::ExportPolicy) -> Result, +) -> Result, ImportProblem> { + let Some(limits) = limits else { + return Ok(None); + }; + let limits = ExportLimits { + prefixes: if limits.prefixes.is_empty() { + None + } else { + Some(parse_prefixes(&limits.prefixes)?) + }, + max_prefixes: Some(limits.max_prefixes).filter(|limit| *limit > 0), + }; + Ok((limits != ExportLimits::default()).then_some(limits)) +} + fn parse_peer( peer: Peer, export: &Arc, + export_limits: &Option>, graceful_restart: GracefulRestartConfig, ) -> Result { let ep = peer.endpoint.as_ref().ok_or(PeerProblem::MissingEndpoint)?; @@ -392,6 +444,12 @@ fn parse_peer( Some(own) => Arc::new(parse_export(own)?), None => Arc::clone(export), }, + export_limits: match peer.export_limits.as_ref() { + Some(own) => parse_export_limits(Some(own)) + .map_err(PeerProblem::ExportLimits)? + .map(Arc::new), + None => export_limits.clone(), + }, graceful_restart: match peer.graceful_restart.as_ref() { Some(own) => graceful_restart.overridden_by(own)?, None => graceful_restart, @@ -1235,4 +1293,111 @@ mod tests { .accept_rpki_invalid ); } + + /// Where learned routes are passed on, every peer in another AS has + /// to have a role: without one, its routes would go to everyone and it + /// would be told of everything. Where they are not, or for our own + /// routers, there is nothing for a role to decide. + #[test] + fn a_peer_in_another_as_needs_a_role_where_learned_routes_go_on() { + let peer = |body: &str| { + format!("peers {{ name: \"p\" endpoint {{ address: \"192.0.2.2\" }} {body} }}") + }; + let learned = "export { learned_routes: true }"; + assert_eq!( + rejected(&format!("{BASE}{learned}{}", peer("remote_asn: 64496"))), + ConfigError::Peer { + peer: "p".into(), + problem: PeerProblem::NoRole + } + ); + // The peer's own export passes them on; or another peer's does. + assert!(matches!( + rejected(&format!( + "{BASE}{}", + peer(&format!("remote_asn: 64496 {learned}")) + )), + ConfigError::Peer { + problem: PeerProblem::NoRole, + .. + } + )); + assert!(matches!( + rejected(&format!( + "{BASE}{} + peers {{ name: \"c\" endpoint {{ address: \"192.0.2.3\" }} remote_asn: 64497 + role: CUSTOMER {learned} }}", + peer("remote_asn: 64496") + )), + ConfigError::Peer { + problem: PeerProblem::NoRole, + .. + } + )); + parse(&format!( + "{BASE}{learned}{}", + peer("remote_asn: 64496 role: PEER") + )) + .unwrap(); + parse(&format!("{BASE}{learned}{}", peer("remote_asn: 65000"))).unwrap(); + parse(&format!("{BASE}{}", peer("remote_asn: 64496"))).unwrap(); + // Peers that are not configured have no role either. + assert_eq!( + rejected(&format!("{BASE}{learned} accept_unconfigured_peers: true")), + ConfigError::UnconfiguredPeersWithoutRole + ); + parse(&format!("{BASE}accept_unconfigured_peers: true")).unwrap(); + } + + /// The export limits at the top hold for every peer that has none of + /// its own; a peer's own take their place, and an empty one lifts + /// them. Which peers they apply to is the role's to say, when a + /// session comes up. + #[test] + fn export_limits_are_the_top_levels_or_a_peers_own() { + let config = parse(&format!( + "{BASE}export_limits {{ + prefixes {{ prefix: \"192.0.2.0/24\" }} + prefixes {{ prefix: \"2001:db8::/32\" max_length: 48 }} + max_prefixes: 10 + }} + peers {{ name: \"a\" endpoint {{ address: \"192.0.2.2\" }} remote_asn: 64496 + role: PEER }} + peers {{ name: \"b\" endpoint {{ address: \"192.0.2.3\" }} remote_asn: 64497 + role: PROVIDER export_limits {{ max_prefixes: 50 }} }} + peers {{ name: \"c\" endpoint {{ address: \"192.0.2.4\" }} remote_asn: 64498 + role: PEER export_limits {{ }} }}" + )) + .unwrap(); + let prefix = |text: &str| IpPrefix::try_from(text).unwrap(); + let top = config.export_limits.as_deref().expect("top-level limits"); + assert_eq!(top.max_prefixes, Some(10)); + assert!(top.allows(&prefix("192.0.2.0/24"))); + assert!(!top.allows(&prefix("192.0.2.0/25"))); + assert!(top.allows(&prefix("2001:db8:1::/48"))); + assert!(!top.allows(&prefix("2001:db8:1::/49"))); + assert!(!top.allows(&prefix("198.51.100.0/24"))); + assert_eq!(config.peers[0].export_limits.as_deref(), Some(top)); + let own = config.peers[1].export_limits.as_deref().expect("its own"); + assert_eq!(own.max_prefixes, Some(50)); + assert!(own.allows(&prefix("198.51.100.0/24"))); + assert_eq!(config.peers[2].export_limits, None); + + assert!(matches!( + rejected(&format!( + "{BASE}export_limits {{ prefixes {{ prefix: \"192.0.2.0/33\" }} }}" + )), + ConfigError::ExportLimits(ImportProblem::Prefix { .. }) + )); + assert!(matches!( + rejected(&format!( + "{BASE}peers {{ name: \"a\" endpoint {{ address: \"192.0.2.2\" }} remote_asn: 1 + export_limits {{ prefixes {{ prefix: \"192.0.2.0/24\" max_length: 16 }} }} }}" + )), + ConfigError::Peer { + problem: PeerProblem::ExportLimits(ImportProblem::MaxLength { .. }), + .. + } + )); + } } diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index 004dbf1..b07f990 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -42,11 +42,13 @@ use crate::error::SessionError; use crate::family::FamilySet; use crate::fsm::{Action, Event, Fsm, FsmConfig, Negotiated, Received}; use crate::metrics::{ - MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ROUTES, PREFIX_LIMIT_EXCEEDED, + EXPORT_LIMIT_EXCEEDED, MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ROUTES, PREFIX_LIMIT_EXCEEDED, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, }; use crate::panic::contained; -use crate::policy::{self, ExportPolicy, HeldDown, HoldDowns, ImportPolicy, PeerRole}; +use crate::policy::{ + self, ExportLimits, ExportPolicy, HeldDown, HoldDowns, ImportPolicy, PeerRole, +}; use crate::reach::{Reach, neighbor_at}; use crate::refusals::{PeerRefusals, RefusalLog}; use crate::restart::{self, Agreed, GracefulRestartConfig, Leave, Restarts}; @@ -114,6 +116,8 @@ pub struct PeerSettings { pub require_role: bool, /// What the peer is told, and how. pub export: Arc, + /// Limits on what the peer is told, if it is one they are for. + pub export_limits: Option>, /// The groups the session is in, for control communities. pub groups: Arc<[u32]>, /// Whether the peer is a route reflector client of ours (RFC 4456). @@ -129,6 +133,7 @@ impl PeerSettings { pub fn unconfigured( hold_time_secs: u16, export: Arc, + export_limits: Option>, graceful_restart: GracefulRestartConfig, ) -> Self { Self { @@ -141,6 +146,7 @@ impl PeerSettings { role: None, require_role: false, export, + export_limits, groups: Arc::default(), reflector_client: false, graceful_restart, @@ -160,6 +166,7 @@ impl From<&PeerConfig> for PeerSettings { role: peer.role, require_role: peer.require_role, export: Arc::clone(&peer.export), + export_limits: peer.export_limits.clone(), groups: peer.groups.as_slice().into(), reflector_client: peer.reflector_client, graceful_restart: peer.graceful_restart, @@ -215,6 +222,19 @@ pub async fn run_session( .with_label_values(&[session.refusals.label()]) .inc(); } + if matches!(result, Err(SessionError::ExportLimitExceeded { .. })) { + // Whatever made us want to send too much will be there on the + // next session too. Until it is put right, the peer is told of + // nothing rather than of too much. + hold_downs.hold( + peer_addr.into(), + session.import.hold_down, + HeldDown::OverExportLimit, + ); + EXPORT_LIMIT_EXCEEDED + .with_label_values(&[session.refusals.label()]) + .inc(); + } // A configured peer whose sessions keep coming up and failing is // slowed down. Not one that is nobody in particular: such peers are // only taken on a test network, where many share an address. @@ -304,6 +324,8 @@ struct Established { rib_peer: PeerIndex, /// Messages the rest of the server queued for the peer. outbound: mpsc::Receiver, + /// Where the rest of the server asks for the session to end, and why. + ending: mpsc::UnboundedReceiver, /// The routes learned from the peer, which go when the session does. routes: PeerRoutes, /// Whose routes the import policy is judging. @@ -575,6 +597,8 @@ struct Session { role: Option, /// What the peer is told, and how. export: Arc, + /// Limits on what the peer is told, if it is one they are for. + export_limits: Option>, /// The groups the session is in, for control communities. groups: Arc<[u32]>, /// Whether the peer is a route reflector client of ours (RFC 4456). @@ -641,6 +665,7 @@ impl Session { import: peer.import, role: peer.role, export: peer.export, + export_limits: peer.export_limits, groups: peer.groups, reflector_client: peer.reflector_client, said: Said::new(), @@ -946,6 +971,12 @@ impl Session { handle.role = self.role; handle.role_confirmed = negotiated.role_confirmed; handle.export = Arc::clone(&self.export); + // To a network above or beside us, or one we know nothing of. + handle.export_limits = self + .export_limits + .clone() + .filter(|_| ExportLimits::apply_to(self.role, negotiated.ibgp)); + let ending = handle.listen_for_end(); handle.groups = Arc::clone(&self.groups); handle.reflector_client = self.reflector_client && negotiated.ibgp; // Routes from other networks are validated, where there is data @@ -987,6 +1018,7 @@ impl Session { peer_id, rib_peer, outbound, + ending, routes: PeerRoutes::new(Arc::clone(&self.server.rib), rib_peer), from: policy::RouteSource { peer_asn: negotiated.peer_asn, @@ -1264,6 +1296,7 @@ fn sweep( async fn from_server(established: &mut Option) -> Wake { let Some(Established { outbound, + ending, validation, stale, swept, @@ -1285,6 +1318,7 @@ async fn from_server(established: &mut Option) -> Wake { } }; tokio::select! { + Some(error) = ending.recv() => Wake::Event(Event::LocalFailure(error)), Some(msg) = outbound.recv() => Wake::Event(Event::Outbound(msg)), Some(suspects) = invalidated => Wake::Revalidate(suspects), Some(outcome) = swept.1.recv() => Wake::Swept(outcome), @@ -1524,7 +1558,7 @@ mod tests { peer_addr, Direction::Inbound, server, - PeerSettings::unconfigured(90, Arc::default(), GracefulRestartConfig::default()), + PeerSettings::unconfigured(90, Arc::default(), None, GracefulRestartConfig::default()), ) .unwrap(); @@ -1575,7 +1609,7 @@ mod tests { peer_addr, Direction::Inbound, server, - PeerSettings::unconfigured(90, Arc::default(), GracefulRestartConfig::default()), + PeerSettings::unconfigured(90, Arc::default(), None, GracefulRestartConfig::default()), ) .unwrap(); session.send_hold_time = Duration::from_millis(100); diff --git a/crates/server/src/error.rs b/crates/server/src/error.rs index a4b7684..f7e9c7d 100644 --- a/crates/server/src/error.rs +++ b/crates/server/src/error.rs @@ -82,6 +82,13 @@ pub enum SessionError { /// down for a while after. #[error("the peer went over its limit of {limit} routes")] PrefixLimitExceeded { limit: u32 }, + /// We were to announce the peer more routes than its export limit + /// allows, and did not. It is kept down for a while after. + #[error( + "{routes} routes were to be advertised to the peer, over its export limit of {limit}; \ + none of them was" + )] + ExportLimitExceeded { limit: u32, routes: usize }, /// The peer took nothing we sent for this long (RFC 9687): it has /// stopped reading, and there is no telling it so. #[error("the peer accepted no data for {0:?}")] @@ -117,6 +124,17 @@ impl SessionError { Self::Collision => BgpError::Cease(CeaseSubcode::ConnCollisionResolution), Self::TableFull(_) => BgpError::Cease(CeaseSubcode::OutOfResources), Self::PrefixLimitExceeded { .. } => BgpError::Cease(CeaseSubcode::MaxPrefixes), + // Nothing is wrong at the peer's end, as far as we know; its + // operators are told what is at ours. + Self::ExportLimitExceeded { limit, .. } => { + return Some(NotificationMessage { + error: BgpError::Cease(CeaseSubcode::AdminShutdown), + data: shutdown_communication(&format!( + "export limit of {limit} routes reached; no more routes were sent, \ + and the session is held down for a while" + )), + }); + } // The session will be set up again; nothing is wrong at the // peer's end. Self::Panicked(_) => BgpError::Cease(CeaseSubcode::AdminReset), @@ -136,6 +154,19 @@ impl SessionError { } } +/// A Shutdown Communication (RFC 9003 §2): text for the peer's operators, +/// at most 255 octets of UTF-8, after its length. +fn shutdown_communication(text: &str) -> Vec { + let mut end = text.len().min(255); + while !text.is_char_boundary(end) { + end -= 1; + } + let mut data = Vec::with_capacity(end + 1); + data.push(end as u8); + data.extend_from_slice(&text.as_bytes()[..end]); + data +} + /// Why the server could not start, or could not go on. #[derive(Debug, Error)] pub enum ServerError { @@ -180,6 +211,14 @@ pub enum ConfigError { RpkiNoCaches, #[error("export: {0}")] Export(#[from] ExportProblem), + #[error("export_limits: {0}")] + ExportLimits(ImportProblem), + #[error( + "accept_unconfigured_peers is set and export.learned_routes passes learned routes on: \ + a peer that is not configured has no role, so the routes it sends would go to every \ + peer, and every route to it" + )] + UnconfiguredPeersWithoutRole, #[error("rpki: the port of cache {0:?} is required")] RpkiCachePort(String), #[error("graceful_restart: {0}")] @@ -226,6 +265,14 @@ pub enum PeerProblem { Group, #[error("route_reflector_client is set for a peer in another AS; it is for routers of our own")] ReflectorClientOutsideAs, + #[error( + "role is not set, and learned routes are passed on (export.learned_routes): without a \ + role, the routes the peer sends would go to every other peer, and it would be told of \ + every route. Set role to PROVIDER, CUSTOMER, PEER, ROUTE_SERVER or ROUTE_SERVER_CLIENT" + )] + NoRole, + #[error("export_limits: {0}")] + ExportLimits(ImportProblem), #[error("import: {0}")] Import(#[from] ImportProblem), #[error("export: {0}")] @@ -356,5 +403,31 @@ mod tests { notification(SessionError::Io(io::ErrorKind::BrokenPipe.into())), None ); + // Going over an export limit is ours to explain: an Administrative + // Shutdown, with the reason for the peer's operators (RFC 9003). + let (error, data) = notification(SessionError::ExportLimitExceeded { + limit: 50, + routes: 900_000, + }) + .unwrap(); + assert_eq!(error, BgpError::Cease(CeaseSubcode::AdminShutdown)); + let text = std::str::from_utf8(&data[1..]).unwrap(); + assert_eq!(usize::from(data[0]), text.len()); + assert!( + text.starts_with("export limit of 50 routes reached"), + "{text}" + ); + } + + /// A Shutdown Communication is at most 255 octets, and whole + /// characters (RFC 9003 §2). + #[test] + fn a_shutdown_communication_is_cut_to_fit() { + let long = "é".repeat(200); + let data = shutdown_communication(&long); + assert_eq!(usize::from(data[0]), data.len() - 1); + assert_eq!(data[0], 254); + assert!(std::str::from_utf8(&data[1..]).is_ok()); + assert_eq!(shutdown_communication(""), [0]); } } diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index d07a4a5..924c019 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -87,6 +87,15 @@ pub static PREFIX_LIMIT_EXCEEDED: LazyLock = LazyLock::new(|| { .expect("metric registration") }); +pub static EXPORT_LIMIT_EXCEEDED: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "bgp_export_limit_exceeded_total", + "Sessions ended because more routes were to be announced to the peer than its export limit allows", + &["peer"] + ) + .expect("metric registration") +}); + pub static RPKI_VRPS: LazyLock = LazyLock::new(|| { register_int_gauge_vec!( "bgp_rpki_vrps", @@ -206,6 +215,7 @@ pub fn init() { LazyLock::force(&PEER_ROUTES); LazyLock::force(&PEER_MAX_PREFIXES); LazyLock::force(&PREFIX_LIMIT_EXCEEDED); + LazyLock::force(&EXPORT_LIMIT_EXCEEDED); LazyLock::force(&RPKI_VRPS); LazyLock::force(&RPKI_ASPAS); LazyLock::force(&RPKI_UPDATES); diff --git a/crates/server/src/policy.rs b/crates/server/src/policy.rs index c3d2342..14ec8de 100644 --- a/crates/server/src/policy.rs +++ b/crates/server/src/policy.rs @@ -269,6 +269,40 @@ pub struct ExportPolicy { pub prepend: u8, } +/// Limits on what is announced to a network that is not our customer, +/// which hold whatever the export policy and the routes say: a check on +/// both, in case either goes wrong. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ExportLimits { + /// The prefixes that may be announced; `None` for any. + pub prefixes: Option, + /// The most routes announced at a time. + pub max_prefixes: Option, +} + +impl ExportLimits { + /// Whether they are for a session to a peer that is `role` to us, in + /// another AS or not (`ibgp`): one above or beside us, or one we know + /// nothing of. Customers, route server clients and our own routers + /// are told of whatever they are to be. + pub fn apply_to(role: Option, ibgp: bool) -> bool { + !ibgp && role.is_none_or(PeerRole::is_above_or_beside) + } + + /// Whether a route for `prefix` may be announced. + pub fn allows(&self, prefix: &IpPrefix) -> bool { + self.prefixes + .as_ref() + .is_none_or(|prefixes| prefixes.allows(prefix)) + } + + /// Whether `routes` routes at a time are more than may be announced. + pub fn exceeded_by(&self, routes: usize) -> bool { + self.max_prefixes + .is_some_and(|limit| routes > limit as usize) + } +} + /// Why a route was not taken. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Rejection { @@ -881,6 +915,8 @@ impl PrefixSet { pub enum HeldDown { /// It went over its limit of routes. OverLimit, + /// We were to announce it more routes than its export limit allows. + OverExportLimit, /// Its sessions keep coming up and failing. Flapping, } @@ -889,6 +925,9 @@ impl HeldDown { pub fn as_str(self) -> &'static str { match self { Self::OverLimit => "held down after going over its limit of routes", + Self::OverExportLimit => { + "held down after more routes were to be advertised to it than its export limit allows" + } Self::Flapping => "held down because its sessions keep failing", } } @@ -1333,6 +1372,31 @@ mod tests { assert!(!Peer.may_be_sent_only_to_customer_routes()); } + /// Export limits are for networks above or beside us, and for one we + /// know nothing of; not for customers, route server clients or our own + /// routers, which are told of what they are to be. + #[test] + fn export_limits_are_for_those_above_or_beside_us() { + use PeerRole::*; + for role in [Provider, Peer, RouteServer] { + assert!(ExportLimits::apply_to(Some(role), false), "{role:?}"); + } + assert!(ExportLimits::apply_to(None, false)); + for role in [Customer, RouteServerClient] { + assert!(!ExportLimits::apply_to(Some(role), false), "{role:?}"); + } + assert!(!ExportLimits::apply_to(None, true)); + + let limits = ExportLimits { + prefixes: None, + max_prefixes: Some(2), + }; + assert!(!limits.exceeded_by(2)); + assert!(limits.exceeded_by(3)); + assert!(limits.allows(&IpPrefix::try_from("203.0.113.0/24").unwrap())); + assert!(!ExportLimits::default().exceeded_by(usize::MAX)); + } + #[test] fn the_path_is_read_with_its_sets() { let mut out = vec![99]; diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 9a85495..875c899 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -27,7 +27,7 @@ use crate::dialer::spawn_dialers; use crate::error::ServerError; use crate::metrics::{CONNECTIONS_REFUSED, PEER_MAX_PREFIXES}; use crate::panic; -use crate::policy::HoldDowns; +use crate::policy::{ExportLimits, HoldDowns}; use crate::reach; use crate::refusals::RefusalLog; use crate::restart::{Leave, Restarts}; @@ -210,6 +210,43 @@ impl BgpServer { ); } } + // The export limits that hold for some peer, each once, with the + // peers they hold for. + let mut limited: Vec<(&ExportLimits, Vec<&str>)> = Vec::new(); + for peer in &self.config.peers { + let ibgp = peer.remote_asn == self.config.local_asn; + if !ExportLimits::apply_to(peer.role, ibgp) { + continue; + } + let Some(limits) = peer.export_limits.as_deref() else { + warn!( + peer = %peer.name, + "Nothing limits what this peer is told (export_limits)" + ); + continue; + }; + match limited + .iter_mut() + .find(|(seen, _)| std::ptr::eq(*seen, limits)) + { + Some((_, names)) => names.push(&peer.name), + None => limited.push((limits, vec![&peer.name])), + } + } + // A route of our own that the limits leave out is likely a mistake + // in one or the other. + for route in &self.config.originate { + for (limits, names) in &limited { + if !limits.allows(&route.prefix) { + warn!( + prefix = %route.prefix, + peers = names.join(", "), + "An originated route is not announced to these peers: export_limits.prefixes \ + leaves it out" + ); + } + } + } if self.config.accept_unconfigured_peers { warn!("Taking sessions and routes from any host (accept_unconfigured_peers)"); } @@ -421,6 +458,7 @@ fn admit( None if config.accept_unconfigured_peers => PeerSettings::unconfigured( config.hold_time_secs, Arc::clone(&config.export), + config.export_limits.clone(), config.graceful_restart, ), None => return Err(Refusal::NotAPeer), diff --git a/crates/server/src/session.rs b/crates/server/src/session.rs index e8b413f..0a848ba 100644 --- a/crates/server/src/session.rs +++ b/crates/server/src/session.rs @@ -18,10 +18,11 @@ use tokio::sync::{broadcast, mpsc, watch}; use crate::address::{Endpoint, PeerAddress, socket_text}; use crate::collision::Direction; +use crate::error::SessionError; use crate::family::FamilySet; use crate::fsm::Negotiated; use crate::metrics::{PEER_UP, SESSIONS_ESTABLISHED}; -use crate::policy::{ExportPolicy, PeerRole}; +use crate::policy::{ExportLimits, ExportPolicy, PeerRole}; use crate::restart::{Agreed, Leave}; use crate::rpki::Subject; use crate::sync::IgnorePoisonRw; @@ -165,6 +166,8 @@ pub struct SessionHandle { pub counters: Arc, /// What the peer is told, and how. pub export: Arc, + /// Limits on what the peer is told, where there are any for it. + pub export_limits: Option>, /// The groups the session is in, for control communities. pub groups: Arc<[u32]>, /// Whether the peer is a route reflector client of ours (RFC 4456). @@ -172,6 +175,8 @@ pub struct SessionHandle { /// The peer's BGP Identifier. pub peer_router_id: Ipv4Addr, outbound: mpsc::Sender, + /// Where the session is asked to end, if it listens for that. + ending: Option>, /// Which of the RPKI's checks the peer's routes are subject to, if /// any, and where the session is told to look at its routes again. validation: Option<(Option, mpsc::UnboundedSender>)>, @@ -209,14 +214,34 @@ impl SessionHandle { graceful_restart: negotiated.graceful_restart.clone(), counters: Arc::default(), export: Arc::default(), + export_limits: None, groups: Arc::default(), reflector_client: false, peer_router_id: negotiated.peer_identifier, outbound, + ending: None, validation: None, } } + /// Has the session listen for being asked to end: what is asked with + /// [`SessionHandle::end`] arrives at the receiver returned. + pub(crate) fn listen_for_end(&mut self) -> mpsc::UnboundedReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + self.ending = Some(tx); + rx + } + + /// Asks the session to end because of `error`: it tells the peer with + /// the NOTIFICATION for it, and closes. What was queued before may go + /// out first; the caller queues nothing more. + pub(crate) fn end(&self, error: SessionError) { + if let Some(ending) = &self.ending { + // Fails only if the session has ended already. + let _ = ending.send(error); + } + } + /// Marks the session's routes as subject to validation with the /// RPKI's data. What is sent with [`SessionHandle::revalidate`] /// arrives at the receiver returned. diff --git a/docs/content/docs/guides/passing-routes-on.md b/docs/content/docs/guides/passing-routes-on.md index 620fe57..fa05e3d 100644 --- a/docs/content/docs/guides/passing-routes-on.md +++ b/docs/content/docs/guides/passing-routes-on.md @@ -84,11 +84,84 @@ peers { name: "transit-a" role: PROVIDER ... } peers { name: "ixp-a-rs" role: ROUTE_SERVER ... } ``` -> [!CAUTION] -> A neighbor without a role is not subject to this restriction. If -> `learned_routes` is enabled for such a neighbor, Ferrix advertises all -> learned routes to it, including routes from providers and peers. Ferrix -> logs a warning at startup for each such neighbor. +If any `export` statement enables `learned_routes`, every EBGP neighbor +must have a role. Ferrix does not start with a configuration in which an +EBGP neighbor has no role, because the routes received from that neighbor +would be advertised to all other neighbors, and all learned routes would +be advertised to it. For the same reason, `accept_unconfigured_peers` is +not permitted with a top-level `export` statement that enables +`learned_routes`. + +A route collector, which receives routes and advertises none, is +configured with the role `CUSTOMER`. + +## Export Limits + +Export limits restrict the routes that Ferrix advertises to providers, +peers, and route servers, regardless of the export policy, the BGP roles, +and the communities of the routes. They protect against route leaks that +the roles do not prevent, for example a route that a customer advertises +for address space that is not its own, or an error in the configuration +of a role. + +Export limits apply to the following neighbors: + +- Neighbors with the role `PROVIDER`, `PEER`, or `ROUTE_SERVER`. +- EBGP neighbors without a role. + +They do not apply to customers, route server clients, or IBGP neighbors, +which receive the full routing table where their export policy permits it. + +### Configuring Export Limits + +The top-level `export_limits` statement applies to every neighbor that +does not have its own `export_limits` statement: + +```textproto +export_limits { + prefixes { prefix: "198.51.100.0/22" max_length: 24 } + prefixes { prefix: "2001:db8::/32" max_length: 48 } + max_prefixes: 50 +} +``` + +| Statement | Description | +|---|---| +| `prefixes` | Specifies the prefixes that can be advertised: typically the address space of the local AS and of its customers. Ferrix does not advertise learned or originated routes for other prefixes to the neighbor. Default: all prefixes. | +| `max_prefixes` | Specifies the maximum number of routes advertised to the neighbor, counting learned and originated routes of both address families. Default: no limit. | + +A neighbor's `export_limits` statement replaces the top-level statement. +An empty statement, `export_limits {}`, removes the limits for the +neighbor. Unlike the `export` statement, the limits are not lost when a +neighbor has its own export policy. + +If more routes are to be advertised to a neighbor than `max_prefixes` +permits, Ferrix advertises none of them. It tears down the session with a +Cease NOTIFICATION message, subcode Administrative Shutdown, that contains +the reason as a shutdown communication (RFC 9003), and places the neighbor +in hold-down for the neighbor's `import.hold_down_seconds` (default: 600 +seconds). Set `max_prefixes` somewhat above the number of routes that the +local AS and its customers originate. + +At startup, Ferrix logs a warning for each neighbor to which export limits +apply but none are configured, and for each originated route that the +`prefixes` of the export limits do not permit. + +### Verifying Export Limits + +When the limit is exceeded, Ferrix logs an error that states the number of +routes and the limit, and increments the metric +`bgp_export_limit_exceeded_total{peer}`. The status page and `/api/peers` +show the reason in the neighbor's last error, for example: + +```text +6 routes were to be advertised to the peer, over its export limit of 5; none of them was +``` + +While the neighbor is in hold-down, its last error is `held down after more +routes were to be advertised to it than its export limit allows`. To restore +the session before the hold-down expires, correct the cause and restart +Ferrix. ## Configuring Export Policy diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md index f5551ef..d1dc92d 100644 --- a/docs/content/docs/operations/monitoring.md +++ b/docs/content/docs/operations/monitoring.md @@ -143,6 +143,13 @@ groups: expr: increase(bgp_prefix_limit_exceeded_total[1h]) > 0 labels: { severity: page } + # A session was torn down because more routes were to be advertised + # to the neighbor than its export limit permits: a probable route + # leak, which Ferrix did not advertise. + - alert: BgpExportLimitExceeded + expr: increase(bgp_export_limit_exceeded_total[1h]) > 0 + labels: { severity: page } + # A configured neighbor has no established session. - alert: BgpPeerDown expr: bgp_peer_up == 0 diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md index db185f0..1cf5e6f 100644 --- a/docs/content/docs/reference/configuration.md +++ b/docs/content/docs/reference/configuration.md @@ -57,6 +57,7 @@ Top-level statements. | `control_communities` bool | `false` | Enables control communities, which restrict the neighbors to which a route is advertised and prepend the local AS number on its way to them. See [Control Communities](@/docs/guides/passing-routes-on.md#control-communities). | | `cluster_id` string | `router_id` | Specifies the cluster ID that the router uses as a route reflector, as a dotted-quad IPv4 address. See [Route Reflection](@/docs/guides/passing-routes-on.md#route-reflection). | | `graceful_restart` [GracefulRestart](#gracefulrestart) | Enabled | Specifies graceful restart for neighbors that have no `graceful_restart` statement. Its `restart_time_seconds` also limits how long Ferrix waits at startup for neighbors to send their routes. | +| `export_limits` [ExportLimits](#exportlimits) | None | Specifies limits on the routes advertised to neighbors whose role is `PROVIDER`, `PEER`, or `ROUTE_SERVER`, and to EBGP neighbors without a role, for neighbors that have no `export_limits` statement. See [Export Limits](@/docs/guides/passing-routes-on.md#export-limits). | ## Peer @@ -72,11 +73,12 @@ Statements for a BGP neighbor. | `address_families` [AddressFamily](#addressfamily), repeated | All | Specifies the address families proposed to the neighbor. A family is enabled only if the neighbor also proposes it (RFC 4760). | | `add_path` [AddPath](#addpath) | `ADD_PATH_OFF` | Specifies the ADD-PATH directions proposed to the neighbor (RFC 7911). | | `import` [ImportPolicy](#importpolicy) | Default checks | Specifies the import policy for routes received from the neighbor. | -| `role` [Role](#role) | None | Specifies the BGP role of the neighbor (RFC 9234). Not permitted for IBGP neighbors. | +| `role` [Role](#role) | None | Specifies the BGP role of the neighbor (RFC 9234). Not permitted for IBGP neighbors. Required for every EBGP neighbor if any `export` statement enables `learned_routes`. | | `require_role` bool | `false` | Rejects the session if the neighbor does not advertise a BGP role. Requires `role`. | | `export` [ExportPolicy](#exportpolicy) | Top-level `export` | Specifies the export policy for the neighbor. Replaces the top-level `export` statement. | | `graceful_restart` [GracefulRestart](#gracefulrestart) | Top-level `graceful_restart` | Specifies graceful restart for the neighbor. Each statement that is set replaces the corresponding top-level statement. | | `route_reflector_client` bool | `false` | Makes the IBGP neighbor a route reflector client: Ferrix advertises to it the routes learned from other IBGP neighbors, and advertises its routes to them. Permitted only for IBGP neighbors. See [Route Reflection](@/docs/guides/passing-routes-on.md#route-reflection). | +| `export_limits` [ExportLimits](#exportlimits) | Top-level `export_limits` | Specifies limits on the routes advertised to the neighbor. Replaces the top-level `export_limits` statement. An empty statement removes the limits for the neighbor. Applies only if the neighbor's role is `PROVIDER`, `PEER`, or `ROUTE_SERVER`, or if it is an EBGP neighbor without a role. | | `groups` uint32, repeated | None | Specifies the groups of the session, by number, for control communities that name sessions by group. Use the same numbers on all routers in the local AS. Range: 1 through 4,294,967,295. See [Session Groups](@/docs/guides/passing-routes-on.md#session-groups). | ## TcpEndpoint @@ -96,7 +98,7 @@ accepted. See [Import Policy](@/docs/guides/import-policy.md). | Statement | Default | Description | |---|---|---| | `max_prefixes` uint32 | 0 (no limit) | Specifies the maximum number of routes that the neighbor can have in the routing table. If the limit is exceeded, Ferrix tears down the session and places the neighbor in hold-down. | -| `hold_down_seconds` uint32 | 600 | Specifies the hold-down time, in seconds, after the neighbor exceeds `max_prefixes`. | +| `hold_down_seconds` uint32 | 600 | Specifies the hold-down time, in seconds, after the neighbor exceeds `max_prefixes`, or after the routes to be advertised to it exceed `export_limits.max_prefixes`. | | `accept_bogons` bool | `false` | Accepts bogon prefixes, and AS paths that contain private, reserved, or documentation AS numbers. | | `accept_default_route` bool | `false` | Accepts the default routes `0.0.0.0/0` and `::/0`. | | `max_ipv4_prefix_length` uint32 | 24 | Specifies the longest IPv4 prefix accepted. | @@ -153,6 +155,17 @@ Statements that determine which routes are advertised to a neighbor. See | `remove_communities` string, repeated | None | Specifies patterns of communities removed from advertised routes. Applied before communities are added. | | `as_path_prepend` uint32 | 0 | Specifies the number of additional times the local AS number is prepended to the AS path. Range: 0 through 16. | +## ExportLimits + +Limits on the routes advertised to a neighbor. The limits apply regardless +of the export policy, BGP roles, and communities of the routes. See +[Export Limits](@/docs/guides/passing-routes-on.md#export-limits). + +| Statement | Default | Description | +|---|---|---| +| `prefixes` [PrefixRange](#prefixrange), repeated | All prefixes | Specifies the prefixes that can be advertised to the neighbor. Ferrix does not advertise learned or originated routes for other prefixes. | +| `max_prefixes` uint32 | 0 (no limit) | Specifies the maximum number of routes advertised to the neighbor. If more routes are to be advertised, Ferrix advertises none of them, tears down the session with a Cease NOTIFICATION message (subcode Administrative Shutdown) that states the reason (RFC 9003), and places the neighbor in hold-down for its `import.hold_down_seconds`. | + ## OriginatedRoute A route that Ferrix originates. diff --git a/docs/content/docs/reference/metrics.md b/docs/content/docs/reference/metrics.md index 2bbd63a..4088f8d 100644 --- a/docs/content/docs/reference/metrics.md +++ b/docs/content/docs/reference/metrics.md @@ -31,6 +31,7 @@ alerting rules based on these metrics, see | `bgp_peer_routes{peer}` | Gauge | Number of the neighbor's routes in the routing table. | | `bgp_peer_max_prefixes{peer}` | Gauge | Prefix limit of the neighbor (`import.max_prefixes`), if configured. | | `bgp_prefix_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the neighbor exceeded its prefix limit. | +| `bgp_export_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the routes to be advertised to the neighbor exceeded its export limit (`export_limits.max_prefixes`). | | `bgp_routes_rejected_total{peer,reason}` | Counter | Number of routes rejected by the import policy, by neighbor and reason. For the reasons, see [Import Policy](@/docs/guides/import-policy.md#default-checks). | | `bgp_routes_exported_total{change}` | Counter | Number of learned routes advertised (`announced`) to or withdrawn (`withdrawn`) from neighbors. | | `bgp_local_routes` | Gauge | Number of originated routes. | diff --git a/docs/examples/transit.textpb b/docs/examples/transit.textpb index b79b87d..4e8cc96 100644 --- a/docs/examples/transit.textpb +++ b/docs/examples/transit.textpb @@ -7,6 +7,17 @@ local_asn: 64500 # Advertise learned routes to all neighbors, as BGP roles permit. export { learned_routes: true } +# Advertise to the providers only routes for the address space of the local +# AS and of the customer, and at most 20 routes. If more routes are to be +# advertised, Ferrix advertises none of them and tears down the session. +export_limits { + prefixes { prefix: "198.51.100.0/24" } + prefixes { prefix: "2001:db8:100::/40" max_length: 48 } + prefixes { prefix: "192.0.2.0/24" } + prefixes { prefix: "2001:db8:a00::/40" max_length: 48 } + max_prefixes: 20 +} + # Customers can restrict the advertisement of their routes with the large # communities 64500:0:P (not to AS P), 64500:1:P (only to AS P), and # 64500:0:0 (to no neighbor), or the standard communities 0:P, 64500:P, and diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh index a864ce6..13a9681 100755 --- a/scripts/interop/lab.sh +++ b/scripts/interop/lab.sh @@ -29,7 +29,7 @@ cd "$(dirname "$0")/../.." bgp="${CARGO_TARGET_DIR:-target}/release/bgp" work=/tmp/ferrix-lab pathspace=ferrixlab -all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit v6only linklocal" +all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit leaks v6only linklocal" phases="${PHASES:-$all_phases}" scale_routes="${SCALE_ROUTES:-200000}" @@ -959,6 +959,57 @@ control_communities: true' ok "back to normal" } +# A customer leaks, and the export limits hold. FRR, now our customer, +# announces a route outside its address space and then more routes than +# BIRD, our provider, is to be told of. A customer's routes go to +# everyone as far as the roles go; the export limits are the check on +# that. The route outside our and the customer's space never goes up to +# BIRD; when more were to go than the limit allows, none of them does, +# and the session with BIRD ends with an Administrative Shutdown that +# says why (RFC 9003). +phase_leaks() { + dut_down TERM + dut_config CUSTOMER 2000000 "" "" "" 'export { learned_routes: true } +export_limits { + prefixes { prefix: "10.1.0.0/16" max_length: 24 } + prefixes { prefix: "10.2.0.0/16" max_length: 24 } + prefixes { prefix: "2001:db8:1::/48" } + prefixes { prefix: "2001:db8:2::/48" } + max_prefixes: 5 +}' + dut_up + frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \ + "address-family ipv4 unicast" "network 10.77.0.0/24" + wait_for "four sessions" established 4 + wait_for "the route outside the customer's space at the daemon" dut_has 10.77.0.0/24 frr4 + + # The customer's route goes up to the provider; the one outside the + # export limits does not. + wait_for "FRR's route at BIRD" bird_has 10.2.0.0/24 + sleep 1 + bird_has 10.77.0.0/24 && fail "BIRD was told a route outside the export limits" + ok "a customer's route outside the export limits is not passed on to the provider" + + # More routes than the limit: ours and the customer's two, and four + # more of the customer's, would be six of five. + frr_configure "router bgp 64602" "address-family ipv4 unicast" \ + "network 10.2.11.0/24" "network 10.2.12.0/24" "network 10.2.13.0/24" "network 10.2.14.0/24" + wait_for "the export limit to end BIRD's session" \ + peer_is bird4 '.state != "Established" and (.last_error // "" | test("export limit"; "i"))' + wait_for "BIRD to be told why" sudo grep -q "export limit of 5 routes reached" "$work/bird.log" + bird_has 10.2.11.0/24 && fail "BIRD was told a route past the export limit" + ok "past its export limit the provider is told nothing more, and why (Cease / Administrative Shutdown): $(peer_is bird4 '.last_error')" + + frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" "address-family ipv4 unicast" \ + "no network 10.77.0.0/24" "no network 10.2.11.0/24" "no network 10.2.12.0/24" \ + "no network 10.2.13.0/24" "no network 10.2.14.0/24" + dut_down TERM + dut_config + dut_up + converged + ok "back to normal" +} + # The exchange LAN loses its IPv4 addresses. One session over IPv6 with # each router carries both families, and IPv4 routes have IPv6 next hops # (RFC 8950), in BGP and in the kernel. Run after the rest: the lab stays