Rework the speaker: sans-I/O FSM, sharded table, RFC 7606, ADD-PATH

The work of the speaker-cleanup branch as one commit; the branch keeps
the step-by-step history.

Sessions
- The state machine is a pure function from events to actions, driven by
  one task per connection. Idle, Connect and Active belong to a dialer
  that keeps configured peers up with backoff.
- Capabilities are negotiated: address families, four-octet AS numbers
  with AS_TRANS and AS4_PATH, route refresh, ADD-PATH per family and
  direction.
- Collision detection per RFC 4271 section 6.8, by address and
  identifier.
- Connections are taken only from configured peers, a few per address.
- A panic in a session is contained and answered with a Cease; a core
  task that stops shuts the server down with a Cease to every peer.
- Writes that make no progress are given up (RFC 9687).

Codec
- Malformed UPDATEs are handled as RFC 7606 says: treat-as-withdraw,
  attribute discard, or session reset, with the NOTIFICATION to send.
- Address families are open types; any can be named and ignored.
- Prefixes are fixed-size values, normalised, with path identifiers.
- Lengths are filled in after writing; one definition of each type code.
- Property tests and fuzz targets for every decoder.

Routing table
- Adj-RIBs-In and Loc-RIB in one sharded table that sessions write to
  directly. Routes are sixteen bytes; attribute sets are interned in
  wire form. Best-path selection per RFC 4271 section 9.1.2.2 with
  deterministic MED.
- A journal per shard lets followers read changes at their own pace;
  one that falls behind is resynchronised.
- On a RIPE RIS dump of 148 peers and 39.3M routes: loaded in 2.0 s on
  12 cores, about 52 bytes a route.

Announcing
- Originated routes come from the configuration. Each session has its
  own exporter with an Adj-RIB-Out, so a slow peer delays only itself.
- Routes to a peer in our own AS have an empty path and a LOCAL_PREF.

Operations
- HTTP status page, JSON API, event stream and Prometheus metrics.
- Benchmarks on synthetic and real tables.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Rayhaan JaufeerallyandClaude Fable 5.1 committed 2026-10-02 12:35:54 +02:00
1 parent 1ec60255ea
commit 630c175918
138 files changed
+21650 -2925

No files matched your search

+451 -43
View File
@@ -1,12 +1,22 @@
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::str::FromStr;
use std::time::Duration;
use eyre::{Result, bail, eyre};
use bgp_packet::community;
use bgp_packet::message::OriginPathAttribute;
use bgp_packet::open::AddPathSendRecv;
use bgp_primitives::constants::AfiSafi;
use bgp_primitives::ip_prefix::IpPrefix;
use bgp_rib::{LocalRoute, LocalRouteAttrs};
use crate::proto::config::BgpConfig;
use crate::error::{ConfigError, OriginateProblem, PeerProblem};
use crate::family::FamilySet;
use crate::proto::config::{self as proto, BgpConfig, OriginatedRoute, Peer};
const DEFAULT_BGP_PORT: u16 = 179;
const DEFAULT_HOLD_TIME_SECS: u16 = 90;
const DEFAULT_CONNECT_RETRY: Duration = Duration::from_secs(5);
const MAX_AS_PATH_PREPEND: u32 = 16;
/// Validated, Rust-native server configuration derived from the proto config.
pub struct Config {
@@ -16,79 +26,477 @@ pub struct Config {
pub hold_time_secs: u16,
pub listen_endpoints: Vec<SocketAddr>,
pub peers: Vec<PeerConfig>,
/// Initial delay between attempts to connect to a configured peer;
/// doubles on each failure.
pub connect_retry: Duration,
/// Resolve collisions against Established sessions by BGP Identifier
/// instead of always closing the new connection (RFC 4271 §8.1.1
/// `CollisionDetectEstablishedState`).
pub collision_detect_established_state: bool,
/// Take connections from addresses that are not those of configured
/// peers. Off unless asked for: with it, any host that can reach us can
/// open a session and announce routes.
pub accept_unconfigured_peers: bool,
/// Routes this router originates.
pub originate: Vec<LocalRoute>,
/// Address for the HTTP API; `None` disables it.
pub http_endpoint: Option<SocketAddr>,
}
#[derive(Debug, Clone)]
pub struct PeerConfig {
pub name: String,
pub endpoint: SocketAddr,
pub remote_asn: u32,
pub hold_time_secs: u16,
/// Source address for connections we open to this peer; `None` lets the
/// operating system choose.
pub local_address: Option<IpAddr>,
/// The address families offered to this peer.
pub families: FamilySet,
/// The directions ADD-PATH (RFC 7911) is offered to this peer for, on
/// every one of `families`. `None` offers it for neither.
pub add_path: Option<AddPathSendRecv>,
}
impl Config {
pub fn from_proto(proto: BgpConfig) -> Result<Self> {
/// The configured peer at `address`, if there is one.
pub fn peer_at(&self, address: IpAddr) -> Option<&PeerConfig> {
self.peers.iter().find(|peer| peer.endpoint.ip() == address)
}
pub fn from_proto(proto: BgpConfig) -> Result<Self, ConfigError> {
// Peers refuse an identifier of 0.0.0.0, as we do theirs.
let router_id = Ipv4Addr::from_str(&proto.router_id)
.map_err(|_| eyre!("Invalid router_id {:?}: must be a dotted-quad IPv4 address", proto.router_id))?;
.ok()
.filter(|id| !id.is_unspecified())
.ok_or_else(|| ConfigError::RouterId(proto.router_id.clone()))?;
if proto.local_asn == 0 {
bail!("local_asn must be non-zero");
return Err(ConfigError::LocalAsn);
}
let listen_endpoints = if proto.listen_endpoints.is_empty() {
vec![
SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), DEFAULT_BGP_PORT),
SocketAddr::new(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED), DEFAULT_BGP_PORT),
SocketAddr::new(
IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED),
DEFAULT_BGP_PORT,
),
]
} else {
proto.listen_endpoints
.into_iter()
proto
.listen_endpoints
.iter()
.map(|ep| parse_endpoint(&ep.address, ep.port))
.collect::<Result<_>>()?
.collect::<Result<_, _>>()?
};
let peers = proto.peers
.into_iter()
.map(|p| {
let ep = p.endpoint.as_ref()
.ok_or_else(|| eyre!("Peer {:?} is missing endpoint", p.name))?;
let endpoint = parse_endpoint(&ep.address, ep.port)?;
let mut peers: Vec<PeerConfig> = Vec::with_capacity(proto.peers.len());
for peer in proto.peers {
let name = peer.name.clone();
let rejected = |problem| ConfigError::Peer {
peer: name.clone(),
problem,
};
let peer = parse_peer(peer).map_err(rejected)?;
// A peer is told from the others by its name in what the
// dialer reports, and by its address when it connects to us.
if peers.iter().any(|earlier| earlier.name == peer.name) {
return Err(rejected(PeerProblem::DuplicateName));
}
if peers
.iter()
.any(|earlier| earlier.endpoint.ip() == peer.endpoint.ip())
{
return Err(rejected(PeerProblem::DuplicateAddress));
}
peers.push(peer);
}
if p.remote_asn == 0 {
bail!("Peer {:?} remote_asn must be non-zero", p.name);
}
let mut originate: Vec<LocalRoute> = Vec::with_capacity(proto.originate.len());
for entry in &proto.originate {
let rejected = |problem| ConfigError::Originate {
prefix: entry.prefix.clone(),
problem,
};
let route = parse_originated_route(entry).map_err(rejected)?;
if originate
.iter()
.any(|earlier| earlier.prefix == route.prefix)
{
return Err(rejected(OriginateProblem::Duplicate));
}
originate.push(route);
}
let hold_time_secs = if p.hold_time_seconds == 0 {
DEFAULT_HOLD_TIME_SECS
} else if p.hold_time_seconds < 3 {
bail!("Peer {:?} hold_time_seconds must be 0 or >= 3", p.name);
} else {
p.hold_time_seconds as u16
};
let http_endpoint = match proto.http_endpoint.into_option() {
Some(ep) if ep.port == 0 => return Err(ConfigError::HttpPort),
Some(ep) => Some(parse_endpoint(&ep.address, ep.port)?),
None => None,
};
Ok(PeerConfig {
name: p.name,
endpoint,
remote_asn: p.remote_asn,
hold_time_secs,
})
})
.collect::<Result<_>>()?;
let connect_retry = if proto.connect_retry_seconds == 0 {
DEFAULT_CONNECT_RETRY
} else {
Duration::from_secs(proto.connect_retry_seconds as u64)
};
Ok(Self { router_id, local_asn: proto.local_asn, hold_time_secs: DEFAULT_HOLD_TIME_SECS, listen_endpoints, peers })
Ok(Self {
router_id,
local_asn: proto.local_asn,
hold_time_secs: DEFAULT_HOLD_TIME_SECS,
listen_endpoints,
peers,
connect_retry,
collision_detect_established_state: proto.collision_detect_established_state,
accept_unconfigured_peers: proto.accept_unconfigured_peers,
originate,
http_endpoint,
})
}
}
fn parse_endpoint(address: &str, port: u32) -> Result<SocketAddr> {
let port = if port == 0 {
DEFAULT_BGP_PORT
} else if port > u16::MAX as u32 {
bail!("Port {port} is out of range");
} else {
port as u16
fn parse_peer(peer: Peer) -> Result<PeerConfig, PeerProblem> {
let ep = peer.endpoint.as_ref().ok_or(PeerProblem::MissingEndpoint)?;
let endpoint =
parse_endpoint(&ep.address, ep.port).map_err(|e| PeerProblem::Endpoint(Box::new(e)))?;
if peer.remote_asn == 0 {
return Err(PeerProblem::RemoteAsn);
}
let hold_time_secs = match peer.hold_time_seconds {
0 => DEFAULT_HOLD_TIME_SECS,
// RFC 4271 §4.2: zero, or at least three seconds.
1 | 2 => return Err(PeerProblem::HoldTime),
secs => u16::try_from(secs).map_err(|_| PeerProblem::HoldTime)?,
};
let ip = IpAddr::from_str(address)
.map_err(|_| eyre!("Invalid IP address {address:?}"))?;
let local_address = if peer.local_address.is_empty() {
None
} else {
let ip = IpAddr::from_str(&peer.local_address)
.map_err(|_| PeerProblem::LocalAddress(peer.local_address.clone()))?;
if ip.is_ipv4() != endpoint.is_ipv4() {
return Err(PeerProblem::AddressFamilyMismatch);
}
Some(ip)
};
let families = if peer.address_families.is_empty() {
FamilySet::supported()
} else {
peer.address_families
.iter()
.map(|family| match family.enum_value() {
Ok(proto::AddressFamily::IPV4_UNICAST) => Ok(AfiSafi::IPV4_UNICAST),
Ok(proto::AddressFamily::IPV6_UNICAST) => Ok(AfiSafi::IPV6_UNICAST),
Ok(proto::AddressFamily::ADDRESS_FAMILY_UNSPECIFIED) | Err(_) => {
Err(PeerProblem::AddressFamily)
}
})
.collect::<Result<_, _>>()?
};
let add_path = match peer.add_path.enum_value() {
Ok(proto::AddPath::ADD_PATH_OFF) => None,
Ok(proto::AddPath::ADD_PATH_RECEIVE) => Some(AddPathSendRecv::Receive),
Ok(proto::AddPath::ADD_PATH_SEND) => Some(AddPathSendRecv::Send),
Ok(proto::AddPath::ADD_PATH_BOTH) => Some(AddPathSendRecv::Both),
Err(_) => return Err(PeerProblem::AddPath),
};
Ok(PeerConfig {
// A peer without a name goes by its address.
name: if peer.name.is_empty() {
endpoint.ip().to_string()
} else {
peer.name
},
endpoint,
remote_asn: peer.remote_asn,
hold_time_secs,
local_address,
families,
add_path,
})
}
fn parse_endpoint(address: &str, port: u32) -> Result<SocketAddr, ConfigError> {
let port = match port {
0 => DEFAULT_BGP_PORT,
port => u16::try_from(port).map_err(|_| ConfigError::Port(port))?,
};
let ip = IpAddr::from_str(address).map_err(|_| ConfigError::Address(address.to_string()))?;
Ok(SocketAddr::new(ip, port))
}
fn parse_originated_route(route: &OriginatedRoute) -> Result<LocalRoute, OriginateProblem> {
let prefix = IpPrefix::try_from(route.prefix.as_str())?;
let origin = match route.origin.to_ascii_uppercase().as_str() {
"" | "IGP" => OriginPathAttribute::IGP,
"EGP" => OriginPathAttribute::EGP,
"INCOMPLETE" => OriginPathAttribute::INCOMPLETE,
_ => return Err(OriginateProblem::Origin(route.origin.clone())),
};
let prepend = u8::try_from(route.as_path_prepend)
.ok()
.filter(|&n| u32::from(n) <= MAX_AS_PATH_PREPEND)
.ok_or(OriginateProblem::Prepend {
max: MAX_AS_PATH_PREPEND,
})?;
let communities = route
.communities
.iter()
.map(|c| community::parse_standard(c))
.collect::<Result<_, _>>()?;
let large_communities = route
.large_communities
.iter()
.map(|c| community::parse_large(c))
.collect::<Result<_, _>>()?;
Ok(LocalRoute::new(
prefix,
LocalRouteAttrs {
origin,
prepend,
med: route.med,
communities,
large_communities,
},
))
}
#[cfg(test)]
mod tests {
use bgp_primitives::ip_prefix::PrefixError;
use super::*;
fn parse(text: &str) -> Result<Config, ConfigError> {
let proto = protobuf::text_format::parse_from_str::<BgpConfig>(text)
.expect("the test's textproto is well formed");
Config::from_proto(proto)
}
fn rejected(text: &str) -> ConfigError {
parse(text).err().expect(text)
}
const BASE: &str = "router_id: \"192.0.2.1\" local_asn: 65000\n";
#[test]
fn originated_routes_are_parsed_with_their_attributes() {
let config = parse(&format!(
"{BASE}originate {{ prefix: \"192.0.2.0/24\" }}
originate {{
prefix: \"2001:db8::/32\" origin: \"incomplete\" as_path_prepend: 2 med: 0
communities: \"65000:100\" communities: \"NO_EXPORT\"
large_communities: \"65000:1:2\"
}}"
))
.unwrap();
assert_eq!(config.originate.len(), 2);
let plain = &config.originate[0];
assert_eq!(plain.prefix, IpPrefix::try_from("192.0.2.0/24").unwrap());
assert_eq!(plain.attrs, LocalRouteAttrs::default());
let full = &config.originate[1].attrs;
assert_eq!(full.origin, OriginPathAttribute::INCOMPLETE);
assert_eq!(full.prepend, 2);
assert_eq!(
full.med,
Some(0),
"an explicit MED of 0 is not the same as no MED"
);
assert_eq!(full.communities, vec![(65000, 100), (65535, 65281)]);
assert_eq!(full.large_communities.len(), 1);
}
#[test]
fn bad_originated_routes_are_rejected_with_the_entry_and_the_reason() {
let problem = |body: &str| match rejected(&format!("{BASE}originate {{ {body} }}")) {
ConfigError::Originate { problem, .. } => problem,
other => panic!("{body}: {other}"),
};
assert_eq!(
problem("prefix: \"not-a-prefix\""),
OriginateProblem::Prefix(PrefixError::Format("not-a-prefix".into()))
);
assert_eq!(
problem("prefix: \"192.0.2.0/24\" origin: \"static\""),
OriginateProblem::Origin("static".into())
);
assert_eq!(
problem("prefix: \"192.0.2.0/24\" as_path_prepend: 17"),
OriginateProblem::Prepend { max: 16 }
);
assert!(matches!(
problem("prefix: \"192.0.2.0/24\" communities: \"70000:1\""),
OriginateProblem::Community(_)
));
assert!(matches!(
problem("prefix: \"192.0.2.0/24\" large_communities: \"1:2\""),
OriginateProblem::Community(_)
));
let duplicate = rejected(&format!(
"{BASE}originate {{ prefix: \"192.0.2.0/24\" }} originate {{ prefix: \"192.0.2.0/24\" med: 5 }}"
));
assert_eq!(
duplicate,
ConfigError::Originate {
prefix: "192.0.2.0/24".into(),
problem: OriginateProblem::Duplicate
}
);
// The message names the entry, so the operator can find it.
assert_eq!(
duplicate.to_string(),
"originate \"192.0.2.0/24\": listed more than once"
);
}
#[test]
fn a_peer_is_offered_every_address_family_unless_some_are_listed() {
let families = |body: &str| {
let config = parse(&format!(
"{BASE}peers {{ name: \"r1\" endpoint {{ address: \"192.0.2.2\" }} remote_asn: 1 {body} }}"
))
.unwrap();
config.peers[0].families.clone()
};
assert_eq!(families(""), FamilySet::supported());
assert_eq!(
families("address_families: IPV6_UNICAST"),
[AfiSafi::IPV6_UNICAST].into_iter().collect()
);
assert_eq!(
families("address_families: IPV4_UNICAST address_families: IPV6_UNICAST"),
FamilySet::supported()
);
}
#[test]
fn add_path_is_off_unless_a_peer_asks_for_it() {
let add_path = |body: &str| {
let config = parse(&format!(
"{BASE}peers {{ name: \"r1\" endpoint {{ address: \"192.0.2.2\" }} remote_asn: 1 {body} }}"
))
.unwrap();
config.peers[0].add_path
};
assert_eq!(add_path(""), None);
assert_eq!(add_path("add_path: ADD_PATH_OFF"), None);
assert_eq!(
add_path("add_path: ADD_PATH_RECEIVE"),
Some(AddPathSendRecv::Receive)
);
assert_eq!(
add_path("add_path: ADD_PATH_BOTH"),
Some(AddPathSendRecv::Both)
);
}
#[test]
fn bad_peers_and_endpoints_are_rejected_with_the_peer_named() {
assert_eq!(
rejected("router_id: \"nope\" local_asn: 1"),
ConfigError::RouterId("nope".into())
);
assert_eq!(rejected("router_id: \"192.0.2.1\""), ConfigError::LocalAsn);
let problem = |body: &str| match rejected(&format!("{BASE}peers {{ name: \"r1\" {body} }}"))
{
ConfigError::Peer { peer, problem } => {
assert_eq!(peer, "r1");
problem
}
other => panic!("{body}: {other}"),
};
let endpoint = "endpoint { address: \"192.0.2.2\" }";
assert_eq!(problem("remote_asn: 1"), PeerProblem::MissingEndpoint);
assert_eq!(problem(endpoint), PeerProblem::RemoteAsn);
assert_eq!(
problem(&format!("{endpoint} remote_asn: 1 hold_time_seconds: 2")),
PeerProblem::HoldTime
);
assert_eq!(
problem(&format!(
"{endpoint} remote_asn: 1 hold_time_seconds: 70000"
)),
PeerProblem::HoldTime
);
assert_eq!(
problem(&format!("{endpoint} remote_asn: 1 local_address: \"::1\"")),
PeerProblem::AddressFamilyMismatch
);
assert_eq!(
problem(&format!(
"{endpoint} remote_asn: 1 address_families: ADDRESS_FAMILY_UNSPECIFIED"
)),
PeerProblem::AddressFamily
);
assert_eq!(
problem("endpoint { address: \"192.0.2.2\" port: 70000 } remote_asn: 1"),
PeerProblem::Endpoint(Box::new(ConfigError::Port(70000)))
);
assert_eq!(
rejected(&format!("{BASE}http_endpoint {{ address: \"127.0.0.1\" }}")),
ConfigError::HttpPort
);
}
/// Peers are told apart by name where the dialer reports on them and by
/// address when they connect to us; two that share either would be
/// taken for each other.
#[test]
fn peers_must_differ_in_name_and_address() {
let peer = |name: &str, address: &str| {
format!(
"peers {{ name: \"{name}\" endpoint {{ address: \"{address}\" }} remote_asn: 1 }}\n"
)
};
let problem = |peers: String| match rejected(&format!("{BASE}{peers}")) {
ConfigError::Peer { problem, .. } => problem,
other => panic!("{other}"),
};
assert_eq!(
problem(peer("r1", "192.0.2.2") + &peer("r1", "192.0.2.3")),
PeerProblem::DuplicateName
);
assert_eq!(
problem(peer("r1", "192.0.2.2") + &peer("r2", "192.0.2.2")),
PeerProblem::DuplicateAddress
);
// A peer without a name goes by its address, so two of them differ.
let config = parse(&format!(
"{BASE}{}{}",
peer("", "192.0.2.2"),
peer("", "192.0.2.3")
))
.unwrap();
assert_eq!(config.peers[0].name, "192.0.2.2");
assert_eq!(config.peers[1].name, "192.0.2.3");
assert_eq!(
config
.peer_at("192.0.2.3".parse().unwrap())
.map(|p| &*p.name),
Some("192.0.2.3")
);
assert!(config.peer_at("192.0.2.4".parse().unwrap()).is_none());
}
/// A peer would refuse our OPEN.
#[test]
fn a_router_id_of_zero_is_rejected() {
assert_eq!(
rejected("router_id: \"0.0.0.0\" local_asn: 1"),
ConfigError::RouterId("0.0.0.0".into())
);
}
}