Rework the speaker: sans-I/O FSM, sharded table, RFC 7606, ADD-PATH
The work of the speaker-cleanup branch as one commit; the branch keeps the step-by-step history. Sessions - The state machine is a pure function from events to actions, driven by one task per connection. Idle, Connect and Active belong to a dialer that keeps configured peers up with backoff. - Capabilities are negotiated: address families, four-octet AS numbers with AS_TRANS and AS4_PATH, route refresh, ADD-PATH per family and direction. - Collision detection per RFC 4271 section 6.8, by address and identifier. - Connections are taken only from configured peers, a few per address. - A panic in a session is contained and answered with a Cease; a core task that stops shuts the server down with a Cease to every peer. - Writes that make no progress are given up (RFC 9687). Codec - Malformed UPDATEs are handled as RFC 7606 says: treat-as-withdraw, attribute discard, or session reset, with the NOTIFICATION to send. - Address families are open types; any can be named and ignored. - Prefixes are fixed-size values, normalised, with path identifiers. - Lengths are filled in after writing; one definition of each type code. - Property tests and fuzz targets for every decoder. Routing table - Adj-RIBs-In and Loc-RIB in one sharded table that sessions write to directly. Routes are sixteen bytes; attribute sets are interned in wire form. Best-path selection per RFC 4271 section 9.1.2.2 with deterministic MED. - A journal per shard lets followers read changes at their own pace; one that falls behind is resynchronised. - On a RIPE RIS dump of 148 peers and 39.3M routes: loaded in 2.0 s on 12 cores, about 52 bytes a route. Announcing - Originated routes come from the configuration. Each session has its own exporter with an Adj-RIB-Out, so a slow peer delays only itself. - Routes to a peer in our own AS have an empty path and a LOCAL_PREF. Operations - HTTP status page, JSON API, event stream and Prometheus metrics. - Benchmarks on synthetic and real tables. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Rayhaan JaufeerallyandClaude Fable 5.1 committed
1 parent
1ec60255ea
commit
630c175918
138 files changed
+21650
-2925
No files matched your search
@@ -2,17 +2,41 @@ use std::net::Ipv4Addr;
|
||||
use std::time::Duration;
|
||||
|
||||
use bgp_integration_tests::test_peer::TestPeer;
|
||||
use bgp_integration_tests::test_server::RouteChange;
|
||||
use bgp_integration_tests::test_server::TestServer;
|
||||
use bgp_packet::errors::{BgpError, CeaseSubcode};
|
||||
use bgp_packet::errors::{BgpError, CeaseSubcode, FsmSubcode, MsgHeaderSubcode, UpdateMsgSubcode};
|
||||
use bgp_packet::message::{
|
||||
Message, MpReachNlriPathAttribute, MpUnreachNlriPathAttribute, NlriNextHop,
|
||||
NotificationMessage, PathAttribute, UpdateMessage,
|
||||
AsPathAttribute, Message, MpReachNlriPathAttribute, MpUnreachNlriPathAttribute,
|
||||
NextHopPathAttribute, NlriNextHop, NotificationMessage, OriginPathAttribute, PathAttribute,
|
||||
UpdateMessage,
|
||||
};
|
||||
use bgp_primitives::constants::{AddressFamilyId, SubsequentAfi};
|
||||
use bgp_primitives::constants::{AfiSafi, IpVersion};
|
||||
use bgp_primitives::ip_prefix::IpPrefix;
|
||||
use bgp_rib::RibEvent;
|
||||
use eyre::Result;
|
||||
|
||||
const TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The attributes every announcement of IPv4 routes must carry. An UPDATE
|
||||
/// without them has its routes withdrawn instead (RFC 7606).
|
||||
fn mandatory_attributes() -> Vec<PathAttribute> {
|
||||
let mut attributes = with_origin_and_as_path(PathAttribute::NextHop(NextHopPathAttribute(
|
||||
Ipv4Addr::new(192, 0, 2, 1),
|
||||
)));
|
||||
// In order of type code: ORIGIN, AS_PATH, NEXT_HOP.
|
||||
attributes.rotate_left(1);
|
||||
attributes
|
||||
}
|
||||
|
||||
/// `first` followed by the attributes every announcement must carry.
|
||||
/// Routes in MP_REACH_NLRI have their next hop there.
|
||||
fn with_origin_and_as_path(first: PathAttribute) -> Vec<PathAttribute> {
|
||||
vec![
|
||||
first,
|
||||
PathAttribute::Origin(OriginPathAttribute::IGP),
|
||||
AsPathAttribute::from_asns(vec![65001]),
|
||||
]
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Connectivity
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -74,6 +98,147 @@ async fn test_bad_marker_does_not_crash_server() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads until a NOTIFICATION arrives, skipping KEEPALIVEs.
|
||||
async fn expect_notification(peer: &mut TestPeer) -> Result<NotificationMessage> {
|
||||
tokio::time::timeout(TIMEOUT, async {
|
||||
loop {
|
||||
match peer.recv().await? {
|
||||
Message::Notification(n) => return Ok(n),
|
||||
Message::KeepAlive => {}
|
||||
other => eyre::bail!("expected NOTIFICATION, got {other:?}"),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// A header with the given length and type, followed by `body`.
|
||||
fn framed(length: u16, msg_type: u8, body: &[u8]) -> Vec<u8> {
|
||||
let mut buf = vec![0xFF; 16];
|
||||
buf.extend_from_slice(&length.to_be_bytes());
|
||||
buf.push(msg_type);
|
||||
buf.extend_from_slice(body);
|
||||
buf
|
||||
}
|
||||
|
||||
/// What the peer sent decides the NOTIFICATION it gets back (RFC 4271 §6),
|
||||
/// in every state, and the session ends.
|
||||
#[tokio::test]
|
||||
async fn test_malformed_messages_are_answered_with_the_matching_notification() -> Result<()> {
|
||||
let server = TestServer::start(65000).await?;
|
||||
let header = |subcode| BgpError::MsgHeader(subcode);
|
||||
let mut bad_marker = framed(19, 4, &[]);
|
||||
bad_marker[0] = 0;
|
||||
|
||||
let cases: Vec<(&str, Vec<u8>, BgpError, Vec<u8>)> = vec![
|
||||
(
|
||||
"length beyond the maximum",
|
||||
framed(5000, 2, &[]),
|
||||
header(MsgHeaderSubcode::BadMessageLength),
|
||||
5000u16.to_be_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
"length below a header",
|
||||
framed(5, 4, &[]),
|
||||
header(MsgHeaderSubcode::BadMessageLength),
|
||||
vec![0, 5],
|
||||
),
|
||||
(
|
||||
"unknown message type",
|
||||
framed(19, 9, &[]),
|
||||
header(MsgHeaderSubcode::BadMessageType),
|
||||
vec![9],
|
||||
),
|
||||
(
|
||||
"marker not all ones",
|
||||
bad_marker,
|
||||
header(MsgHeaderSubcode::ConnNotSynchronized),
|
||||
vec![],
|
||||
),
|
||||
(
|
||||
"UPDATE with attributes running past its end",
|
||||
framed(23, 2, &[0, 0, 0, 40]),
|
||||
BgpError::UpdateMsg(UpdateMsgSubcode::MalformedAttrs),
|
||||
vec![],
|
||||
),
|
||||
];
|
||||
|
||||
for (i, (what, bytes, error, data)) in cases.into_iter().enumerate() {
|
||||
// Established: a distinct router ID each time, so that a session
|
||||
// still closing does not collide with the next one.
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 1, i as u8 + 1), 90)
|
||||
.await?;
|
||||
peer.send_raw(&bytes).await?;
|
||||
let notification = expect_notification(&mut peer).await?;
|
||||
assert_eq!(
|
||||
(notification.error, notification.data),
|
||||
(error.clone(), data.clone()),
|
||||
"{what}, Established"
|
||||
);
|
||||
|
||||
// OpenSent: the same bytes instead of an OPEN.
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
let _ = peer.recv_open().await?;
|
||||
peer.send_raw(&bytes).await?;
|
||||
let notification = expect_notification(&mut peer).await?;
|
||||
assert_eq!(
|
||||
(notification.error, notification.data),
|
||||
(error, data),
|
||||
"{what}, OpenSent"
|
||||
);
|
||||
}
|
||||
|
||||
// A well-formed message at the wrong time is an FSM error, not a
|
||||
// message error.
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
let _ = peer.recv_open().await?;
|
||||
peer.send(&Message::KeepAlive).await?;
|
||||
let notification = expect_notification(&mut peer).await?;
|
||||
assert_eq!(
|
||||
notification.error,
|
||||
BgpError::FsmError(FsmSubcode::UnexpectedOpenSent)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A message may arrive in pieces, with the server's own timers firing in
|
||||
/// between. The server once threw away the piece it had already read when
|
||||
/// that happened, so the message was silently lost (and the stream was only
|
||||
/// back in step if the remainder happened to be thrown away as well).
|
||||
#[tokio::test]
|
||||
async fn test_message_split_across_a_keepalive_tick_is_still_one_message() -> Result<()> {
|
||||
// Hold time 3 s: the server sends a KEEPALIVE every second.
|
||||
let server = TestServer::start_with_hold_time(65000, 3).await?;
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 3).await?;
|
||||
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv4, vec![10], 8)?;
|
||||
let mut update = bytes::BytesMut::new();
|
||||
Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: mandatory_attributes(),
|
||||
announced_routes: vec![prefix.into()],
|
||||
})
|
||||
.to_wire(&peer.ctx, &mut update)?;
|
||||
|
||||
// The first half, a pause long enough for the server's keepalive timer
|
||||
// to fire, then the rest.
|
||||
peer.send_raw(&update[..10]).await?;
|
||||
tokio::time::sleep(Duration::from_millis(1300)).await;
|
||||
peer.send_raw(&update[10..]).await?;
|
||||
|
||||
tokio::time::timeout(TIMEOUT, async {
|
||||
while !server.received_routes().iter().any(|e| e.prefix == prefix) {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| eyre::eyre!("the split UPDATE never reached the RIB"))?;
|
||||
assert_eq!(server.registry.list().len(), 1, "session was dropped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OpenConfirm → Established
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -173,9 +338,9 @@ async fn test_server_accepts_update() -> Result<()> {
|
||||
|
||||
let update = Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![],
|
||||
path_attributes: mandatory_attributes(),
|
||||
// 10.0.0.0/8 — one byte of prefix for a /8.
|
||||
announced_routes: vec![IpPrefix::new(AddressFamilyId::Ipv4, vec![10], 8)?],
|
||||
announced_routes: vec![IpPrefix::new(IpVersion::Ipv4, vec![10], 8)?.into()],
|
||||
});
|
||||
peer.send(&update).await?;
|
||||
|
||||
@@ -193,15 +358,16 @@ async fn test_server_accepts_ipv6_update() -> Result<()> {
|
||||
.await?;
|
||||
|
||||
// 2001:db8::/32
|
||||
let prefix = IpPrefix::new(AddressFamilyId::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
let update = Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![PathAttribute::MpReachNlri(MpReachNlriPathAttribute {
|
||||
afi: AddressFamilyId::Ipv6,
|
||||
safi: SubsequentAfi::Unicast,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix],
|
||||
})],
|
||||
path_attributes: with_origin_and_as_path(PathAttribute::MpReachNlri(
|
||||
MpReachNlriPathAttribute {
|
||||
family: AfiSafi::IPV6_UNICAST,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix.into()],
|
||||
},
|
||||
)),
|
||||
announced_routes: vec![],
|
||||
});
|
||||
peer.send(&update).await?;
|
||||
@@ -219,26 +385,27 @@ async fn test_server_accepts_ipv6_update() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn test_rib_records_ipv4_announce() -> Result<()> {
|
||||
let server = TestServer::start(65000).await?;
|
||||
let mut events = server.adj_rib_in.subscribe();
|
||||
let mut events = server.route_changes();
|
||||
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90)
|
||||
.await?;
|
||||
|
||||
let prefix = IpPrefix::new(AddressFamilyId::Ipv4, vec![10], 8)?;
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv4, vec![10], 8)?;
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![],
|
||||
announced_routes: vec![prefix.clone()],
|
||||
path_attributes: mandatory_attributes(),
|
||||
announced_routes: vec![prefix.into()],
|
||||
}))
|
||||
.await?;
|
||||
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), events.recv()).await??;
|
||||
assert!(
|
||||
matches!(&event, RibEvent::Announced { prefix: p, .. } if p == &prefix),
|
||||
matches!(&event, RouteChange::Announced { prefix: p, .. } if p == &prefix),
|
||||
"Expected Announced for 10.0.0.0/8, got {event:?}",
|
||||
);
|
||||
|
||||
let snapshot = server.adj_rib_in.snapshot().await;
|
||||
let snapshot = server.received_routes();
|
||||
assert!(
|
||||
snapshot.iter().any(|e| e.prefix == prefix),
|
||||
"10.0.0.0/8 missing from RIB snapshot",
|
||||
@@ -250,25 +417,26 @@ async fn test_rib_records_ipv4_announce() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn test_rib_records_ipv4_withdraw() -> Result<()> {
|
||||
let server = TestServer::start(65000).await?;
|
||||
let mut events = server.adj_rib_in.subscribe();
|
||||
let mut events = server.route_changes();
|
||||
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90)
|
||||
.await?;
|
||||
|
||||
let prefix = IpPrefix::new(AddressFamilyId::Ipv4, vec![10], 8)?;
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv4, vec![10], 8)?;
|
||||
|
||||
// Announce first.
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![],
|
||||
announced_routes: vec![prefix.clone()],
|
||||
path_attributes: mandatory_attributes(),
|
||||
announced_routes: vec![prefix.into()],
|
||||
}))
|
||||
.await?;
|
||||
tokio::time::timeout(Duration::from_secs(1), events.recv()).await??;
|
||||
|
||||
// Now withdraw.
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![prefix.clone()],
|
||||
withdrawn_routes: vec![prefix.into()],
|
||||
path_attributes: vec![],
|
||||
announced_routes: vec![],
|
||||
}))
|
||||
@@ -276,11 +444,11 @@ async fn test_rib_records_ipv4_withdraw() -> Result<()> {
|
||||
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), events.recv()).await??;
|
||||
assert!(
|
||||
matches!(&event, RibEvent::Withdrawn { prefix: p, .. } if p == &prefix),
|
||||
matches!(&event, RouteChange::Withdrawn { prefix: p, .. } if p == &prefix),
|
||||
"Expected Withdrawn for 10.0.0.0/8, got {event:?}",
|
||||
);
|
||||
|
||||
let snapshot = server.adj_rib_in.snapshot().await;
|
||||
let snapshot = server.received_routes();
|
||||
assert!(
|
||||
!snapshot.iter().any(|e| e.prefix == prefix),
|
||||
"10.0.0.0/8 should have been removed from RIB snapshot",
|
||||
@@ -292,31 +460,33 @@ async fn test_rib_records_ipv4_withdraw() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn test_rib_records_ipv6_announce() -> Result<()> {
|
||||
let server = TestServer::start(65000).await?;
|
||||
let mut events = server.adj_rib_in.subscribe();
|
||||
let mut events = server.route_changes();
|
||||
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90)
|
||||
.await?;
|
||||
|
||||
let prefix = IpPrefix::new(AddressFamilyId::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![PathAttribute::MpReachNlri(MpReachNlriPathAttribute {
|
||||
afi: AddressFamilyId::Ipv6,
|
||||
safi: SubsequentAfi::Unicast,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix.clone()],
|
||||
})],
|
||||
path_attributes: with_origin_and_as_path(PathAttribute::MpReachNlri(
|
||||
MpReachNlriPathAttribute {
|
||||
family: AfiSafi::IPV6_UNICAST,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix.into()],
|
||||
},
|
||||
)),
|
||||
announced_routes: vec![],
|
||||
}))
|
||||
.await?;
|
||||
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), events.recv()).await??;
|
||||
assert!(
|
||||
matches!(&event, RibEvent::Announced { prefix: p, .. } if p == &prefix),
|
||||
matches!(&event, RouteChange::Announced { prefix: p, .. } if p == &prefix),
|
||||
"Expected Announced for 2001:db8::/32, got {event:?}",
|
||||
);
|
||||
|
||||
let snapshot = server.adj_rib_in.snapshot().await;
|
||||
let snapshot = server.received_routes();
|
||||
assert!(
|
||||
snapshot.iter().any(|e| e.prefix == prefix),
|
||||
"2001:db8::/32 missing from RIB snapshot",
|
||||
@@ -328,22 +498,24 @@ async fn test_rib_records_ipv6_announce() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn test_rib_records_ipv6_withdraw() -> Result<()> {
|
||||
let server = TestServer::start(65000).await?;
|
||||
let mut events = server.adj_rib_in.subscribe();
|
||||
let mut events = server.route_changes();
|
||||
|
||||
let mut peer = TestPeer::connect(server.addr).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?;
|
||||
peer.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90)
|
||||
.await?;
|
||||
|
||||
let prefix = IpPrefix::new(AddressFamilyId::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
let prefix = IpPrefix::new(IpVersion::Ipv6, vec![0x20, 0x01, 0x0d, 0xb8], 32)?;
|
||||
|
||||
// Announce first.
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![PathAttribute::MpReachNlri(MpReachNlriPathAttribute {
|
||||
afi: AddressFamilyId::Ipv6,
|
||||
safi: SubsequentAfi::Unicast,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix.clone()],
|
||||
})],
|
||||
path_attributes: with_origin_and_as_path(PathAttribute::MpReachNlri(
|
||||
MpReachNlriPathAttribute {
|
||||
family: AfiSafi::IPV6_UNICAST,
|
||||
next_hop: NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()),
|
||||
prefixes: vec![prefix.into()],
|
||||
},
|
||||
)),
|
||||
announced_routes: vec![],
|
||||
}))
|
||||
.await?;
|
||||
@@ -353,9 +525,8 @@ async fn test_rib_records_ipv6_withdraw() -> Result<()> {
|
||||
peer.send(&Message::Update(UpdateMessage {
|
||||
withdrawn_routes: vec![],
|
||||
path_attributes: vec![PathAttribute::MpUnreachNlri(MpUnreachNlriPathAttribute {
|
||||
afi: AddressFamilyId::Ipv6,
|
||||
safi: SubsequentAfi::Unicast,
|
||||
prefixes: vec![prefix.clone()],
|
||||
family: AfiSafi::IPV6_UNICAST,
|
||||
prefixes: vec![prefix.into()],
|
||||
})],
|
||||
announced_routes: vec![],
|
||||
}))
|
||||
@@ -363,11 +534,11 @@ async fn test_rib_records_ipv6_withdraw() -> Result<()> {
|
||||
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), events.recv()).await??;
|
||||
assert!(
|
||||
matches!(&event, RibEvent::Withdrawn { prefix: p, .. } if p == &prefix),
|
||||
matches!(&event, RouteChange::Withdrawn { prefix: p, .. } if p == &prefix),
|
||||
"Expected Withdrawn for 2001:db8::/32, got {event:?}",
|
||||
);
|
||||
|
||||
let snapshot = server.adj_rib_in.snapshot().await;
|
||||
let snapshot = server.received_routes();
|
||||
assert!(
|
||||
!snapshot.iter().any(|e| e.prefix == prefix),
|
||||
"2001:db8::/32 should have been removed from RIB snapshot",
|
||||
|
||||
Reference in new issue
Block a user