From bd674002a2f09cdca22d2878d0585c53e961e8a9 Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Sun, 4 Oct 2026 01:03:16 +0200 Subject: [PATCH] Keep the path attributes of rejected routes, and show them A rejected route was recorded with its prefix, reason and AS path, which is not enough to tell why a neighbor sent it: whether a /32 is a blackhole request, which communities a leaked route carries, which network marked it Only to Customer. Each recorded rejection now keeps the attributes of the UPDATE it came in, without the NLRI of MP_REACH_NLRI and MP_UNREACH_NLRI, and its next hop. The attributes are copied once per UPDATE, only when a route of it is rejected, and shared by its rejected routes. A route rejected later, when the RPKI data changes, has the attributes it was stored with. /api/routes/rejected gives them as `summary` and `attributes`, as for received routes, and the status page shows them under "show" in each row of the refused routes. Co-Authored-By: Claude Opus 5.5 --- crates/integration-tests/tests/debugging.rs | 14 +++++ crates/server/src/refusals.rs | 63 ++++++++++++++++++--- crates/server/src/routes.rs | 52 +++++++++++++---- crates/web/src/api.rs | 8 +++ crates/web/static/statusz.html | 28 +++++++-- docs/content/docs/operations/monitoring.md | 10 ++-- docs/content/docs/reference/http-api.md | 20 ++++++- 7 files changed, 166 insertions(+), 29 deletions(-) diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs index 5807713..b976a26 100644 --- a/crates/integration-tests/tests/debugging.rs +++ b/crates/integration-tests/tests/debugging.rs @@ -250,6 +250,20 @@ async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { row("10.0.0.0/8", "bogon prefix", "[3333]"), ] ); + // And with all its attributes, as for a route that was taken, to see + // what came along with it. + let latest = &record["recent"][0]; + assert_eq!(latest["summary"]["next_hop"], "192.0.2.1"); + assert_eq!(latest["summary"]["origin"], "IGP"); + assert_eq!( + latest["summary"]["as_path"], + serde_json::json!([3333, 2121]) + ); + assert_eq!( + latest["attributes"].as_array().map(Vec::len), + Some(3), + "ORIGIN, AS_PATH and NEXT_HOP: {latest}" + ); // Asked for by peer, by name or by address. for peer in ["refused-demo", "127.0.0.1"] { let one: Value = get(&format!("/api/routes/rejected?peer={peer}")) diff --git a/crates/server/src/refusals.rs b/crates/server/src/refusals.rs index 2279400..d4d04ef 100644 --- a/crates/server/src/refusals.rs +++ b/crates/server/src/refusals.rs @@ -12,6 +12,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime}; +use bgp_packet::message::{NlriNextHop, PathAttribute}; use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; use prometheus::IntCounter; use tracing::warn; @@ -42,9 +43,25 @@ pub struct RefusedRoute { pub reason: Rejection, /// The AS path it came with. pub as_path: Arc<[u32]>, + /// All its path attributes, without MP_REACH_NLRI and MP_UNREACH_NLRI: + /// their prefixes are routes of their own, and the next hop is below. + /// As the peer sent them; for a route refused on a second look (the + /// RPKI's data changed), as they were kept, with what the import policy + /// put on. Shared by the routes refused from one UPDATE. + pub attributes: Arc<[PathAttribute]>, + /// Its next hop, if it came in MP_REACH_NLRI. That of an IPv4 route in + /// the body of the UPDATE is in the NEXT_HOP attribute. + pub next_hop: Option, pub at: SystemTime, } +/// What the routes of one UPDATE came with, for [`PeerRefusals::note`]. +pub(crate) struct Came<'a> { + pub(crate) as_path: &'a Arc<[u32]>, + pub(crate) attributes: &'a Arc<[PathAttribute]>, + pub(crate) next_hop: Option<&'a NlriNextHop>, +} + /// The refusals of one peer. pub struct PeerRefusals { /// What the peer is called in the log and in metrics: its configured @@ -78,9 +95,9 @@ impl PeerRefusals { } } - /// Notes that `routes`, which came with the AS path `as_path`, were - /// refused for `reason`. - pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, as_path: &Arc<[u32]>) { + /// Notes that `routes`, which came with what `came` says, were refused + /// for `reason`. + pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, came: &Came<'_>) { let Some(latest) = routes.last() else { return; }; @@ -100,7 +117,9 @@ impl PeerRefusals { prefix: route.prefix, path_id: route.path_id, reason, - as_path: Arc::clone(as_path), + as_path: Arc::clone(came.as_path), + attributes: Arc::clone(came.attributes), + next_hop: came.next_hop.copied(), at, }); } @@ -118,7 +137,7 @@ impl PeerRefusals { peer = %self.label, prefix = %latest.prefix, reason = reason.as_str(), - as_path = ?&as_path[..], + as_path = ?&came.as_path[..], others_since_last_report = state.unlogged + count - 1, "Refused a route" ); @@ -193,6 +212,8 @@ impl RefusalLog { #[cfg(test)] mod tests { + use bgp_packet::message::{AsPathAttribute, OriginPathAttribute}; + use super::*; fn route(n: u32) -> Nlri { @@ -210,8 +231,19 @@ mod tests { assert!(log.peers().is_empty(), "nothing refused yet"); let path: Arc<[u32]> = vec![3333, 64512].into(); - record.note(&[route(1), route(2)], Rejection::BogonAs, &path); - record.note(&[route(3)], Rejection::TooSpecific, &path); + let attributes: Arc<[PathAttribute]> = vec![ + PathAttribute::Origin(OriginPathAttribute::IGP), + AsPathAttribute::from_asns(vec![3333, 64512]), + ] + .into(); + let next_hop = NlriNextHop::Ipv6("2001:db8::1".parse().unwrap()); + let came = Came { + as_path: &path, + attributes: &attributes, + next_hop: Some(&next_hop), + }; + record.note(&[route(1), route(2)], Rejection::BogonAs, &came); + record.note(&[route(3)], Rejection::TooSpecific, &came); assert_eq!(record.total(), 3); assert_eq!( @@ -226,6 +258,13 @@ mod tests { ); assert_eq!(recent[0].reason, Rejection::TooSpecific); assert_eq!(&recent[2].as_path[..], [3333, 64512]); + // The whole of what the route came with is kept, to be looked at. + assert_eq!(recent[2].attributes, attributes); + assert_eq!(recent[2].next_hop, Some(next_hop)); + assert!( + Arc::ptr_eq(&recent[0].attributes, &recent[2].attributes), + "routes from one UPDATE share their attributes" + ); // The peer's next session finds the same record. let again = log.of_peer(address, "another-name"); @@ -240,10 +279,16 @@ mod tests { fn only_so_many_refused_routes_are_kept() { let record = PeerRefusals::new("refusals-test-2"); let path: Arc<[u32]> = vec![3333].into(); + let attributes: Arc<[PathAttribute]> = Vec::new().into(); + let came = Came { + as_path: &path, + attributes: &attributes, + next_hop: None, + }; let many: Vec = (0..1000).map(route).collect(); - record.note(&many, Rejection::BogonPrefix, &path); + record.note(&many, Rejection::BogonPrefix, &came); for n in 1000..1010 { - record.note(&[route(n)], Rejection::BogonPrefix, &path); + record.note(&[route(n)], Rejection::BogonPrefix, &came); } assert_eq!(record.total(), 1010); diff --git a/crates/server/src/routes.rs b/crates/server/src/routes.rs index 0b8c32c..fd1e10c 100644 --- a/crates/server/src/routes.rs +++ b/crates/server/src/routes.rs @@ -14,7 +14,7 @@ use bgp_rib::table::{AttrSet, Outcome, PeerIndex, RibTable}; use bgp_rpki::{PathValidity, Validity}; use crate::policy::{self, ImportPolicy, Rejection, RouteSource}; -use crate::refusals::PeerRefusals; +use crate::refusals::{Came, PeerRefusals}; use crate::rpki::Checks; /// The most routes of one peer that are held aside as invalid. Far more @@ -127,6 +127,7 @@ pub(crate) fn apply( .check_path(&update.path_attributes, &import.from), path_invalid: OnceCell::new(), as_path: OnceCell::new(), + refused_attributes: OnceCell::new(), mark: only_to_customer_mark(import, update), changed: OnceCell::new(), }; @@ -177,9 +178,10 @@ struct Applying<'a> { /// Whether the ASPAs find the path invalid: asked only if a route /// gets as far as that mattering. path_invalid: OnceCell, - /// The path the routes came with, for the record of what is refused: - /// made only if something is. + /// The path the routes came with, and all their attributes, for the + /// record of what is refused: made only if something is. as_path: OnceCell>, + refused_attributes: OnceCell>, /// See [`only_to_customer_mark`]. mark: Option, /// The UPDATE's attributes as the policy changes them, if it does: @@ -224,7 +226,7 @@ impl Applying<'_> { let (origin, used_next_hop) = match self.judge_together(family, next_hop) { Ok(judged) => judged, Err(rejection) => { - self.refuse(nlri, rejection); + self.refuse(nlri, rejection, next_hop); return Ok(()); } }; @@ -251,7 +253,7 @@ impl Applying<'_> { if rejection.may_pass() { invalid.push(*route); } - self.refuse(std::slice::from_ref(route), rejection); + self.refuse(std::slice::from_ref(route), rejection, next_hop); } } } @@ -327,11 +329,18 @@ impl Applying<'_> { Ok(()) } - /// Notes that `refused` were not taken, and takes out what the peer - /// had announced for them before. - fn refuse(&mut self, refused: &[Nlri], rejection: Rejection) { - let as_path = self.as_path.get_or_init(|| path_of(self.update)); - self.import.refusals.note(refused, rejection, as_path); + /// Notes that `refused`, whose next hop is `next_hop` if they came in + /// MP_REACH_NLRI, were not taken, and takes out what the peer had + /// announced for them before. + fn refuse(&mut self, refused: &[Nlri], rejection: Rejection, next_hop: Option<&NlriNextHop>) { + let came = Came { + as_path: self.as_path.get_or_init(|| path_of(self.update)), + attributes: self + .refused_attributes + .get_or_init(|| attributes_of(self.update)), + next_hop, + }; + self.import.refusals.note(refused, rejection, &came); self.applied.rejected += refused.len(); self.applied.outcome += self.table.withdraw(self.import.peer, refused); } @@ -398,6 +407,22 @@ fn path_is_invalid(import: &Import<'_>, update: &UpdateMessage) -> bool { check.verify(aspas, &path, has_set) == PathValidity::Invalid } +/// The UPDATE's path attributes without its prefixes: MP_REACH_NLRI and +/// MP_UNREACH_NLRI, which can hold thousands, are left out. +fn attributes_of(update: &UpdateMessage) -> Arc<[PathAttribute]> { + update + .path_attributes + .iter() + .filter(|attribute| { + !matches!( + attribute, + PathAttribute::MpReachNlri(_) | PathAttribute::MpUnreachNlri(_) + ) + }) + .cloned() + .collect() +} + /// The AS numbers of the UPDATE's AS_PATH. fn path_of(update: &UpdateMessage) -> Arc<[u32]> { let path = update @@ -444,10 +469,15 @@ pub(crate) fn revalidate( applied.outcome += table.withdraw(import.peer, std::slice::from_ref(suspect)); applied.rejected += 1; route.attrs.as_path(&mut path); + let (attributes, next_hop) = route.attrs.attributes().unwrap_or_default(); import.refusals.note( std::slice::from_ref(suspect), rejection, - &Arc::from(&path[..]), + &Came { + as_path: &Arc::from(&path[..]), + attributes: &attributes.into(), + next_hop: next_hop.as_ref(), + }, ); held.hold(*suspect, &route.attrs); } diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 8c61abf..941687b 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -950,6 +950,12 @@ pub struct RefusedRouteView { pub reason: &'static str, pub as_path: Vec, pub at_unix: u64, + /// What the route came with, as for a received route: its next hop, + /// origin, MED, local preference and communities. + pub summary: AttrSummary, + /// The decoded attribute list, as for a received route. MP_REACH_NLRI + /// is not in it: its next hop is in the summary. + pub attributes: Vec, } #[derive(Debug, Default, Deserialize)] @@ -992,6 +998,8 @@ pub async fn rejected_routes( reason: route.reason.as_str(), as_path: route.as_path.to_vec(), at_unix: unix_secs(route.at), + summary: summarize(&route.attributes, route.next_hop.as_ref()), + attributes: route.attributes.to_vec(), }) .collect(), }) diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index 883901f..39b4edb 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -27,6 +27,10 @@ table.data th { background: #dde; font-weight: bold; } #event-log { font-family: monospace; font-size: 12px; max-height: 22em; overflow-y: auto; } #event-log div { white-space: pre-wrap; } details { clear: both; margin: 8px 0 0; } +details.refused-detail { margin: 0; } +details.refused-detail summary { cursor: pointer; } +details.refused-detail table td { padding: 1px 8px 1px 0; border: 0; } +details.refused-detail pre { max-height: 20em; overflow: auto; margin: 4px 0 0; white-space: pre-wrap; overflow-wrap: anywhere; } summary { cursor: pointer; color: #00c; } summary:hover { text-decoration: underline; } #config-text { border: 1px solid #99a; padding: 4px 6px; margin: 4px 0; font-size: 12px; overflow-x: auto; } @@ -100,7 +104,7 @@ from

Refused routes (...)

Routes that peers announced and their import policies did not take. The counts are since the daemon started; the latest refusals of each peer are kept.
- +
peerprefixAS pathreasonwhen
peerprefixAS pathreasonwhenattributes
@@ -325,21 +329,37 @@ from (interfaces ? `
Interfaces: ${interfaces}` : '')); } + // Everything a refused route came with, folded away: the fields of the + // received routes' table, then the attributes as decoded. + function refusedDetail(r) { + const s = r.summary; + const field = (name, value) => `${name}${value}`; + return `
show` + + field('next hop', s.next_hop ? esc(s.next_hop) : DASH) + + field('origin', s.origin ? esc(s.origin) : DASH) + + field('MED', s.med ?? DASH) + + field('local pref', s.local_pref ?? DASH) + + field('communities', commList(s.communities, true)) + + field('large communities', commList(s.large_communities)) + + `
${esc(r.attributes.map(a => JSON.stringify(a)).join('\n'))}
`; + } + // For each peer a line that says how many and why, then its latest few. const REFUSED_SHOWN = 20; function renderRefused() { const b = $('refused-body'); const total = state.refused.reduce((n, p) => n + p.total, 0); $('refused-count').textContent = total ? `${total} from ${state.refused.length} peer${state.refused.length === 1 ? '' : 's'}` : '0'; - if (!state.refused.length) { setHTML(b, 'Nothing a peer announced has been refused.'); return; } + if (!state.refused.length) { setHTML(b, 'Nothing a peer announced has been refused.'); return; } setHTML(b, state.refused.map(p => { const why = Object.entries(p.by_reason).sort((a, c) => c[1] - a[1]).map(([reason, n]) => `${n} ${esc(reason)}`).join(', '); const more = p.recent.length > REFUSED_SHOWN ? ` Latest ${REFUSED_SHOWN} shown; the latest ${p.recent.length} as json.` : ''; - return `${esc(p.peer)} ${esc(p.peer_addr)}: ${p.total} refused (${why}).${more}` + + return `${esc(p.peer)} ${esc(p.peer_addr)}: ${p.total} refused (${why}).${more}` + p.recent.slice(0, REFUSED_SHOWN).map(r => ` ${esc(r.prefix)}${r.path_id ? ` path ${r.path_id}` : ''} - ${asPath(r.as_path)}${esc(r.reason)}${age(r.at_unix)} ago`).join(''); + ${asPath(r.as_path)}${esc(r.reason)}${age(r.at_unix)} ago + ${refusedDetail(r)}`).join(''); }).join('')); } diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md index 19fae22..1186df3 100644 --- a/docs/content/docs/operations/monitoring.md +++ b/docs/content/docs/operations/monitoring.md @@ -70,10 +70,12 @@ curl -s '127.0.0.1:8080/api/routes/received?address=193.0.0.1&best=true' | jq Ferrix does not store rejected routes in the routing table. For each neighbor, it records the number of rejected routes for each reason, and the -200 most recent rejected routes with their AS paths. The status page -displays this information under "Refused routes", and the HTTP endpoint -`/api/routes/rejected` returns it as JSON. To display the records for one -neighbor, add `?peer=NAME`. +200 most recent rejected routes with all the path attributes they were +received with. The status page displays this information under "Refused +routes"; to display the attributes of a rejected route, select **show** in +its row. The HTTP endpoint `/api/routes/rejected` returns the same +information as JSON. To display the records for one neighbor, add +`?peer=NAME`. ```json [{"peer":"transit-a","total":3,"by_reason":{"RPKI invalid":2,"bogon prefix":1}, diff --git a/docs/content/docs/reference/http-api.md b/docs/content/docs/reference/http-api.md index 4397841..00b5482 100644 --- a/docs/content/docs/reference/http-api.md +++ b/docs/content/docs/reference/http-api.md @@ -143,11 +143,29 @@ routes. To return the records for one neighbor, add `?peer=NAME`. "path_id": 0, "reason": "own AS in path", "as_path": [65001, 64500], - "at_unix": 1791020702 + "at_unix": 1791020702, + "summary": { + "origin": "IGP", + "as_path": [65001, 64500], + "next_hop": "10.0.0.2", + "med": null, + "local_pref": null, + "communities": [], + "large_communities": [] + }, + "attributes": [ + { "Origin": 0 }, + { "AsPath": { "segments": [{ "ordered": true, "path": [65001, 64500] }] } }, + { "NextHop": "10.0.0.2" } + ] }] }] ``` +| Field | Description | +|---|---| +| `summary`, `attributes` | The path attributes of the rejected route, in the same format as for received routes. For a route rejected when RPKI data changed, the attributes as stored, including those added by the import policy. | + ## Kernel `/api/kernel` returns the state of kernel route installation: