Keep the path attributes of rejected routes, and show them

A rejected route was recorded with its prefix, reason and AS path, which
is not enough to tell why a neighbor sent it: whether a /32 is a
blackhole request, which communities a leaked route carries, which
network marked it Only to Customer.

Each recorded rejection now keeps the attributes of the UPDATE it came
in, without the NLRI of MP_REACH_NLRI and MP_UNREACH_NLRI, and its next
hop. The attributes are copied once per UPDATE, only when a route of it
is rejected, and shared by its rejected routes. A route rejected later,
when the RPKI data changes, has the attributes it was stored with.

/api/routes/rejected gives them as `summary` and `attributes`, as for
received routes, and the status page shows them under "show" in each
row of the refused routes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/integration-tests/tests/debugging.rs

@@ -250,6 +250,20 @@ async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> {
             row("10.0.0.0/8", "bogon prefix", "[3333]"),
         ]
     );
+    // And with all its attributes, as for a route that was taken, to see
+    // what came along with it.
+    let latest = &record["recent"][0];
+    assert_eq!(latest["summary"]["next_hop"], "192.0.2.1");
+    assert_eq!(latest["summary"]["origin"], "IGP");
+    assert_eq!(
+        latest["summary"]["as_path"],
+        serde_json::json!([3333, 2121])
+    );
+    assert_eq!(
+        latest["attributes"].as_array().map(Vec::len),
+        Some(3),
+        "ORIGIN, AS_PATH and NEXT_HOP: {latest}"
+    );
     // Asked for by peer, by name or by address.
     for peer in ["refused-demo", "127.0.0.1"] {
         let one: Value = get(&format!("/api/routes/rejected?peer={peer}"))

crates/server/src/refusals.rs

@@ -12,6 +12,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant, SystemTime};
 
+use bgp_packet::message::{NlriNextHop, PathAttribute};
 use bgp_primitives::ip_prefix::{IpPrefix, Nlri};
 use prometheus::IntCounter;
 use tracing::warn;
@@ -42,9 +43,25 @@ pub struct RefusedRoute {
     pub reason: Rejection,
     /// The AS path it came with.
     pub as_path: Arc<[u32]>,
+    /// All its path attributes, without MP_REACH_NLRI and MP_UNREACH_NLRI:
+    /// their prefixes are routes of their own, and the next hop is below.
+    /// As the peer sent them; for a route refused on a second look (the
+    /// RPKI's data changed), as they were kept, with what the import policy
+    /// put on. Shared by the routes refused from one UPDATE.
+    pub attributes: Arc<[PathAttribute]>,
+    /// Its next hop, if it came in MP_REACH_NLRI. That of an IPv4 route in
+    /// the body of the UPDATE is in the NEXT_HOP attribute.
+    pub next_hop: Option<NlriNextHop>,
     pub at: SystemTime,
 }
 
+/// What the routes of one UPDATE came with, for [`PeerRefusals::note`].
+pub(crate) struct Came<'a> {
+    pub(crate) as_path: &'a Arc<[u32]>,
+    pub(crate) attributes: &'a Arc<[PathAttribute]>,
+    pub(crate) next_hop: Option<&'a NlriNextHop>,
+}
+
 /// The refusals of one peer.
 pub struct PeerRefusals {
     /// What the peer is called in the log and in metrics: its configured
@@ -78,9 +95,9 @@ impl PeerRefusals {
         }
     }
 
-    /// Notes that `routes`, which came with the AS path `as_path`, were
+    /// Notes that `routes`, which came with what `came` says, were refused
-    /// refused for `reason`.
+    /// for `reason`.
-    pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, as_path: &Arc<[u32]>) {
+    pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, came: &Came<'_>) {
         let Some(latest) = routes.last() else {
             return;
         };
@@ -100,7 +117,9 @@ impl PeerRefusals {
                 prefix: route.prefix,
                 path_id: route.path_id,
                 reason,
-                as_path: Arc::clone(as_path),
+                as_path: Arc::clone(came.as_path),
+                attributes: Arc::clone(came.attributes),
+                next_hop: came.next_hop.copied(),
                 at,
             });
         }
@@ -118,7 +137,7 @@ impl PeerRefusals {
             peer = %self.label,
             prefix = %latest.prefix,
             reason = reason.as_str(),
-            as_path = ?&as_path[..],
+            as_path = ?&came.as_path[..],
             others_since_last_report = state.unlogged + count - 1,
             "Refused a route"
         );
@@ -193,6 +212,8 @@ impl RefusalLog {
 
 #[cfg(test)]
 mod tests {
+    use bgp_packet::message::{AsPathAttribute, OriginPathAttribute};
+
     use super::*;
 
     fn route(n: u32) -> Nlri {
@@ -210,8 +231,19 @@ mod tests {
         assert!(log.peers().is_empty(), "nothing refused yet");
 
         let path: Arc<[u32]> = vec![3333, 64512].into();
-        record.note(&[route(1), route(2)], Rejection::BogonAs, &path);
+        let attributes: Arc<[PathAttribute]> = vec![
-        record.note(&[route(3)], Rejection::TooSpecific, &path);
+            PathAttribute::Origin(OriginPathAttribute::IGP),
+            AsPathAttribute::from_asns(vec![3333, 64512]),
+        ]
+        .into();
+        let next_hop = NlriNextHop::Ipv6("2001:db8::1".parse().unwrap());
+        let came = Came {
+            as_path: &path,
+            attributes: &attributes,
+            next_hop: Some(&next_hop),
+        };
+        record.note(&[route(1), route(2)], Rejection::BogonAs, &came);
+        record.note(&[route(3)], Rejection::TooSpecific, &came);
 
         assert_eq!(record.total(), 3);
         assert_eq!(
@@ -226,6 +258,13 @@ mod tests {
         );
         assert_eq!(recent[0].reason, Rejection::TooSpecific);
         assert_eq!(&recent[2].as_path[..], [3333, 64512]);
+        // The whole of what the route came with is kept, to be looked at.
+        assert_eq!(recent[2].attributes, attributes);
+        assert_eq!(recent[2].next_hop, Some(next_hop));
+        assert!(
+            Arc::ptr_eq(&recent[0].attributes, &recent[2].attributes),
+            "routes from one UPDATE share their attributes"
+        );
 
         // The peer's next session finds the same record.
         let again = log.of_peer(address, "another-name");
@@ -240,10 +279,16 @@ mod tests {
     fn only_so_many_refused_routes_are_kept() {
         let record = PeerRefusals::new("refusals-test-2");
         let path: Arc<[u32]> = vec![3333].into();
+        let attributes: Arc<[PathAttribute]> = Vec::new().into();
+        let came = Came {
+            as_path: &path,
+            attributes: &attributes,
+            next_hop: None,
+        };
         let many: Vec<Nlri> = (0..1000).map(route).collect();
-        record.note(&many, Rejection::BogonPrefix, &path);
+        record.note(&many, Rejection::BogonPrefix, &came);
         for n in 1000..1010 {
-            record.note(&[route(n)], Rejection::BogonPrefix, &path);
+            record.note(&[route(n)], Rejection::BogonPrefix, &came);
         }
 
         assert_eq!(record.total(), 1010);

crates/server/src/routes.rs

@@ -14,7 +14,7 @@ use bgp_rib::table::{AttrSet, Outcome, PeerIndex, RibTable};
 use bgp_rpki::{PathValidity, Validity};
 
 use crate::policy::{self, ImportPolicy, Rejection, RouteSource};
-use crate::refusals::PeerRefusals;
+use crate::refusals::{Came, PeerRefusals};
 use crate::rpki::Checks;
 
 /// The most routes of one peer that are held aside as invalid. Far more
@@ -127,6 +127,7 @@ pub(crate) fn apply(
             .check_path(&update.path_attributes, &import.from),
         path_invalid: OnceCell::new(),
         as_path: OnceCell::new(),
+        refused_attributes: OnceCell::new(),
         mark: only_to_customer_mark(import, update),
         changed: OnceCell::new(),
     };
@@ -177,9 +178,10 @@ struct Applying<'a> {
     /// Whether the ASPAs find the path invalid: asked only if a route
     /// gets as far as that mattering.
     path_invalid: OnceCell<bool>,
-    /// The path the routes came with, for the record of what is refused:
+    /// The path the routes came with, and all their attributes, for the
-    /// made only if something is.
+    /// record of what is refused: made only if something is.
     as_path: OnceCell<Arc<[u32]>>,
+    refused_attributes: OnceCell<Arc<[PathAttribute]>>,
     /// See [`only_to_customer_mark`].
     mark: Option<u32>,
     /// The UPDATE's attributes as the policy changes them, if it does:
@@ -224,7 +226,7 @@ impl Applying<'_> {
         let (origin, used_next_hop) = match self.judge_together(family, next_hop) {
             Ok(judged) => judged,
             Err(rejection) => {
-                self.refuse(nlri, rejection);
+                self.refuse(nlri, rejection, next_hop);
                 return Ok(());
             }
         };
@@ -251,7 +253,7 @@ impl Applying<'_> {
                     if rejection.may_pass() {
                         invalid.push(*route);
                     }
-                    self.refuse(std::slice::from_ref(route), rejection);
+                    self.refuse(std::slice::from_ref(route), rejection, next_hop);
                 }
             }
         }
@@ -327,11 +329,18 @@ impl Applying<'_> {
         Ok(())
     }
 
-    /// Notes that `refused` were not taken, and takes out what the peer
+    /// Notes that `refused`, whose next hop is `next_hop` if they came in
-    /// had announced for them before.
+    /// MP_REACH_NLRI, were not taken, and takes out what the peer had
-    fn refuse(&mut self, refused: &[Nlri], rejection: Rejection) {
+    /// announced for them before.
-        let as_path = self.as_path.get_or_init(|| path_of(self.update));
+    fn refuse(&mut self, refused: &[Nlri], rejection: Rejection, next_hop: Option<&NlriNextHop>) {
-        self.import.refusals.note(refused, rejection, as_path);
+        let came = Came {
+            as_path: self.as_path.get_or_init(|| path_of(self.update)),
+            attributes: self
+                .refused_attributes
+                .get_or_init(|| attributes_of(self.update)),
+            next_hop,
+        };
+        self.import.refusals.note(refused, rejection, &came);
         self.applied.rejected += refused.len();
         self.applied.outcome += self.table.withdraw(self.import.peer, refused);
     }
@@ -398,6 +407,22 @@ fn path_is_invalid(import: &Import<'_>, update: &UpdateMessage) -> bool {
     check.verify(aspas, &path, has_set) == PathValidity::Invalid
 }
 
+/// The UPDATE's path attributes without its prefixes: MP_REACH_NLRI and
+/// MP_UNREACH_NLRI, which can hold thousands, are left out.
+fn attributes_of(update: &UpdateMessage) -> Arc<[PathAttribute]> {
+    update
+        .path_attributes
+        .iter()
+        .filter(|attribute| {
+            !matches!(
+                attribute,
+                PathAttribute::MpReachNlri(_) | PathAttribute::MpUnreachNlri(_)
+            )
+        })
+        .cloned()
+        .collect()
+}
+
 /// The AS numbers of the UPDATE's AS_PATH.
 fn path_of(update: &UpdateMessage) -> Arc<[u32]> {
     let path = update
@@ -444,10 +469,15 @@ pub(crate) fn revalidate(
         applied.outcome += table.withdraw(import.peer, std::slice::from_ref(suspect));
         applied.rejected += 1;
         route.attrs.as_path(&mut path);
+        let (attributes, next_hop) = route.attrs.attributes().unwrap_or_default();
         import.refusals.note(
             std::slice::from_ref(suspect),
             rejection,
-            &Arc::from(&path[..]),
+            &Came {
+                as_path: &Arc::from(&path[..]),
+                attributes: &attributes.into(),
+                next_hop: next_hop.as_ref(),
+            },
         );
         held.hold(*suspect, &route.attrs);
     }

crates/web/src/api.rs

@@ -950,6 +950,12 @@ pub struct RefusedRouteView {
     pub reason: &'static str,
     pub as_path: Vec<u32>,
     pub at_unix: u64,
+    /// What the route came with, as for a received route: its next hop,
+    /// origin, MED, local preference and communities.
+    pub summary: AttrSummary,
+    /// The decoded attribute list, as for a received route. MP_REACH_NLRI
+    /// is not in it: its next hop is in the summary.
+    pub attributes: Vec<PathAttribute>,
 }
 
 #[derive(Debug, Default, Deserialize)]
@@ -992,6 +998,8 @@ pub async fn rejected_routes(
                     reason: route.reason.as_str(),
                     as_path: route.as_path.to_vec(),
                     at_unix: unix_secs(route.at),
+                    summary: summarize(&route.attributes, route.next_hop.as_ref()),
+                    attributes: route.attributes.to_vec(),
                 })
                 .collect(),
         })

crates/web/static/statusz.html

@@ -27,6 +27,10 @@ table.data th { background: #dde; font-weight: bold; }
 #event-log { font-family: monospace; font-size: 12px; max-height: 22em; overflow-y: auto; }
 #event-log div { white-space: pre-wrap; }
 details { clear: both; margin: 8px 0 0; }
+details.refused-detail { margin: 0; }
+details.refused-detail summary { cursor: pointer; }
+details.refused-detail table td { padding: 1px 8px 1px 0; border: 0; }
+details.refused-detail pre { max-height: 20em; overflow: auto; margin: 4px 0 0; white-space: pre-wrap; overflow-wrap: anywhere; }
 summary { cursor: pointer; color: #00c; }
 summary:hover { text-decoration: underline; }
 #config-text { border: 1px solid #99a; padding: 4px 6px; margin: 4px 0; font-size: 12px; overflow-x: auto; }
@@ -100,7 +104,7 @@ from <select id="lookup-peer"><option value="">any peer</option></select>
 <h1 id="refused">Refused routes (<span id="refused-count">...</span>)</h1>
 <div class="dim">Routes that peers announced and their import policies did not take. The counts are since the daemon started; the latest refusals of each peer are kept.</div>
 <table class="data">
-<thead><tr><th>peer</th><th>prefix</th><th>AS path</th><th>reason</th><th>when</th></tr></thead>
+<thead><tr><th>peer</th><th>prefix</th><th>AS path</th><th>reason</th><th>when</th><th>attributes</th></tr></thead>
 <tbody id="refused-body"></tbody>
 </table>
 
@@ -325,21 +329,37 @@ from <select id="lookup-peer"><option value="">any peer</option></select>
       (interfaces ? `<br>Interfaces: ${interfaces}` : ''));
   }
 
+  // Everything a refused route came with, folded away: the fields of the
+  // received routes' table, then the attributes as decoded.
+  function refusedDetail(r) {
+    const s = r.summary;
+    const field = (name, value) => `<tr><td class="dim">${name}</td><td class="mono">${value}</td></tr>`;
+    return `<details class="refused-detail"><summary>show</summary><table>` +
+      field('next hop', s.next_hop ? esc(s.next_hop) : DASH) +
+      field('origin', s.origin ? esc(s.origin) : DASH) +
+      field('MED', s.med ?? DASH) +
+      field('local pref', s.local_pref ?? DASH) +
+      field('communities', commList(s.communities, true)) +
+      field('large communities', commList(s.large_communities)) +
+      `</table><pre class="mono">${esc(r.attributes.map(a => JSON.stringify(a)).join('\n'))}</pre></details>`;
+  }
+
   // For each peer a line that says how many and why, then its latest few.
   const REFUSED_SHOWN = 20;
   function renderRefused() {
     const b = $('refused-body');
     const total = state.refused.reduce((n, p) => n + p.total, 0);
     $('refused-count').textContent = total ? `${total} from ${state.refused.length} peer${state.refused.length === 1 ? '' : 's'}` : '0';
-    if (!state.refused.length) { setHTML(b, '<tr><td colspan="5" class="dim">Nothing a peer announced has been refused.</td></tr>'); return; }
+    if (!state.refused.length) { setHTML(b, '<tr><td colspan="6" class="dim">Nothing a peer announced has been refused.</td></tr>'); return; }
     setHTML(b, state.refused.map(p => {
       const why = Object.entries(p.by_reason).sort((a, c) => c[1] - a[1]).map(([reason, n]) => `${n} ${esc(reason)}`).join(', ');
       const more = p.recent.length > REFUSED_SHOWN
         ? ` <span class="dim">Latest ${REFUSED_SHOWN} shown; <a href="/api/routes/rejected?peer=${encodeURIComponent(p.peer)}">the latest ${p.recent.length} as json</a>.</span>` : '';
-      return `<tr><td colspan="5"><b>${esc(p.peer)}</b> <span class="dim">${esc(p.peer_addr)}</span>: <span class="bad">${p.total} refused</span> (${why}).${more}</td></tr>` +
+      return `<tr><td colspan="6"><b>${esc(p.peer)}</b> <span class="dim">${esc(p.peer_addr)}</span>: <span class="bad">${p.total} refused</span> (${why}).${more}</td></tr>` +
         p.recent.slice(0, REFUSED_SHOWN).map(r => `<tr><td></td>
           <td class="mono">${esc(r.prefix)}${r.path_id ? ` <span class="dim">path ${r.path_id}</span>` : ''}</td>
-          <td class="mono">${asPath(r.as_path)}</td><td>${esc(r.reason)}</td><td>${age(r.at_unix)} ago</td></tr>`).join('');
+          <td class="mono">${asPath(r.as_path)}</td><td>${esc(r.reason)}</td><td>${age(r.at_unix)} ago</td>
+          <td>${refusedDetail(r)}</td></tr>`).join('');
     }).join(''));
   }
 

docs/content/docs/operations/monitoring.md

@@ -70,10 +70,12 @@ curl -s '127.0.0.1:8080/api/routes/received?address=193.0.0.1&best=true' | jq
 
 Ferrix does not store rejected routes in the routing table. For each
 neighbor, it records the number of rejected routes for each reason, and the
-200 most recent rejected routes with their AS paths. The status page
+200 most recent rejected routes with all the path attributes they were
-displays this information under "Refused routes", and the HTTP endpoint
+received with. The status page displays this information under "Refused
-`/api/routes/rejected` returns it as JSON. To display the records for one
+routes"; to display the attributes of a rejected route, select **show** in
-neighbor, add `?peer=NAME`.
+its row. The HTTP endpoint `/api/routes/rejected` returns the same
+information as JSON. To display the records for one neighbor, add
+`?peer=NAME`.
 
 ```json
 [{"peer":"transit-a","total":3,"by_reason":{"RPKI invalid":2,"bogon prefix":1},

docs/content/docs/reference/http-api.md

@@ -143,11 +143,29 @@ routes. To return the records for one neighbor, add `?peer=NAME`.
     "path_id": 0,
     "reason": "own AS in path",
     "as_path": [65001, 64500],
-    "at_unix": 1791020702
+    "at_unix": 1791020702,
+    "summary": {
+      "origin": "IGP",
+      "as_path": [65001, 64500],
+      "next_hop": "10.0.0.2",
+      "med": null,
+      "local_pref": null,
+      "communities": [],
+      "large_communities": []
+    },
+    "attributes": [
+      { "Origin": 0 },
+      { "AsPath": { "segments": [{ "ordered": true, "path": [65001, 64500] }] } },
+      { "NextHop": "10.0.0.2" }
+    ]
   }]
 }]
 ```
 
+| Field | Description |
+|---|---|
+| `summary`, `attributes` | The path attributes of the rejected route, in the same format as for received routes. For a route rejected when RPKI data changed, the attributes as stored, including those added by the import policy. |
+
 ## Kernel
 
 `/api/kernel` returns the state of kernel route installation: