From 599cd0ad8b1d4d9b68426ea1792f146954e5be94 Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Fri, 2 Oct 2026 13:14:41 +0200 Subject: [PATCH] Look routes up, and see and alert on the ones that were refused Two things were hard to find out from the status page and the API: what the table has for a given prefix or address, and what a peer announced that was refused and why. Looking up. /api/routes/received takes a query, and the status page has a box for it: - prefix: every route for that prefix, the best first; - address: every prefix that covers the address, most specific first; - within: a prefix and everything inside it; - peer (by name or address) and best narrow any of these. A query that makes no sense is answered 400 with the reason, which the page shows. Results come in address order. Routes are shown with the peer's configured name. Refused routes. A refused route is not stored, so until now there was only a count. For each peer the server now keeps how many routes were refused for each reason, and the latest 200 with the AS path they came with. The record is kept by the peer's address and outlives the session, so that a peer that was cut off can still be looked into. It is at /api/routes/rejected and in a new section of the status page; the peers table links to it. Alerting. bgp_routes_rejected_total is now by peer as well as reason. New: bgp_peer_routes and bgp_peer_max_prefixes, to see a peer nearing its limit, and bgp_prefix_limit_exceeded_total. The README has rules to start from. The log has a line for a peer's first refused route and then at most one a minute, with the count of the others. Co-Authored-By: Claude Fable 5.1 --- bin/bgp/README.md | 67 ++++- bin/bgp/src/main.rs | 1 + crates/integration-tests/src/test_server.rs | 6 + crates/integration-tests/tests/debugging.rs | 290 ++++++++++++++++++++ crates/rib/src/table/mod.rs | 11 +- crates/rib/src/table/tests.rs | 29 ++ crates/server/src/connection.rs | 28 +- crates/server/src/lib.rs | 1 + crates/server/src/metrics.rs | 35 ++- crates/server/src/policy.rs | 7 +- crates/server/src/refusals.rs | 258 +++++++++++++++++ crates/server/src/routes.rs | 28 +- crates/server/src/server.rs | 13 +- crates/web/src/api.rs | 250 +++++++++++++++-- crates/web/src/lib.rs | 9 +- crates/web/static/statusz.html | 126 ++++++++- 16 files changed, 1110 insertions(+), 49 deletions(-) create mode 100644 crates/integration-tests/tests/debugging.rs create mode 100644 crates/server/src/refusals.rs diff --git a/bin/bgp/README.md b/bin/bgp/README.md index f2fe027..077435d 100644 --- a/bin/bgp/README.md +++ b/bin/bgp/README.md @@ -21,7 +21,8 @@ When `http_endpoint` is set the binary also serves: | `GET /api/config` | The configuration file as read at startup | | `GET /api/peers` | Configured peers and established sessions | | `GET /api/routes/local` | Originated routes | -| `GET /api/routes/received` | Adj-RIB-In contents | +| `GET /api/routes/received` | Routes learned from peers; see "Finding a route" | +| `GET /api/routes/rejected` | Routes peers announced that were refused, by peer | | `GET /api/events` | Server-sent events for session and route changes | | `GET /metricz` | Prometheus metrics | @@ -63,6 +64,70 @@ A refused route is dropped, along with the route it would have replaced. `bgp_routes_rejected_total` counts them by reason, and the status page shows how many routes each peer has and how many were refused. +## Finding a route + +The status page has a look-up box over the received routes, and +`/api/routes/received` takes the same questions: + +| Query | Answer | +| --- | --- | +| `?prefix=193.0.0.0/21` | Every route for that prefix, the best first | +| `?address=193.0.0.1` | Every prefix that covers the address, most specific first | +| `?within=193.0.0.0/16` | That prefix and everything inside it | +| `&peer=NAME` | Only routes from that peer (its name or its address) | +| `&best=true` | Only the route chosen for each prefix | +| `&limit=N` | At most N routes (1000 unless given, 100000 at most) | + +`within`, `peer` and `best` on their own look through the whole table, which +on a full table takes a moment. + +## Seeing and alerting on refused routes + +A refused route is not in the table. What is kept instead, for each peer, is +how many were refused for each reason and the latest 200 of them with their +AS paths. The status page lists them under "Refused routes"; +`/api/routes/rejected[?peer=NAME]` has them as JSON. The record is kept when +the session ends, so a peer that was cut off can still be looked into. The +log has a line for a peer's first refused route and then at most one a +minute. + +For alerting, `/metricz` has: + +| Metric | Meaning | +| --- | --- | +| `bgp_routes_rejected_total{peer,reason}` | Routes refused, by peer and why | +| `bgp_peer_routes{peer}` | Routes an established peer has in the table | +| `bgp_peer_max_prefixes{peer}` | The peer's limit, where it has one | +| `bgp_prefix_limit_exceeded_total{peer}` | Sessions ended for going over the limit | +| `bgp_connections_refused_total{reason}` | Connections closed without a session | + +Rules to start from: + +```yaml +groups: + - name: bgp + rules: + # A peer is announcing what its policy does not allow: a leak or a + # hijack on its side, or a list on ours that is out of date. + - alert: BgpRoutesRefused + expr: sum by (peer, reason) (increase(bgp_routes_rejected_total[10m])) > 0 + labels: { severity: warning } + # A route that has been through our own AS came back: a loop, or + # someone announcing our AS number. + - alert: BgpOwnAsInPath + expr: increase(bgp_routes_rejected_total{reason="own AS in path"}[10m]) > 0 + labels: { severity: page } + # A peer is close to its limit; the limit may need raising before the + # session is cut. + - alert: BgpPeerNearPrefixLimit + expr: bgp_peer_routes / on (peer) bgp_peer_max_prefixes > 0.8 + for: 10m + labels: { severity: warning } + - alert: BgpPrefixLimitExceeded + expr: increase(bgp_prefix_limit_exceeded_total[1h]) > 0 + labels: { severity: page } +``` + ## Trying it without a router `fake_peer` is a minimal BGP speaker that connects to the daemon, announces a diff --git a/bin/bgp/src/main.rs b/bin/bgp/src/main.rs index a00176f..4678e23 100644 --- a/bin/bgp/src/main.rs +++ b/bin/bgp/src/main.rs @@ -50,6 +50,7 @@ async fn main() -> Result<()> { registry: server.registry.clone(), local_rib: server.local_rib.clone(), rib: Arc::clone(&server.rib), + refusals: server.refusals.clone(), }; tokio::spawn(async move { if let Err(e) = bgp_web::serve_on(state, listener).await { diff --git a/crates/integration-tests/src/test_server.rs b/crates/integration-tests/src/test_server.rs index 04e4726..e1b1f32 100644 --- a/crates/integration-tests/src/test_server.rs +++ b/crates/integration-tests/src/test_server.rs @@ -7,6 +7,7 @@ use bgp_rib::LocalRib; use bgp_rib::table::{Event, PeerIndex, Reader, RibTable}; use bgp_server::config::Config; use bgp_server::error::ServerError; +use bgp_server::refusals::RefusalLog; use bgp_server::server::BgpServer; use bgp_server::session::SessionRegistry; use bgp_web::{AppState, RouterInfo}; @@ -26,6 +27,8 @@ pub struct TestServer { pub local_rib: LocalRib, /// Live view of the server's established sessions. pub registry: SessionRegistry, + /// What peers announced that was not taken. + pub refusals: RefusalLog, pub local_asn: u32, } @@ -68,6 +71,7 @@ impl TestServer { let rib = Arc::clone(&bound.rib); let local_rib = bound.local_rib.clone(); let registry = bound.registry.clone(); + let refusals = bound.refusals.clone(); let handle = tokio::spawn(bound.run()); Ok(Self { @@ -76,6 +80,7 @@ impl TestServer { rib, local_rib, registry, + refusals, local_asn, }) } @@ -193,6 +198,7 @@ impl TestServer { registry: self.registry.clone(), local_rib: self.local_rib.clone(), rib: Arc::clone(&self.rib), + refusals: self.refusals.clone(), }; let handle = tokio::spawn(bgp_web::serve_on(state, listener)); Ok((base_url, HttpGuard { handle })) diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs new file mode 100644 index 0000000..cd8aba8 --- /dev/null +++ b/crates/integration-tests/tests/debugging.rs @@ -0,0 +1,290 @@ +//! Finding out what the server has and why: looking routes up, and seeing +//! what was refused. + +use std::net::Ipv4Addr; +use std::time::Duration; + +use bgp_integration_tests::test_peer::TestPeer; +use bgp_integration_tests::test_server::TestServer; +use bgp_packet::message::{ + AsPathAttribute, Message, NextHopPathAttribute, OriginPathAttribute, PathAttribute, + UpdateMessage, +}; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_server::config::PeerConfig; +use bgp_server::family::FamilySet; +use bgp_server::policy::ImportPolicy; +use eyre::Result; +use serde_json::Value; +use tokio::net::TcpListener; + +const TIMEOUT: Duration = Duration::from_secs(5); + +fn announcement(prefixes: &[&str], as_path: &[u32]) -> Message { + Message::Update(UpdateMessage { + withdrawn_routes: vec![], + path_attributes: vec![ + PathAttribute::Origin(OriginPathAttribute::IGP), + AsPathAttribute::from_asns(as_path.to_vec()), + PathAttribute::NextHop(NextHopPathAttribute(Ipv4Addr::new(192, 0, 2, 1))), + ], + announced_routes: prefixes + .iter() + .map(|p| IpPrefix::try_from(*p).unwrap().into()) + .collect(), + }) +} + +/// The routes a query returns, as "prefix from AS, best or not". +async fn routes(client: &reqwest::Client, base: &str, query: &str) -> Result> { + let reply = client + .get(format!("{base}/api/routes/received?{query}")) + .send() + .await? + .error_for_status()?; + let routes: Value = reply.json().await?; + Ok(routes + .as_array() + .unwrap() + .iter() + .map(|r| { + format!( + "{} AS{}{}", + r["prefix"].as_str().unwrap(), + r["peer_asn"], + if r["best"] == true { " best" } else { "" } + ) + }) + .collect()) +} + +/// The questions asked when something is wrong with a prefix: who sends it +/// and which route won, what covers an address, what is inside a block. +#[tokio::test] +async fn test_routes_can_be_looked_up() -> Result<()> { + let server = TestServer::start(65000).await?; + let (base, _http) = server.serve_http().await?; + let client = reqwest::Client::new(); + + let mut one = TestPeer::connect(server.addr).await?; + one.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?; + let mut two = TestPeer::connect(server.addr).await?; + two.handshake(65002, Ipv4Addr::new(10, 0, 0, 2), 90).await?; + one.send(&announcement( + &["10.0.0.0/8", "10.1.0.0/16", "10.1.2.0/24", "192.0.2.0/24"], + &[65001], + )) + .await?; + // A longer path for one of the same prefixes: it loses. + two.send(&announcement(&["10.1.0.0/16"], &[65002, 65003])) + .await?; + tokio::time::timeout(TIMEOUT, async { + while server.rib.len() != 5 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + + // One prefix: every route for it, the best first. + assert_eq!( + routes(&client, &base, "prefix=10.1.0.0/16").await?, + ["10.1.0.0/16 AS65001 best", "10.1.0.0/16 AS65002"] + ); + assert!( + routes(&client, &base, "prefix=10.2.0.0/16") + .await? + .is_empty() + ); + + // An address: every prefix that covers it, the most specific first. + assert_eq!( + routes(&client, &base, "address=10.1.2.3").await?, + [ + "10.1.2.0/24 AS65001 best", + "10.1.0.0/16 AS65001 best", + "10.1.0.0/16 AS65002", + "10.0.0.0/8 AS65001 best" + ] + ); + assert_eq!( + routes(&client, &base, "address=10.200.0.1").await?, + ["10.0.0.0/8 AS65001 best"] + ); + + // A block: itself and everything inside it. + assert_eq!( + routes(&client, &base, "within=10.1.0.0/16").await?, + [ + "10.1.0.0/16 AS65001 best", + "10.1.0.0/16 AS65002", + "10.1.2.0/24 AS65001 best" + ] + ); + + // Only what was chosen; and everything, in address order. + assert_eq!( + routes(&client, &base, "within=10.0.0.0/8&best=true").await?, + [ + "10.0.0.0/8 AS65001 best", + "10.1.0.0/16 AS65001 best", + "10.1.2.0/24 AS65001 best" + ] + ); + assert_eq!(routes(&client, &base, "").await?.len(), 5); + assert_eq!(routes(&client, &base, "limit=2").await?.len(), 2); + + // One peer's routes, by its address; a peer that is not there has none. + assert_eq!(routes(&client, &base, "peer=127.0.0.1").await?.len(), 5); + assert!(routes(&client, &base, "peer=192.0.2.9").await?.is_empty()); + + // A query that makes no sense says so. + for (query, complaint) in [ + ("prefix=10.1.0.0", "is not a prefix"), + ("address=example", "is not an IP address"), + ("prefix=10.0.0.0/8&address=10.0.0.1", "at most one of"), + ] { + let reply = client + .get(format!("{base}/api/routes/received?{query}")) + .send() + .await?; + assert_eq!(reply.status(), 400, "{query}"); + let text = reply.text().await?; + assert!(text.contains(complaint), "{query}: {text}"); + } + Ok(()) +} + +/// A refused route is not in the table, so it has to be findable somewhere +/// else: which peer, which prefix, with what path, and why. And there is +/// something to alert on. +#[tokio::test] +async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let endpoint = listener.local_addr()?; + let server = TestServer::start_with(1299, |c| { + c.accept_unconfigured_peers = false; + c.peers = vec![PeerConfig { + name: "refused-demo".into(), + endpoint, + remote_asn: 3333, + hold_time_secs: 90, + local_address: None, + families: FamilySet::supported(), + add_path: None, + import: ImportPolicy { + max_prefixes: Some(100), + ..ImportPolicy::default() + }, + }]; + }) + .await?; + let (base, _http) = server.serve_http().await?; + let client = reqwest::Client::new(); + let get = |path: &str| { + let request = client.get(format!("{base}{path}")); + async move { Ok::<_, eyre::Report>(request.send().await?.error_for_status()?) } + }; + + let refused: Value = get("/api/routes/rejected").await?.json().await?; + assert_eq!(refused, serde_json::json!([]), "nothing refused yet"); + + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake(3333, Ipv4Addr::new(10, 0, 0, 2), 90).await?; + peer.send(&announcement(&["193.0.0.0/21", "10.0.0.0/8"], &[3333])) + .await?; + peer.send(&announcement( + &["194.0.1.0/24", "194.0.2.0/24"], + &[3333, 64512], + )) + .await?; + peer.send(&announcement(&["193.0.9.0/25"], &[3333, 2121])) + .await?; + tokio::time::timeout(TIMEOUT, async { + while server + .registry + .list() + .first() + .map(|s| s.counters.rejected()) + != Some(4) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + + let refused: Value = get("/api/routes/rejected").await?.json().await?; + let record = &refused[0]; + assert_eq!(record["peer"], "refused-demo"); + assert_eq!(record["peer_addr"], "127.0.0.1"); + assert_eq!(record["total"], 4); + assert_eq!( + record["by_reason"], + serde_json::json!({"bogon AS in path": 2, "bogon prefix": 1, "prefix too specific": 1}) + ); + // The latest first, each with the path it came with. + let recent: Vec<(String, String, String)> = record["recent"] + .as_array() + .unwrap() + .iter() + .map(|r| { + ( + r["prefix"].as_str().unwrap().to_string(), + r["reason"].as_str().unwrap().to_string(), + r["as_path"].to_string(), + ) + }) + .collect(); + let row = |p: &str, why: &str, path: &str| (p.to_string(), why.to_string(), path.to_string()); + assert_eq!( + recent, + [ + row("193.0.9.0/25", "prefix too specific", "[3333,2121]"), + row("194.0.2.0/24", "bogon AS in path", "[3333,64512]"), + row("194.0.1.0/24", "bogon AS in path", "[3333,64512]"), + row("10.0.0.0/8", "bogon prefix", "[3333]"), + ] + ); + // Asked for by peer, by name or by address. + for peer in ["refused-demo", "127.0.0.1"] { + let one: Value = get(&format!("/api/routes/rejected?peer={peer}")) + .await? + .json() + .await?; + assert_eq!(one.as_array().unwrap().len(), 1, "{peer}"); + } + let none: Value = get("/api/routes/rejected?peer=someone-else") + .await? + .json() + .await?; + assert_eq!(none, serde_json::json!([])); + + // The peer's line in the list of peers has the numbers too. + let peers: Value = get("/api/peers").await?.json().await?; + assert_eq!(peers[0]["routes"], 1); + assert_eq!(peers[0]["rejected_routes"], 4); + + // What an alert would be made from. + let metrics = get("/metricz").await?.text().await?; + for line in [ + r#"bgp_routes_rejected_total{peer="refused-demo",reason="bogon AS in path"} 2"#, + r#"bgp_routes_rejected_total{peer="refused-demo",reason="bogon prefix"} 1"#, + r#"bgp_peer_routes{peer="refused-demo"} 1"#, + r#"bgp_peer_max_prefixes{peer="refused-demo"} 100"#, + ] { + assert!(metrics.contains(line), "{line} is not in:\n{metrics}"); + } + + // The record outlives the session; the count of routes does not. + drop(peer); + tokio::time::timeout(TIMEOUT, async { + while !server.registry.list().is_empty() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + let refused: Value = get("/api/routes/rejected").await?.json().await?; + assert_eq!(refused[0]["total"], 4); + let metrics = get("/metricz").await?.text().await?; + assert!(!metrics.contains(r#"bgp_peer_routes{peer="refused-demo"}"#)); + Ok(()) +} diff --git a/crates/rib/src/table/mod.rs b/crates/rib/src/table/mod.rs index dc71ae7..443570e 100644 --- a/crates/rib/src/table/mod.rs +++ b/crates/rib/src/table/mod.rs @@ -533,10 +533,19 @@ impl RibTable { /// particular order. For showing some of a table that is too large to /// show all of. pub fn some_routes(&self, limit: usize) -> Vec { + self.routes_where(limit, |_| true) + } + + /// Up to `limit` of the routes that `wanted` picks, in no particular + /// order. This looks at routes until it has enough, which for a choosy + /// `wanted` is every route in the table: it is for looking into the + /// table by hand, not for anything done often. `wanted` is called with + /// a shard held and must not call into the table. + pub fn routes_where(&self, limit: usize, wanted: impl Fn(&RouteRef<'_>) -> bool) -> Vec { let mut routes = Vec::new(); for shard in &self.shards { for row in &read(shard).rows { - for route in row.routes() { + for route in row.routes().filter(&wanted) { if routes.len() == limit { return routes; } diff --git a/crates/rib/src/table/tests.rs b/crates/rib/src/table/tests.rs index 32266f4..5b8291d 100644 --- a/crates/rib/src/table/tests.rs +++ b/crates/rib/src/table/tests.rs @@ -763,3 +763,32 @@ fn a_row_can_hold_more_routes_than_a_sixteen_bit_index_counts() { assert!(select::after_remove(&mut row, count, &peers, &mut scratch)); assert_eq!(row.route(row.best()).path_id, 1); } + +#[test] +fn routes_can_be_picked_out_of_the_table() { + let fixture = Fixture::new(&[external(1), external(2)]); + for n in 0..50u8 { + let p = prefix(&format!("10.{n}.0.0/16")); + fixture.announce(0, &p, &Route::via(&[1])); + if n % 2 == 0 { + fixture.announce(1, &p, &Route::via(&[2, 3])); + } + } + let table = &fixture.table; + let second = fixture.peers[1]; + assert_eq!(table.routes_where(1000, |_| true).len(), 75); + assert_eq!( + table.routes_where(1000, |route| route.peer == second).len(), + 25 + ); + assert_eq!(table.routes_where(1000, |route| route.best).len(), 50); + let inside = prefix("10.16.0.0/12"); + let found = table.routes_where(1000, |route| inside.contains(&route.prefix)); + assert_eq!(found.len(), 16 + 8); + // No more than asked for, however many there are. + assert_eq!( + table.routes_where(10, |route| route.peer == second).len(), + 10 + ); + assert!(table.routes_where(0, |_| true).is_empty()); +} diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index a6b30e1..73c6141 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -22,6 +22,7 @@ use bgp_packet::update_decode::UpdateError; use bgp_primitives::parser::ParserContext; use bgp_rib::table::{Outcome, PeerIndex, PeerInfo, RibTable}; use bytes::BytesMut; +use prometheus::IntGauge; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; @@ -35,10 +36,12 @@ use crate::error::SessionError; use crate::family::FamilySet; use crate::fsm::{Action, Event, Fsm, FsmConfig, Negotiated, Received}; use crate::metrics::{ - MESSAGES_RECEIVED, MESSAGES_SENT, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, + MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ROUTES, PREFIX_LIMIT_EXCEEDED, + SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, }; use crate::panic::contained; use crate::policy::{self, HoldDowns, ImportPolicy}; +use crate::refusals::{PeerRefusals, RefusalLog}; use crate::routes; use crate::session::{PeerId, SessionCounters, SessionGuard, SessionHandle, SessionRegistry}; @@ -69,6 +72,8 @@ pub struct ServerContext { pub collisions: CollisionTable, /// Peers that are being kept down for going over their limits. pub hold_downs: HoldDowns, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } /// What is configured for the peer at the other end of a connection. @@ -159,6 +164,9 @@ pub async fn run_session( .server .hold_downs .hold(peer_addr.ip(), session.import.hold_down); + PREFIX_LIMIT_EXCEEDED + .with_label_values(&[session.refusals.label()]) + .inc(); } // Whatever ended the session, routes learned over it are gone @@ -172,6 +180,8 @@ pub async fn run_session( .. } = established; let outcome = routes.remove().await; + // The peer has no routes now, and no session to count them for. + let _ = PEER_ROUTES.remove_label_values(&[session.refusals.label()]); info!( peer = %peer_id, removed_routes = outcome.removed, @@ -199,6 +209,8 @@ struct Established { from: policy::From, /// How many routes the peer has, and how many were refused. counters: Arc, + /// The same number of routes, where metrics are read from. + routes_metric: IntGauge, /// Keeps the session listed in the registry. _registration: SessionGuard, } @@ -302,6 +314,8 @@ struct Session { peer_name: Option, /// Which of the peer's routes are taken. import: Arc, + /// Where what is refused is noted. + refusals: Arc, } impl Session { @@ -324,7 +338,13 @@ impl Session { families: peer.families, add_path: peer.add_path, }); + // The peer goes by its configured name, or by its address. + let label = peer + .name + .clone() + .unwrap_or_else(|| peer_addr.ip().to_string()); let session = Session { + refusals: server.refusals.of_peer(peer_addr.ip(), &label), fsm, frames: FrameReader::new(reader), writer, @@ -604,6 +624,7 @@ impl Session { four_octet_asn: negotiated.four_octet_asn, }, counters: Arc::clone(&handle.counters), + routes_metric: PEER_ROUTES.with_label_values(&[self.refusals.label()]), _registration: self.server.registry.register(handle), }); Ok(()) @@ -622,6 +643,7 @@ impl Session { peer: established.rib_peer, from: established.from, policy: &self.import, + refusals: &self.refusals, }; let routes::Applied { outcome, rejected } = routes::apply(&self.server.rib, &import, update, attributes_wire)?; @@ -639,6 +661,7 @@ impl Session { let held = established .counters .note(outcome.added, outcome.removed, rejected); + established.routes_metric.set(held as i64); match self.import.max_prefixes { Some(limit) if held > u64::from(limit) => { Err(SessionError::PrefixLimitExceeded { limit }) @@ -907,6 +930,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), + refusals: RefusalLog::default(), }); let (mut session, _) = Session::open( stream, @@ -953,6 +977,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), + refusals: RefusalLog::default(), }); let (mut session, _) = Session::open( stream, @@ -1016,6 +1041,7 @@ mod tests { four_octet_asn: true, }, policy: &ImportPolicy::permissive(), + refusals: &RefusalLog::default().of_peer(Ipv4Addr::new(192, 0, 2, 1).into(), "test"), }; routes::apply(&table, &import, &update, None).unwrap(); assert_eq!(table.len(), 1); diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index d3edefb..e117bec 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -9,6 +9,7 @@ pub mod fsm; pub mod metrics; mod panic; pub mod policy; +pub mod refusals; mod routes; pub mod server; pub mod session; diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index 1f873e8..b17ff6e 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -4,8 +4,8 @@ use std::sync::LazyLock; use bgp_packet::message::Message; use prometheus::{ - IntCounter, IntCounterVec, IntGauge, register_int_counter, register_int_counter_vec, - register_int_gauge, + IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_int_counter, + register_int_counter_vec, register_int_gauge, register_int_gauge_vec, }; pub static MESSAGES_SENT: LazyLock = LazyLock::new(|| { @@ -54,8 +54,35 @@ pub static SESSIONS_ESTABLISHED: LazyLock = LazyLock::new(|| { pub static ROUTES_REJECTED: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "bgp_routes_rejected_total", - "Routes announced by peers and refused by their import policy, by why", - &["reason"] + "Routes announced by a peer and refused by its import policy, by why", + &["peer", "reason"] + ) + .expect("metric registration") +}); + +pub static PEER_ROUTES: LazyLock = LazyLock::new(|| { + register_int_gauge_vec!( + "bgp_peer_routes", + "Routes of an established peer that are in the routing table", + &["peer"] + ) + .expect("metric registration") +}); + +pub static PEER_MAX_PREFIXES: LazyLock = LazyLock::new(|| { + register_int_gauge_vec!( + "bgp_peer_max_prefixes", + "The most routes a configured peer may have (import.max_prefixes), where there is a limit", + &["peer"] + ) + .expect("metric registration") +}); + +pub static PREFIX_LIMIT_EXCEEDED: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "bgp_prefix_limit_exceeded_total", + "Sessions ended because the peer went over its limit of routes", + &["peer"] ) .expect("metric registration") }); diff --git a/crates/server/src/policy.rs b/crates/server/src/policy.rs index 6125a6a..39c2bec 100644 --- a/crates/server/src/policy.rs +++ b/crates/server/src/policy.rs @@ -151,6 +151,11 @@ impl Rejection { Self::OriginNotAllowed, ]; + /// Where the reason is in [`Rejection::ALL`]. + pub(crate) fn index(self) -> usize { + self as usize + } + pub fn as_str(self) -> &'static str { match self { Self::BogonPrefix => "bogon prefix", @@ -300,7 +305,7 @@ impl PathFacts { } /// The AS numbers of a path in order, sets and all. -fn asns(path: Option<&AsPathAttribute>) -> impl Iterator + '_ { +pub(crate) fn asns(path: Option<&AsPathAttribute>) -> impl Iterator + '_ { path.into_iter() .flat_map(|path| &path.segments) .flat_map(|segment| segment.path.iter().copied()) diff --git a/crates/server/src/refusals.rs b/crates/server/src/refusals.rs new file mode 100644 index 0000000..2326902 --- /dev/null +++ b/crates/server/src/refusals.rs @@ -0,0 +1,258 @@ +//! What peers announced that was not taken, kept for looking at. +//! +//! A refused route is not stored, so without this there would be nothing +//! to show for it but a number. For each peer the reasons are counted and +//! the latest refusals are kept: enough to see what a peer is doing wrong, +//! or what in our policy is. They are kept by the peer's address and +//! outlive the session, since the session a peer was cut off on is the one +//! worth looking at. + +use std::collections::{HashMap, VecDeque}; +use std::net::IpAddr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; +use prometheus::IntCounter; +use tracing::warn; + +use crate::metrics::ROUTES_REJECTED; +use crate::policy::Rejection; +use crate::sync::IgnorePoison; + +/// How many of a peer's latest refused routes are kept. +pub const RECENT_REFUSALS: usize = 200; + +/// How many peers refusals are kept for. More than any configuration has; +/// it bounds what hosts that are let in without being configured (on a +/// test network) can make us keep. +const MAX_PEERS: usize = 8192; + +/// At most one line in the log about a peer's refused routes in this long. +/// The first of them is logged at once. +const LOG_INTERVAL: Duration = Duration::from_secs(60); + +/// A route a peer announced that its import policy refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefusedRoute { + pub prefix: IpPrefix, + /// The peer's identifier for the route; zero without ADD-PATH. + pub path_id: u32, + pub reason: Rejection, + /// The AS path it came with. + pub as_path: Arc<[u32]>, + pub at: SystemTime, +} + +/// The refusals of one peer. +pub struct PeerRefusals { + /// What the peer is called in the log and in metrics: its configured + /// name, or its address. + label: String, + by_reason: [AtomicU64; Rejection::ALL.len()], + metrics: [IntCounter; Rejection::ALL.len()], + state: Mutex, +} + +struct State { + /// The latest refused routes, oldest first. + recent: VecDeque, + logged: Option, + /// How many were refused since the log last said so. + unlogged: u64, +} + +impl PeerRefusals { + fn new(label: &str) -> Self { + Self { + label: label.to_string(), + by_reason: Default::default(), + metrics: Rejection::ALL + .map(|reason| ROUTES_REJECTED.with_label_values(&[label, reason.as_str()])), + state: Mutex::new(State { + recent: VecDeque::new(), + logged: None, + unlogged: 0, + }), + } + } + + /// Notes that `routes`, which came with the AS path `as_path`, were + /// refused for `reason`. + pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, as_path: &Arc<[u32]>) { + let Some(latest) = routes.last() else { + return; + }; + let count = routes.len() as u64; + self.by_reason[reason.index()].fetch_add(count, Ordering::Relaxed); + self.metrics[reason.index()].inc_by(count); + + let at = SystemTime::now(); + let mut state = self.state.locked(); + // Only the last of them can still be in the list afterwards. + let kept = &routes[routes.len().saturating_sub(RECENT_REFUSALS)..]; + for route in kept { + if state.recent.len() == RECENT_REFUSALS { + state.recent.pop_front(); + } + state.recent.push_back(RefusedRoute { + prefix: route.prefix, + path_id: route.path_id, + reason, + as_path: Arc::clone(as_path), + at, + }); + } + + // A peer that sends a table of routes we refuse must not write a + // table of lines to the log. + if state + .logged + .is_some_and(|logged| logged.elapsed() < LOG_INTERVAL) + { + state.unlogged += count; + return; + } + warn!( + peer = %self.label, + prefix = %latest.prefix, + reason = reason.as_str(), + as_path = ?&as_path[..], + others_since_last_report = state.unlogged + count - 1, + "Refused a route" + ); + state.logged = Some(Instant::now()); + state.unlogged = 0; + } + + pub fn label(&self) -> &str { + &self.label + } + + /// How many routes were refused, since the server started. + pub fn total(&self) -> u64 { + self.by_reason + .iter() + .map(|count| count.load(Ordering::Relaxed)) + .sum() + } + + /// How many routes were refused for each reason that came up. + pub fn by_reason(&self) -> Vec<(Rejection, u64)> { + Rejection::ALL + .into_iter() + .map(|reason| { + ( + reason, + self.by_reason[reason.index()].load(Ordering::Relaxed), + ) + }) + .filter(|(_, count)| *count > 0) + .collect() + } + + /// The latest refused routes, newest first. + pub fn recent(&self) -> Vec { + self.state.locked().recent.iter().rev().cloned().collect() + } +} + +/// The refusals of every peer, by address. Cheap to clone; clones share +/// one record. +#[derive(Clone, Default)] +pub struct RefusalLog(Arc>>>); + +impl RefusalLog { + /// The record for the peer at `address`, made if there was none. + /// `label` is what to call the peer; see [`PeerRefusals::label`]. + pub(crate) fn of_peer(&self, address: IpAddr, label: &str) -> Arc { + let mut peers = self.0.locked(); + if let Some(existing) = peers.get(&address) { + return Arc::clone(existing); + } + let record = Arc::new(PeerRefusals::new(label)); + // Past the bound the record is still made, for the session to + // count in, but it is not kept after. + if peers.len() < MAX_PEERS { + peers.insert(address, Arc::clone(&record)); + } + record + } + + /// Every peer that has had a route refused, with its record. + pub fn peers(&self) -> Vec<(IpAddr, Arc)> { + self.0 + .locked() + .iter() + .filter(|(_, record)| record.total() > 0) + .map(|(address, record)| (*address, Arc::clone(record))) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn route(n: u32) -> Nlri { + let bytes = (0xc100_0000u32 + (n << 8)).to_be_bytes(); + IpPrefix::new(bgp_primitives::constants::IpVersion::Ipv4, bytes, 24) + .unwrap() + .into() + } + + #[test] + fn refusals_are_counted_by_reason_and_the_latest_are_kept() { + let log = RefusalLog::default(); + let address: IpAddr = "192.0.2.1".parse().unwrap(); + let record = log.of_peer(address, "refusals-test-1"); + assert!(log.peers().is_empty(), "nothing refused yet"); + + let path: Arc<[u32]> = vec![3333, 64512].into(); + record.note(&[route(1), route(2)], Rejection::BogonAs, &path); + record.note(&[route(3)], Rejection::TooSpecific, &path); + + assert_eq!(record.total(), 3); + assert_eq!( + record.by_reason(), + [(Rejection::TooSpecific, 1), (Rejection::BogonAs, 2)] + ); + let recent = record.recent(); + assert_eq!( + recent.iter().map(|r| r.prefix).collect::>(), + [route(3).prefix, route(2).prefix, route(1).prefix], + "newest first" + ); + assert_eq!(recent[0].reason, Rejection::TooSpecific); + assert_eq!(&recent[2].as_path[..], [3333, 64512]); + + // The peer's next session finds the same record. + let again = log.of_peer(address, "another-name"); + assert_eq!(again.total(), 3); + assert_eq!(again.label(), "refusals-test-1"); + assert_eq!(log.peers().len(), 1); + } + + /// A peer that sends a great many routes we refuse costs a fixed + /// amount of memory. + #[test] + fn only_so_many_refused_routes_are_kept() { + let record = PeerRefusals::new("refusals-test-2"); + let path: Arc<[u32]> = vec![3333].into(); + let many: Vec = (0..1000).map(route).collect(); + record.note(&many, Rejection::BogonPrefix, &path); + for n in 1000..1010 { + record.note(&[route(n)], Rejection::BogonPrefix, &path); + } + + assert_eq!(record.total(), 1010); + let recent = record.recent(); + assert_eq!(recent.len(), RECENT_REFUSALS); + assert_eq!(recent[0].prefix, route(1009).prefix); + assert_eq!( + recent[RECENT_REFUSALS - 1].prefix, + route(1010 - RECENT_REFUSALS as u32).prefix + ); + } +} diff --git a/crates/server/src/routes.rs b/crates/server/src/routes.rs index 0210644..24f3a00 100644 --- a/crates/server/src/routes.rs +++ b/crates/server/src/routes.rs @@ -2,6 +2,7 @@ //! as the peer's import policy lets them in. use std::borrow::Cow; +use std::cell::OnceCell; use std::sync::Arc; use bgp_packet::message::{NlriNextHop, PathAttribute, UpdateMessage}; @@ -9,8 +10,8 @@ use bgp_primitives::ip_prefix::Nlri; use bgp_primitives::parser::EncodeError; use bgp_rib::table::{AttrSet, Outcome, PeerIndex, RibTable}; -use crate::metrics::ROUTES_REJECTED; -use crate::policy::{From, ImportPolicy, Rejection}; +use crate::policy::{self, From, ImportPolicy, Rejection}; +use crate::refusals::PeerRefusals; /// What applying an UPDATE did. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -26,6 +27,8 @@ pub(crate) struct Import<'a> { pub(crate) peer: PeerIndex, pub(crate) from: From, pub(crate) policy: &'a ImportPolicy, + /// Where what is refused is noted. + pub(crate) refusals: &'a PeerRefusals, } /// Applies `update` to `table`: its withdrawals first and then its @@ -62,6 +65,9 @@ pub(crate) fn apply( } } + // The path the routes came with, for the record of what is refused: + // made only if something is. + let as_path = OnceCell::new(); // What the routes have in common is judged once. let path_verdict = import .policy @@ -80,9 +86,9 @@ pub(crate) fn apply( return Ok(()); } let mut refuse = |refused: &[Nlri], rejection: Rejection| { - ROUTES_REJECTED - .with_label_values(&[rejection.as_str()]) - .inc_by(refused.len() as u64); + import + .refusals + .note(refused, rejection, as_path.get_or_init(|| path_of(update))); applied.rejected += refused.len(); applied.outcome += table.withdraw(peer, refused); }; @@ -108,6 +114,18 @@ pub(crate) fn apply( Ok(applied) } +/// The AS numbers of the UPDATE's AS_PATH. +fn path_of(update: &UpdateMessage) -> Arc<[u32]> { + let path = update + .path_attributes + .iter() + .find_map(|attribute| match attribute { + PathAttribute::AsPath(path) => Some(path), + _ => None, + }); + policy::asns(path).collect() +} + /// The routes of `nlri` whose prefixes the policy takes; the others are /// given to `refuse`. Nearly always that is all of them, and nothing is /// copied. diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 73bf06e..a66788d 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -22,9 +22,10 @@ use crate::config::Config; use crate::connection::{PeerSettings, ServerContext, run_session}; use crate::dialer::spawn_dialers; use crate::error::ServerError; -use crate::metrics::CONNECTIONS_REFUSED; +use crate::metrics::{CONNECTIONS_REFUSED, PEER_MAX_PREFIXES}; use crate::panic; use crate::policy::HoldDowns; +use crate::refusals::RefusalLog; use crate::session::{ConfiguredPeer, SessionRegistry}; use crate::sync::IgnorePoison; @@ -68,6 +69,8 @@ pub struct BgpServer { pub local_rib: LocalRib, /// Live view of established sessions. pub registry: SessionRegistry, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } impl BgpServer { @@ -102,6 +105,7 @@ impl BgpServer { rib: Arc::new(RibTable::new(table_shards())), local_rib, registry: SessionRegistry::with_configured(configured), + refusals: RefusalLog::default(), config, }) } @@ -127,7 +131,11 @@ impl BgpServer { ); for peer in &self.config.peers { - if peer.import.max_prefixes.is_none() { + if let Some(limit) = peer.import.max_prefixes { + PEER_MAX_PREFIXES + .with_label_values(&[&peer.name]) + .set(limit.into()); + } else { warn!( peer = %peer.name, "No limit on the routes taken from this peer (import.max_prefixes)" @@ -145,6 +153,7 @@ impl BgpServer { registry: self.registry.clone(), collisions: CollisionTable::new(self.config.collision_detect_established_state), hold_downs: HoldDowns::default(), + refusals: self.refusals.clone(), }); // The tasks the server cannot do without. None of them ends while diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index c3ad194..04dd2ed 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -1,17 +1,20 @@ //! JSON handlers and their view types. Everything here is read-only: the //! daemon is configured through its configuration file. +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; use std::time::{SystemTime, UNIX_EPOCH}; use axum::Json; -use std::collections::HashMap; use axum::extract::{Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use bgp_packet::community; use bgp_packet::message::{NlriNextHop, OriginPathAttribute, PathAttribute}; -use bgp_rib::table::{PeerIndex, Route}; +use bgp_primitives::constants::IpVersion; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_rib::table::{PeerIndex, RibTable, Route}; use bgp_rib::{LocalRoute, RibStopped}; use bgp_server::session::{ConfiguredPeer, DialState, PeerId, SessionHandle, SessionRegistry}; use serde::{Deserialize, Serialize}; @@ -249,6 +252,8 @@ pub fn summarize(attributes: &[PathAttribute], next_hop: Option<&NlriNextHop>) - #[derive(Debug, Clone, Serialize)] pub struct ReceivedRouteView { + /// The peer's configured name, if it has one. + pub peer_name: Option, pub peer_addr: String, pub peer_asn: u32, pub prefix: String, @@ -265,7 +270,7 @@ pub struct ReceivedRouteView { /// Who the routes in the table are from: the table knows peers by number, /// the sessions know which number is whose. -pub(crate) struct PeerNames(HashMap); +pub(crate) struct PeerNames(HashMap)>); impl PeerNames { pub(crate) fn of(registry: &SessionRegistry) -> Self { @@ -273,18 +278,31 @@ impl PeerNames { registry .list() .into_iter() - .map(|session| (session.rib_peer, session.peer)) + .map(|session| (session.rib_peer, (session.peer, session.name))) .collect(), ) } + /// The peers that `wanted` names: by configured name, or by address. + fn matching(&self, wanted: &str) -> Vec { + let address: Option = wanted.parse().ok(); + self.0 + .iter() + .filter(|(_, (peer, name))| { + name.as_deref() == Some(wanted) || address == Some(peer.addr.ip()) + }) + .map(|(index, _)| *index) + .collect() + } + /// A view of one route. `None` if the route's session has just ended: /// the route is on its way out of the table. pub(crate) fn view(&self, route: &Route) -> Option { - let peer = self.0.get(&route.peer)?; + let (peer, name) = self.0.get(&route.peer)?; // What a session stored was accepted by the decoder, and parses. let (attributes, next_hop) = route.attrs.attributes().unwrap_or_default(); Some(ReceivedRouteView { + peer_name: name.clone(), peer_addr: peer.addr.to_string(), peer_asn: peer.asn, prefix: route.prefix.to_string(), @@ -379,37 +397,221 @@ const DEFAULT_ROUTE_LIMIT: usize = 1000; /// each is copied out of the table and written as JSON. const MAX_ROUTE_LIMIT: usize = 100_000; -#[derive(Debug, Deserialize)] +/// Which routes to return. At most one of `prefix`, `address` and `within` +/// says where to look; `peer` and `best` narrow what is found there. +#[derive(Debug, Default, Deserialize)] pub struct RouteQuery { /// At most this many routes. limit: Option, + /// The routes for exactly this prefix, e.g. `193.0.0.0/21`. + prefix: Option, + /// The routes for every prefix that covers this address, the most + /// specific prefix first: what a packet to the address could follow. + address: Option, + /// The routes for this prefix and every prefix inside it. + within: Option, + /// Only routes from this peer: its configured name, or its address. + peer: Option, + /// Only the best route of each prefix. + #[serde(default)] + best: bool, } -/// Some of the routes learned from peers: all the routes of as many -/// prefixes as fit in the limit. `/api/status` has the totals. +/// Why a query was not answered. +pub enum QueryError { + /// The query makes no sense; the text says how. + Bad(String), + /// Answering it failed here. + Failed, +} + +impl IntoResponse for QueryError { + fn into_response(self) -> Response { + match self { + Self::Bad(why) => (StatusCode::BAD_REQUEST, why).into_response(), + Self::Failed => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } +} + +/// Where in the table a query looks. +enum Scope { + /// Everywhere. + All, + Exactly(IpPrefix), + Covering(IpPrefix), + Within(IpPrefix), +} + +impl Scope { + fn of(query: &RouteQuery) -> Result { + let prefix = |text: &str| { + IpPrefix::try_from(text) + .map_err(|e| QueryError::Bad(format!("{text:?} is not a prefix: {e}"))) + }; + match (&query.prefix, &query.address, &query.within) { + (None, None, None) => Ok(Self::All), + (Some(text), None, None) => Ok(Self::Exactly(prefix(text)?)), + (None, Some(text), None) => { + let address: IpAddr = text + .parse() + .map_err(|_| QueryError::Bad(format!("{text:?} is not an IP address")))?; + // The address as a prefix as long as an address is. + let host = match address { + IpAddr::V4(v4) => IpPrefix::new(IpVersion::Ipv4, v4.octets(), 32), + IpAddr::V6(v6) => IpPrefix::new(IpVersion::Ipv6, v6.octets(), 128), + }; + host.map(Self::Covering) + .map_err(|e| QueryError::Bad(e.to_string())) + } + (None, None, Some(text)) => Ok(Self::Within(prefix(text)?)), + _ => Err(QueryError::Bad( + "give at most one of prefix, address and within".into(), + )), + } + } +} + +/// The routes a query asks for, in the order to show them. +fn find_routes( + table: &RibTable, + names: &PeerNames, + scope: &Scope, + query: &RouteQuery, + limit: usize, +) -> Vec { + let peers = query.peer.as_deref().map(|peer| names.matching(peer)); + let wanted = |peer: PeerIndex, best: bool| { + (!query.best || best) && peers.as_ref().is_none_or(|peers| peers.contains(&peer)) + }; + let mut routes = match scope { + Scope::Exactly(prefix) => table.routes(prefix), + // From the longest prefix that could cover the address to the + // shortest: one lookup for each length. + Scope::Covering(host) => (0..=host.length()) + .rev() + .flat_map(|length| table.routes(&host.shortened(length))) + .collect(), + Scope::Within(outer) => table.routes_where(limit, |route| { + outer.contains(&route.prefix) && wanted(route.peer, route.best) + }), + Scope::All => table.routes_where(limit, |route| wanted(route.peer, route.best)), + }; + routes.retain(|route| wanted(route.peer, route.best)); + routes.truncate(limit); + + // For each prefix the best route first. The prefixes in address + // order; for a lookup by address, which finds one prefix of each + // length, the most specific first. + if matches!(scope, Scope::Covering(_)) { + routes.sort_by_key(|route| (std::cmp::Reverse(route.prefix.length()), !route.best)); + } else { + routes.sort_by_key(|route| (sort_key(&route.prefix), !route.best)); + } + routes + .iter() + .filter_map(|route| names.view(route)) + .collect() +} + +/// Orders prefixes by IP version, then address, then shorter before +/// longer. +fn sort_key(prefix: &IpPrefix) -> (u16, [u8; 16], u8) { + let mut address = [0u8; 16]; + address[..prefix.bytes().len()].copy_from_slice(prefix.bytes()); + (prefix.address_family() as u16, address, prefix.length()) +} + +/// Routes learned from peers. Without a query, all the routes of as many +/// prefixes as fit in the limit; see [`RouteQuery`] for looking up +/// particular ones. `/api/status` has the totals. pub async fn received_routes( State(state): State, Query(query): Query, -) -> Result>, StatusCode> { +) -> Result>, QueryError> { let limit = query .limit .unwrap_or(DEFAULT_ROUTE_LIMIT) .min(MAX_ROUTE_LIMIT); + let scope = Scope::of(&query)?; let names = PeerNames::of(&state.registry); - // Copying, parsing and sorting many routes takes long enough to keep - // off the threads that sessions run on. - let routes = tokio::task::spawn_blocking(move || { - let mut routes: Vec = state - .rib - .some_routes(limit) - .iter() - .filter_map(|route| names.view(route)) - .collect(); - routes.sort_by(|a, b| (&a.prefix, &a.peer_addr).cmp(&(&b.prefix, &b.peer_addr))); - routes - }) - .await - // Only if making the views panicked. - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + // Looking through the table, copying, parsing and sorting many routes + // takes long enough to keep off the threads that sessions run on. + let routes = + tokio::task::spawn_blocking(move || find_routes(&state.rib, &names, &scope, &query, limit)) + .await + // Only if making the views panicked. + .map_err(|_| QueryError::Failed)?; Ok(Json(routes)) } + +/// What one peer announced that was refused. +#[derive(Debug, Clone, Serialize)] +pub struct RefusalsView { + /// The peer's configured name, or its address if it has none. + pub peer: String, + pub peer_addr: String, + /// How many routes were refused since the daemon started. + pub total: u64, + /// The same, by reason. + pub by_reason: BTreeMap<&'static str, u64>, + /// The latest of them, newest first. + pub recent: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RefusedRouteView { + pub prefix: String, + pub path_id: u32, + pub reason: &'static str, + pub as_path: Vec, + pub at_unix: u64, +} + +#[derive(Debug, Default, Deserialize)] +pub struct RefusalQuery { + /// Only this peer: its configured name, or its address. + peer: Option, +} + +/// The routes that peers announced and their import policies refused: for +/// each peer that had any, how many and why, and the latest of them. +pub async fn rejected_routes( + State(state): State, + Query(query): Query, +) -> Json> { + let mut peers: Vec = state + .refusals + .peers() + .into_iter() + .filter(|(address, record)| { + query + .peer + .as_deref() + .is_none_or(|wanted| record.label() == wanted || wanted.parse() == Ok(*address)) + }) + .map(|(address, record)| RefusalsView { + peer: record.label().to_string(), + peer_addr: address.to_string(), + total: record.total(), + by_reason: record + .by_reason() + .into_iter() + .map(|(reason, count)| (reason.as_str(), count)) + .collect(), + recent: record + .recent() + .into_iter() + .map(|route| RefusedRouteView { + prefix: route.prefix.to_string(), + path_id: route.path_id, + reason: route.reason.as_str(), + as_path: route.as_path.to_vec(), + at_unix: unix_secs(route.at), + }) + .collect(), + }) + .collect(); + peers.sort_by(|a, b| a.peer.cmp(&b.peer)); + Json(peers) +} diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index 5e65f24..0c82043 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -19,11 +19,15 @@ use axum::response::{Html, IntoResponse, Redirect}; use axum::routing::get; use bgp_rib::LocalRib; use bgp_rib::table::RibTable; +use bgp_server::refusals::RefusalLog; use bgp_server::session::SessionRegistry; use tokio::net::TcpListener; use tracing::info; -pub use api::{AttrSummary, ConfigView, LocalRouteView, PeerView, ReceivedRouteView, StatusView}; +pub use api::{ + AttrSummary, ConfigView, LocalRouteView, PeerView, ReceivedRouteView, RefusalsView, + RefusedRouteView, StatusView, +}; /// Static facts about the router shown in status output. #[derive(Debug, Clone)] @@ -81,6 +85,8 @@ pub struct AppState { pub local_rib: LocalRib, /// The routing table: routes learned from peers and the best of them. pub rib: Arc, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } /// The status page, embedded so the binary is self-contained. @@ -103,6 +109,7 @@ pub fn router(state: AppState) -> Router { .route("/api/peers", get(api::peers)) .route("/api/routes/local", get(api::local_routes)) .route("/api/routes/received", get(api::received_routes)) + .route("/api/routes/rejected", get(api::rejected_routes)) .route("/api/events", get(events::stream)) .route("/metricz", get(metrics::metricz)) .with_state(state) diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index eaa5724..8fc2cf7 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -31,6 +31,8 @@ summary { cursor: pointer; color: #00c; } summary:hover { text-decoration: underline; } #config-text { border: 1px solid #99a; padding: 4px 6px; margin: 4px 0; font-size: 12px; overflow-x: auto; } address { font-size: smaller; margin-top: 2em; color: #444; } +form.lookup { margin: 4px 0; } +form.lookup input[type=text] { font-family: monospace; } @@ -45,7 +47,7 @@ Router ID ..., local AS ...
Running on ...
-View: variables, status json, peers json, originated routes json, received routes json
+View: variables, status json, peers json, originated routes json, received routes json, refused routes json

@@ -68,11 +70,30 @@ View: variables, status json, <

Received routes (...)

+
+Look up + +from + + +show all + +
prefixfrom peerAS pathnext hoporiginMEDlocal prefcommunitieslarge communitiesbest
+

Refused routes (...)

+
Routes that peers announced and their import policies did not take. The counts are since the daemon started; the latest refusals of each peer are kept.
+ + + +
peerprefixAS pathreasonwhen
+

Event log

@@ -90,7 +111,7 @@ View: variables, status json, < const $ = id => document.getElementById(id); const esc = s => String(s).replace(/[&<>"]/g, c => ({'&':'&','<':'<','>':'>','"':'"'}[c])); - const state = { status: null, peers: [], local: [], received: [] }; + const state = { status: null, peers: [], local: [], received: [], refused: [] }; const DASH = '-'; // ---- helpers ----------------------------------------------------------- @@ -156,8 +177,55 @@ View: variables, status json, < ${up ? esc(p.local_addr) : DASH}${up ? fmtTime(p.established_at_unix) : DASH} ${up ? age(p.established_at_unix) : DASH}${up ? (p.four_octet_asn ? 'yes' : 'no') : DASH} ${up ? esc((p.address_families || []).join(', ')) + addPath(p) : DASH} - ${up ? p.routes : DASH}${up ? p.rejected_routes : DASH}${detail}`; + ${up ? p.routes : DASH} + ${up ? (p.rejected_routes ? `${p.rejected_routes}` : 0) : DASH}${detail}`; }).join('')); + renderPeerChoices(); + } + + // ---- looking routes up ------------------------------------------------- + // The query lives in the page's address, so that it survives a reload and + // auto-refresh, and can be passed on. + const QUERY_KEYS = ['q', 'mode', 'peer', 'best']; + function lookup() { + const here = new URLSearchParams(location.search); + return { q: (here.get('q') || '').trim(), mode: here.get('mode') || 'exact', peer: here.get('peer') || '', best: here.get('best') === '1' }; + } + function isLookup() { const l = lookup(); return !!(l.q || l.peer || l.best); } + // What to ask the API for: an address finds every prefix that covers it. + function receivedUrl() { + const l = lookup(), ask = new URLSearchParams(); + if (l.q) ask.set(l.mode === 'within' ? 'within' : l.q.includes('/') ? 'prefix' : 'address', l.q); + if (l.peer) ask.set('peer', l.peer); + if (l.best) ask.set('best', 'true'); + const query = ask.toString(); + return '/api/routes/received' + (query ? `?${query}` : ''); + } + function renderPeerChoices() { + const chosen = lookup().peer; + const names = [...new Set(state.peers.filter(p => p.state === 'Established') + .map(p => p.name || p.peer_addr.replace(/:\d+$/, '').replace(/^\[|\]$/g, '')))]; + if (chosen && !names.includes(chosen)) names.push(chosen); + setHTML($('lookup-peer'), '' + + names.map(n => ``).join('')); + } + function showLookup() { + const l = lookup(); + $('lookup-q').value = l.q; $('lookup-mode').value = l.mode; $('lookup-best').checked = l.best; + renderPeerChoices(); + } + function setLookup(l) { + const url = new URL(location.href); + QUERY_KEYS.forEach(k => url.searchParams.delete(k)); + if (l) { + if (l.q) url.searchParams.set('q', l.q); + if (l.q && l.mode === 'within') url.searchParams.set('mode', 'within'); + if (l.peer) url.searchParams.set('peer', l.peer); + if (l.best) url.searchParams.set('best', '1'); + } + history.replaceState(null, '', url.pathname + url.search + '#received'); + showLookup(); renderRefreshControl(); + loadReceived().catch(console.error); } function renderLocal() { @@ -172,23 +240,57 @@ View: variables, status json, < } function renderReceived() { - const b = $('received-body'), st = state.status; - $('received-count').textContent = st && st.received_routes > state.received.length + const b = $('received-body'), st = state.status, looking = isLookup(); + $('received-count').textContent = looking + ? `${state.received.length} found${st ? ` among ${st.received_routes}` : ''}` + : st && st.received_routes > state.received.length ? `showing ${state.received.length} of ${st.received_routes}, for ${st.received_prefixes} prefixes` : `${state.received.length}${st ? `, for ${st.received_prefixes} prefixes` : ''}`; - if (!state.received.length) { setHTML(b, 'No routes received from peers.'); return; } + if (!state.received.length) { + setHTML(b, `${looking ? 'No route matches.' : 'No routes received from peers.'}`); return; + } setHTML(b, state.received.map(r => { const s = r.summary; return ` - ${esc(r.prefix)}${esc(r.peer_addr)} AS${r.peer_asn}${r.path_id ? ` path ${r.path_id}` : ''}${asPath(s.as_path)} + ${esc(r.prefix)}${r.peer_name ? esc(r.peer_name) + ' ' : ''}${esc(r.peer_addr)} AS${r.peer_asn}${r.path_id ? ` path ${r.path_id}` : ''}${asPath(s.as_path)} ${s.next_hop ? esc(s.next_hop) : DASH}${s.origin ? esc(s.origin) : DASH}${s.med ?? DASH}${s.local_pref ?? DASH} ${commList(s.communities, true)}${commList(s.large_communities)}${r.best ? 'best' : ''}`; }).join('')); } + // For each peer a line that says how many and why, then its latest few. + const REFUSED_SHOWN = 20; + function renderRefused() { + const b = $('refused-body'); + const total = state.refused.reduce((n, p) => n + p.total, 0); + $('refused-count').textContent = total ? `${total} from ${state.refused.length} peer${state.refused.length === 1 ? '' : 's'}` : '0'; + if (!state.refused.length) { setHTML(b, 'Nothing a peer announced has been refused.'); return; } + setHTML(b, state.refused.map(p => { + const why = Object.entries(p.by_reason).sort((a, c) => c[1] - a[1]).map(([reason, n]) => `${n} ${esc(reason)}`).join(', '); + const more = p.recent.length > REFUSED_SHOWN + ? ` Latest ${REFUSED_SHOWN} shown; the latest ${p.recent.length} as json.` : ''; + return `${esc(p.peer)} ${esc(p.peer_addr)}: ${p.total} refused (${why}).${more}` + + p.recent.slice(0, REFUSED_SHOWN).map(r => ` + ${esc(r.prefix)}${r.path_id ? ` path ${r.path_id}` : ''} + ${asPath(r.as_path)}${esc(r.reason)}${age(r.at_unix)} ago`).join(''); + }).join('')); + } + // ---- data loading ------------------------------------------------------ async function loadStatus() { state.status = await getJSON('/api/status'); renderStatus(); renderLocal(); renderReceived(); } async function loadPeers() { state.peers = await getJSON('/api/peers'); renderPeers(); } async function loadLocal() { state.local = await getJSON('/api/routes/local'); renderLocal(); } - async function loadReceived() { state.received = await getJSON('/api/routes/received'); renderReceived(); } - async function loadAll() { await Promise.all([loadStatus(), loadPeers(), loadLocal(), loadReceived()]); } + async function loadReceived() { + const note = $('lookup-note'); + try { + state.received = await getJSON(receivedUrl()); + note.textContent = ''; + } catch (e) { + // A query the server could not make sense of: say what it said. + state.received = []; + note.innerHTML = `${esc(e.message.replace(/^[^:]*: \d+ /, ''))}`; + } + renderReceived(); + } + async function loadRefused() { state.refused = await getJSON('/api/routes/rejected'); renderRefused(); } + async function loadAll() { await Promise.all([loadStatus(), loadPeers(), loadLocal(), loadReceived(), loadRefused()]); } // ---- events ------------------------------------------------------------ function logEvent(html) { @@ -246,6 +348,12 @@ View: variables, status json, < $('config-zippy').hidden = false; }).catch(console.error); renderRefreshControl(); + showLookup(); + $('lookup').addEventListener('submit', e => { + e.preventDefault(); + setLookup({ q: $('lookup-q').value.trim(), mode: $('lookup-mode').value, peer: $('lookup-peer').value, best: $('lookup-best').checked }); + }); + $('lookup-clear').addEventListener('click', e => { e.preventDefault(); setLookup(null); }); loadAll().catch(e => logEvent(`${esc(e.message)}`)); if (REFRESH) { connect();