diff --git a/bin/bgp/README.md b/bin/bgp/README.md index f2fe027..077435d 100644 --- a/bin/bgp/README.md +++ b/bin/bgp/README.md @@ -21,7 +21,8 @@ When `http_endpoint` is set the binary also serves: | `GET /api/config` | The configuration file as read at startup | | `GET /api/peers` | Configured peers and established sessions | | `GET /api/routes/local` | Originated routes | -| `GET /api/routes/received` | Adj-RIB-In contents | +| `GET /api/routes/received` | Routes learned from peers; see "Finding a route" | +| `GET /api/routes/rejected` | Routes peers announced that were refused, by peer | | `GET /api/events` | Server-sent events for session and route changes | | `GET /metricz` | Prometheus metrics | @@ -63,6 +64,70 @@ A refused route is dropped, along with the route it would have replaced. `bgp_routes_rejected_total` counts them by reason, and the status page shows how many routes each peer has and how many were refused. +## Finding a route + +The status page has a look-up box over the received routes, and +`/api/routes/received` takes the same questions: + +| Query | Answer | +| --- | --- | +| `?prefix=193.0.0.0/21` | Every route for that prefix, the best first | +| `?address=193.0.0.1` | Every prefix that covers the address, most specific first | +| `?within=193.0.0.0/16` | That prefix and everything inside it | +| `&peer=NAME` | Only routes from that peer (its name or its address) | +| `&best=true` | Only the route chosen for each prefix | +| `&limit=N` | At most N routes (1000 unless given, 100000 at most) | + +`within`, `peer` and `best` on their own look through the whole table, which +on a full table takes a moment. + +## Seeing and alerting on refused routes + +A refused route is not in the table. What is kept instead, for each peer, is +how many were refused for each reason and the latest 200 of them with their +AS paths. The status page lists them under "Refused routes"; +`/api/routes/rejected[?peer=NAME]` has them as JSON. The record is kept when +the session ends, so a peer that was cut off can still be looked into. The +log has a line for a peer's first refused route and then at most one a +minute. + +For alerting, `/metricz` has: + +| Metric | Meaning | +| --- | --- | +| `bgp_routes_rejected_total{peer,reason}` | Routes refused, by peer and why | +| `bgp_peer_routes{peer}` | Routes an established peer has in the table | +| `bgp_peer_max_prefixes{peer}` | The peer's limit, where it has one | +| `bgp_prefix_limit_exceeded_total{peer}` | Sessions ended for going over the limit | +| `bgp_connections_refused_total{reason}` | Connections closed without a session | + +Rules to start from: + +```yaml +groups: + - name: bgp + rules: + # A peer is announcing what its policy does not allow: a leak or a + # hijack on its side, or a list on ours that is out of date. + - alert: BgpRoutesRefused + expr: sum by (peer, reason) (increase(bgp_routes_rejected_total[10m])) > 0 + labels: { severity: warning } + # A route that has been through our own AS came back: a loop, or + # someone announcing our AS number. + - alert: BgpOwnAsInPath + expr: increase(bgp_routes_rejected_total{reason="own AS in path"}[10m]) > 0 + labels: { severity: page } + # A peer is close to its limit; the limit may need raising before the + # session is cut. + - alert: BgpPeerNearPrefixLimit + expr: bgp_peer_routes / on (peer) bgp_peer_max_prefixes > 0.8 + for: 10m + labels: { severity: warning } + - alert: BgpPrefixLimitExceeded + expr: increase(bgp_prefix_limit_exceeded_total[1h]) > 0 + labels: { severity: page } +``` + ## Trying it without a router `fake_peer` is a minimal BGP speaker that connects to the daemon, announces a diff --git a/bin/bgp/src/main.rs b/bin/bgp/src/main.rs index a00176f..4678e23 100644 --- a/bin/bgp/src/main.rs +++ b/bin/bgp/src/main.rs @@ -50,6 +50,7 @@ async fn main() -> Result<()> { registry: server.registry.clone(), local_rib: server.local_rib.clone(), rib: Arc::clone(&server.rib), + refusals: server.refusals.clone(), }; tokio::spawn(async move { if let Err(e) = bgp_web::serve_on(state, listener).await { diff --git a/crates/integration-tests/src/test_server.rs b/crates/integration-tests/src/test_server.rs index 04e4726..e1b1f32 100644 --- a/crates/integration-tests/src/test_server.rs +++ b/crates/integration-tests/src/test_server.rs @@ -7,6 +7,7 @@ use bgp_rib::LocalRib; use bgp_rib::table::{Event, PeerIndex, Reader, RibTable}; use bgp_server::config::Config; use bgp_server::error::ServerError; +use bgp_server::refusals::RefusalLog; use bgp_server::server::BgpServer; use bgp_server::session::SessionRegistry; use bgp_web::{AppState, RouterInfo}; @@ -26,6 +27,8 @@ pub struct TestServer { pub local_rib: LocalRib, /// Live view of the server's established sessions. pub registry: SessionRegistry, + /// What peers announced that was not taken. + pub refusals: RefusalLog, pub local_asn: u32, } @@ -68,6 +71,7 @@ impl TestServer { let rib = Arc::clone(&bound.rib); let local_rib = bound.local_rib.clone(); let registry = bound.registry.clone(); + let refusals = bound.refusals.clone(); let handle = tokio::spawn(bound.run()); Ok(Self { @@ -76,6 +80,7 @@ impl TestServer { rib, local_rib, registry, + refusals, local_asn, }) } @@ -193,6 +198,7 @@ impl TestServer { registry: self.registry.clone(), local_rib: self.local_rib.clone(), rib: Arc::clone(&self.rib), + refusals: self.refusals.clone(), }; let handle = tokio::spawn(bgp_web::serve_on(state, listener)); Ok((base_url, HttpGuard { handle })) diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs new file mode 100644 index 0000000..cd8aba8 --- /dev/null +++ b/crates/integration-tests/tests/debugging.rs @@ -0,0 +1,290 @@ +//! Finding out what the server has and why: looking routes up, and seeing +//! what was refused. + +use std::net::Ipv4Addr; +use std::time::Duration; + +use bgp_integration_tests::test_peer::TestPeer; +use bgp_integration_tests::test_server::TestServer; +use bgp_packet::message::{ + AsPathAttribute, Message, NextHopPathAttribute, OriginPathAttribute, PathAttribute, + UpdateMessage, +}; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_server::config::PeerConfig; +use bgp_server::family::FamilySet; +use bgp_server::policy::ImportPolicy; +use eyre::Result; +use serde_json::Value; +use tokio::net::TcpListener; + +const TIMEOUT: Duration = Duration::from_secs(5); + +fn announcement(prefixes: &[&str], as_path: &[u32]) -> Message { + Message::Update(UpdateMessage { + withdrawn_routes: vec![], + path_attributes: vec![ + PathAttribute::Origin(OriginPathAttribute::IGP), + AsPathAttribute::from_asns(as_path.to_vec()), + PathAttribute::NextHop(NextHopPathAttribute(Ipv4Addr::new(192, 0, 2, 1))), + ], + announced_routes: prefixes + .iter() + .map(|p| IpPrefix::try_from(*p).unwrap().into()) + .collect(), + }) +} + +/// The routes a query returns, as "prefix from AS, best or not". +async fn routes(client: &reqwest::Client, base: &str, query: &str) -> Result> { + let reply = client + .get(format!("{base}/api/routes/received?{query}")) + .send() + .await? + .error_for_status()?; + let routes: Value = reply.json().await?; + Ok(routes + .as_array() + .unwrap() + .iter() + .map(|r| { + format!( + "{} AS{}{}", + r["prefix"].as_str().unwrap(), + r["peer_asn"], + if r["best"] == true { " best" } else { "" } + ) + }) + .collect()) +} + +/// The questions asked when something is wrong with a prefix: who sends it +/// and which route won, what covers an address, what is inside a block. +#[tokio::test] +async fn test_routes_can_be_looked_up() -> Result<()> { + let server = TestServer::start(65000).await?; + let (base, _http) = server.serve_http().await?; + let client = reqwest::Client::new(); + + let mut one = TestPeer::connect(server.addr).await?; + one.handshake(65001, Ipv4Addr::new(10, 0, 0, 1), 90).await?; + let mut two = TestPeer::connect(server.addr).await?; + two.handshake(65002, Ipv4Addr::new(10, 0, 0, 2), 90).await?; + one.send(&announcement( + &["10.0.0.0/8", "10.1.0.0/16", "10.1.2.0/24", "192.0.2.0/24"], + &[65001], + )) + .await?; + // A longer path for one of the same prefixes: it loses. + two.send(&announcement(&["10.1.0.0/16"], &[65002, 65003])) + .await?; + tokio::time::timeout(TIMEOUT, async { + while server.rib.len() != 5 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + + // One prefix: every route for it, the best first. + assert_eq!( + routes(&client, &base, "prefix=10.1.0.0/16").await?, + ["10.1.0.0/16 AS65001 best", "10.1.0.0/16 AS65002"] + ); + assert!( + routes(&client, &base, "prefix=10.2.0.0/16") + .await? + .is_empty() + ); + + // An address: every prefix that covers it, the most specific first. + assert_eq!( + routes(&client, &base, "address=10.1.2.3").await?, + [ + "10.1.2.0/24 AS65001 best", + "10.1.0.0/16 AS65001 best", + "10.1.0.0/16 AS65002", + "10.0.0.0/8 AS65001 best" + ] + ); + assert_eq!( + routes(&client, &base, "address=10.200.0.1").await?, + ["10.0.0.0/8 AS65001 best"] + ); + + // A block: itself and everything inside it. + assert_eq!( + routes(&client, &base, "within=10.1.0.0/16").await?, + [ + "10.1.0.0/16 AS65001 best", + "10.1.0.0/16 AS65002", + "10.1.2.0/24 AS65001 best" + ] + ); + + // Only what was chosen; and everything, in address order. + assert_eq!( + routes(&client, &base, "within=10.0.0.0/8&best=true").await?, + [ + "10.0.0.0/8 AS65001 best", + "10.1.0.0/16 AS65001 best", + "10.1.2.0/24 AS65001 best" + ] + ); + assert_eq!(routes(&client, &base, "").await?.len(), 5); + assert_eq!(routes(&client, &base, "limit=2").await?.len(), 2); + + // One peer's routes, by its address; a peer that is not there has none. + assert_eq!(routes(&client, &base, "peer=127.0.0.1").await?.len(), 5); + assert!(routes(&client, &base, "peer=192.0.2.9").await?.is_empty()); + + // A query that makes no sense says so. + for (query, complaint) in [ + ("prefix=10.1.0.0", "is not a prefix"), + ("address=example", "is not an IP address"), + ("prefix=10.0.0.0/8&address=10.0.0.1", "at most one of"), + ] { + let reply = client + .get(format!("{base}/api/routes/received?{query}")) + .send() + .await?; + assert_eq!(reply.status(), 400, "{query}"); + let text = reply.text().await?; + assert!(text.contains(complaint), "{query}: {text}"); + } + Ok(()) +} + +/// A refused route is not in the table, so it has to be findable somewhere +/// else: which peer, which prefix, with what path, and why. And there is +/// something to alert on. +#[tokio::test] +async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let endpoint = listener.local_addr()?; + let server = TestServer::start_with(1299, |c| { + c.accept_unconfigured_peers = false; + c.peers = vec![PeerConfig { + name: "refused-demo".into(), + endpoint, + remote_asn: 3333, + hold_time_secs: 90, + local_address: None, + families: FamilySet::supported(), + add_path: None, + import: ImportPolicy { + max_prefixes: Some(100), + ..ImportPolicy::default() + }, + }]; + }) + .await?; + let (base, _http) = server.serve_http().await?; + let client = reqwest::Client::new(); + let get = |path: &str| { + let request = client.get(format!("{base}{path}")); + async move { Ok::<_, eyre::Report>(request.send().await?.error_for_status()?) } + }; + + let refused: Value = get("/api/routes/rejected").await?.json().await?; + assert_eq!(refused, serde_json::json!([]), "nothing refused yet"); + + let mut peer = TestPeer::connect(server.addr).await?; + peer.handshake(3333, Ipv4Addr::new(10, 0, 0, 2), 90).await?; + peer.send(&announcement(&["193.0.0.0/21", "10.0.0.0/8"], &[3333])) + .await?; + peer.send(&announcement( + &["194.0.1.0/24", "194.0.2.0/24"], + &[3333, 64512], + )) + .await?; + peer.send(&announcement(&["193.0.9.0/25"], &[3333, 2121])) + .await?; + tokio::time::timeout(TIMEOUT, async { + while server + .registry + .list() + .first() + .map(|s| s.counters.rejected()) + != Some(4) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + + let refused: Value = get("/api/routes/rejected").await?.json().await?; + let record = &refused[0]; + assert_eq!(record["peer"], "refused-demo"); + assert_eq!(record["peer_addr"], "127.0.0.1"); + assert_eq!(record["total"], 4); + assert_eq!( + record["by_reason"], + serde_json::json!({"bogon AS in path": 2, "bogon prefix": 1, "prefix too specific": 1}) + ); + // The latest first, each with the path it came with. + let recent: Vec<(String, String, String)> = record["recent"] + .as_array() + .unwrap() + .iter() + .map(|r| { + ( + r["prefix"].as_str().unwrap().to_string(), + r["reason"].as_str().unwrap().to_string(), + r["as_path"].to_string(), + ) + }) + .collect(); + let row = |p: &str, why: &str, path: &str| (p.to_string(), why.to_string(), path.to_string()); + assert_eq!( + recent, + [ + row("193.0.9.0/25", "prefix too specific", "[3333,2121]"), + row("194.0.2.0/24", "bogon AS in path", "[3333,64512]"), + row("194.0.1.0/24", "bogon AS in path", "[3333,64512]"), + row("10.0.0.0/8", "bogon prefix", "[3333]"), + ] + ); + // Asked for by peer, by name or by address. + for peer in ["refused-demo", "127.0.0.1"] { + let one: Value = get(&format!("/api/routes/rejected?peer={peer}")) + .await? + .json() + .await?; + assert_eq!(one.as_array().unwrap().len(), 1, "{peer}"); + } + let none: Value = get("/api/routes/rejected?peer=someone-else") + .await? + .json() + .await?; + assert_eq!(none, serde_json::json!([])); + + // The peer's line in the list of peers has the numbers too. + let peers: Value = get("/api/peers").await?.json().await?; + assert_eq!(peers[0]["routes"], 1); + assert_eq!(peers[0]["rejected_routes"], 4); + + // What an alert would be made from. + let metrics = get("/metricz").await?.text().await?; + for line in [ + r#"bgp_routes_rejected_total{peer="refused-demo",reason="bogon AS in path"} 2"#, + r#"bgp_routes_rejected_total{peer="refused-demo",reason="bogon prefix"} 1"#, + r#"bgp_peer_routes{peer="refused-demo"} 1"#, + r#"bgp_peer_max_prefixes{peer="refused-demo"} 100"#, + ] { + assert!(metrics.contains(line), "{line} is not in:\n{metrics}"); + } + + // The record outlives the session; the count of routes does not. + drop(peer); + tokio::time::timeout(TIMEOUT, async { + while !server.registry.list().is_empty() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await?; + let refused: Value = get("/api/routes/rejected").await?.json().await?; + assert_eq!(refused[0]["total"], 4); + let metrics = get("/metricz").await?.text().await?; + assert!(!metrics.contains(r#"bgp_peer_routes{peer="refused-demo"}"#)); + Ok(()) +} diff --git a/crates/rib/src/table/mod.rs b/crates/rib/src/table/mod.rs index dc71ae7..443570e 100644 --- a/crates/rib/src/table/mod.rs +++ b/crates/rib/src/table/mod.rs @@ -533,10 +533,19 @@ impl RibTable { /// particular order. For showing some of a table that is too large to /// show all of. pub fn some_routes(&self, limit: usize) -> Vec { + self.routes_where(limit, |_| true) + } + + /// Up to `limit` of the routes that `wanted` picks, in no particular + /// order. This looks at routes until it has enough, which for a choosy + /// `wanted` is every route in the table: it is for looking into the + /// table by hand, not for anything done often. `wanted` is called with + /// a shard held and must not call into the table. + pub fn routes_where(&self, limit: usize, wanted: impl Fn(&RouteRef<'_>) -> bool) -> Vec { let mut routes = Vec::new(); for shard in &self.shards { for row in &read(shard).rows { - for route in row.routes() { + for route in row.routes().filter(&wanted) { if routes.len() == limit { return routes; } diff --git a/crates/rib/src/table/tests.rs b/crates/rib/src/table/tests.rs index 32266f4..5b8291d 100644 --- a/crates/rib/src/table/tests.rs +++ b/crates/rib/src/table/tests.rs @@ -763,3 +763,32 @@ fn a_row_can_hold_more_routes_than_a_sixteen_bit_index_counts() { assert!(select::after_remove(&mut row, count, &peers, &mut scratch)); assert_eq!(row.route(row.best()).path_id, 1); } + +#[test] +fn routes_can_be_picked_out_of_the_table() { + let fixture = Fixture::new(&[external(1), external(2)]); + for n in 0..50u8 { + let p = prefix(&format!("10.{n}.0.0/16")); + fixture.announce(0, &p, &Route::via(&[1])); + if n % 2 == 0 { + fixture.announce(1, &p, &Route::via(&[2, 3])); + } + } + let table = &fixture.table; + let second = fixture.peers[1]; + assert_eq!(table.routes_where(1000, |_| true).len(), 75); + assert_eq!( + table.routes_where(1000, |route| route.peer == second).len(), + 25 + ); + assert_eq!(table.routes_where(1000, |route| route.best).len(), 50); + let inside = prefix("10.16.0.0/12"); + let found = table.routes_where(1000, |route| inside.contains(&route.prefix)); + assert_eq!(found.len(), 16 + 8); + // No more than asked for, however many there are. + assert_eq!( + table.routes_where(10, |route| route.peer == second).len(), + 10 + ); + assert!(table.routes_where(0, |_| true).is_empty()); +} diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index a6b30e1..73c6141 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -22,6 +22,7 @@ use bgp_packet::update_decode::UpdateError; use bgp_primitives::parser::ParserContext; use bgp_rib::table::{Outcome, PeerIndex, PeerInfo, RibTable}; use bytes::BytesMut; +use prometheus::IntGauge; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; @@ -35,10 +36,12 @@ use crate::error::SessionError; use crate::family::FamilySet; use crate::fsm::{Action, Event, Fsm, FsmConfig, Negotiated, Received}; use crate::metrics::{ - MESSAGES_RECEIVED, MESSAGES_SENT, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, + MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ROUTES, PREFIX_LIMIT_EXCEEDED, + SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, }; use crate::panic::contained; use crate::policy::{self, HoldDowns, ImportPolicy}; +use crate::refusals::{PeerRefusals, RefusalLog}; use crate::routes; use crate::session::{PeerId, SessionCounters, SessionGuard, SessionHandle, SessionRegistry}; @@ -69,6 +72,8 @@ pub struct ServerContext { pub collisions: CollisionTable, /// Peers that are being kept down for going over their limits. pub hold_downs: HoldDowns, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } /// What is configured for the peer at the other end of a connection. @@ -159,6 +164,9 @@ pub async fn run_session( .server .hold_downs .hold(peer_addr.ip(), session.import.hold_down); + PREFIX_LIMIT_EXCEEDED + .with_label_values(&[session.refusals.label()]) + .inc(); } // Whatever ended the session, routes learned over it are gone @@ -172,6 +180,8 @@ pub async fn run_session( .. } = established; let outcome = routes.remove().await; + // The peer has no routes now, and no session to count them for. + let _ = PEER_ROUTES.remove_label_values(&[session.refusals.label()]); info!( peer = %peer_id, removed_routes = outcome.removed, @@ -199,6 +209,8 @@ struct Established { from: policy::From, /// How many routes the peer has, and how many were refused. counters: Arc, + /// The same number of routes, where metrics are read from. + routes_metric: IntGauge, /// Keeps the session listed in the registry. _registration: SessionGuard, } @@ -302,6 +314,8 @@ struct Session { peer_name: Option, /// Which of the peer's routes are taken. import: Arc, + /// Where what is refused is noted. + refusals: Arc, } impl Session { @@ -324,7 +338,13 @@ impl Session { families: peer.families, add_path: peer.add_path, }); + // The peer goes by its configured name, or by its address. + let label = peer + .name + .clone() + .unwrap_or_else(|| peer_addr.ip().to_string()); let session = Session { + refusals: server.refusals.of_peer(peer_addr.ip(), &label), fsm, frames: FrameReader::new(reader), writer, @@ -604,6 +624,7 @@ impl Session { four_octet_asn: negotiated.four_octet_asn, }, counters: Arc::clone(&handle.counters), + routes_metric: PEER_ROUTES.with_label_values(&[self.refusals.label()]), _registration: self.server.registry.register(handle), }); Ok(()) @@ -622,6 +643,7 @@ impl Session { peer: established.rib_peer, from: established.from, policy: &self.import, + refusals: &self.refusals, }; let routes::Applied { outcome, rejected } = routes::apply(&self.server.rib, &import, update, attributes_wire)?; @@ -639,6 +661,7 @@ impl Session { let held = established .counters .note(outcome.added, outcome.removed, rejected); + established.routes_metric.set(held as i64); match self.import.max_prefixes { Some(limit) if held > u64::from(limit) => { Err(SessionError::PrefixLimitExceeded { limit }) @@ -907,6 +930,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), + refusals: RefusalLog::default(), }); let (mut session, _) = Session::open( stream, @@ -953,6 +977,7 @@ mod tests { registry: SessionRegistry::new(), collisions: CollisionTable::new(false), hold_downs: HoldDowns::default(), + refusals: RefusalLog::default(), }); let (mut session, _) = Session::open( stream, @@ -1016,6 +1041,7 @@ mod tests { four_octet_asn: true, }, policy: &ImportPolicy::permissive(), + refusals: &RefusalLog::default().of_peer(Ipv4Addr::new(192, 0, 2, 1).into(), "test"), }; routes::apply(&table, &import, &update, None).unwrap(); assert_eq!(table.len(), 1); diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index d3edefb..e117bec 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -9,6 +9,7 @@ pub mod fsm; pub mod metrics; mod panic; pub mod policy; +pub mod refusals; mod routes; pub mod server; pub mod session; diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index 1f873e8..b17ff6e 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -4,8 +4,8 @@ use std::sync::LazyLock; use bgp_packet::message::Message; use prometheus::{ - IntCounter, IntCounterVec, IntGauge, register_int_counter, register_int_counter_vec, - register_int_gauge, + IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_int_counter, + register_int_counter_vec, register_int_gauge, register_int_gauge_vec, }; pub static MESSAGES_SENT: LazyLock = LazyLock::new(|| { @@ -54,8 +54,35 @@ pub static SESSIONS_ESTABLISHED: LazyLock = LazyLock::new(|| { pub static ROUTES_REJECTED: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "bgp_routes_rejected_total", - "Routes announced by peers and refused by their import policy, by why", - &["reason"] + "Routes announced by a peer and refused by its import policy, by why", + &["peer", "reason"] + ) + .expect("metric registration") +}); + +pub static PEER_ROUTES: LazyLock = LazyLock::new(|| { + register_int_gauge_vec!( + "bgp_peer_routes", + "Routes of an established peer that are in the routing table", + &["peer"] + ) + .expect("metric registration") +}); + +pub static PEER_MAX_PREFIXES: LazyLock = LazyLock::new(|| { + register_int_gauge_vec!( + "bgp_peer_max_prefixes", + "The most routes a configured peer may have (import.max_prefixes), where there is a limit", + &["peer"] + ) + .expect("metric registration") +}); + +pub static PREFIX_LIMIT_EXCEEDED: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "bgp_prefix_limit_exceeded_total", + "Sessions ended because the peer went over its limit of routes", + &["peer"] ) .expect("metric registration") }); diff --git a/crates/server/src/policy.rs b/crates/server/src/policy.rs index 6125a6a..39c2bec 100644 --- a/crates/server/src/policy.rs +++ b/crates/server/src/policy.rs @@ -151,6 +151,11 @@ impl Rejection { Self::OriginNotAllowed, ]; + /// Where the reason is in [`Rejection::ALL`]. + pub(crate) fn index(self) -> usize { + self as usize + } + pub fn as_str(self) -> &'static str { match self { Self::BogonPrefix => "bogon prefix", @@ -300,7 +305,7 @@ impl PathFacts { } /// The AS numbers of a path in order, sets and all. -fn asns(path: Option<&AsPathAttribute>) -> impl Iterator + '_ { +pub(crate) fn asns(path: Option<&AsPathAttribute>) -> impl Iterator + '_ { path.into_iter() .flat_map(|path| &path.segments) .flat_map(|segment| segment.path.iter().copied()) diff --git a/crates/server/src/refusals.rs b/crates/server/src/refusals.rs new file mode 100644 index 0000000..2326902 --- /dev/null +++ b/crates/server/src/refusals.rs @@ -0,0 +1,258 @@ +//! What peers announced that was not taken, kept for looking at. +//! +//! A refused route is not stored, so without this there would be nothing +//! to show for it but a number. For each peer the reasons are counted and +//! the latest refusals are kept: enough to see what a peer is doing wrong, +//! or what in our policy is. They are kept by the peer's address and +//! outlive the session, since the session a peer was cut off on is the one +//! worth looking at. + +use std::collections::{HashMap, VecDeque}; +use std::net::IpAddr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; +use prometheus::IntCounter; +use tracing::warn; + +use crate::metrics::ROUTES_REJECTED; +use crate::policy::Rejection; +use crate::sync::IgnorePoison; + +/// How many of a peer's latest refused routes are kept. +pub const RECENT_REFUSALS: usize = 200; + +/// How many peers refusals are kept for. More than any configuration has; +/// it bounds what hosts that are let in without being configured (on a +/// test network) can make us keep. +const MAX_PEERS: usize = 8192; + +/// At most one line in the log about a peer's refused routes in this long. +/// The first of them is logged at once. +const LOG_INTERVAL: Duration = Duration::from_secs(60); + +/// A route a peer announced that its import policy refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefusedRoute { + pub prefix: IpPrefix, + /// The peer's identifier for the route; zero without ADD-PATH. + pub path_id: u32, + pub reason: Rejection, + /// The AS path it came with. + pub as_path: Arc<[u32]>, + pub at: SystemTime, +} + +/// The refusals of one peer. +pub struct PeerRefusals { + /// What the peer is called in the log and in metrics: its configured + /// name, or its address. + label: String, + by_reason: [AtomicU64; Rejection::ALL.len()], + metrics: [IntCounter; Rejection::ALL.len()], + state: Mutex, +} + +struct State { + /// The latest refused routes, oldest first. + recent: VecDeque, + logged: Option, + /// How many were refused since the log last said so. + unlogged: u64, +} + +impl PeerRefusals { + fn new(label: &str) -> Self { + Self { + label: label.to_string(), + by_reason: Default::default(), + metrics: Rejection::ALL + .map(|reason| ROUTES_REJECTED.with_label_values(&[label, reason.as_str()])), + state: Mutex::new(State { + recent: VecDeque::new(), + logged: None, + unlogged: 0, + }), + } + } + + /// Notes that `routes`, which came with the AS path `as_path`, were + /// refused for `reason`. + pub(crate) fn note(&self, routes: &[Nlri], reason: Rejection, as_path: &Arc<[u32]>) { + let Some(latest) = routes.last() else { + return; + }; + let count = routes.len() as u64; + self.by_reason[reason.index()].fetch_add(count, Ordering::Relaxed); + self.metrics[reason.index()].inc_by(count); + + let at = SystemTime::now(); + let mut state = self.state.locked(); + // Only the last of them can still be in the list afterwards. + let kept = &routes[routes.len().saturating_sub(RECENT_REFUSALS)..]; + for route in kept { + if state.recent.len() == RECENT_REFUSALS { + state.recent.pop_front(); + } + state.recent.push_back(RefusedRoute { + prefix: route.prefix, + path_id: route.path_id, + reason, + as_path: Arc::clone(as_path), + at, + }); + } + + // A peer that sends a table of routes we refuse must not write a + // table of lines to the log. + if state + .logged + .is_some_and(|logged| logged.elapsed() < LOG_INTERVAL) + { + state.unlogged += count; + return; + } + warn!( + peer = %self.label, + prefix = %latest.prefix, + reason = reason.as_str(), + as_path = ?&as_path[..], + others_since_last_report = state.unlogged + count - 1, + "Refused a route" + ); + state.logged = Some(Instant::now()); + state.unlogged = 0; + } + + pub fn label(&self) -> &str { + &self.label + } + + /// How many routes were refused, since the server started. + pub fn total(&self) -> u64 { + self.by_reason + .iter() + .map(|count| count.load(Ordering::Relaxed)) + .sum() + } + + /// How many routes were refused for each reason that came up. + pub fn by_reason(&self) -> Vec<(Rejection, u64)> { + Rejection::ALL + .into_iter() + .map(|reason| { + ( + reason, + self.by_reason[reason.index()].load(Ordering::Relaxed), + ) + }) + .filter(|(_, count)| *count > 0) + .collect() + } + + /// The latest refused routes, newest first. + pub fn recent(&self) -> Vec { + self.state.locked().recent.iter().rev().cloned().collect() + } +} + +/// The refusals of every peer, by address. Cheap to clone; clones share +/// one record. +#[derive(Clone, Default)] +pub struct RefusalLog(Arc>>>); + +impl RefusalLog { + /// The record for the peer at `address`, made if there was none. + /// `label` is what to call the peer; see [`PeerRefusals::label`]. + pub(crate) fn of_peer(&self, address: IpAddr, label: &str) -> Arc { + let mut peers = self.0.locked(); + if let Some(existing) = peers.get(&address) { + return Arc::clone(existing); + } + let record = Arc::new(PeerRefusals::new(label)); + // Past the bound the record is still made, for the session to + // count in, but it is not kept after. + if peers.len() < MAX_PEERS { + peers.insert(address, Arc::clone(&record)); + } + record + } + + /// Every peer that has had a route refused, with its record. + pub fn peers(&self) -> Vec<(IpAddr, Arc)> { + self.0 + .locked() + .iter() + .filter(|(_, record)| record.total() > 0) + .map(|(address, record)| (*address, Arc::clone(record))) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn route(n: u32) -> Nlri { + let bytes = (0xc100_0000u32 + (n << 8)).to_be_bytes(); + IpPrefix::new(bgp_primitives::constants::IpVersion::Ipv4, bytes, 24) + .unwrap() + .into() + } + + #[test] + fn refusals_are_counted_by_reason_and_the_latest_are_kept() { + let log = RefusalLog::default(); + let address: IpAddr = "192.0.2.1".parse().unwrap(); + let record = log.of_peer(address, "refusals-test-1"); + assert!(log.peers().is_empty(), "nothing refused yet"); + + let path: Arc<[u32]> = vec![3333, 64512].into(); + record.note(&[route(1), route(2)], Rejection::BogonAs, &path); + record.note(&[route(3)], Rejection::TooSpecific, &path); + + assert_eq!(record.total(), 3); + assert_eq!( + record.by_reason(), + [(Rejection::TooSpecific, 1), (Rejection::BogonAs, 2)] + ); + let recent = record.recent(); + assert_eq!( + recent.iter().map(|r| r.prefix).collect::>(), + [route(3).prefix, route(2).prefix, route(1).prefix], + "newest first" + ); + assert_eq!(recent[0].reason, Rejection::TooSpecific); + assert_eq!(&recent[2].as_path[..], [3333, 64512]); + + // The peer's next session finds the same record. + let again = log.of_peer(address, "another-name"); + assert_eq!(again.total(), 3); + assert_eq!(again.label(), "refusals-test-1"); + assert_eq!(log.peers().len(), 1); + } + + /// A peer that sends a great many routes we refuse costs a fixed + /// amount of memory. + #[test] + fn only_so_many_refused_routes_are_kept() { + let record = PeerRefusals::new("refusals-test-2"); + let path: Arc<[u32]> = vec![3333].into(); + let many: Vec = (0..1000).map(route).collect(); + record.note(&many, Rejection::BogonPrefix, &path); + for n in 1000..1010 { + record.note(&[route(n)], Rejection::BogonPrefix, &path); + } + + assert_eq!(record.total(), 1010); + let recent = record.recent(); + assert_eq!(recent.len(), RECENT_REFUSALS); + assert_eq!(recent[0].prefix, route(1009).prefix); + assert_eq!( + recent[RECENT_REFUSALS - 1].prefix, + route(1010 - RECENT_REFUSALS as u32).prefix + ); + } +} diff --git a/crates/server/src/routes.rs b/crates/server/src/routes.rs index 0210644..24f3a00 100644 --- a/crates/server/src/routes.rs +++ b/crates/server/src/routes.rs @@ -2,6 +2,7 @@ //! as the peer's import policy lets them in. use std::borrow::Cow; +use std::cell::OnceCell; use std::sync::Arc; use bgp_packet::message::{NlriNextHop, PathAttribute, UpdateMessage}; @@ -9,8 +10,8 @@ use bgp_primitives::ip_prefix::Nlri; use bgp_primitives::parser::EncodeError; use bgp_rib::table::{AttrSet, Outcome, PeerIndex, RibTable}; -use crate::metrics::ROUTES_REJECTED; -use crate::policy::{From, ImportPolicy, Rejection}; +use crate::policy::{self, From, ImportPolicy, Rejection}; +use crate::refusals::PeerRefusals; /// What applying an UPDATE did. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -26,6 +27,8 @@ pub(crate) struct Import<'a> { pub(crate) peer: PeerIndex, pub(crate) from: From, pub(crate) policy: &'a ImportPolicy, + /// Where what is refused is noted. + pub(crate) refusals: &'a PeerRefusals, } /// Applies `update` to `table`: its withdrawals first and then its @@ -62,6 +65,9 @@ pub(crate) fn apply( } } + // The path the routes came with, for the record of what is refused: + // made only if something is. + let as_path = OnceCell::new(); // What the routes have in common is judged once. let path_verdict = import .policy @@ -80,9 +86,9 @@ pub(crate) fn apply( return Ok(()); } let mut refuse = |refused: &[Nlri], rejection: Rejection| { - ROUTES_REJECTED - .with_label_values(&[rejection.as_str()]) - .inc_by(refused.len() as u64); + import + .refusals + .note(refused, rejection, as_path.get_or_init(|| path_of(update))); applied.rejected += refused.len(); applied.outcome += table.withdraw(peer, refused); }; @@ -108,6 +114,18 @@ pub(crate) fn apply( Ok(applied) } +/// The AS numbers of the UPDATE's AS_PATH. +fn path_of(update: &UpdateMessage) -> Arc<[u32]> { + let path = update + .path_attributes + .iter() + .find_map(|attribute| match attribute { + PathAttribute::AsPath(path) => Some(path), + _ => None, + }); + policy::asns(path).collect() +} + /// The routes of `nlri` whose prefixes the policy takes; the others are /// given to `refuse`. Nearly always that is all of them, and nothing is /// copied. diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 73bf06e..a66788d 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -22,9 +22,10 @@ use crate::config::Config; use crate::connection::{PeerSettings, ServerContext, run_session}; use crate::dialer::spawn_dialers; use crate::error::ServerError; -use crate::metrics::CONNECTIONS_REFUSED; +use crate::metrics::{CONNECTIONS_REFUSED, PEER_MAX_PREFIXES}; use crate::panic; use crate::policy::HoldDowns; +use crate::refusals::RefusalLog; use crate::session::{ConfiguredPeer, SessionRegistry}; use crate::sync::IgnorePoison; @@ -68,6 +69,8 @@ pub struct BgpServer { pub local_rib: LocalRib, /// Live view of established sessions. pub registry: SessionRegistry, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } impl BgpServer { @@ -102,6 +105,7 @@ impl BgpServer { rib: Arc::new(RibTable::new(table_shards())), local_rib, registry: SessionRegistry::with_configured(configured), + refusals: RefusalLog::default(), config, }) } @@ -127,7 +131,11 @@ impl BgpServer { ); for peer in &self.config.peers { - if peer.import.max_prefixes.is_none() { + if let Some(limit) = peer.import.max_prefixes { + PEER_MAX_PREFIXES + .with_label_values(&[&peer.name]) + .set(limit.into()); + } else { warn!( peer = %peer.name, "No limit on the routes taken from this peer (import.max_prefixes)" @@ -145,6 +153,7 @@ impl BgpServer { registry: self.registry.clone(), collisions: CollisionTable::new(self.config.collision_detect_established_state), hold_downs: HoldDowns::default(), + refusals: self.refusals.clone(), }); // The tasks the server cannot do without. None of them ends while diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index c3ad194..04dd2ed 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -1,17 +1,20 @@ //! JSON handlers and their view types. Everything here is read-only: the //! daemon is configured through its configuration file. +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; use std::time::{SystemTime, UNIX_EPOCH}; use axum::Json; -use std::collections::HashMap; use axum::extract::{Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use bgp_packet::community; use bgp_packet::message::{NlriNextHop, OriginPathAttribute, PathAttribute}; -use bgp_rib::table::{PeerIndex, Route}; +use bgp_primitives::constants::IpVersion; +use bgp_primitives::ip_prefix::IpPrefix; +use bgp_rib::table::{PeerIndex, RibTable, Route}; use bgp_rib::{LocalRoute, RibStopped}; use bgp_server::session::{ConfiguredPeer, DialState, PeerId, SessionHandle, SessionRegistry}; use serde::{Deserialize, Serialize}; @@ -249,6 +252,8 @@ pub fn summarize(attributes: &[PathAttribute], next_hop: Option<&NlriNextHop>) - #[derive(Debug, Clone, Serialize)] pub struct ReceivedRouteView { + /// The peer's configured name, if it has one. + pub peer_name: Option, pub peer_addr: String, pub peer_asn: u32, pub prefix: String, @@ -265,7 +270,7 @@ pub struct ReceivedRouteView { /// Who the routes in the table are from: the table knows peers by number, /// the sessions know which number is whose. -pub(crate) struct PeerNames(HashMap); +pub(crate) struct PeerNames(HashMap)>); impl PeerNames { pub(crate) fn of(registry: &SessionRegistry) -> Self { @@ -273,18 +278,31 @@ impl PeerNames { registry .list() .into_iter() - .map(|session| (session.rib_peer, session.peer)) + .map(|session| (session.rib_peer, (session.peer, session.name))) .collect(), ) } + /// The peers that `wanted` names: by configured name, or by address. + fn matching(&self, wanted: &str) -> Vec { + let address: Option = wanted.parse().ok(); + self.0 + .iter() + .filter(|(_, (peer, name))| { + name.as_deref() == Some(wanted) || address == Some(peer.addr.ip()) + }) + .map(|(index, _)| *index) + .collect() + } + /// A view of one route. `None` if the route's session has just ended: /// the route is on its way out of the table. pub(crate) fn view(&self, route: &Route) -> Option { - let peer = self.0.get(&route.peer)?; + let (peer, name) = self.0.get(&route.peer)?; // What a session stored was accepted by the decoder, and parses. let (attributes, next_hop) = route.attrs.attributes().unwrap_or_default(); Some(ReceivedRouteView { + peer_name: name.clone(), peer_addr: peer.addr.to_string(), peer_asn: peer.asn, prefix: route.prefix.to_string(), @@ -379,37 +397,221 @@ const DEFAULT_ROUTE_LIMIT: usize = 1000; /// each is copied out of the table and written as JSON. const MAX_ROUTE_LIMIT: usize = 100_000; -#[derive(Debug, Deserialize)] +/// Which routes to return. At most one of `prefix`, `address` and `within` +/// says where to look; `peer` and `best` narrow what is found there. +#[derive(Debug, Default, Deserialize)] pub struct RouteQuery { /// At most this many routes. limit: Option, + /// The routes for exactly this prefix, e.g. `193.0.0.0/21`. + prefix: Option, + /// The routes for every prefix that covers this address, the most + /// specific prefix first: what a packet to the address could follow. + address: Option, + /// The routes for this prefix and every prefix inside it. + within: Option, + /// Only routes from this peer: its configured name, or its address. + peer: Option, + /// Only the best route of each prefix. + #[serde(default)] + best: bool, } -/// Some of the routes learned from peers: all the routes of as many -/// prefixes as fit in the limit. `/api/status` has the totals. +/// Why a query was not answered. +pub enum QueryError { + /// The query makes no sense; the text says how. + Bad(String), + /// Answering it failed here. + Failed, +} + +impl IntoResponse for QueryError { + fn into_response(self) -> Response { + match self { + Self::Bad(why) => (StatusCode::BAD_REQUEST, why).into_response(), + Self::Failed => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } +} + +/// Where in the table a query looks. +enum Scope { + /// Everywhere. + All, + Exactly(IpPrefix), + Covering(IpPrefix), + Within(IpPrefix), +} + +impl Scope { + fn of(query: &RouteQuery) -> Result { + let prefix = |text: &str| { + IpPrefix::try_from(text) + .map_err(|e| QueryError::Bad(format!("{text:?} is not a prefix: {e}"))) + }; + match (&query.prefix, &query.address, &query.within) { + (None, None, None) => Ok(Self::All), + (Some(text), None, None) => Ok(Self::Exactly(prefix(text)?)), + (None, Some(text), None) => { + let address: IpAddr = text + .parse() + .map_err(|_| QueryError::Bad(format!("{text:?} is not an IP address")))?; + // The address as a prefix as long as an address is. + let host = match address { + IpAddr::V4(v4) => IpPrefix::new(IpVersion::Ipv4, v4.octets(), 32), + IpAddr::V6(v6) => IpPrefix::new(IpVersion::Ipv6, v6.octets(), 128), + }; + host.map(Self::Covering) + .map_err(|e| QueryError::Bad(e.to_string())) + } + (None, None, Some(text)) => Ok(Self::Within(prefix(text)?)), + _ => Err(QueryError::Bad( + "give at most one of prefix, address and within".into(), + )), + } + } +} + +/// The routes a query asks for, in the order to show them. +fn find_routes( + table: &RibTable, + names: &PeerNames, + scope: &Scope, + query: &RouteQuery, + limit: usize, +) -> Vec { + let peers = query.peer.as_deref().map(|peer| names.matching(peer)); + let wanted = |peer: PeerIndex, best: bool| { + (!query.best || best) && peers.as_ref().is_none_or(|peers| peers.contains(&peer)) + }; + let mut routes = match scope { + Scope::Exactly(prefix) => table.routes(prefix), + // From the longest prefix that could cover the address to the + // shortest: one lookup for each length. + Scope::Covering(host) => (0..=host.length()) + .rev() + .flat_map(|length| table.routes(&host.shortened(length))) + .collect(), + Scope::Within(outer) => table.routes_where(limit, |route| { + outer.contains(&route.prefix) && wanted(route.peer, route.best) + }), + Scope::All => table.routes_where(limit, |route| wanted(route.peer, route.best)), + }; + routes.retain(|route| wanted(route.peer, route.best)); + routes.truncate(limit); + + // For each prefix the best route first. The prefixes in address + // order; for a lookup by address, which finds one prefix of each + // length, the most specific first. + if matches!(scope, Scope::Covering(_)) { + routes.sort_by_key(|route| (std::cmp::Reverse(route.prefix.length()), !route.best)); + } else { + routes.sort_by_key(|route| (sort_key(&route.prefix), !route.best)); + } + routes + .iter() + .filter_map(|route| names.view(route)) + .collect() +} + +/// Orders prefixes by IP version, then address, then shorter before +/// longer. +fn sort_key(prefix: &IpPrefix) -> (u16, [u8; 16], u8) { + let mut address = [0u8; 16]; + address[..prefix.bytes().len()].copy_from_slice(prefix.bytes()); + (prefix.address_family() as u16, address, prefix.length()) +} + +/// Routes learned from peers. Without a query, all the routes of as many +/// prefixes as fit in the limit; see [`RouteQuery`] for looking up +/// particular ones. `/api/status` has the totals. pub async fn received_routes( State(state): State, Query(query): Query, -) -> Result>, StatusCode> { +) -> Result>, QueryError> { let limit = query .limit .unwrap_or(DEFAULT_ROUTE_LIMIT) .min(MAX_ROUTE_LIMIT); + let scope = Scope::of(&query)?; let names = PeerNames::of(&state.registry); - // Copying, parsing and sorting many routes takes long enough to keep - // off the threads that sessions run on. - let routes = tokio::task::spawn_blocking(move || { - let mut routes: Vec = state - .rib - .some_routes(limit) - .iter() - .filter_map(|route| names.view(route)) - .collect(); - routes.sort_by(|a, b| (&a.prefix, &a.peer_addr).cmp(&(&b.prefix, &b.peer_addr))); - routes - }) - .await - // Only if making the views panicked. - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + // Looking through the table, copying, parsing and sorting many routes + // takes long enough to keep off the threads that sessions run on. + let routes = + tokio::task::spawn_blocking(move || find_routes(&state.rib, &names, &scope, &query, limit)) + .await + // Only if making the views panicked. + .map_err(|_| QueryError::Failed)?; Ok(Json(routes)) } + +/// What one peer announced that was refused. +#[derive(Debug, Clone, Serialize)] +pub struct RefusalsView { + /// The peer's configured name, or its address if it has none. + pub peer: String, + pub peer_addr: String, + /// How many routes were refused since the daemon started. + pub total: u64, + /// The same, by reason. + pub by_reason: BTreeMap<&'static str, u64>, + /// The latest of them, newest first. + pub recent: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RefusedRouteView { + pub prefix: String, + pub path_id: u32, + pub reason: &'static str, + pub as_path: Vec, + pub at_unix: u64, +} + +#[derive(Debug, Default, Deserialize)] +pub struct RefusalQuery { + /// Only this peer: its configured name, or its address. + peer: Option, +} + +/// The routes that peers announced and their import policies refused: for +/// each peer that had any, how many and why, and the latest of them. +pub async fn rejected_routes( + State(state): State, + Query(query): Query, +) -> Json> { + let mut peers: Vec = state + .refusals + .peers() + .into_iter() + .filter(|(address, record)| { + query + .peer + .as_deref() + .is_none_or(|wanted| record.label() == wanted || wanted.parse() == Ok(*address)) + }) + .map(|(address, record)| RefusalsView { + peer: record.label().to_string(), + peer_addr: address.to_string(), + total: record.total(), + by_reason: record + .by_reason() + .into_iter() + .map(|(reason, count)| (reason.as_str(), count)) + .collect(), + recent: record + .recent() + .into_iter() + .map(|route| RefusedRouteView { + prefix: route.prefix.to_string(), + path_id: route.path_id, + reason: route.reason.as_str(), + as_path: route.as_path.to_vec(), + at_unix: unix_secs(route.at), + }) + .collect(), + }) + .collect(); + peers.sort_by(|a, b| a.peer.cmp(&b.peer)); + Json(peers) +} diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index 5e65f24..0c82043 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -19,11 +19,15 @@ use axum::response::{Html, IntoResponse, Redirect}; use axum::routing::get; use bgp_rib::LocalRib; use bgp_rib::table::RibTable; +use bgp_server::refusals::RefusalLog; use bgp_server::session::SessionRegistry; use tokio::net::TcpListener; use tracing::info; -pub use api::{AttrSummary, ConfigView, LocalRouteView, PeerView, ReceivedRouteView, StatusView}; +pub use api::{ + AttrSummary, ConfigView, LocalRouteView, PeerView, ReceivedRouteView, RefusalsView, + RefusedRouteView, StatusView, +}; /// Static facts about the router shown in status output. #[derive(Debug, Clone)] @@ -81,6 +85,8 @@ pub struct AppState { pub local_rib: LocalRib, /// The routing table: routes learned from peers and the best of them. pub rib: Arc, + /// What peers announced that was not taken. + pub refusals: RefusalLog, } /// The status page, embedded so the binary is self-contained. @@ -103,6 +109,7 @@ pub fn router(state: AppState) -> Router { .route("/api/peers", get(api::peers)) .route("/api/routes/local", get(api::local_routes)) .route("/api/routes/received", get(api::received_routes)) + .route("/api/routes/rejected", get(api::rejected_routes)) .route("/api/events", get(events::stream)) .route("/metricz", get(metrics::metricz)) .with_state(state) diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index eaa5724..8fc2cf7 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -31,6 +31,8 @@ summary { cursor: pointer; color: #00c; } summary:hover { text-decoration: underline; } #config-text { border: 1px solid #99a; padding: 4px 6px; margin: 4px 0; font-size: 12px; overflow-x: auto; } address { font-size: smaller; margin-top: 2em; color: #444; } +form.lookup { margin: 4px 0; } +form.lookup input[type=text] { font-family: monospace; } @@ -45,7 +47,7 @@ Router ID ..., local AS ...
@@ -68,11 +70,30 @@ View: variables, status json, <

Received routes (...)

+
+Look up + +from + + +show all + +
prefixfrom peerAS pathnext hoporiginMEDlocal prefcommunitieslarge communitiesbest
+

Refused routes (...)

+
Routes that peers announced and their import policies did not take. The counts are since the daemon started; the latest refusals of each peer are kept.
+ + + +
peerprefixAS pathreasonwhen
+

Event log

@@ -90,7 +111,7 @@ View: variables, status json, < const $ = id => document.getElementById(id); const esc = s => String(s).replace(/[&<>"]/g, c => ({'&':'&','<':'<','>':'>','"':'"'}[c])); - const state = { status: null, peers: [], local: [], received: [] }; + const state = { status: null, peers: [], local: [], received: [], refused: [] }; const DASH = '-'; // ---- helpers ----------------------------------------------------------- @@ -156,8 +177,55 @@ View: variables, status json, < ${up ? esc(p.local_addr) : DASH}${up ? fmtTime(p.established_at_unix) : DASH} ${up ? age(p.established_at_unix) : DASH}${up ? (p.four_octet_asn ? 'yes' : 'no') : DASH} ${up ? esc((p.address_families || []).join(', ')) + addPath(p) : DASH} - ${up ? p.routes : DASH}${up ? p.rejected_routes : DASH}${detail}`; + ${up ? p.routes : DASH} + ${up ? (p.rejected_routes ? `${p.rejected_routes}` : 0) : DASH}${detail}`; }).join('')); + renderPeerChoices(); + } + + // ---- looking routes up ------------------------------------------------- + // The query lives in the page's address, so that it survives a reload and + // auto-refresh, and can be passed on. + const QUERY_KEYS = ['q', 'mode', 'peer', 'best']; + function lookup() { + const here = new URLSearchParams(location.search); + return { q: (here.get('q') || '').trim(), mode: here.get('mode') || 'exact', peer: here.get('peer') || '', best: here.get('best') === '1' }; + } + function isLookup() { const l = lookup(); return !!(l.q || l.peer || l.best); } + // What to ask the API for: an address finds every prefix that covers it. + function receivedUrl() { + const l = lookup(), ask = new URLSearchParams(); + if (l.q) ask.set(l.mode === 'within' ? 'within' : l.q.includes('/') ? 'prefix' : 'address', l.q); + if (l.peer) ask.set('peer', l.peer); + if (l.best) ask.set('best', 'true'); + const query = ask.toString(); + return '/api/routes/received' + (query ? `?${query}` : ''); + } + function renderPeerChoices() { + const chosen = lookup().peer; + const names = [...new Set(state.peers.filter(p => p.state === 'Established') + .map(p => p.name || p.peer_addr.replace(/:\d+$/, '').replace(/^\[|\]$/g, '')))]; + if (chosen && !names.includes(chosen)) names.push(chosen); + setHTML($('lookup-peer'), '' + + names.map(n => ``).join('')); + } + function showLookup() { + const l = lookup(); + $('lookup-q').value = l.q; $('lookup-mode').value = l.mode; $('lookup-best').checked = l.best; + renderPeerChoices(); + } + function setLookup(l) { + const url = new URL(location.href); + QUERY_KEYS.forEach(k => url.searchParams.delete(k)); + if (l) { + if (l.q) url.searchParams.set('q', l.q); + if (l.q && l.mode === 'within') url.searchParams.set('mode', 'within'); + if (l.peer) url.searchParams.set('peer', l.peer); + if (l.best) url.searchParams.set('best', '1'); + } + history.replaceState(null, '', url.pathname + url.search + '#received'); + showLookup(); renderRefreshControl(); + loadReceived().catch(console.error); } function renderLocal() { @@ -172,23 +240,57 @@ View: variables, status json, < } function renderReceived() { - const b = $('received-body'), st = state.status; - $('received-count').textContent = st && st.received_routes > state.received.length + const b = $('received-body'), st = state.status, looking = isLookup(); + $('received-count').textContent = looking + ? `${state.received.length} found${st ? ` among ${st.received_routes}` : ''}` + : st && st.received_routes > state.received.length ? `showing ${state.received.length} of ${st.received_routes}, for ${st.received_prefixes} prefixes` : `${state.received.length}${st ? `, for ${st.received_prefixes} prefixes` : ''}`; - if (!state.received.length) { setHTML(b, 'No routes received from peers.'); return; } + if (!state.received.length) { + setHTML(b, `${looking ? 'No route matches.' : 'No routes received from peers.'}`); return; + } setHTML(b, state.received.map(r => { const s = r.summary; return ` - ${esc(r.prefix)}${esc(r.peer_addr)} AS${r.peer_asn}${r.path_id ? ` path ${r.path_id}` : ''}${asPath(s.as_path)} + ${esc(r.prefix)}${r.peer_name ? esc(r.peer_name) + ' ' : ''}${esc(r.peer_addr)} AS${r.peer_asn}${r.path_id ? ` path ${r.path_id}` : ''}${asPath(s.as_path)} ${s.next_hop ? esc(s.next_hop) : DASH}${s.origin ? esc(s.origin) : DASH}${s.med ?? DASH}${s.local_pref ?? DASH} ${commList(s.communities, true)}${commList(s.large_communities)}${r.best ? 'best' : ''}`; }).join('')); } + // For each peer a line that says how many and why, then its latest few. + const REFUSED_SHOWN = 20; + function renderRefused() { + const b = $('refused-body'); + const total = state.refused.reduce((n, p) => n + p.total, 0); + $('refused-count').textContent = total ? `${total} from ${state.refused.length} peer${state.refused.length === 1 ? '' : 's'}` : '0'; + if (!state.refused.length) { setHTML(b, 'Nothing a peer announced has been refused.'); return; } + setHTML(b, state.refused.map(p => { + const why = Object.entries(p.by_reason).sort((a, c) => c[1] - a[1]).map(([reason, n]) => `${n} ${esc(reason)}`).join(', '); + const more = p.recent.length > REFUSED_SHOWN + ? ` Latest ${REFUSED_SHOWN} shown; the latest ${p.recent.length} as json.` : ''; + return `${esc(p.peer)} ${esc(p.peer_addr)}: ${p.total} refused (${why}).${more}` + + p.recent.slice(0, REFUSED_SHOWN).map(r => ` + ${esc(r.prefix)}${r.path_id ? ` path ${r.path_id}` : ''} + ${asPath(r.as_path)}${esc(r.reason)}${age(r.at_unix)} ago`).join(''); + }).join('')); + } + // ---- data loading ------------------------------------------------------ async function loadStatus() { state.status = await getJSON('/api/status'); renderStatus(); renderLocal(); renderReceived(); } async function loadPeers() { state.peers = await getJSON('/api/peers'); renderPeers(); } async function loadLocal() { state.local = await getJSON('/api/routes/local'); renderLocal(); } - async function loadReceived() { state.received = await getJSON('/api/routes/received'); renderReceived(); } - async function loadAll() { await Promise.all([loadStatus(), loadPeers(), loadLocal(), loadReceived()]); } + async function loadReceived() { + const note = $('lookup-note'); + try { + state.received = await getJSON(receivedUrl()); + note.textContent = ''; + } catch (e) { + // A query the server could not make sense of: say what it said. + state.received = []; + note.innerHTML = `${esc(e.message.replace(/^[^:]*: \d+ /, ''))}`; + } + renderReceived(); + } + async function loadRefused() { state.refused = await getJSON('/api/routes/rejected'); renderRefused(); } + async function loadAll() { await Promise.all([loadStatus(), loadPeers(), loadLocal(), loadReceived(), loadRefused()]); } // ---- events ------------------------------------------------------------ function logEvent(html) { @@ -246,6 +348,12 @@ View: variables, status json, < $('config-zippy').hidden = false; }).catch(console.error); renderRefreshControl(); + showLookup(); + $('lookup').addEventListener('submit', e => { + e.preventDefault(); + setLookup({ q: $('lookup-q').value.trim(), mode: $('lookup-mode').value, peer: $('lookup-peer').value, best: $('lookup-best').checked }); + }); + $('lookup-clear').addEventListener('click', e => { e.preventDefault(); setLookup(null); }); loadAll().catch(e => logEvent(`${esc(e.message)}`)); if (REFRESH) { connect();