From 99a47da2af73918fe1cd7c6aa70048fcaf5e4a04 Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Fri, 9 Oct 2026 11:41:47 +0200 Subject: [PATCH] status: a session's TCP connection as the kernel sees it; a route in full When a session advertised a full table to a route collector at two thousand routes a second, nothing on the status page said whether the daemon or the peer set that pace. The answer is in the kernel: the peer's receive window, the queues, and how long the connection was held up by the window or by the send buffer, which `ss -ti` shows. This change asks the kernel the same way `ss` does, over the socket diagnostics netlink family (`NETLINK_SOCK_DIAG`, `SOCK_DIAG_BY_FAMILY` with the connection's addresses and ports), and reads `tcp_info` and the congestion control name from the answer. No external program is run and no unsafe code is needed: the request and the answer are byte layouts of the kernel's ABI, read field by field, with fields an older kernel does not send left at zero. A socket on a link-local address is found by its interface, which the request names. * `bgp_fib::sockdiag`: the request, the parser, `TcpConnection`, and `tcp_connection(local, peer)`; `Ok(None)` where there is no such connection or the system is not Linux. * `GET /api/tcp?peer=ADDRESS` in the HTTP API: the connection of the session with that peer, with times in milliseconds, the two windows named by what they allow, and the held-up times also as shares of the busy time. * The status page: a "tcp" link on each established session, which expands a row below it with the figures, re-asked on every refresh while open. The group rows' span is corrected for the "sent" column. Also on the status page: the attributes of a received route that have no column of their own were badges with their value in a tooltip, which is hard to read. Clicking the attrs cell now expands a row under the route with every attribute by its name (ORIGIN, AS_PATH, NEXT_HOP, MED, LOCAL_PREF, COMMUNITIES, LARGE_COMMUNITIES, AGGREGATOR and the rest), the path identifier, the selection rank and the RPKI and ASPA states; the badges stay. Both expansions use the terms of `ss` and the RFCs rather than sentences. Documentation: the endpoint in the HTTP API reference, and the link and the expansion in the monitoring guide. Testing: unit tests of the request and of the parser on a reply built field by field, and of a "no such socket" and a refused answer; a Linux-only test that asks the kernel about a loopback connection of the test's own and about one that does not exist; the lab's basics phase checks the endpoint for the FRR session; the page was checked by hand against the daemon in the Linux VM. Co-Authored-By: Claude Fable 5.1 --- crates/fib/src/lib.rs | 1 + crates/fib/src/sockdiag.rs | 426 +++++++++++++++++++++ crates/web/src/api.rs | 152 ++++++++ crates/web/src/lib.rs | 1 + crates/web/static/statusz.html | 148 ++++++- docs/content/docs/operations/monitoring.md | 14 +- docs/content/docs/reference/http-api.md | 69 ++++ scripts/interop/lab.sh | 6 + 8 files changed, 807 insertions(+), 10 deletions(-) create mode 100644 crates/fib/src/sockdiag.rs diff --git a/crates/fib/src/lib.rs b/crates/fib/src/lib.rs index ca0be02..2fd57df 100644 --- a/crates/fib/src/lib.rs +++ b/crates/fib/src/lib.rs @@ -43,6 +43,7 @@ mod kernel; pub mod netlink; pub mod plan; pub mod routes; +pub mod sockdiag; pub mod topology; use std::net::IpAddr; diff --git a/crates/fib/src/sockdiag.rs b/crates/fib/src/sockdiag.rs new file mode 100644 index 0000000..636badc --- /dev/null +++ b/crates/fib/src/sockdiag.rs @@ -0,0 +1,426 @@ +//! What the kernel knows about one of our TCP connections: the round-trip +//! time, the windows, the queues and what has held the connection up, +//! asked over the socket diagnostics netlink family (`NETLINK_SOCK_DIAG`, +//! as `ss -ti` asks). For the status page, when someone wants to know +//! why a session is slow. + +use std::io; +use std::net::{IpAddr, SocketAddr}; + +/// A TCP connection as the kernel sees it. Times are in microseconds, +/// sizes in octets, rates in octets per second, unless said otherwise. +/// Fields the running kernel does not report are zero. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TcpConnection { + /// The TCP state, by its number (1 is ESTABLISHED); see + /// [`TcpConnection::state_name`]. + pub state: u8, + /// The congestion control algorithm, if the kernel named it. + pub congestion: Option, + /// Octets received and not yet read by us. + pub recv_queue: u32, + /// Octets written by us and not yet acknowledged by the peer. + pub send_queue: u32, + pub rtt: u32, + pub rtt_variance: u32, + pub min_rtt: u32, + pub rto: u32, + pub mss: u32, + pub pmtu: u32, + /// The congestion window, in segments. + pub cwnd: u32, + /// The slow start threshold, in segments; `None` if there is none. + pub ssthresh: Option, + /// What the peer lets us send: its receive window. + pub send_window: u32, + /// What we let the peer send. + pub receive_window: u32, + pub receive_space: u32, + pub receive_ssthresh: u32, + pub send_window_scale: u8, + pub receive_window_scale: u8, + /// Segments sent and not acknowledged. + pub unacked: u32, + /// Segments retransmitted and still out. + pub retrans: u32, + pub lost: u32, + pub total_retrans: u32, + pub bytes_retrans: u64, + /// Octets in the socket's buffer that are not sent yet. + pub not_sent: u32, + pub pacing_rate: u64, + pub delivery_rate: u64, + pub bytes_sent: u64, + pub bytes_acked: u64, + pub bytes_received: u64, + pub segments_out: u32, + pub segments_in: u32, + /// How long the connection had something to send. + pub busy: u64, + /// Of that, how long it was held up by the peer's receive window. + pub limited_by_receive_window: u64, + /// And how long by our own send buffer being full. + pub limited_by_send_buffer: u64, + pub delivered: u32, + pub reordering: u32, + /// Milliseconds since data was last sent, received, and since the + /// last acknowledgement came. + pub last_data_sent_ms: u32, + pub last_data_received_ms: u32, + pub last_ack_received_ms: u32, +} + +impl TcpConnection { + pub fn state_name(&self) -> &'static str { + match self.state { + 1 => "established", + 2 => "syn-sent", + 3 => "syn-received", + 4 => "fin-wait-1", + 5 => "fin-wait-2", + 6 => "time-wait", + 7 => "close", + 8 => "close-wait", + 9 => "last-ack", + 10 => "listen", + 11 => "closing", + _ => "unknown", + } + } +} + +const SOCK_DIAG_BY_FAMILY: u16 = 20; +const NLMSG_ERROR: u16 = 2; +const NLM_F_REQUEST: u16 = 1; +const IPPROTO_TCP: u8 = 6; +const AF_INET: u8 = 2; +const AF_INET6: u8 = 10; +const INET_DIAG_INFO: u16 = 2; +const INET_DIAG_CONG: u16 = 4; +/// `idiag_ext` bits asking for those two. +const EXTENSIONS: u8 = (1 << (INET_DIAG_INFO - 1)) | (1 << (INET_DIAG_CONG - 1)); +const NO_COOKIE: u32 = u32::MAX; +const HEADER_LEN: usize = 16; +const REQUEST_LEN: usize = HEADER_LEN + 56; +const DIAG_MSG_LEN: usize = 72; + +/// The request for the connection from `local` to `peer`, with `sequence`. +pub fn request(local: SocketAddr, peer: SocketAddr, sequence: u32) -> Vec { + let family = match local { + SocketAddr::V4(_) => AF_INET, + SocketAddr::V6(_) => AF_INET6, + }; + let mut buf = Vec::with_capacity(REQUEST_LEN); + buf.extend_from_slice(&(REQUEST_LEN as u32).to_ne_bytes()); + buf.extend_from_slice(&SOCK_DIAG_BY_FAMILY.to_ne_bytes()); + buf.extend_from_slice(&NLM_F_REQUEST.to_ne_bytes()); + buf.extend_from_slice(&sequence.to_ne_bytes()); + buf.extend_from_slice(&0u32.to_ne_bytes()); + // inet_diag_req_v2 + buf.extend_from_slice(&[family, IPPROTO_TCP, EXTENSIONS, 0]); + buf.extend_from_slice(&u32::MAX.to_ne_bytes()); // any state + // inet_diag_sockid: ports in network order, addresses as 16 octets. + buf.extend_from_slice(&local.port().to_be_bytes()); + buf.extend_from_slice(&peer.port().to_be_bytes()); + buf.extend_from_slice(&address_octets(local.ip())); + buf.extend_from_slice(&address_octets(peer.ip())); + // The interface, which a socket on a link-local address is bound to + // and is found by. + let interface = match local { + SocketAddr::V6(v6) => v6.scope_id(), + SocketAddr::V4(_) => 0, + }; + buf.extend_from_slice(&interface.to_ne_bytes()); + buf.extend_from_slice(&NO_COOKIE.to_ne_bytes()); + buf.extend_from_slice(&NO_COOKIE.to_ne_bytes()); + debug_assert_eq!(buf.len(), REQUEST_LEN); + buf +} + +fn address_octets(address: IpAddr) -> [u8; 16] { + let mut octets = [0u8; 16]; + match address { + IpAddr::V4(v4) => octets[..4].copy_from_slice(&v4.octets()), + IpAddr::V6(v6) => octets = v6.octets(), + } + octets +} + +/// What the kernel answered. +#[derive(Debug, PartialEq, Eq)] +pub enum Answer { + Connection(Box), + /// No such connection. + None, + /// The kernel refused, with this error number. + Refused(i32), +} + +/// Reads the kernel's answer to a [`request`]. +pub fn parse(buf: &[u8]) -> io::Result { + let header = buf + .get(..HEADER_LEN) + .ok_or_else(|| io::Error::other("short netlink answer"))?; + let length = ne_u32(&header[0..4]) as usize; + let message_type = ne_u16(&header[4..6]); + let message = buf + .get(HEADER_LEN..length.min(buf.len())) + .ok_or_else(|| io::Error::other("short netlink answer"))?; + if message_type == NLMSG_ERROR { + let errno = -(message + .first_chunk::<4>() + .map_or(0, |e| i32::from_ne_bytes(*e))); + // ENOENT: no such socket. + return Ok(if errno == 2 { + Answer::None + } else { + Answer::Refused(errno) + }); + } + if message_type != SOCK_DIAG_BY_FAMILY || message.len() < DIAG_MSG_LEN { + return Err(io::Error::other("not a socket diagnostic")); + } + let mut connection = TcpConnection { + state: message[1], + recv_queue: ne_u32(&message[56..60]), + send_queue: ne_u32(&message[60..64]), + ..TcpConnection::default() + }; + let mut attributes = &message[DIAG_MSG_LEN..]; + while attributes.len() >= 4 { + let length = usize::from(ne_u16(&attributes[0..2])); + let kind = ne_u16(&attributes[2..4]); + if length < 4 || length > attributes.len() { + break; + } + let value = &attributes[4..length]; + match kind { + INET_DIAG_INFO => fill_from_tcp_info(&mut connection, value), + INET_DIAG_CONG => { + let name = value.split(|b| *b == 0).next().unwrap_or(&[]); + connection.congestion = Some(String::from_utf8_lossy(name).into_owned()); + } + _ => {} + } + attributes = &attributes[(length + 3) & !3..]; + } + Ok(Answer::Connection(Box::new(connection))) +} + +/// Fills in what `tcp_info` (linux/tcp.h) has, as far as the kernel sent +/// it: the structure has grown over the years, and a field that is not +/// there is left at zero. +fn fill_from_tcp_info(c: &mut TcpConnection, info: &[u8]) { + let u32_at = |at: usize| info.get(at..at + 4).map_or(0, ne_u32); + let u64_at = |at: usize| info.get(at..at + 8).map_or(0, ne_u64); + c.send_window_scale = info.get(6).map_or(0, |b| b & 0x0f); + c.receive_window_scale = info.get(6).map_or(0, |b| b >> 4); + c.rto = u32_at(8); + c.mss = u32_at(16); + c.unacked = u32_at(24); + c.lost = u32_at(32); + c.retrans = u32_at(36); + c.last_data_sent_ms = u32_at(44); + c.last_data_received_ms = u32_at(52); + c.last_ack_received_ms = u32_at(56); + c.pmtu = u32_at(60); + c.receive_ssthresh = u32_at(64); + c.rtt = u32_at(68); + c.rtt_variance = u32_at(72); + c.ssthresh = Some(u32_at(76)).filter(|t| *t < 0x7fff_ffff); + c.cwnd = u32_at(80); + c.reordering = u32_at(88); + c.receive_space = u32_at(96); + c.total_retrans = u32_at(100); + c.pacing_rate = u64_at(104); + c.bytes_acked = u64_at(120); + c.bytes_received = u64_at(128); + c.segments_out = u32_at(136); + c.segments_in = u32_at(140); + c.not_sent = u32_at(144); + c.min_rtt = u32_at(148); + c.delivery_rate = u64_at(160); + c.busy = u64_at(168); + c.limited_by_receive_window = u64_at(176); + c.limited_by_send_buffer = u64_at(184); + c.delivered = u32_at(192); + c.bytes_sent = u64_at(200); + c.bytes_retrans = u64_at(208); + c.send_window = u32_at(228); + c.receive_window = u32_at(232); +} + +fn ne_u16(bytes: &[u8]) -> u16 { + u16::from_ne_bytes([bytes[0], bytes[1]]) +} + +fn ne_u32(bytes: &[u8]) -> u32 { + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +fn ne_u64(bytes: &[u8]) -> u64 { + let mut array = [0u8; 8]; + array.copy_from_slice(bytes); + u64::from_ne_bytes(array) +} + +/// Asks the kernel about the TCP connection from `local` to `peer`. +/// `Ok(None)` if there is no such connection, or on an operating system +/// that cannot be asked. +#[cfg(target_os = "linux")] +pub async fn tcp_connection( + local: SocketAddr, + peer: SocketAddr, +) -> io::Result> { + use netlink_sys::protocols::NETLINK_SOCK_DIAG; + use netlink_sys::{AsyncSocket, AsyncSocketExt, TokioSocket}; + + let mut socket = TokioSocket::new(NETLINK_SOCK_DIAG)?; + socket + .socket_mut() + .bind(&netlink_sys::SocketAddr::new(0, 0))?; + let sequence = 1; + socket.send(&request(local, peer, sequence)).await?; + let mut buf = bytes::BytesMut::with_capacity(4096); + tokio::time::timeout(std::time::Duration::from_secs(2), socket.recv(&mut buf)) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "the kernel did not answer"))??; + match parse(&buf)? { + Answer::Connection(connection) => Ok(Some(*connection)), + Answer::None => Ok(None), + Answer::Refused(errno) => Err(io::Error::from_raw_os_error(errno)), + } +} + +#[cfg(not(target_os = "linux"))] +pub async fn tcp_connection( + _local: SocketAddr, + _peer: SocketAddr, +) -> io::Result> { + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A reply as the kernel writes it, for an established connection + /// with a `tcp_info` of the full modern length. + fn reply() -> Vec { + let mut info = vec![0u8; 248]; + info[0] = 1; // established + info[6] = 0x7a; // snd_wscale 10, rcv_wscale 7 + let put32 = |info: &mut Vec, at: usize, v: u32| { + info[at..at + 4].copy_from_slice(&v.to_ne_bytes()) + }; + let put64 = |info: &mut Vec, at: usize, v: u64| { + info[at..at + 8].copy_from_slice(&v.to_ne_bytes()) + }; + put32(&mut info, 16, 1448); // mss + put32(&mut info, 24, 12); // unacked + put32(&mut info, 68, 2300); // rtt + put32(&mut info, 76, 0x7fff_ffff); // no ssthresh + put32(&mut info, 80, 42); // cwnd + put32(&mut info, 144, 65536); // notsent + put64(&mut info, 176, 900_000); // rwnd limited + put64(&mut info, 168, 1_000_000); // busy + put32(&mut info, 228, 131072); // snd_wnd + put32(&mut info, 232, 262144); // rcv_wnd + let cong = b"cubic\0"; + let mut message = vec![0u8; DIAG_MSG_LEN]; + message[1] = 1; + message[60..64].copy_from_slice(&80_000u32.to_ne_bytes()); // wqueue + // attributes: INFO, then CONG + let mut attrs = Vec::new(); + attrs.extend_from_slice(&((4 + info.len()) as u16).to_ne_bytes()); + attrs.extend_from_slice(&INET_DIAG_INFO.to_ne_bytes()); + attrs.extend_from_slice(&info); + attrs.extend_from_slice(&((4 + cong.len()) as u16).to_ne_bytes()); + attrs.extend_from_slice(&INET_DIAG_CONG.to_ne_bytes()); + attrs.extend_from_slice(cong); + while attrs.len() % 4 != 0 { + attrs.push(0); + } + let length = HEADER_LEN + message.len() + attrs.len(); + let mut buf = Vec::new(); + buf.extend_from_slice(&(length as u32).to_ne_bytes()); + buf.extend_from_slice(&SOCK_DIAG_BY_FAMILY.to_ne_bytes()); + buf.extend_from_slice(&[0, 0, 1, 0, 0, 0, 0, 0, 0, 0]); + buf.extend_from_slice(&message); + buf.extend_from_slice(&attrs); + buf + } + + #[test] + fn a_reply_is_read_field_by_field() { + let Answer::Connection(c) = parse(&reply()).unwrap() else { + panic!("a connection"); + }; + assert_eq!(c.state_name(), "established"); + assert_eq!(c.congestion.as_deref(), Some("cubic")); + assert_eq!(c.send_queue, 80_000); + assert_eq!((c.send_window_scale, c.receive_window_scale), (10, 7)); + assert_eq!((c.mss, c.unacked, c.rtt, c.cwnd), (1448, 12, 2300, 42)); + assert_eq!(c.ssthresh, None); + assert_eq!(c.not_sent, 65536); + assert_eq!((c.busy, c.limited_by_receive_window), (1_000_000, 900_000)); + assert_eq!((c.send_window, c.receive_window), (131072, 262144)); + } + + #[test] + fn no_such_connection_and_a_refusal_are_told_apart() { + let error = |errno: i32| { + let mut buf = vec![0u8; 20]; + buf[0..4].copy_from_slice(&20u32.to_ne_bytes()); + buf[4..6].copy_from_slice(&NLMSG_ERROR.to_ne_bytes()); + buf[16..20].copy_from_slice(&(-errno).to_ne_bytes()); + buf + }; + assert_eq!(parse(&error(2)).unwrap(), Answer::None); + assert_eq!(parse(&error(13)).unwrap(), Answer::Refused(13)); + } + + #[test] + fn a_request_names_the_connection() { + let local: SocketAddr = "[2001:db8::1]:179".parse().unwrap(); + let peer: SocketAddr = "[2001:db8::2]:40000".parse().unwrap(); + let request = request(local, peer, 7); + assert_eq!(request.len(), REQUEST_LEN); + assert_eq!(request[16], AF_INET6); + assert_eq!(&request[24..26], &179u16.to_be_bytes()); + assert_eq!(&request[26..28], &40000u16.to_be_bytes()); + assert_eq!( + &request[28..44], + &"2001:db8::1" + .parse::() + .unwrap() + .octets() + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn the_kernel_tells_of_a_connection_of_ours() { + use tokio::io::AsyncWriteExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut client = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (_server, _) = listener.accept().await.unwrap(); + client.write_all(&[0u8; 1000]).await.unwrap(); + let connection = tcp_connection(client.local_addr().unwrap(), client.peer_addr().unwrap()) + .await + .unwrap() + .expect("the kernel knows it"); + assert_eq!(connection.state_name(), "established"); + assert!(connection.mss > 0 && connection.cwnd > 0); + assert!(connection.bytes_sent >= 1000 || connection.bytes_acked >= 1000); + let none = tcp_connection( + "127.0.0.1:1".parse().unwrap(), + "127.0.0.1:2".parse().unwrap(), + ) + .await + .unwrap(); + assert!(none.is_none()); + } +} diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 8fdf357..4824b82 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -1577,3 +1577,155 @@ mod tests { assert_eq!(other, expected); } } + +/// `GET /api/tcp?peer=ADDRESS:PORT`: what the kernel knows of the TCP +/// connection of the session with that peer, as `/api/peers` names it. +#[derive(Debug, Deserialize)] +pub struct TcpQuery { + pub peer: String, +} + +/// A session's TCP connection, as the kernel reports it (Linux only). +/// Times in milliseconds, sizes in octets, rates in octets per second. +#[derive(Debug, Serialize)] +pub struct TcpView { + pub state: &'static str, + pub congestion: Option, + pub rtt_ms: f64, + pub rtt_variance_ms: f64, + pub min_rtt_ms: f64, + pub rto_ms: f64, + pub mss: u32, + pub pmtu: u32, + pub cwnd_segments: u32, + pub ssthresh_segments: Option, + /// The peer's receive window: what it lets us send. + pub send_window: u32, + /// Our receive window: what we let the peer send. + pub receive_window: u32, + pub receive_space: u32, + pub send_window_scale: u8, + pub receive_window_scale: u8, + /// Octets written by us and not yet acknowledged, and of those the + /// octets not even sent yet. + pub send_queue: u32, + pub not_sent: u32, + pub recv_queue: u32, + pub unacked_segments: u32, + pub retrans_segments: u32, + pub lost_segments: u32, + pub total_retrans: u32, + pub bytes_retrans: u64, + pub pacing_rate: u64, + pub delivery_rate: u64, + pub bytes_sent: u64, + pub bytes_acked: u64, + pub bytes_received: u64, + pub segments_out: u32, + pub segments_in: u32, + /// How long the connection had something to send, and of that how + /// long it was held up by the peer's window and by our send buffer. + pub busy_ms: f64, + pub limited_by_receive_window_ms: f64, + pub limited_by_send_buffer_ms: f64, + /// The same two, as shares of the busy time, 0 to 1. + pub limited_by_receive_window_share: f64, + pub limited_by_send_buffer_share: f64, + pub last_data_sent_ms: u32, + pub last_data_received_ms: u32, + pub last_ack_received_ms: u32, +} + +/// The answer to `/api/tcp`. +#[derive(Debug, Serialize)] +pub struct TcpAnswer { + /// Whether the kernel could be asked and knew the connection. + pub available: bool, + /// Why not, if not. + pub reason: Option, + pub connection: Option, +} + +pub async fn tcp( + State(state): State, + Query(query): Query, +) -> Result, QueryError> { + let session = state + .registry + .list() + .into_iter() + .find(|s| socket_text(s.peer.addr) == query.peer) + .ok_or_else(|| QueryError::Bad(format!("no session with {}", query.peer)))?; + let answer = + match bgp_fib::sockdiag::tcp_connection(session.local_addr, session.peer.addr).await { + Ok(Some(c)) => { + let ms = |us: u64| us as f64 / 1000.0; + let share = |part: u64| { + if c.busy == 0 { + 0.0 + } else { + part as f64 / c.busy as f64 + } + }; + TcpAnswer { + available: true, + reason: None, + connection: Some(TcpView { + state: c.state_name(), + congestion: c.congestion.clone(), + rtt_ms: ms(c.rtt.into()), + rtt_variance_ms: ms(c.rtt_variance.into()), + min_rtt_ms: ms(c.min_rtt.into()), + rto_ms: ms(c.rto.into()), + mss: c.mss, + pmtu: c.pmtu, + cwnd_segments: c.cwnd, + ssthresh_segments: c.ssthresh, + send_window: c.send_window, + receive_window: c.receive_window, + receive_space: c.receive_space, + send_window_scale: c.send_window_scale, + receive_window_scale: c.receive_window_scale, + send_queue: c.send_queue, + not_sent: c.not_sent, + recv_queue: c.recv_queue, + unacked_segments: c.unacked, + retrans_segments: c.retrans, + lost_segments: c.lost, + total_retrans: c.total_retrans, + bytes_retrans: c.bytes_retrans, + pacing_rate: c.pacing_rate, + delivery_rate: c.delivery_rate, + bytes_sent: c.bytes_sent, + bytes_acked: c.bytes_acked, + bytes_received: c.bytes_received, + segments_out: c.segments_out, + segments_in: c.segments_in, + busy_ms: ms(c.busy), + limited_by_receive_window_ms: ms(c.limited_by_receive_window), + limited_by_send_buffer_ms: ms(c.limited_by_send_buffer), + limited_by_receive_window_share: share(c.limited_by_receive_window), + limited_by_send_buffer_share: share(c.limited_by_send_buffer), + last_data_sent_ms: c.last_data_sent_ms, + last_data_received_ms: c.last_data_received_ms, + last_ack_received_ms: c.last_ack_received_ms, + }), + } + } + Ok(None) => TcpAnswer { + available: false, + reason: Some(if cfg!(target_os = "linux") { + "the kernel has no such connection".to_string() + } else { + "only on Linux".to_string() + }), + connection: None, + }, + Err(e) => TcpAnswer { + available: false, + reason: Some(format!("the kernel could not be asked: {e}")), + connection: None, + }, + }; + Ok(Json(answer)) +} diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index 5db1421..2b467e5 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -144,6 +144,7 @@ pub fn router(state: AppState) -> Router { .route("/api/routes/rejected", get(api::rejected_routes)) .route("/api/rpki", get(api::rpki)) .route("/api/kernel", get(api::kernel)) + .route("/api/tcp", get(api::tcp)) .route("/api/events", get(events::stream)) .route("/metricz", get(metrics::metricz)) .with_state(state) diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index 818761b..6893900 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -68,6 +68,14 @@ table.data th { background: var(--head); font-weight: bold; position: sticky; to table.data > tbody > tr:hover > td { background: var(--hover); } table.data td.num, table.data th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } table.data tr.group > td { background: var(--band); font-weight: bold; padding: 3px 6px; } +table.data tr.tcp-row > td { background: var(--band); padding: 6px 10px 8px 28px; } +table.data td.attrs { cursor: pointer; } +table.data td.attrs:hover { background: var(--hover); } +table.data tr.route-detail > td { background: var(--band); padding: 6px 10px 8px 40px; } +.tcp-grid { display: grid; grid-template-columns: max-content 1fr; gap: 1px 16px; font-size: 12px; } +.tcp-grid > div:nth-child(odd) { color: var(--fg-soft); white-space: nowrap; } +a.tcp-link { color: var(--dim); font-size: 11.5px; text-decoration: none; border: 1px solid var(--line); border-radius: 3px; padding: 0 4px; margin-left: 4px; } +a.tcp-link:hover, a.tcp-link.open { color: var(--link); border-color: var(--link); } table.data tr.best > td { background: var(--best-bg); } table.data tr.dim td { color: var(--dim); } .asn { cursor: help; } @@ -241,7 +249,7 @@ from const $ = id => document.getElementById(id); const esc = s => String(s).replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); - const state = { status: null, peers: [], local: [], received: [], receivedPage: {}, rejected: [], rpki: null, kernel: null }; + const state = { status: null, peers: [], local: [], received: [], receivedPage: {}, rejected: [], rpki: null, kernel: null, tcpOpen: new Set(), routeOpen: new Set() }; const DASH = '–'; const fmtCount = n => Number(n).toLocaleString('en-US'); // A glyph: a small badge with its meaning as the tooltip. @@ -532,7 +540,7 @@ from const down = peers.filter(p => peerRank(p) === 0).length; const waiting = peers.filter(p => p.state === 'Passive').length; $('peers-count').textContent = `${up} of ${peers.length} established` + (down ? `, ${down} down` : '') + (waiting ? `, ${waiting} waiting` : ''); - if (!peers.length) { setHTML(b, 'No peers configured and none connected.'); return; } + if (!peers.length) { setHTML(b, 'No peers configured and none connected.'); return; } const problemsOnly = $('peers-problems').checked; // The IPv4 and IPv6 sessions of one neighbour, named -v4 and // -v6, are one neighbour to the reader: listed together, IPv6 @@ -549,7 +557,7 @@ from const expected = up && p.expected_routes > 0 ? ` of ~${fmtCount(p.expected_routes)}` : ''; const classes = [peerRank(p) === 3 ? 'dim' : '', pair || ''].filter(Boolean).join(' '); - return ` + return ` ${sinceGlyph(p)}${name} ${esc(p.peer_addr)} AS${asn(p.peer_asn)} ${roleGlyph(p)} ${esc(p.state)} @@ -559,7 +567,7 @@ from ${up ? trendGlyph(p) + fmtCount(p.routes) + expected + stale : DASH} ${up ? fmtCount(p.advertised_routes) : DASH} ${rejected}${held} - ${peerDetail(p)}`; + ${peerDetail(p)}${up ? tcpLink(p) : ''}`; }; // Rows for a sorted list, with the v4/v6 pairs in it marked. const rows = list => list.map((p, i) => { @@ -575,23 +583,93 @@ from // they are to us. const problems = peers.filter(p => peerRank(p) <= 1).sort(byName); if (problems.length) { - html += `Needing attention (${problems.length})` + rows(problems); + html += `Needing attention (${problems.length})` + rows(problems); } if (!problemsOnly) { for (const group of ROLE_ORDER) { const members = peers.filter(p => peerRank(p) >= 2 && peerGroup(p) === group).sort(byName); if (!members.length) continue; - html += `${GROUP_TITLE[group]} (${members.length})` + rows(members); + html += `${GROUP_TITLE[group]} (${members.length})` + rows(members); } } else if (!problems.length) { - html += 'Nothing needs attention.'; + html += 'Nothing needs attention.'; } setHTML(b, html); + renderTcpRows(); const marked = peers.filter(changedSince).length; setHTML($('since-ctl'), marked ? `${glyph('●', 'Rows that changed while this tab was in the background', 'since')} ${marked} changed since you last looked · clear` : ''); renderPeerChoices(); } + // The TCP connection of a session, as the kernel sees it: a link on the + // row, and a row below it while it is open, asked for afresh on every + // refresh. + function tcpLink(p) { + const open = state.tcpOpen.has(p.peer_addr); + return `tcp`; + } + const fmtOctets = n => n >= 1048576 ? `${(n / 1048576).toFixed(1)} MiB` : n >= 1024 ? `${(n / 1024).toFixed(1)} KiB` : `${n} B`; + const fmtBits = n => { const b = n * 8; return b >= 1e9 ? `${(b / 1e9).toFixed(2)} Gbit/s` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} Mbit/s` : `${(b / 1e3).toFixed(0)} kbit/s`; }; + const fmtMs = n => n >= 1000 ? `${(n / 1000).toFixed(1)} s` : `${n.toFixed(n < 10 ? 2 : 0)} ms`; + const ago = ms => ms >= 60000 ? `${Math.round(ms / 60000)}m` : ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`; + function tcpDetail(a) { + if (!a.available) return `${esc(a.reason || 'n/a')}`; + const c = a.connection; + const pct = share => `${(share * 100).toFixed(0)}%`; + const held = share => share > 0.5 ? 'red' : share > 0.1 ? 'amber' : ''; + const kv = (k, v) => `${k} ${v}`; + const rows = [ + ['state', `${esc(c.state.toUpperCase())} ${c.congestion ? esc(c.congestion) : ''}`], + ['rtt', `${fmtMs(c.rtt_ms)} ${kv('minrtt', fmtMs(c.min_rtt_ms))} ${kv('rttvar', fmtMs(c.rtt_variance_ms))} ${kv('rto', fmtMs(c.rto_ms))}`], + ['cwnd', `${fmtCount(c.cwnd_segments)} ${kv('mss', `${c.mss}`)} ${kv('=', fmtOctets(c.cwnd_segments * c.mss))} ${kv('ssthresh', c.ssthresh_segments == null ? '∞' : fmtCount(c.ssthresh_segments))} ${kv('pmtu', `${c.pmtu}`)}`], + ['snd_wnd', `${fmtOctets(c.send_window)} ${kv('wscale', `${c.send_window_scale}`)}`], + ['rcv_wnd', `${fmtOctets(c.receive_window)} ${kv('rcv_space', fmtOctets(c.receive_space))} ${kv('wscale', `${c.receive_window_scale}`)}`], + ['send-q', `${fmtOctets(c.send_queue)} ${kv('notsent', fmtOctets(c.not_sent))} ${kv('unacked', fmtCount(c.unacked_segments))}`], + ['recv-q', fmtOctets(c.recv_queue)], + ['retrans', `${fmtCount(c.retrans_segments)} ${kv('lost', fmtCount(c.lost_segments))} ${kv('total', `${fmtCount(c.total_retrans)} / ${fmtOctets(c.bytes_retrans)}`)}`], + ['rate', `${kv('delivery', fmtBits(c.delivery_rate))} ${kv('pacing', fmtBits(c.pacing_rate))}`], + ['limited', `${kv('busy', fmtMs(c.busy_ms))} ${kv('rwnd', pct(c.limited_by_receive_window_share))} ${kv('sndbuf', pct(c.limited_by_send_buffer_share))}`], + ['bytes', `${kv('sent', fmtOctets(c.bytes_sent))} ${kv('acked', fmtOctets(c.bytes_acked))} ${kv('received', fmtOctets(c.bytes_received))} ${kv('segs', `${fmtCount(c.segments_out)}↑ ${fmtCount(c.segments_in)}↓`)}`], + ['last', `${kv('snd', ago(c.last_data_sent_ms))} ${kv('rcv', ago(c.last_data_received_ms))} ${kv('ack', ago(c.last_ack_received_ms))}`], + ]; + return `
${rows.map(([k, v]) => `
${k}
${v}
`).join('')}
`; + } + async function renderTcpRows() { + for (const addr of state.tcpOpen) { + const tr = document.querySelector(`#peers-body tr[data-peer-addr="${CSS.escape(addr)}"]`); + if (!tr) continue; + let row = tr.nextElementSibling; + if (!row || !row.classList.contains('tcp-row')) { + row = document.createElement('tr'); + row.className = 'tcp-row'; + row.innerHTML = 'asking the kernel…'; + tr.after(row); + } + try { + const answer = await getJSON(`/api/tcp?peer=${encodeURIComponent(addr)}`); + row.innerHTML = `${tcpDetail(answer)}`; + } catch (e) { + row.innerHTML = `${esc(String(e))}`; + } + } + } + document.addEventListener('click', e => { + const link = e.target.closest('a.tcp-link'); + if (!link) return; + e.preventDefault(); + const addr = link.dataset.peer; + if (state.tcpOpen.has(addr)) { + state.tcpOpen.delete(addr); + link.classList.remove('open'); + const tr = link.closest('tr'), next = tr && tr.nextElementSibling; + if (next && next.classList.contains('tcp-row')) next.remove(); + } else { + state.tcpOpen.add(addr); + link.classList.add('open'); + renderTcpRows(); + } + }); + // The strip: what needs attention, in one line, or that nothing does. function renderStrip() { const items = []; @@ -748,18 +826,70 @@ from if (!state.received.length) { setHTML(b, `${looking ? 'No route matches.' : 'No routes received from peers.'}`); return; } - setHTML(b, state.received.map(r => { const s = r.summary; return ` + setHTML(b, state.received.map(r => { const s = r.summary; return ` ${rankGlyph(r)}${r.stale ? glyph('stale', 'Kept from a session of the peer’s that ended (graceful restart), and not sent again since', 'amber') : ''} ${esc(r.prefix)} ${r.peer_name ? `${esc(r.peer_name)} ` : ''}${esc(r.peer_addr)} AS${asn(r.peer_asn)}${r.path_id ? ` path ${r.path_id}` : ''} ${clamp(asPath(s.as_path))} ${s.next_hop ? esc(s.next_hop) : DASH}${s.origin ? esc(s.origin) : DASH} ${s.med ?? DASH}${s.local_pref ?? DASH} - ${clamp(commList(s.communities))}${clamp(commList(s.large_communities, true))}${otherList(s.other)} + ${clamp(commList(s.communities))}${clamp(commList(s.large_communities, true))}${otherList(s.other)} ${validity('RPKI', r.rpki)} ${validity('ASPA', r.aspa)}`; }).join('')); markTall(b); + renderRouteDetails(); } + // A route, in full, under its row: every attribute by its name, as the + // RFCs write it, opened by a click on the attrs cell. + const routeKey = r => `${r.prefix}|${r.peer_addr}|${r.path_id}`; + const ATTR_NAMES = { + 'atomic aggregate': 'ATOMIC_AGGREGATE', 'aggregator': 'AGGREGATOR', 'extended communities': 'EXTENDED_COMMUNITIES', + 'only to customer': 'OTC', 'originator ID': 'ORIGINATOR_ID', 'cluster list': 'CLUSTER_LIST', + }; + function routeDetail(r) { + const s = r.summary; + const rows = [ + ['ORIGIN', s.origin ? esc(s.origin) : DASH], + ['AS_PATH', s.as_path.length ? s.as_path.map(asn).join(' ') : '(empty)'], + ['NEXT_HOP', s.next_hop ? esc(s.next_hop) : DASH], + ['MED', s.med ?? DASH], + ['LOCAL_PREF', s.local_pref ?? DASH], + ['COMMUNITIES', s.communities.length ? s.communities.map(esc).join(' ') : DASH], + ['LARGE_COMMUNITIES', s.large_communities.length ? s.large_communities.map(esc).join(' ') : DASH], + ...(s.other || []).map(a => [esc(ATTR_NAMES[a.name] || a.name), esc(a.value) || 'present']), + ]; + if (r.path_id) rows.push(['PATH_ID', `${r.path_id}`]); + rows.push(['selection', `${r.best ? 'best' : `rank ${r.rank}`}${r.stale ? ', stale' : ''}`]); + rows.push(['rpki', r.rpki ? esc(r.rpki) : DASH], ['aspa', r.aspa ? esc(r.aspa) : DASH]); + return `
${rows.map(([k, v]) => `
${k}
${v}
`).join('')}
`; + } + function renderRouteDetails() { + for (const key of state.routeOpen) { + const r = state.received.find(r => routeKey(r) === key); + const tr = document.querySelector(`#received-body tr[data-route="${CSS.escape(key)}"]`); + if (!r || !tr) continue; + const next = tr.nextElementSibling; + if (next && next.classList.contains('route-detail')) continue; + const row = document.createElement('tr'); + row.className = 'route-detail'; + row.innerHTML = `${routeDetail(r)}`; + tr.after(row); + } + } + document.addEventListener('click', e => { + const cell = e.target.closest('#received-body td.attrs'); + if (!cell || e.target.closest('a')) return; + const tr = cell.closest('tr'), key = tr.dataset.route; + if (state.routeOpen.has(key)) { + state.routeOpen.delete(key); + const next = tr.nextElementSibling; + if (next && next.classList.contains('route-detail')) next.remove(); + } else { + state.routeOpen.add(key); + renderRouteDetails(); + } + }); + const RPKI_STATE_CLASS = { synchronised: 'green', synchronising: 'amber', connecting: 'red' }; function renderRpki() { const r = state.rpki, b = $('rpki-body'); diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md index 8edc754..fec9bb4 100644 --- a/docs/content/docs/operations/monitoring.md +++ b/docs/content/docs/operations/monitoring.md @@ -39,7 +39,13 @@ Below the status line, the page displays the following information: - Each neighbor, with its session state, time established, next hop, session properties, number of accepted routes, number of routes advertised to it (**sent**), rejected announcements, and held routes, - and its last error. Neighbors that need attention are + and its last error. The **tcp** link on an established session expands + a row with the connection as the kernel reports it: round-trip time, + congestion window, the neighbor's receive window and Ferrix's, queues, + retransmissions, throughput, and the share of time the connection was + held up by the neighbor's window or by the local send buffer. This + shows whether a slow advertisement of a table is paced by the neighbor. + Linux only. Neighbors that need attention are listed first; the others are grouped by role. The IPv4 and IPv6 sessions of one neighbor, named `NAME-v4` and `NAME-v6`, are listed together, IPv6 first, and joined by a bar. The **problems only** option hides the neighbors that @@ -153,6 +159,12 @@ advertise the same routes can therefore have different counts. The number of routes currently held, because they are RPKI or ASPA invalid or their next hop is unreachable, is displayed in the neighbor table as **held**. +In the received routes table, the **attrs** column shows the attributes +that have no column of their own as badges. Clicking the cell expands a +row below the route that lists every attribute of the route by its name, +with the path identifier, the selection rank, and the RPKI and ASPA +states. + ```json [{"peer":"transit-a","total":3,"by_reason":{"RPKI invalid":2,"bogon prefix":1}, "recent":[{"prefix":"203.0.113.0/24","reason":"RPKI invalid","as_path":[64496,64511]}]}] diff --git a/docs/content/docs/reference/http-api.md b/docs/content/docs/reference/http-api.md index 70bf96b..30de5a7 100644 --- a/docs/content/docs/reference/http-api.md +++ b/docs/content/docs/reference/http-api.md @@ -37,6 +37,7 @@ server returns the following error statuses: | `/api/routes/rejected` | Rejected routes, by neighbor. | | `/api/rpki` | RPKI cache, data, and data age. | | `/api/kernel` | Kernel route installation state and interfaces. | +| `/api/tcp` | The TCP connection of one session, as the kernel reports it. | | `/api/events` | Session and route changes, as server-sent events. | | `/metricz` | Prometheus metrics. See [Metrics Reference](@/docs/reference/metrics.md). | @@ -230,6 +231,74 @@ routes. To return the records for one neighbor, add `?peer=NAME`. | `refused` | Number of prefixes whose active route the kernel rejected. | | `errors` | Number of requests that the kernel rejected. | +## TCP Connection + +`/api/tcp?peer=ADDRESS` returns what the kernel reports about the TCP +connection of the session with the neighbor at `ADDRESS`, given as +`peer_addr` in `/api/peers` shows it: the round-trip time, the congestion +and receive windows, the queues, the retransmissions, and how long the +connection has been held up by the neighbor's receive window or by the +local send buffer. Linux only; on other systems, and for a session the +kernel has no connection for, `available` is `false` and `reason` says +why. + +```json +{ + "available": true, + "reason": null, + "connection": { + "state": "established", + "congestion": "cubic", + "rtt_ms": 0.62, + "rtt_variance_ms": 0.11, + "min_rtt_ms": 0.4, + "rto_ms": 201, + "mss": 1428, + "pmtu": 1500, + "cwnd_segments": 10, + "ssthresh_segments": null, + "send_window": 1048576, + "receive_window": 3145728, + "receive_space": 65536, + "send_window_scale": 7, + "receive_window_scale": 10, + "send_queue": 0, + "not_sent": 0, + "recv_queue": 0, + "unacked_segments": 0, + "retrans_segments": 0, + "lost_segments": 0, + "total_retrans": 0, + "bytes_retrans": 0, + "pacing_rate": 2309920000, + "delivery_rate": 184860000, + "bytes_sent": 312993, + "bytes_acked": 312994, + "bytes_received": 190, + "segments_out": 230, + "segments_in": 118, + "busy_ms": 41, + "limited_by_receive_window_ms": 0, + "limited_by_send_buffer_ms": 0, + "limited_by_receive_window_share": 0, + "limited_by_send_buffer_share": 0, + "last_data_sent_ms": 1234, + "last_data_received_ms": 1300, + "last_ack_received_ms": 1234 + } +} +``` + +Times are in milliseconds, sizes in octets, rates in octets per second. +`send_window` is the neighbor's receive window, which bounds what Ferrix +can send; `receive_window` is what Ferrix lets the neighbor send. The two +`_share` values are the fractions of the busy time, 0 to 1, in which the +connection was held up by the neighbor's window and by the local send +buffer. A session that advertises a table to a neighbor that reads it +slowly shows a high `limited_by_receive_window_share`. The status page +shows the same information under the **tcp** link of each established +session. + ## Events `/api/events` is a stream of diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh index 54f2ce2..fcfd9a9 100755 --- a/scripts/interop/lab.sh +++ b/scripts/interop/lab.sh @@ -489,6 +489,12 @@ phase_basics() { in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan" in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan" ok "packets get through to both, over IPv4 and IPv6" + + # The kernel's view of a session's connection, as the page asks for it. + frr4_addr=$(api /api/peers | jq -r '.[] | select(.name == "frr4") | .peer_addr') + api "/api/tcp?peer=$frr4_addr" | jq -e '.available and .connection.state == "established" and .connection.mss > 0' >/dev/null \ + || fail "no TCP details for frr4: $(api "/api/tcp?peer=$frr4_addr")" + ok "the kernel tells of a session's TCP connection" } # RFC 9234: each end says what it is to the other, and routes are marked