From d7d7d39bb9a315df39bdf14b438e4414322259c1 Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Fri, 9 Oct 2026 12:12:53 +0200 Subject: [PATCH] metrics: export each session's TCP connection figures The kernel's view of a session's TCP connection, which the status page shows on request since the last change, could not be looked at after the fact: a feed that crawled overnight left no record of whether the peer's window or our own side held it up. This change samples it at every scrape of `/metricz`: for each established session, one `NETLINK_SOCK_DIAG` round trip, and twenty gauges labelled by peer, `bgp_peer_tcp_*`: smoothed, least and variance of the round-trip time; the congestion window and segment size; the peer's receive window and ours; the send queue, the part of it not yet sent, the receive queue and the segments in flight; retransmitted and lost segments; the busy time and the time held up by the peer's window and by our send buffer, cumulative, so that their rates give the share of time a session is held up; octets sent and received; and the delivery and pacing rates. A session the kernel has no connection for loses its series, as does one that ends. `SessionHandle::metric_label` is the one place a peer's metric label is made (its name, or its address), used by the advertiser's gauge and here. Documentation: the metrics reference lists them; the monitoring guide gives the query for the share of time a session is held up by the neighbor's receive window, which is the question the overnight feed raised. Testing: a daemon in the Linux VM with one session shows the series and drops them when the session ends; the lab's basics phase checks the scrape for the FRR session's round-trip time; the workspace tests pass. Co-Authored-By: Claude Fable 5.1 --- crates/server/src/advertise.rs | 8 +- crates/server/src/connection.rs | 3 +- crates/server/src/metrics.rs | 138 ++++++++++++++++++++- crates/server/src/session.rs | 8 ++ crates/web/src/metrics.rs | 81 +++++++++++- docs/content/docs/operations/monitoring.md | 3 + docs/content/docs/reference/metrics.md | 13 ++ scripts/interop/lab.sh | 1 + 8 files changed, 244 insertions(+), 11 deletions(-) diff --git a/crates/server/src/advertise.rs b/crates/server/src/advertise.rs index 0bdda15..ba059fd 100644 --- a/crates/server/src/advertise.rs +++ b/crates/server/src/advertise.rs @@ -51,7 +51,6 @@ use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, watch}; use tokio::task::JoinHandle; use tracing::{debug, error, info, warn}; -use crate::address::PeerAddress; use crate::error::SessionError; use crate::export::{self, Egress, Outgoing, Reflection, Source}; use crate::family::FamilySet; @@ -175,12 +174,7 @@ impl Advertiser { .as_ref() .map(|_| session.families.clone()), converged: Some(self.converged.clone()).filter(|converged| !*converged.borrow()), - // The peer goes by its configured name, or by its address, as - // in its other metrics. - advertised_metric: PEER_ADVERTISED.with_label_values(&[&session - .name - .clone() - .unwrap_or_else(|| PeerAddress::from(session.peer.addr).to_string())]), + advertised_metric: PEER_ADVERTISED.with_label_values(&[&session.metric_label()]), }; let running = Running { session: session.clone(), diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index bf1753a..15a650b 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -44,7 +44,7 @@ use crate::fsm::{Action, Event, Fsm, FsmConfig, Negotiated, Received}; use crate::holddown::{HeldDown, HoldDowns}; use crate::metrics::{ EXPORT_LIMIT_EXCEEDED, MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ADVERTISED, PEER_ROUTES, - PREFIX_LIMIT_EXCEEDED, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, + PEER_TCP, PREFIX_LIMIT_EXCEEDED, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name, }; use crate::panic::contained; use crate::policy::{self, ExportLimits, ExportPolicy, ImportPolicy, PeerRole}; @@ -316,6 +316,7 @@ async fn release_routes( // Fails only if the session never set it, which is nothing to do. let _ = PEER_ROUTES.remove_label_values(&[session.rejections.label()]); let _ = PEER_ADVERTISED.remove_label_values(&[session.rejections.label()]); + PEER_TCP.remove(session.rejections.label()); if *session.leave.borrow() == Some(Leave::Restart) { // The daemon is about to go, and its routes with it; the kernel // keeps its own. Taking them out first would only hold the diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index f7e2c06..9e599c6 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -4,8 +4,8 @@ use std::sync::LazyLock; use bgp_packet::message::Message; use prometheus::{ - IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_int_counter, - register_int_counter_vec, register_int_gauge, register_int_gauge_vec, + GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_gauge_vec, + register_int_counter, register_int_counter_vec, register_int_gauge, register_int_gauge_vec, }; pub static MESSAGES_SENT: LazyLock = LazyLock::new(|| { @@ -78,6 +78,139 @@ pub static PEER_ADVERTISED: LazyLock = LazyLock::new(|| { .expect("metric registration") }); +/// A session's TCP connection as the kernel reports it (`TCP_INFO`), +/// sampled when the metrics are scraped. Seconds, octets, segments, and +/// octets per second; the busy, limited and byte figures are cumulative +/// over the connection's life. +pub struct PeerTcp { + pub rtt_seconds: GaugeVec, + pub min_rtt_seconds: GaugeVec, + pub rtt_variance_seconds: GaugeVec, + pub cwnd_segments: IntGaugeVec, + pub mss_bytes: IntGaugeVec, + pub send_window_bytes: IntGaugeVec, + pub receive_window_bytes: IntGaugeVec, + pub send_queue_bytes: IntGaugeVec, + pub not_sent_bytes: IntGaugeVec, + pub receive_queue_bytes: IntGaugeVec, + pub unacked_segments: IntGaugeVec, + pub retransmitted_segments: IntGaugeVec, + pub lost_segments: IntGaugeVec, + pub busy_seconds: GaugeVec, + pub receive_window_limited_seconds: GaugeVec, + pub send_buffer_limited_seconds: GaugeVec, + pub sent_bytes: IntGaugeVec, + pub received_bytes: IntGaugeVec, + pub delivery_rate_bytes_per_second: IntGaugeVec, + pub pacing_rate_bytes_per_second: IntGaugeVec, +} + +impl PeerTcp { + /// Forgets the peer's series, when its session has ended. + pub fn remove(&self, label: &str) { + for gauge in [ + &self.rtt_seconds, + &self.min_rtt_seconds, + &self.rtt_variance_seconds, + &self.busy_seconds, + &self.receive_window_limited_seconds, + &self.send_buffer_limited_seconds, + ] { + let _ = gauge.remove_label_values(&[label]); + } + for gauge in [ + &self.cwnd_segments, + &self.mss_bytes, + &self.send_window_bytes, + &self.receive_window_bytes, + &self.send_queue_bytes, + &self.not_sent_bytes, + &self.receive_queue_bytes, + &self.unacked_segments, + &self.retransmitted_segments, + &self.lost_segments, + &self.sent_bytes, + &self.received_bytes, + &self.delivery_rate_bytes_per_second, + &self.pacing_rate_bytes_per_second, + ] { + let _ = gauge.remove_label_values(&[label]); + } + } +} + +pub static PEER_TCP: LazyLock = LazyLock::new(|| { + let seconds = |name: &str, help: &str| { + register_gauge_vec!(format!("bgp_peer_tcp_{name}"), help, &["peer"]) + .expect("metric registration") + }; + let count = |name: &str, help: &str| { + register_int_gauge_vec!(format!("bgp_peer_tcp_{name}"), help, &["peer"]) + .expect("metric registration") + }; + PeerTcp { + rtt_seconds: seconds( + "rtt_seconds", + "Smoothed round-trip time of the session's TCP connection", + ), + min_rtt_seconds: seconds( + "min_rtt_seconds", + "Least round-trip time seen on the connection", + ), + rtt_variance_seconds: seconds( + "rtt_variance_seconds", + "Round-trip time variance of the connection", + ), + cwnd_segments: count("cwnd_segments", "Congestion window, in segments"), + mss_bytes: count("mss_bytes", "Maximum segment size"), + send_window_bytes: count( + "send_window_bytes", + "The peer's receive window: what it lets us send", + ), + receive_window_bytes: count( + "receive_window_bytes", + "Our receive window: what we let the peer send", + ), + send_queue_bytes: count( + "send_queue_bytes", + "Octets written and not yet acknowledged by the peer", + ), + not_sent_bytes: count("not_sent_bytes", "Octets written and not yet sent"), + receive_queue_bytes: count("receive_queue_bytes", "Octets received and not yet read"), + unacked_segments: count("unacked_segments", "Segments sent and not yet acknowledged"), + retransmitted_segments: count( + "retransmitted_segments", + "Segments retransmitted over the connection's life", + ), + lost_segments: count("lost_segments", "Segments thought lost now"), + busy_seconds: seconds( + "busy_seconds", + "Time the connection had something to send, over its life", + ), + receive_window_limited_seconds: seconds( + "receive_window_limited_seconds", + "Of the busy time, time held up by the peer's receive window", + ), + send_buffer_limited_seconds: seconds( + "send_buffer_limited_seconds", + "Of the busy time, time held up by our send buffer being full", + ), + sent_bytes: count("sent_bytes", "Octets sent over the connection's life"), + received_bytes: count( + "received_bytes", + "Octets received over the connection's life", + ), + delivery_rate_bytes_per_second: count( + "delivery_rate_bytes_per_second", + "Delivery rate the kernel measured most recently", + ), + pacing_rate_bytes_per_second: count( + "pacing_rate_bytes_per_second", + "Rate the kernel paces the connection at", + ), + } +}); + pub static PEER_MAX_PREFIXES: LazyLock = LazyLock::new(|| { register_int_gauge_vec!( "bgp_peer_max_prefixes", @@ -238,6 +371,7 @@ pub fn init() { LazyLock::force(&ROUTES_EXPORTED); LazyLock::force(&PEER_UP); LazyLock::force(&PEER_ADVERTISED); + LazyLock::force(&PEER_TCP); BUILD_INFO .with_label_values(&[env!("CARGO_PKG_VERSION")]) .set(1); diff --git a/crates/server/src/session.rs b/crates/server/src/session.rs index 92425e7..3b7fd66 100644 --- a/crates/server/src/session.rs +++ b/crates/server/src/session.rs @@ -296,6 +296,14 @@ impl SessionHandle { /// Whether `other` is a handle to this same session, and not to another /// session with the same peer. + /// What the peer goes by in the metrics: its configured name, or its + /// address. + pub fn metric_label(&self) -> String { + self.name + .clone() + .unwrap_or_else(|| crate::address::PeerAddress::from(self.peer.addr).to_string()) + } + pub fn is_same_session(&self, other: &SessionHandle) -> bool { self.outbound.same_channel(&other.outbound) } diff --git a/crates/web/src/metrics.rs b/crates/web/src/metrics.rs index 6d54677..6a645fe 100644 --- a/crates/web/src/metrics.rs +++ b/crates/web/src/metrics.rs @@ -5,7 +5,7 @@ use axum::extract::State; use axum::http::{StatusCode, header}; use axum::response::IntoResponse; use bgp_server::metrics::{ - LOCAL_ROUTES, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, RIB_ROUTES, + LOCAL_ROUTES, PEER_TCP, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, RIB_ROUTES, }; use prometheus::{Encoder, TextEncoder}; @@ -24,6 +24,7 @@ pub async fn metricz(State(state): State) -> impl IntoResponse { RIB_PREFIXES.set(gauge(state.rib.prefix_count())); RIB_ATTRIBUTE_SETS.set(gauge(state.rib.attrs().len())); RIB_JOURNAL_ENTRIES.set(gauge(state.rib.journal_len())); + sample_tcp(&state).await; let mut buf = Vec::new(); let encoder = TextEncoder::new(); @@ -37,3 +38,81 @@ pub async fn metricz(State(state): State) -> impl IntoResponse { Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), } } + +/// Asks the kernel about every established session's TCP connection and +/// sets the peer's gauges: one netlink round trip a session, at scrape +/// time, so that there is history to look back on. A session the kernel +/// has no connection for any more loses its series. +async fn sample_tcp(state: &AppState) { + let tcp = &*PEER_TCP; + for session in state.registry.list() { + let label = session.metric_label(); + let connection = + bgp_fib::sockdiag::tcp_connection(session.local_addr, session.peer.addr).await; + let Ok(Some(c)) = connection else { + tcp.remove(&label); + continue; + }; + let seconds = |us: u64| us as f64 / 1e6; + let int = |n: u64| i64::try_from(n).unwrap_or(i64::MAX); + let label = [label.as_str()]; + tcp.rtt_seconds + .with_label_values(&label) + .set(seconds(c.rtt.into())); + tcp.min_rtt_seconds + .with_label_values(&label) + .set(seconds(c.min_rtt.into())); + tcp.rtt_variance_seconds + .with_label_values(&label) + .set(seconds(c.rtt_variance.into())); + tcp.cwnd_segments + .with_label_values(&label) + .set(c.cwnd.into()); + tcp.mss_bytes.with_label_values(&label).set(c.mss.into()); + tcp.send_window_bytes + .with_label_values(&label) + .set(c.send_window.into()); + tcp.receive_window_bytes + .with_label_values(&label) + .set(c.receive_window.into()); + tcp.send_queue_bytes + .with_label_values(&label) + .set(c.send_queue.into()); + tcp.not_sent_bytes + .with_label_values(&label) + .set(c.not_sent.into()); + tcp.receive_queue_bytes + .with_label_values(&label) + .set(c.recv_queue.into()); + tcp.unacked_segments + .with_label_values(&label) + .set(c.unacked.into()); + tcp.retransmitted_segments + .with_label_values(&label) + .set(c.total_retrans.into()); + tcp.lost_segments + .with_label_values(&label) + .set(c.lost.into()); + tcp.busy_seconds + .with_label_values(&label) + .set(seconds(c.busy)); + tcp.receive_window_limited_seconds + .with_label_values(&label) + .set(seconds(c.limited_by_receive_window)); + tcp.send_buffer_limited_seconds + .with_label_values(&label) + .set(seconds(c.limited_by_send_buffer)); + tcp.sent_bytes + .with_label_values(&label) + .set(int(c.bytes_sent)); + tcp.received_bytes + .with_label_values(&label) + .set(int(c.bytes_received)); + tcp.delivery_rate_bytes_per_second + .with_label_values(&label) + .set(int(c.delivery_rate)); + tcp.pacing_rate_bytes_per_second + .with_label_values(&label) + .set(int(c.pacing_rate)); + } +} diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md index fec9bb4..25a738d 100644 --- a/docs/content/docs/operations/monitoring.md +++ b/docs/content/docs/operations/monitoring.md @@ -252,6 +252,9 @@ Table 3 lists queries for a monitoring dashboard. | Established sessions | `sum(bgp_peer_up)`, of `count(bgp_peer_up)` | | Routes by neighbor | `bgp_peer_routes` | | Routes advertised to each neighbor | `bgp_peer_advertised_routes` | +| Share of time a session is held up by the neighbor's receive window | `rate(bgp_peer_tcp_receive_window_limited_seconds[5m]) / rate(bgp_peer_tcp_busy_seconds[5m])` | +| Round-trip time to each neighbor | `bgp_peer_tcp_rtt_seconds` | +| Octets queued towards a neighbor | `bgp_peer_tcp_send_queue_bytes` | | Prefixes with a best path | `bgp_rib_prefixes` | | Rejected routes by reason | `sum by (reason) (rate(bgp_routes_rejected_total[5m]))` | | UPDATE messages received and sent | `rate(bgp_messages_received_total{type="update"}[5m])`, `rate(bgp_messages_sent_total{type="update"}[5m])` | diff --git a/docs/content/docs/reference/metrics.md b/docs/content/docs/reference/metrics.md index 5592706..47c2cdf 100644 --- a/docs/content/docs/reference/metrics.md +++ b/docs/content/docs/reference/metrics.md @@ -30,6 +30,19 @@ alerting rules based on these metrics, see |---|---|---| | `bgp_peer_routes{peer}` | Gauge | Number of the neighbor's routes in the routing table. | | `bgp_peer_advertised_routes{peer}` | Gauge | Number of routes advertised to the neighbor and not withdrawn since. | +| `bgp_peer_tcp_rtt_seconds{peer}` | Gauge | Smoothed round-trip time of the session's TCP connection. Also `bgp_peer_tcp_min_rtt_seconds` and `bgp_peer_tcp_rtt_variance_seconds`. Linux only; sampled from the kernel at each scrape, as are all `bgp_peer_tcp_*` metrics. | +| `bgp_peer_tcp_cwnd_segments{peer}` | Gauge | Congestion window, in segments. `bgp_peer_tcp_mss_bytes` is the segment size. | +| `bgp_peer_tcp_send_window_bytes{peer}` | Gauge | The neighbor's receive window: what it lets Ferrix send. Zero while the neighbor is not reading. | +| `bgp_peer_tcp_receive_window_bytes{peer}` | Gauge | Ferrix's receive window: what it lets the neighbor send. | +| `bgp_peer_tcp_send_queue_bytes{peer}` | Gauge | Octets written and not yet acknowledged by the neighbor; `bgp_peer_tcp_not_sent_bytes` is the part not yet sent. | +| `bgp_peer_tcp_receive_queue_bytes{peer}` | Gauge | Octets received and not yet read. | +| `bgp_peer_tcp_unacked_segments{peer}` | Gauge | Segments in flight. | +| `bgp_peer_tcp_retransmitted_segments{peer}` | Gauge | Segments retransmitted over the connection's life. `bgp_peer_tcp_lost_segments` are those thought lost now. | +| `bgp_peer_tcp_busy_seconds{peer}` | Gauge | Time the connection had data to send, over its life. | +| `bgp_peer_tcp_receive_window_limited_seconds{peer}` | Gauge | Of the busy time, time the connection was held up by the neighbor's receive window. | +| `bgp_peer_tcp_send_buffer_limited_seconds{peer}` | Gauge | Of the busy time, time the connection was held up by the local send buffer being full. | +| `bgp_peer_tcp_sent_bytes{peer}` | Gauge | Octets sent over the connection's life. `bgp_peer_tcp_received_bytes` likewise. | +| `bgp_peer_tcp_delivery_rate_bytes_per_second{peer}` | Gauge | Delivery rate the kernel measured most recently; `bgp_peer_tcp_pacing_rate_bytes_per_second` is the rate it paces the connection at. | | `bgp_peer_max_prefixes{peer}` | Gauge | Prefix limit of the neighbor (`import.max_prefixes`), if configured. | | `bgp_prefix_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the neighbor exceeded its prefix limit. | | `bgp_export_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the routes to be advertised to the neighbor exceeded its export limit (`export_limits.max_prefixes`). | diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh index fcfd9a9..655ab7f 100755 --- a/scripts/interop/lab.sh +++ b/scripts/interop/lab.sh @@ -494,6 +494,7 @@ phase_basics() { frr4_addr=$(api /api/peers | jq -r '.[] | select(.name == "frr4") | .peer_addr') api "/api/tcp?peer=$frr4_addr" | jq -e '.available and .connection.state == "established" and .connection.mss > 0' >/dev/null \ || fail "no TCP details for frr4: $(api "/api/tcp?peer=$frr4_addr")" + api /metricz | grep -q "^bgp_peer_tcp_rtt_seconds{peer=\"frr4\"}" || fail "no TCP metrics for frr4" ok "the kernel tells of a session's TCP connection" }