metrics: export each session's TCP connection figures

The kernel's view of a session's TCP connection, which the status page
shows on request since the last change, could not be looked at after
the fact: a feed that crawled overnight left no record of whether the
peer's window or our own side held it up.

This change samples it at every scrape of `/metricz`: for each
established session, one `NETLINK_SOCK_DIAG` round trip, and twenty
gauges labelled by peer, `bgp_peer_tcp_*`: smoothed, least and variance
of the round-trip time; the congestion window and segment size; the
peer's receive window and ours; the send queue, the part of it not yet
sent, the receive queue and the segments in flight; retransmitted and
lost segments; the busy time and the time held up by the peer's window
and by our send buffer, cumulative, so that their rates give the share
of time a session is held up; octets sent and received; and the
delivery and pacing rates. A session the kernel has no connection for
loses its series, as does one that ends.

`SessionHandle::metric_label` is the one place a peer's metric label is
made (its name, or its address), used by the advertiser's gauge and here.

Documentation: the metrics reference lists them; the monitoring guide
gives the query for the share of time a session is held up by the
neighbor's receive window, which is the question the overnight feed
raised.

Testing: a daemon in the Linux VM with one session shows the series and
drops them when the session ends; the lab's basics phase checks the
scrape for the FRR session's round-trip time; the workspace tests pass.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

crates/server/src/advertise.rs

@@ -51,7 +51,6 @@ use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, watch};
 use tokio::task::JoinHandle;
 use tracing::{debug, error, info, warn};
 
-use crate::address::PeerAddress;
 use crate::error::SessionError;
 use crate::export::{self, Egress, Outgoing, Reflection, Source};
 use crate::family::FamilySet;
@@ -175,12 +174,7 @@ impl Advertiser {
                 .as_ref()
                 .map(|_| session.families.clone()),
             converged: Some(self.converged.clone()).filter(|converged| !*converged.borrow()),
-            // The peer goes by its configured name, or by its address, as
-            // in its other metrics.
-            advertised_metric: PEER_ADVERTISED.with_label_values(&[&session
-                .name
-                .clone()
-                .unwrap_or_else(|| PeerAddress::from(session.peer.addr).to_string())]),
+            advertised_metric: PEER_ADVERTISED.with_label_values(&[&session.metric_label()]),
         };
         let running = Running {
             session: session.clone(),

crates/server/src/connection.rs

@@ -44,7 +44,7 @@ use crate::fsm::{Action, Event, Fsm, FsmConfig, Negotiated, Received};
 use crate::holddown::{HeldDown, HoldDowns};
 use crate::metrics::{
     EXPORT_LIMIT_EXCEEDED, MESSAGES_RECEIVED, MESSAGES_SENT, PEER_ADVERTISED, PEER_ROUTES,
-    PREFIX_LIMIT_EXCEEDED, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name,
+    PEER_TCP, PREFIX_LIMIT_EXCEEDED, SESSIONS_ESTABLISHED_TOTAL, UPDATE_ERRORS, message_type_name,
 };
 use crate::panic::contained;
 use crate::policy::{self, ExportLimits, ExportPolicy, ImportPolicy, PeerRole};
@@ -316,6 +316,7 @@ async fn release_routes(
     // Fails only if the session never set it, which is nothing to do.
     let _ = PEER_ROUTES.remove_label_values(&[session.rejections.label()]);
     let _ = PEER_ADVERTISED.remove_label_values(&[session.rejections.label()]);
+    PEER_TCP.remove(session.rejections.label());
     if *session.leave.borrow() == Some(Leave::Restart) {
         // The daemon is about to go, and its routes with it; the kernel
         // keeps its own. Taking them out first would only hold the

crates/server/src/metrics.rs

@@ -4,8 +4,8 @@ use std::sync::LazyLock;
 
 use bgp_packet::message::Message;
 use prometheus::{
-    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_int_counter,
-    register_int_counter_vec, register_int_gauge, register_int_gauge_vec,
+    GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, register_gauge_vec,
+    register_int_counter, register_int_counter_vec, register_int_gauge, register_int_gauge_vec,
 };
 
 pub static MESSAGES_SENT: LazyLock<IntCounterVec> = LazyLock::new(|| {
@@ -78,6 +78,139 @@ pub static PEER_ADVERTISED: LazyLock<IntGaugeVec> = LazyLock::new(|| {
     .expect("metric registration")
 });
 
+/// A session's TCP connection as the kernel reports it (`TCP_INFO`),
+/// sampled when the metrics are scraped. Seconds, octets, segments, and
+/// octets per second; the busy, limited and byte figures are cumulative
+/// over the connection's life.
+pub struct PeerTcp {
+    pub rtt_seconds: GaugeVec,
+    pub min_rtt_seconds: GaugeVec,
+    pub rtt_variance_seconds: GaugeVec,
+    pub cwnd_segments: IntGaugeVec,
+    pub mss_bytes: IntGaugeVec,
+    pub send_window_bytes: IntGaugeVec,
+    pub receive_window_bytes: IntGaugeVec,
+    pub send_queue_bytes: IntGaugeVec,
+    pub not_sent_bytes: IntGaugeVec,
+    pub receive_queue_bytes: IntGaugeVec,
+    pub unacked_segments: IntGaugeVec,
+    pub retransmitted_segments: IntGaugeVec,
+    pub lost_segments: IntGaugeVec,
+    pub busy_seconds: GaugeVec,
+    pub receive_window_limited_seconds: GaugeVec,
+    pub send_buffer_limited_seconds: GaugeVec,
+    pub sent_bytes: IntGaugeVec,
+    pub received_bytes: IntGaugeVec,
+    pub delivery_rate_bytes_per_second: IntGaugeVec,
+    pub pacing_rate_bytes_per_second: IntGaugeVec,
+}
+
+impl PeerTcp {
+    /// Forgets the peer's series, when its session has ended.
+    pub fn remove(&self, label: &str) {
+        for gauge in [
+            &self.rtt_seconds,
+            &self.min_rtt_seconds,
+            &self.rtt_variance_seconds,
+            &self.busy_seconds,
+            &self.receive_window_limited_seconds,
+            &self.send_buffer_limited_seconds,
+        ] {
+            let _ = gauge.remove_label_values(&[label]);
+        }
+        for gauge in [
+            &self.cwnd_segments,
+            &self.mss_bytes,
+            &self.send_window_bytes,
+            &self.receive_window_bytes,
+            &self.send_queue_bytes,
+            &self.not_sent_bytes,
+            &self.receive_queue_bytes,
+            &self.unacked_segments,
+            &self.retransmitted_segments,
+            &self.lost_segments,
+            &self.sent_bytes,
+            &self.received_bytes,
+            &self.delivery_rate_bytes_per_second,
+            &self.pacing_rate_bytes_per_second,
+        ] {
+            let _ = gauge.remove_label_values(&[label]);
+        }
+    }
+}
+
+pub static PEER_TCP: LazyLock<PeerTcp> = LazyLock::new(|| {
+    let seconds = |name: &str, help: &str| {
+        register_gauge_vec!(format!("bgp_peer_tcp_{name}"), help, &["peer"])
+            .expect("metric registration")
+    };
+    let count = |name: &str, help: &str| {
+        register_int_gauge_vec!(format!("bgp_peer_tcp_{name}"), help, &["peer"])
+            .expect("metric registration")
+    };
+    PeerTcp {
+        rtt_seconds: seconds(
+            "rtt_seconds",
+            "Smoothed round-trip time of the session's TCP connection",
+        ),
+        min_rtt_seconds: seconds(
+            "min_rtt_seconds",
+            "Least round-trip time seen on the connection",
+        ),
+        rtt_variance_seconds: seconds(
+            "rtt_variance_seconds",
+            "Round-trip time variance of the connection",
+        ),
+        cwnd_segments: count("cwnd_segments", "Congestion window, in segments"),
+        mss_bytes: count("mss_bytes", "Maximum segment size"),
+        send_window_bytes: count(
+            "send_window_bytes",
+            "The peer's receive window: what it lets us send",
+        ),
+        receive_window_bytes: count(
+            "receive_window_bytes",
+            "Our receive window: what we let the peer send",
+        ),
+        send_queue_bytes: count(
+            "send_queue_bytes",
+            "Octets written and not yet acknowledged by the peer",
+        ),
+        not_sent_bytes: count("not_sent_bytes", "Octets written and not yet sent"),
+        receive_queue_bytes: count("receive_queue_bytes", "Octets received and not yet read"),
+        unacked_segments: count("unacked_segments", "Segments sent and not yet acknowledged"),
+        retransmitted_segments: count(
+            "retransmitted_segments",
+            "Segments retransmitted over the connection's life",
+        ),
+        lost_segments: count("lost_segments", "Segments thought lost now"),
+        busy_seconds: seconds(
+            "busy_seconds",
+            "Time the connection had something to send, over its life",
+        ),
+        receive_window_limited_seconds: seconds(
+            "receive_window_limited_seconds",
+            "Of the busy time, time held up by the peer's receive window",
+        ),
+        send_buffer_limited_seconds: seconds(
+            "send_buffer_limited_seconds",
+            "Of the busy time, time held up by our send buffer being full",
+        ),
+        sent_bytes: count("sent_bytes", "Octets sent over the connection's life"),
+        received_bytes: count(
+            "received_bytes",
+            "Octets received over the connection's life",
+        ),
+        delivery_rate_bytes_per_second: count(
+            "delivery_rate_bytes_per_second",
+            "Delivery rate the kernel measured most recently",
+        ),
+        pacing_rate_bytes_per_second: count(
+            "pacing_rate_bytes_per_second",
+            "Rate the kernel paces the connection at",
+        ),
+    }
+});
+
 pub static PEER_MAX_PREFIXES: LazyLock<IntGaugeVec> = LazyLock::new(|| {
     register_int_gauge_vec!(
         "bgp_peer_max_prefixes",
@@ -238,6 +371,7 @@ pub fn init() {
     LazyLock::force(&ROUTES_EXPORTED);
     LazyLock::force(&PEER_UP);
     LazyLock::force(&PEER_ADVERTISED);
+    LazyLock::force(&PEER_TCP);
     BUILD_INFO
         .with_label_values(&[env!("CARGO_PKG_VERSION")])
         .set(1);

crates/server/src/session.rs

@@ -296,6 +296,14 @@ impl SessionHandle {
 
     /// Whether `other` is a handle to this same session, and not to another
     /// session with the same peer.
+    /// What the peer goes by in the metrics: its configured name, or its
+    /// address.
+    pub fn metric_label(&self) -> String {
+        self.name
+            .clone()
+            .unwrap_or_else(|| crate::address::PeerAddress::from(self.peer.addr).to_string())
+    }
+
     pub fn is_same_session(&self, other: &SessionHandle) -> bool {
         self.outbound.same_channel(&other.outbound)
     }

crates/web/src/metrics.rs

@@ -5,7 +5,7 @@ use axum::extract::State;
 use axum::http::{StatusCode, header};
 use axum::response::IntoResponse;
 use bgp_server::metrics::{
-    LOCAL_ROUTES, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, RIB_ROUTES,
+    LOCAL_ROUTES, PEER_TCP, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, RIB_ROUTES,
 };
 use prometheus::{Encoder, TextEncoder};
 
@@ -24,6 +24,7 @@ pub async fn metricz(State(state): State<AppState>) -> impl IntoResponse {
     RIB_PREFIXES.set(gauge(state.rib.prefix_count()));
     RIB_ATTRIBUTE_SETS.set(gauge(state.rib.attrs().len()));
     RIB_JOURNAL_ENTRIES.set(gauge(state.rib.journal_len()));
+    sample_tcp(&state).await;
 
     let mut buf = Vec::new();
     let encoder = TextEncoder::new();
@@ -37,3 +38,81 @@ pub async fn metricz(State(state): State<AppState>) -> impl IntoResponse {
         Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(),
     }
 }
+
+/// Asks the kernel about every established session's TCP connection and
+/// sets the peer's gauges: one netlink round trip a session, at scrape
+/// time, so that there is history to look back on. A session the kernel
+/// has no connection for any more loses its series.
+async fn sample_tcp(state: &AppState) {
+    let tcp = &*PEER_TCP;
+    for session in state.registry.list() {
+        let label = session.metric_label();
+        let connection =
+            bgp_fib::sockdiag::tcp_connection(session.local_addr, session.peer.addr).await;
+        let Ok(Some(c)) = connection else {
+            tcp.remove(&label);
+            continue;
+        };
+        let seconds = |us: u64| us as f64 / 1e6;
+        let int = |n: u64| i64::try_from(n).unwrap_or(i64::MAX);
+        let label = [label.as_str()];
+        tcp.rtt_seconds
+            .with_label_values(&label)
+            .set(seconds(c.rtt.into()));
+        tcp.min_rtt_seconds
+            .with_label_values(&label)
+            .set(seconds(c.min_rtt.into()));
+        tcp.rtt_variance_seconds
+            .with_label_values(&label)
+            .set(seconds(c.rtt_variance.into()));
+        tcp.cwnd_segments
+            .with_label_values(&label)
+            .set(c.cwnd.into());
+        tcp.mss_bytes.with_label_values(&label).set(c.mss.into());
+        tcp.send_window_bytes
+            .with_label_values(&label)
+            .set(c.send_window.into());
+        tcp.receive_window_bytes
+            .with_label_values(&label)
+            .set(c.receive_window.into());
+        tcp.send_queue_bytes
+            .with_label_values(&label)
+            .set(c.send_queue.into());
+        tcp.not_sent_bytes
+            .with_label_values(&label)
+            .set(c.not_sent.into());
+        tcp.receive_queue_bytes
+            .with_label_values(&label)
+            .set(c.recv_queue.into());
+        tcp.unacked_segments
+            .with_label_values(&label)
+            .set(c.unacked.into());
+        tcp.retransmitted_segments
+            .with_label_values(&label)
+            .set(c.total_retrans.into());
+        tcp.lost_segments
+            .with_label_values(&label)
+            .set(c.lost.into());
+        tcp.busy_seconds
+            .with_label_values(&label)
+            .set(seconds(c.busy));
+        tcp.receive_window_limited_seconds
+            .with_label_values(&label)
+            .set(seconds(c.limited_by_receive_window));
+        tcp.send_buffer_limited_seconds
+            .with_label_values(&label)
+            .set(seconds(c.limited_by_send_buffer));
+        tcp.sent_bytes
+            .with_label_values(&label)
+            .set(int(c.bytes_sent));
+        tcp.received_bytes
+            .with_label_values(&label)
+            .set(int(c.bytes_received));
+        tcp.delivery_rate_bytes_per_second
+            .with_label_values(&label)
+            .set(int(c.delivery_rate));
+        tcp.pacing_rate_bytes_per_second
+            .with_label_values(&label)
+            .set(int(c.pacing_rate));
+    }
+}

docs/content/docs/operations/monitoring.md

@@ -252,6 +252,9 @@ Table 3 lists queries for a monitoring dashboard.
 | Established sessions | `sum(bgp_peer_up)`, of `count(bgp_peer_up)` |
 | Routes by neighbor | `bgp_peer_routes` |
 | Routes advertised to each neighbor | `bgp_peer_advertised_routes` |
+| Share of time a session is held up by the neighbor's receive window | `rate(bgp_peer_tcp_receive_window_limited_seconds[5m]) / rate(bgp_peer_tcp_busy_seconds[5m])` |
+| Round-trip time to each neighbor | `bgp_peer_tcp_rtt_seconds` |
+| Octets queued towards a neighbor | `bgp_peer_tcp_send_queue_bytes` |
 | Prefixes with a best path | `bgp_rib_prefixes` |
 | Rejected routes by reason | `sum by (reason) (rate(bgp_routes_rejected_total[5m]))` |
 | UPDATE messages received and sent | `rate(bgp_messages_received_total{type="update"}[5m])`, `rate(bgp_messages_sent_total{type="update"}[5m])` |

docs/content/docs/reference/metrics.md

@@ -30,6 +30,19 @@ alerting rules based on these metrics, see
 |---|---|---|
 | `bgp_peer_routes{peer}` | Gauge | Number of the neighbor's routes in the routing table. |
 | `bgp_peer_advertised_routes{peer}` | Gauge | Number of routes advertised to the neighbor and not withdrawn since. |
+| `bgp_peer_tcp_rtt_seconds{peer}` | Gauge | Smoothed round-trip time of the session's TCP connection. Also `bgp_peer_tcp_min_rtt_seconds` and `bgp_peer_tcp_rtt_variance_seconds`. Linux only; sampled from the kernel at each scrape, as are all `bgp_peer_tcp_*` metrics. |
+| `bgp_peer_tcp_cwnd_segments{peer}` | Gauge | Congestion window, in segments. `bgp_peer_tcp_mss_bytes` is the segment size. |
+| `bgp_peer_tcp_send_window_bytes{peer}` | Gauge | The neighbor's receive window: what it lets Ferrix send. Zero while the neighbor is not reading. |
+| `bgp_peer_tcp_receive_window_bytes{peer}` | Gauge | Ferrix's receive window: what it lets the neighbor send. |
+| `bgp_peer_tcp_send_queue_bytes{peer}` | Gauge | Octets written and not yet acknowledged by the neighbor; `bgp_peer_tcp_not_sent_bytes` is the part not yet sent. |
+| `bgp_peer_tcp_receive_queue_bytes{peer}` | Gauge | Octets received and not yet read. |
+| `bgp_peer_tcp_unacked_segments{peer}` | Gauge | Segments in flight. |
+| `bgp_peer_tcp_retransmitted_segments{peer}` | Gauge | Segments retransmitted over the connection's life. `bgp_peer_tcp_lost_segments` are those thought lost now. |
+| `bgp_peer_tcp_busy_seconds{peer}` | Gauge | Time the connection had data to send, over its life. |
+| `bgp_peer_tcp_receive_window_limited_seconds{peer}` | Gauge | Of the busy time, time the connection was held up by the neighbor's receive window. |
+| `bgp_peer_tcp_send_buffer_limited_seconds{peer}` | Gauge | Of the busy time, time the connection was held up by the local send buffer being full. |
+| `bgp_peer_tcp_sent_bytes{peer}` | Gauge | Octets sent over the connection's life. `bgp_peer_tcp_received_bytes` likewise. |
+| `bgp_peer_tcp_delivery_rate_bytes_per_second{peer}` | Gauge | Delivery rate the kernel measured most recently; `bgp_peer_tcp_pacing_rate_bytes_per_second` is the rate it paces the connection at. |
 | `bgp_peer_max_prefixes{peer}` | Gauge | Prefix limit of the neighbor (`import.max_prefixes`), if configured. |
 | `bgp_prefix_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the neighbor exceeded its prefix limit. |
 | `bgp_export_limit_exceeded_total{peer}` | Counter | Number of sessions torn down because the routes to be advertised to the neighbor exceeded its export limit (`export_limits.max_prefixes`). |

scripts/interop/lab.sh

@@ -494,6 +494,7 @@ phase_basics() {
     frr4_addr=$(api /api/peers | jq -r '.[] | select(.name == "frr4") | .peer_addr')
     api "/api/tcp?peer=$frr4_addr" | jq -e '.available and .connection.state == "established" and .connection.mss > 0' >/dev/null \
         || fail "no TCP details for frr4: $(api "/api/tcp?peer=$frr4_addr")"
+    api /metricz | grep -q "^bgp_peer_tcp_rtt_seconds{peer=\"frr4\"}" || fail "no TCP metrics for frr4"
     ok "the kernel tells of a session's TCP connection"
 }