diff --git a/Cargo.lock b/Cargo.lock index 2fdaef4..6f83b14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,6 +315,7 @@ dependencies = [ "nix", "prometheus", "serde", + "tikv-jemalloc-ctl", "tokio", "tokio-stream", "tracing", @@ -1098,6 +1099,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1679,6 +1686,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" diff --git a/Cargo.toml b/Cargo.toml index 020b661..fea4275 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,8 +63,10 @@ hashbrown = "0.17" axum = "0.8.9" tokio-stream = { version = "0.1.19", features = ["sync"] } reqwest = { version = "0.13.5", default-features = false, features = ["json"] } -# The daemon's memory allocator; see bin/bgp/src/main.rs. -tikv-jemallocator = "0.7.0" +# The daemon's memory allocator; see bin/bgp/src/main.rs. With its +# statistics, which the status page and the metrics show. +tikv-jemallocator = { version = "0.7.0", features = ["stats", "background_threads"] } +tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] } # `cargo build --profile profiling`: the release build, with the line tables # a profiler needs to name functions. See scripts/profile/. diff --git a/bin/bgp/Cargo.toml b/bin/bgp/Cargo.toml index 45e1820..c41fef0 100644 --- a/bin/bgp/Cargo.toml +++ b/bin/bgp/Cargo.toml @@ -23,5 +23,10 @@ tracing-subscriber.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator.workspace = true +[features] +# jemalloc's heap profiler, for scripts/profile/full-table.sh with +# HEAP=1: not for a production build, where it costs memory and time. +heap-profiling = ["tikv-jemallocator/profiling"] + [lints] workspace = true diff --git a/crates/bench/examples/feed.rs b/crates/bench/examples/feed.rs index 132e05f..4a50945 100644 --- a/crates/bench/examples/feed.rs +++ b/crates/bench/examples/feed.rs @@ -17,6 +17,12 @@ //! follow on from it. //! - `FEED_HOLD_SECS`: how long the sessions stay open after the table is //! sent (default 30). +//! - `FEED_COLLECTOR=1`: before the peers, one more session, as a route +//! collector would have: over IPv6, to `FEED_COLLECTOR_TO` (default +//! `[::1]:1790`) from `FEED_COLLECTOR_FROM` if set, offering to receive +//! several routes for a prefix (ADD-PATH) for both families and IPv4 +//! routes with IPv6 next hops, and reading and dropping whatever it is +//! sent. Its AS number is 212232. //! //! What is printed: when the sessions are up and how long the writes took. //! The writes end when the kernel's socket buffers have taken the last @@ -28,6 +34,9 @@ use std::time::{Duration, Instant}; use bgp_bench::{Prefixes, Shape, announcements, encoded, env}; use bgp_integration_tests::test_peer::TestPeer; +use bgp_packet::open::{AddPathEntry, Capability, ExtendedNextHopEntry, SendReceive}; +use bgp_primitives::constants::{Afi, AfiSafi, Safi}; +use bgp_server::family::SUPPORTED; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpSocket; use tokio::net::tcp::OwnedWriteHalf; @@ -81,6 +90,61 @@ async fn main() { started.elapsed().as_secs_f64() ); + // The collector first, so that it is there as the table arrives, as a + // router's collector session is after a restart. + let collector = env("FEED_COLLECTOR", 0) == 1; + let mut draining = Vec::new(); + let mut collector_writer = None; + if collector { + let to: SocketAddr = std::env::var("FEED_COLLECTOR_TO") + .unwrap_or_else(|_| "[::1]:1790".into()) + .parse() + .expect("FEED_COLLECTOR_TO is an IPv6 address and port"); + let socket = TcpSocket::new_v6().expect("a socket"); + if let Ok(from) = std::env::var("FEED_COLLECTOR_FROM") { + let address: std::net::Ipv6Addr = from.parse().expect("an IPv6 address"); + socket + .bind(SocketAddr::new(address.into(), 0)) + .expect("the collector's address is ours to connect from"); + } + let stream = socket.connect(to).await.expect("the daemon listens"); + let mut session = TestPeer::from_stream(stream); + let mut capabilities = vec![Capability::FourByteAsn { asn: 212_232 }]; + capabilities.extend(SUPPORTED.into_iter().map(Capability::multiprotocol)); + capabilities.push(Capability::AddPath { + entries: [AfiSafi::IPV4_UNICAST, AfiSafi::IPV6_UNICAST] + .into_iter() + .map(|family| AddPathEntry { + afi: family.afi, + safi: family.safi, + send_recv: SendReceive::Receive, + }) + .collect(), + }); + capabilities.push(Capability::ExtendedNextHop { + entries: vec![ExtendedNextHopEntry { + afi: Afi::IPV4, + safi: Safi::UNICAST, + nh_afi: Afi::IPV6, + }], + }); + session + .handshake_with(212_232, Ipv4Addr::new(10, 0, 0, 200), 90, capabilities) + .await + .expect("the daemon takes the collector's session"); + let (mut reader, writer) = session.into_stream().into_split(); + draining.push(tokio::spawn(async move { + let mut sink = [0u8; 65536]; + while let Ok(n) = reader.read(&mut sink).await { + if n == 0 { + break; + } + } + })); + println!("collector session established"); + collector_writer = Some(writer); + } + let mut sessions = Vec::new(); for peer in 1..=peers { let socket = TcpSocket::new_v4().expect("a socket"); @@ -129,7 +193,7 @@ async fn main() { }) }) .collect(); - let mut writers: Vec = Vec::new(); + let mut writers: Vec = collector_writer.into_iter().collect(); for (index, sender) in senders.into_iter().enumerate() { let (writer, took) = sender.await.expect("the sender runs"); println!( @@ -155,4 +219,5 @@ async fn main() { } } println!("sessions closed after {} s", hold.as_secs()); + drop(draining); } diff --git a/crates/server/src/advertise.rs b/crates/server/src/advertise.rs index ba059fd..78128dd 100644 --- a/crates/server/src/advertise.rs +++ b/crates/server/src/advertise.rs @@ -253,8 +253,9 @@ const COALESCE: Duration = Duration::from_millis(50); const RESYNCS_AT_ONCE: usize = 4; /// How many routes ready to go out are remembered, by the attributes they -/// came with. A table has a few hundred thousand distinct sets. -const DRESSED_KEPT: usize = 262_144; +/// came with. A table has a few hundred thousand distinct sets, each a +/// kilobyte or so in this form; what a flush is sending holds its own. +const DRESSED_KEPT: usize = 65_536; fn version_index(version: IpVersion) -> usize { match version { @@ -305,8 +306,11 @@ struct Sent { /// The path identifier it went out with (ADD-PATH, RFC 7911), which /// stays its own for as long as it is announced. id: u32, - /// As it went out, to tell whether it has to go out again. - dressed: Arc, + /// The attributes it went out with, as the table has them: what it + /// goes out as follows from those, so a route whose set is the same + /// has not changed. The table holds the set anyway; the form it goes + /// out in is a kilobyte a set and is not kept here. + attrs: Arc, } /// What the peer is to end up with for a prefix. @@ -314,9 +318,10 @@ enum Verdict { /// The best route as it goes out, or `None` for a prefix the peer is /// not to have. Best(Option>), - /// Every route that goes out, with what tells it apart; empty for a - /// prefix the peer is not to have. - All(Vec<(PathKey, Arc)>), + /// Every route that goes out, with what tells it apart and the + /// attributes it came with; empty for a prefix the peer is not to + /// have. + All(Vec<(PathKey, Arc, Arc)>), } /// What the peer is to end up with for each prefix heard of. @@ -826,7 +831,7 @@ impl Exporter { }; if let Some(dressed) = learned.dressed(&attrs, source, version, &egress) { - wanted.push(((peer, path_id), dressed)); + wanted.push(((peer, path_id), attrs, dressed)); } } } @@ -898,11 +903,11 @@ impl Exporter { // again only if it changed; a new one gets the lowest // identifier not in use for the prefix. let mut has: Vec = Vec::with_capacity(wanted.len()); - for (key, dressed) in wanted { + for (key, attrs, dressed) in wanted { let id = match had.iter().position(|sent| sent.key == key) { Some(at) => { let sent = had.swap_remove(at); - if Arc::ptr_eq(&sent.dressed, &dressed) { + if Arc::ptr_eq(&sent.attrs, &attrs) { has.push(sent); continue; } @@ -919,7 +924,7 @@ impl Exporter { path_id: id, }, ); - has.push(Sent { key, id, dressed }); + has.push(Sent { key, id, attrs }); } withdrawn.extend(had.iter().map(|sent| Nlri { prefix, diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index 9e599c6..313643b 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -211,6 +211,17 @@ pub static PEER_TCP: LazyLock = LazyLock::new(|| { } }); +/// The allocator's account of the daemon's memory, by `kind`: allocated, +/// active, metadata, resident, mapped, retained (jemalloc's `stats.*`). +pub static MEMORY: LazyLock = LazyLock::new(|| { + register_int_gauge_vec!( + "bgp_memory_bytes", + "The daemon's memory as its allocator accounts for it, by kind", + &["kind"] + ) + .expect("metric registration") +}); + pub static PEER_MAX_PREFIXES: LazyLock = LazyLock::new(|| { register_int_gauge_vec!( "bgp_peer_max_prefixes", @@ -372,6 +383,7 @@ pub fn init() { LazyLock::force(&PEER_UP); LazyLock::force(&PEER_ADVERTISED); LazyLock::force(&PEER_TCP); + LazyLock::force(&MEMORY); BUILD_INFO .with_label_values(&[env!("CARGO_PKG_VERSION")]) .set(1); diff --git a/crates/web/Cargo.toml b/crates/web/Cargo.toml index c047d36..5b79f1a 100644 --- a/crates/web/Cargo.toml +++ b/crates/web/Cargo.toml @@ -24,5 +24,10 @@ tokio-stream.workspace = true tracing.workspace = true nix = { workspace = true, features = ["hostname"] } +# The allocator's own account of memory, for the status page and the +# metrics. The allocator itself is the daemon's choice (bin/bgp). +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemalloc-ctl.workspace = true + [lints] workspace = true diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 4824b82..2f417f3 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -39,6 +39,9 @@ fn unix_secs(t: SystemTime) -> u64 { #[derive(Debug, Serialize)] pub struct StatusView { + /// The allocator's account of the daemon's memory; `None` where the + /// allocator keeps none. + pub memory: Option, pub router_id: String, pub local_asn: u32, pub version: String, @@ -606,6 +609,7 @@ impl IntoResponse for Unavailable { pub async fn status(State(state): State) -> Result, Unavailable> { Ok(Json(StatusView { + memory: crate::memory::memory_use(), router_id: state.info.router_id.to_string(), local_asn: state.info.local_asn, version: state.info.version.clone(), diff --git a/crates/web/src/lib.rs b/crates/web/src/lib.rs index 2b467e5..1ccb99b 100644 --- a/crates/web/src/lib.rs +++ b/crates/web/src/lib.rs @@ -7,6 +7,7 @@ mod api; mod events; +mod memory; mod metrics; use std::net::Ipv4Addr; diff --git a/crates/web/src/memory.rs b/crates/web/src/memory.rs new file mode 100644 index 0000000..f8708a4 --- /dev/null +++ b/crates/web/src/memory.rs @@ -0,0 +1,46 @@ +//! What the allocator says the daemon holds: jemalloc's own statistics, +//! for the status page and the metrics, where the kernel's resident size +//! alone would not say what is in use and what the allocator keeps back. + +use serde::Serialize; + +/// jemalloc's account of memory, in octets. +#[derive(Debug, Clone, Copy, Serialize)] +pub struct MemoryUse { + /// Octets in allocations the program holds. + pub allocated: u64, + /// Octets in the pages those allocations are in: `allocated` plus + /// the fragmentation within pages. + pub active: u64, + /// Octets of the allocator's own bookkeeping. + pub metadata: u64, + /// Octets in pages mapped and physically resident, as far as the + /// allocator knows: `active`, the metadata, and pages freed but not + /// yet returned to the kernel. + pub resident: u64, + /// Octets mapped, resident or not. + pub mapped: u64, + /// Octets of address space kept back after being unmapped, which + /// cost nothing physical. + pub retained: u64, +} + +#[cfg(not(target_env = "msvc"))] +pub fn memory_use() -> Option { + use tikv_jemalloc_ctl::{epoch, stats}; + // The statistics are refreshed only when the epoch is advanced. + epoch::advance().ok()?; + Some(MemoryUse { + allocated: stats::allocated::read().ok()? as u64, + active: stats::active::read().ok()? as u64, + metadata: stats::metadata::read().ok()? as u64, + resident: stats::resident::read().ok()? as u64, + mapped: stats::mapped::read().ok()? as u64, + retained: stats::retained::read().ok()? as u64, + }) +} + +#[cfg(target_env = "msvc")] +pub fn memory_use() -> Option { + None +} diff --git a/crates/web/src/metrics.rs b/crates/web/src/metrics.rs index 6a645fe..d117a3c 100644 --- a/crates/web/src/metrics.rs +++ b/crates/web/src/metrics.rs @@ -5,7 +5,8 @@ use axum::extract::State; use axum::http::{StatusCode, header}; use axum::response::IntoResponse; use bgp_server::metrics::{ - LOCAL_ROUTES, PEER_TCP, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, RIB_ROUTES, + LOCAL_ROUTES, MEMORY, PEER_TCP, RIB_ATTRIBUTE_SETS, RIB_JOURNAL_ENTRIES, RIB_PREFIXES, + RIB_ROUTES, }; use prometheus::{Encoder, TextEncoder}; @@ -25,6 +26,19 @@ pub async fn metricz(State(state): State) -> impl IntoResponse { RIB_ATTRIBUTE_SETS.set(gauge(state.rib.attrs().len())); RIB_JOURNAL_ENTRIES.set(gauge(state.rib.journal_len())); sample_tcp(&state).await; + if let Some(m) = crate::memory::memory_use() { + let int = |n: u64| i64::try_from(n).unwrap_or(i64::MAX); + for (kind, octets) in [ + ("allocated", m.allocated), + ("active", m.active), + ("metadata", m.metadata), + ("resident", m.resident), + ("mapped", m.mapped), + ("retained", m.retained), + ] { + MEMORY.with_label_values(&[kind]).set(int(octets)); + } + } let mut buf = Vec::new(); let encoder = TextEncoder::new(); diff --git a/crates/web/static/statusz.html b/crates/web/static/statusz.html index 6893900..33671df 100644 --- a/crates/web/static/statusz.html +++ b/crates/web/static/statusz.html @@ -165,7 +165,7 @@ input, select, button { color-scheme: light dark; }
...: router ID ..., AS ..., Ferrix ...
-Started ..., up ...
+Started ..., up ...
On this page: peers · RPKI · originated · kernel · received, and looking routes up · rejected · events
@@ -410,6 +410,8 @@ from $('s-asn').className = 'asn'; $('s-asn').dataset.asn = s.local_asn; $('s-version').textContent = s.version; $('s-hostname').textContent = s.hostname; $('s-started').textContent = new Date(s.started_at_unix * 1000).toString().replace(/ \(.*\)$/, ''); + const m = s.memory; + setHTML($('s-memory'), m ? ` · memory ${fmtOctets(m.allocated)} allocated, ${fmtOctets(m.resident)} resident` : ''); tick(); } diff --git a/docs/content/docs/development/testing.md b/docs/content/docs/development/testing.md index 0f54b5e..a4f80f5 100644 --- a/docs/content/docs/development/testing.md +++ b/docs/content/docs/development/testing.md @@ -214,6 +214,8 @@ environment variables that control the run. | `SCALE_DOWN` | Divides the size of the table | 1 | | `HOLD_SECS` | How long the sessions stay open after the table is sent | 30 | | `FREQ` | The sampling frequency in hertz | 1999 | +| `COLLECTOR` | `1` adds a route collector session that is sent every path (`all_paths`) | 0 | +| `HEAP` | `1` profiles memory instead: the daemon is built with jemalloc's heap profiler, and `heap.svg` and `heap-top.txt` show where the allocations are | 0 | | `OUT` | The directory for the results | `~/profile` | | `JEMALLOC_CONF` | jemalloc options for the daemon (its `MALLOC_CONF`), for example `thp:always` | none | diff --git a/docs/content/docs/operations/monitoring.md b/docs/content/docs/operations/monitoring.md index 25a738d..ff7b8c3 100644 --- a/docs/content/docs/operations/monitoring.md +++ b/docs/content/docs/operations/monitoring.md @@ -252,6 +252,7 @@ Table 3 lists queries for a monitoring dashboard. | Established sessions | `sum(bgp_peer_up)`, of `count(bgp_peer_up)` | | Routes by neighbor | `bgp_peer_routes` | | Routes advertised to each neighbor | `bgp_peer_advertised_routes` | +| Memory in use and resident | `bgp_memory_bytes{kind="allocated"}`, `bgp_memory_bytes{kind="resident"}` | | Share of time a session is held up by the neighbor's receive window | `rate(bgp_peer_tcp_receive_window_limited_seconds[5m]) / rate(bgp_peer_tcp_busy_seconds[5m])` | | Round-trip time to each neighbor | `bgp_peer_tcp_rtt_seconds` | | Octets queued towards a neighbor | `bgp_peer_tcp_send_queue_bytes` | diff --git a/docs/content/docs/reference/http-api.md b/docs/content/docs/reference/http-api.md index 30de5a7..fccb45f 100644 --- a/docs/content/docs/reference/http-api.md +++ b/docs/content/docs/reference/http-api.md @@ -29,7 +29,7 @@ server returns the following error statuses: |---|---| | `/` | Redirects to `/statusz`. | | `/statusz` | Status page. | -| `/api/status` | Router ID, AS number, uptime, table sizes, and whether control communities are enabled (`control_communities`). | +| `/api/status` | Router ID, AS number, uptime, table sizes, memory use (`memory`, as jemalloc accounts for it: `allocated`, `active`, `metadata`, `resident`, `mapped`, `retained` octets), and whether control communities are enabled (`control_communities`). | | `/api/config` | The configuration that was read at startup. | | `/api/peers` | Configured neighbors and their sessions. | | `/api/routes/local` | Originated routes. | diff --git a/docs/content/docs/reference/metrics.md b/docs/content/docs/reference/metrics.md index 47c2cdf..cb71162 100644 --- a/docs/content/docs/reference/metrics.md +++ b/docs/content/docs/reference/metrics.md @@ -30,6 +30,7 @@ alerting rules based on these metrics, see |---|---|---| | `bgp_peer_routes{peer}` | Gauge | Number of the neighbor's routes in the routing table. | | `bgp_peer_advertised_routes{peer}` | Gauge | Number of routes advertised to the neighbor and not withdrawn since. | +| `bgp_memory_bytes{kind}` | Gauge | The daemon's memory as jemalloc accounts for it. `kind` is `allocated` (octets in live allocations), `active` (the pages they occupy), `metadata` (the allocator's own), `resident` (active, metadata and freed pages not yet returned to the kernel), `mapped`, or `retained` (address space kept back, which costs nothing physical). | | `bgp_peer_tcp_rtt_seconds{peer}` | Gauge | Smoothed round-trip time of the session's TCP connection. Also `bgp_peer_tcp_min_rtt_seconds` and `bgp_peer_tcp_rtt_variance_seconds`. Linux only; sampled from the kernel at each scrape, as are all `bgp_peer_tcp_*` metrics. | | `bgp_peer_tcp_cwnd_segments{peer}` | Gauge | Congestion window, in segments. `bgp_peer_tcp_mss_bytes` is the segment size. | | `bgp_peer_tcp_send_window_bytes{peer}` | Gauge | The neighbor's receive window: what it lets Ferrix send. Zero while the neighbor is not reading. | diff --git a/scripts/profile/full-table.sh b/scripts/profile/full-table.sh index 1318454..46e46cb 100755 --- a/scripts/profile/full-table.sh +++ b/scripts/profile/full-table.sh @@ -7,6 +7,8 @@ # PEERS=4 scripts/profile/full-table.sh four peers send it at once # KERNEL=0 scripts/profile/full-table.sh the routes stay in the table # SCALE_DOWN=10 scripts/profile/full-table.sh a tenth of the table +# COLLECTOR=1 scripts/profile/full-table.sh a route collector is sent every path +# HEAP=1 scripts/profile/full-table.sh where the memory goes, not the time # # From a machine that is not Linux: scripts/linux-vm.sh scripts/profile/full-table.sh # @@ -26,7 +28,10 @@ set -eu peers=${PEERS:-1} scale=${SCALE_DOWN:-1} kernel=${KERNEL:-1} +collector=${COLLECTOR:-0} +heap=${HEAP:-0} hold=${HOLD_SECS:-30} +level=${LOG_LEVEL:-warn} freq=${FREQ:-1999} out=${OUT:-$HOME/profile} ns=ferrix-profile @@ -47,7 +52,12 @@ done # allocator is built with them too). cd "$repository" export RUSTFLAGS="-C force-frame-pointers=yes" CFLAGS="-fno-omit-frame-pointer" -cargo build --profile profiling --locked -p bgp --bin bgp +# With HEAP=1 the daemon is built with jemalloc's heap profiler and told +# to sample allocations and to write a profile at exit; jeprof turns the +# profile into collapsed stacks for a flame graph of where the memory is. +features=() +[ "$heap" = 1 ] && features=(--features heap-profiling) +cargo build --profile profiling --locked -p bgp --bin bgp "${features[@]}" cargo build --profile profiling --locked -p bgp-bench --example feed mkdir -p "$out" @@ -66,12 +76,12 @@ sudo ip netns add "$ns_peer" sudo ip link add bench0 netns "$ns" type veth peer name peer0 netns "$ns_peer" run ip link set lo up run ip addr add 192.0.2.254/24 dev bench0 -run ip -6 addr add 2001:db8::ffff/64 dev bench0 +run ip -6 addr add 2001:db8::ffff/64 dev bench0 nodad run ip link set bench0 up peer ip link set lo up -for n in $(seq "$peers"); do +for n in $(seq "$((peers + collector))"); do peer ip addr add "192.0.2.$n/24" dev peer0 - peer ip -6 addr add "2001:db8::$n/64" dev peer0 + peer ip -6 addr add "2001:db8::$n/64" dev peer0 nodad done peer ip link set peer0 up peer ping -c 1 -W 2 192.0.2.254 > /dev/null || fail "the namespaces do not reach each other" @@ -84,11 +94,13 @@ cat > "$out/daemon.textpb" <> "$out/daemon.textpb" <> "$out/daemon.textpb" <> "$out/daemon.textpb" fi # `JEMALLOC_CONF`, if set, is jemalloc's MALLOC_CONF for the daemon: # "thp:always", say. The symbols are prefixed, hence the variable's name. -run env _RJEM_MALLOC_CONF="${JEMALLOC_CONF:-}" "$daemon_bin" "$out/daemon.textpb" --level warn > "$out/daemon.log" 2>&1 & +malloc_conf=${JEMALLOC_CONF:-} +if [ "$heap" = 1 ]; then + # One sample every 512 KiB allocated, and a profile every 1 GiB + # allocated: the last of them is the nearest to the table being in. + rm -f "$out"/heap.*.heap + malloc_conf="prof:true,prof_active:true,lg_prof_sample:19,lg_prof_interval:30,prof_prefix:$out/heap${malloc_conf:+,$malloc_conf}" +fi +run env _RJEM_MALLOC_CONF="$malloc_conf" "$daemon_bin" "$out/daemon.textpb" --level "$level" > "$out/daemon.log" 2>&1 & for _ in $(seq 100); do run curl -sf http://127.0.0.1:8080/api/status > /dev/null 2>&1 && break sleep 0.1 @@ -121,7 +154,8 @@ perf_pid=$! sleep 1 peer env FEED_TO=192.0.2.254:1790 FEED_PEERS="$peers" FEED_SCALE_DOWN="$scale" \ - FEED_FROM=192.0.2 FEED_HOLD_SECS="$hold" "$feed_bin" > "$out/feed.log" 2>&1 & + FEED_FROM=192.0.2 FEED_HOLD_SECS="$hold" FEED_COLLECTOR="$collector" \ + FEED_COLLECTOR_TO="[2001:db8::ffff]:1790" FEED_COLLECTOR_FROM="2001:db8::$((peers + 1))" "$feed_bin" > "$out/feed.log" 2>&1 & feed_pid=$! feeding() { kill -0 "$feed_pid" 2>/dev/null || fail "the load generator stopped; see $out/feed.log"; } @@ -147,8 +181,31 @@ if [ "$kernel" = 1 ]; then installed=$(since "$start") echo "$prefixes routes in the kernel after $installed s ($(echo "$prefixes / $installed / 1000" | bc -l | cut -c1-3) k routes/s)" fi -rss=$(awk '/VmRSS/ { printf "%.0f", $2 / 1024 }' "/proc/$daemon_pid/status") -echo "daemon: $rss MiB resident with the table in" +if [ "$collector" = 1 ]; then + # Nearly every taken route goes to the collector: not one with its AS + # on the path or a well-known community that says not to. The count + # has settled when it has not moved for a few seconds. + advertised() { status peers '[.[] | select(.name == "collector") | .advertised_routes] | add // 0'; } + last=-1 + while :; do + feeding; sleep 3 + now_held=$(advertised) + echo " collector holds $now_held of $received after $(since "$start") s" + [ "$now_held" -gt 0 ] && [ "$now_held" = "$last" ] && break + last=$now_held + done + echo "collector: $(advertised) routes advertised after $(since "$start") s" +fi +memory_report() { + rss=$(awk '/VmRSS/ { printf "%.0f", $2 / 1024 }' "/proc/$daemon_pid/status") + echo "daemon: $rss MiB resident $1" + status status '.memory | to_entries | map("\(.key) \(.value / 1048576 | floor) MiB") | join(", ")' | sed 's/^/jemalloc: /' +} +memory_report "with the table in" +# And once the allocator has had its decay time to give back what the +# load allocated and freed: the figure a router shows for the long run. +sleep 30 +memory_report "30 s later" run curl -sf http://127.0.0.1:8080/metricz | grep '^bgp_kernel_route_changes_total' | sed 's/^/kernel: /' sleep 1 sudo kill -INT "$perf_pid" @@ -156,6 +213,21 @@ wait "$perf_pid" || true cat "$out/feed.log" sudo chown "$(id -u)" "$out/perf.data" +if [ "$heap" = 1 ]; then + sudo chown "$(id -u)" "$out"/heap.*.heap 2>/dev/null || true + # jeprof comes with jemalloc's sources as a template; the prefix the + # allocator's symbols carry is filled in. + template=$(find "$HOME/.cargo/registry/src" -path "*tikv-jemalloc-sys*" -name jeprof.in | head -1) + [ -n "$template" ] || fail "jeprof.in not found in the cargo registry" + sed -e 's/@JEMALLOC_PREFIX@/_rjem_/g' -e 's/@jemalloc_version@/5.3/' "$template" > "$out/jeprof" + # The latest profile is the one nearest to the table being in. + profile=$(ls -t "$out"/heap.*.heap | head -1) + perl "$out/jeprof" --collapsed "$daemon_bin" "$profile" > "$out/heap.folded" 2> "$out/jeprof.log" \ + || fail "jeprof failed; see $out/jeprof.log" + inferno-flamegraph --title "ferrix heap: $peers peer(s), $prefixes prefixes each$([ "$collector" = 1 ] && echo ", a collector")" --countname bytes --colors mem --width 1600 "$out/heap.folded" > "$out/heap.svg" + perl "$out/jeprof" --text "$daemon_bin" "$profile" 2>/dev/null | head -40 > "$out/heap-top.txt" + echo "heap profile $profile: heap.svg heap-top.txt" +fi # Without the period field, so that each sample counts one whatever event # perf sampled by (cpu-clock on a machine without performance counters). perf script -i "$out/perf.data" -F comm,pid,tid,time,event,ip,sym,dso 2> /dev/null \