diff --git a/Cargo.toml b/Cargo.toml index 044c905..020b661 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,3 +65,10 @@ tokio-stream = { version = "0.1.19", features = ["sync"] } reqwest = { version = "0.13.5", default-features = false, features = ["json"] } # The daemon's memory allocator; see bin/bgp/src/main.rs. tikv-jemallocator = "0.7.0" + +# `cargo build --profile profiling`: the release build, with the line tables +# a profiler needs to name functions. See scripts/profile/. +[profile.profiling] +inherits = "release" +debug = "line-tables-only" +strip = "none" diff --git a/crates/bench/examples/feed.rs b/crates/bench/examples/feed.rs new file mode 100644 index 0000000..132e05f --- /dev/null +++ b/crates/bench/examples/feed.rs @@ -0,0 +1,158 @@ +//! Feeds the synthetic Internet routing table (see the crate documentation) +//! to a running daemon over TCP, as one or more peers would, and holds the +//! sessions open afterwards. This is the load for profiling the daemon as +//! it takes in a full table; `scripts/profile/full-table.sh` runs it under +//! `perf`. +//! +//! Settings, from the environment: +//! +//! - `FEED_TO`: where the daemon listens (default `127.0.0.1:1790`). +//! - `FEED_PEERS`: how many peers send the table (default 1). +//! - `FEED_SCALE_DOWN`: as `BENCH_SCALE_DOWN`; 10 makes the table ten +//! times smaller (default 1). +//! - `FEED_FROM`: the first three octets of an IPv4 /24, "192.0.2". Peer N +//! connects from .N, which is also the next hop of its IPv4 routes. +//! Without it, peers connect from wherever the system puts them. +//! - `FEED_ASN`: the first peer's AS number (default 64513); the others +//! follow on from it. +//! - `FEED_HOLD_SECS`: how long the sessions stay open after the table is +//! sent (default 30). +//! +//! What is printed: when the sessions are up and how long the writes took. +//! The writes end when the kernel's socket buffers have taken the last +//! bytes, which is before the daemon has read them: when the table is in +//! is for the daemon's API to say. + +use std::net::{Ipv4Addr, SocketAddr}; +use std::time::{Duration, Instant}; + +use bgp_bench::{Prefixes, Shape, announcements, encoded, env}; +use bgp_integration_tests::test_peer::TestPeer; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpSocket; +use tokio::net::tcp::OwnedWriteHalf; + +/// A KEEPALIVE message: the marker, the length (19) and the type (4). +const KEEPALIVE: [u8; 19] = { + let mut bytes = [0xff; 19]; + bytes[16] = 0; + bytes[17] = 19; + bytes[18] = 4; + bytes +}; + +/// How often KEEPALIVEs go out while the sessions are held open: well +/// inside the 90 s hold time. +const KEEPALIVE_EVERY: Duration = Duration::from_secs(20); + +fn mib(bytes: usize) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +#[tokio::main] +async fn main() { + let to: SocketAddr = std::env::var("FEED_TO") + .unwrap_or_else(|_| "127.0.0.1:1790".into()) + .parse() + .expect("FEED_TO is an address and port"); + let peers = env("FEED_PEERS", 1) as u32; + let shape = Shape::INTERNET.scaled_down(env("FEED_SCALE_DOWN", 1)); + let from = std::env::var("FEED_FROM").ok(); + let first_asn = env("FEED_ASN", 64513) as u32; + let hold = Duration::from_secs(env("FEED_HOLD_SECS", 30) as u64); + + let started = Instant::now(); + let prefixes = Prefixes::generate(shape); + let tables: Vec> = (1..=peers) + .map(|peer| { + encoded(&announcements(&prefixes, shape, peer)) + .iter() + .flat_map(|message| message.iter().copied()) + .collect() + }) + .collect(); + let routes = shape.prefixes() * peers as usize; + let bytes: usize = tables.iter().map(Vec::len).sum(); + println!( + "table: {} IPv4 + {} IPv6 prefixes from each of {peers} peer(s): {routes} routes, {:.1} MiB on the wire, made in {:.2} s", + shape.ipv4_prefixes, + shape.ipv6_prefixes, + mib(bytes), + started.elapsed().as_secs_f64() + ); + + let mut sessions = Vec::new(); + for peer in 1..=peers { + let socket = TcpSocket::new_v4().expect("a socket"); + if let Some(from) = &from { + let address: Ipv4Addr = format!("{from}.{peer}") + .parse() + .expect("FEED_FROM is the first three octets of an IPv4 address"); + socket + .bind(SocketAddr::new(address.into(), 0)) + .expect("the peer's address is ours to connect from"); + } + let stream = socket.connect(to).await.expect("the daemon listens"); + let mut session = TestPeer::from_stream(stream); + session + .handshake( + first_asn + peer - 1, + Ipv4Addr::new(10, 0, 0, peer as u8), + 90, + ) + .await + .expect("the daemon takes the session"); + sessions.push(session.into_stream()); + } + println!("{peers} session(s) established; sending"); + + let started = Instant::now(); + let senders: Vec<_> = sessions + .into_iter() + .zip(tables) + .map(|(stream, table)| { + tokio::spawn(async move { + let (mut reader, mut writer) = stream.into_split(); + // Whatever the daemon sends back, KEEPALIVEs as a rule, is + // read and dropped, so that its writer is never held up. + tokio::spawn(async move { + let mut sink = [0u8; 65536]; + while let Ok(n) = reader.read(&mut sink).await { + if n == 0 { + break; + } + } + }); + let started = Instant::now(); + writer.write_all(&table).await.expect("the daemon reads"); + (writer, started.elapsed()) + }) + }) + .collect(); + let mut writers: Vec = Vec::new(); + for (index, sender) in senders.into_iter().enumerate() { + let (writer, took) = sender.await.expect("the sender runs"); + println!( + "peer {}: table written in {:.3} s", + index + 1, + took.as_secs_f64() + ); + writers.push(writer); + } + let elapsed = started.elapsed(); + println!( + "all {routes} routes written in {:.3} s: {:.2} M routes/s into the sockets", + elapsed.as_secs_f64(), + routes as f64 / elapsed.as_secs_f64() / 1e6 + ); + + let deadline = Instant::now() + hold; + while Instant::now() < deadline { + tokio::time::sleep(KEEPALIVE_EVERY.min(deadline - Instant::now())).await; + for writer in &mut writers { + // A session the daemon closed is left alone. + let _ = writer.write_all(&KEEPALIVE).await; + } + } + println!("sessions closed after {} s", hold.as_secs()); +} diff --git a/crates/fib/src/plan.rs b/crates/fib/src/plan.rs index 0946b68..02c403e 100644 --- a/crates/fib/src/plan.rs +++ b/crates/fib/src/plan.rs @@ -317,6 +317,14 @@ pub struct Plan { unresolved_counts: [usize; Unresolved::ALL.len()], /// Prefixes whose route the kernel refused. refused: HashSet, + /// How many entries of `installed` are blackhole routes, and how many + /// are kept from an earlier run without having been confirmed. Kept in + /// step with the map by [`Plan::put`] and [`Plan::take`], so that + /// asking costs nothing however large the table: the kernel task asks + /// after every batch of changes, and a scan of a million entries each + /// time was a third of its work. + blackholes: usize, + kept_unconfirmed: usize, } impl Plan { @@ -327,10 +335,38 @@ impl Plan { /// How many of the routes the kernel has drop their packets. pub fn blackholes(&self) -> usize { - self.installed - .values() - .filter(|entry| entry.target == Some(Target::Discard)) - .count() + self.blackholes + } + + /// What `entry` counts for: one if it is a blackhole route, and one if + /// it is kept from an earlier run without having been confirmed. + fn weigh(entry: &Entry) -> (usize, usize) { + ( + usize::from(entry.target == Some(Target::Discard)), + usize::from(entry.kept && !entry.confirmed), + ) + } + + /// Puts `entry` in as the route of `prefix`, in place of what was + /// there, keeping the counts in step. + fn put(&mut self, prefix: IpPrefix, entry: Entry) { + let (blackhole, kept) = Self::weigh(&entry); + self.blackholes += blackhole; + self.kept_unconfirmed += kept; + if let Some(old) = self.installed.insert(prefix, entry) { + let (blackhole, kept) = Self::weigh(&old); + self.blackholes -= blackhole; + self.kept_unconfirmed -= kept; + } + } + + /// Takes the route of `prefix` out, keeping the counts in step. + fn take(&mut self, prefix: &IpPrefix) -> Option { + let old = self.installed.remove(prefix)?; + let (blackhole, kept) = Self::weigh(&old); + self.blackholes -= blackhole; + self.kept_unconfirmed -= kept; + Some(old) } /// How many prefixes have a best route that could not be given to the @@ -383,7 +419,7 @@ impl Plan { /// have sent theirs again, and to remove everything meanwhile would /// stop the machine forwarding for as long as that takes. pub fn adopt(&mut self, prefix: IpPrefix, target: Option, keep: bool) { - self.installed.insert( + self.put( prefix, Entry { target, @@ -396,32 +432,36 @@ impl Plan { /// How many routes are being kept from an earlier run without the /// routing table having confirmed them. pub fn kept(&self) -> usize { - self.installed - .values() - .filter(|entry| entry.kept && !entry.confirmed) - .count() + self.kept_unconfirmed } /// The time for keeping unconfirmed routes of an earlier run is over: /// those the routing table still does not have go. pub fn release(&mut self) -> Vec { let mut stale = Vec::new(); + let mut blackholes_gone = 0; self.installed.retain(|prefix, entry| { let goes = entry.kept && !entry.confirmed; entry.kept = false; if goes { stale.push(Op::Remove { prefix: *prefix }); + blackholes_gone += usize::from(entry.target == Some(Target::Discard)); } !goes }); + // What stays was confirmed, and is now not kept either. + self.kept_unconfirmed = 0; + self.blackholes -= blackholes_gone; stale } /// The whole table follows: nothing is known to be wanted until it is /// heard of again. pub fn reset(&mut self) { + self.kept_unconfirmed = 0; for entry in self.installed.values_mut() { entry.confirmed = false; + self.kept_unconfirmed += usize::from(entry.kept); } self.forget_unresolved_and_refused(); } @@ -442,20 +482,26 @@ impl Plan { confirmed: true, kept: false, }); + let before = Self::weigh(entry); entry.confirmed = true; - if entry.target == Some(target) { - return None; - } - entry.target = Some(target); - Some(Op::Install { prefix, target }) + let op = if entry.target == Some(target) { + None + } else { + entry.target = Some(target); + Some(Op::Install { prefix, target }) + }; + let after = Self::weigh(entry); + self.blackholes += after.0; + self.blackholes -= before.0; + self.kept_unconfirmed += after.1; + self.kept_unconfirmed -= before.1; + op } Err(reason) => { self.set_unresolved(prefix, Some(reason)); // Packets must not go on following a route that is no // longer the best. - self.installed - .remove(&prefix) - .map(|_| Op::Remove { prefix }) + self.take(&prefix).map(|_| Op::Remove { prefix }) } } } @@ -464,9 +510,7 @@ impl Plan { pub fn withdrawn(&mut self, prefix: IpPrefix) -> Option { self.set_unresolved(prefix, None); self.refused.remove(&prefix); - self.installed - .remove(&prefix) - .map(|_| Op::Remove { prefix }) + self.take(&prefix).map(|_| Op::Remove { prefix }) } /// The whole table has been given: what was not in it goes. @@ -480,7 +524,7 @@ impl Plan { stale .into_iter() .map(|prefix| { - self.installed.remove(&prefix); + self.take(&prefix); Op::Remove { prefix } }) .collect() @@ -495,7 +539,7 @@ impl Plan { if self.installed.get(prefix)?.target != Some(*target) { return None; } - self.installed.remove(prefix); + self.take(prefix); self.refused.insert(*prefix); // What the kernel refused to replace, it still has. Some(Op::Remove { prefix: *prefix }) @@ -508,6 +552,8 @@ impl Plan { /// Everything goes: the changes that empty the kernel of our routes. pub fn clear(&mut self) -> Vec { self.forget_unresolved_and_refused(); + self.blackholes = 0; + self.kept_unconfirmed = 0; self.installed .drain() .map(|(prefix, _)| Op::Remove { prefix }) @@ -1163,6 +1209,60 @@ mod tests { ] } + /// What the counts would be if the entries were scanned. + fn scanned(plan: &Plan) -> (usize, usize) { + plan.installed + .values() + .map(Plan::weigh) + .fold((0, 0), |(b, k), (eb, ek)| (b + eb, k + ek)) + } + + #[test] + fn the_counts_keep_step_with_the_routes() { + let mut plan = Plan::default(); + let p = |n: u8| prefix(&format!("203.0.113.{n}/32")); + let via = target("192.0.2.2", 2); + let check = |plan: &Plan| assert_eq!((plan.blackholes(), plan.kept()), scanned(plan)); + + // Adopted from an earlier run: two kept, one a blackhole. + plan.adopt(p(1), Some(Target::Discard), true); + plan.adopt(p(2), Some(via), true); + plan.adopt(p(3), Some(via), false); + check(&plan); + // Heard of again, as a blackhole and as a route by way of a + // gateway; one changes kind. + plan.best(p(1), Ok(Target::Discard)); + plan.best(p(2), Ok(Target::Discard)); + plan.best(p(4), Ok(Target::Discard)); + check(&plan); + plan.best(p(4), Ok(via)); + check(&plan); + // The whole table again: everything unconfirmed, then confirmed. + plan.reset(); + check(&plan); + plan.best(p(1), Ok(Target::Discard)); + plan.synced(); + check(&plan); + plan.reset(); + plan.release(); + check(&plan); + // Gone for want of a next hop, withdrawn, refused, adopted over. + plan.best(p(1), Err(Unresolved::Missing)); + plan.best(p(5), Ok(Target::Discard)); + plan.withdrawn(p(5)); + plan.best(p(6), Ok(Target::Discard)); + plan.refused_by_kernel(&Op::Install { + prefix: p(6), + target: Target::Discard, + }); + plan.adopt(p(7), Some(Target::Discard), true); + plan.adopt(p(7), Some(via), true); + check(&plan); + let had = plan.installed(); + assert_eq!(plan.clear().len(), had); + assert_eq!((plan.blackholes(), plan.kept()), (0, 0)); + } + proptest! { /// Whatever happens to the best routes, and whatever an earlier /// run left behind, a kernel that does as it is asked ends up with diff --git a/crates/integration-tests/src/test_peer.rs b/crates/integration-tests/src/test_peer.rs index 1b99bb8..747d721 100644 --- a/crates/integration-tests/src/test_peer.rs +++ b/crates/integration-tests/src/test_peer.rs @@ -51,6 +51,12 @@ impl TestPeer { } } + /// The connection itself, for a test or a tool that goes on without + /// the framing, once the handshake is done. + pub fn into_stream(self) -> TcpStream { + self.stream + } + /// Our end of the connection: what the server sees the peer as. pub fn local_addr(&self) -> Result { Ok(self.stream.local_addr()?) diff --git a/docs/content/docs/development/testing.md b/docs/content/docs/development/testing.md index e6f1959..3e94f48 100644 --- a/docs/content/docs/development/testing.md +++ b/docs/content/docs/development/testing.md @@ -172,11 +172,83 @@ performance: approximately 1 million IPv4 prefixes, mostly /24, and in UPDATE messages; AS paths of several hops; and communities on most routes. -Table 5 lists results on an Apple M-series computer with a release build. +Table 5 lists `full_table` results on an Apple M-series computer with a +release build: 1.23 million prefixes from each of four neighbors, 4.92 +million routes. | Measurement | Result | |---|---| -| 1.23 million routes from one neighbor, accepted into the routing table | 0.47 seconds | -| The same routes received by a second neighbor (`transit`) | 2.1 seconds, approximately 4 routes per UPDATE message | -| 1 million IPv4 routes installed in the kernel (Linux virtual machine) | Slightly more than 1 second | -| The same routes removed from the kernel | 0.5 seconds | +| Decoding the UPDATE messages, one thread | 0.43 seconds, 11 million routes per second | +| Decoding, interning attributes, inserting and selecting, one thread | 1.9 seconds, 2.5 million routes per second | +| The same, 12 threads | 0.7 seconds, 6.9 million routes per second | +| Four neighbors send the table over TCP to the in-process server | 0.9 seconds, 5.2 million routes per second | +| Memory per route, attribute sets included (`--features count-memory`) | 77 bytes | + +## Profiling + +The `scripts/profile/full-table.sh` script measures the daemon itself, as +built for production, while one or more neighbors send it a full routing +table, and records where its CPU time goes. It requires Linux with `perf`, +`inferno` (`cargo install inferno`), `curl`, `jq`, and `sudo`. On another +operating system, run it in the Linux virtual machine: + +```bash +scripts/linux-vm.sh scripts/profile/full-table.sh +``` + +The script builds the daemon with the `profiling` Cargo profile, which is +the release profile with line tables, and with frame pointers, so that +`perf` can attribute samples to functions. It starts the daemon in a +network namespace of its own, connected by a virtual Ethernet pair to a +second namespace in which the load generator +(`crates/bench/examples/feed.rs`) connects as the neighbors and sends the +synthetic table. Routes that the daemon installs go into the first +namespace's routing table and do not affect the host. Table 6 lists the +environment variables that control the run. + +| Variable | Effect | Default | +|---|---|---| +| `PEERS` | The number of neighbors that send the table | 1 | +| `KERNEL` | `1` installs the best routes in the kernel; `0` keeps them in the routing table only | 1 | +| `SCALE_DOWN` | Divides the size of the table | 1 | +| `HOLD_SECS` | How long the sessions stay open after the table is sent | 30 | +| `FREQ` | The sampling frequency in hertz | 1999 | +| `OUT` | The directory for the results | `~/profile` | + +When run through `scripts/linux-vm.sh`, pass the variables with `env`, +because the virtual machine does not receive the host's environment: + +```bash +scripts/linux-vm.sh env PEERS=4 KERNEL=0 scripts/profile/full-table.sh +``` + +The script prints how long the routes take to arrive in the routing table +and in the kernel, and the daemon's resident memory. It writes the +following files to the results directory: + +| File | Contents | +|---|---| +| `flame.svg` | An interactive flame graph of the daemon's threads | +| `top-self.txt` | Functions by the time spent in their own code | +| `top-children.txt` | Functions by the time spent in them and everything they call | +| `stacks.folded` | The collapsed stacks the flame graph is drawn from | +| `perf.data`, `daemon.log`, `feed.log` | The raw samples and the logs | + +Table 7 lists results in the Linux virtual machine (8 virtual CPUs on an +Apple M-series computer) with the synthetic table of 1.23 million +prefixes. + +| Measurement | Result | +|---|---| +| One neighbor; routes in the routing table | 0.7 seconds, 1.7 million routes per second | +| One neighbor; the best routes installed in the kernel | 2.0 seconds after the first route | +| One neighbor, without the kernel; routes in the routing table | 0.6 seconds, 2.0 million routes per second | +| Four neighbors; 4.92 million routes in the routing table | 1.1 seconds, 4.5 million routes per second | +| Four neighbors; the best routes installed in the kernel | 3.0 seconds after the first route | +| Resident memory with one neighbor's table installed | 520 MB | +| Resident memory with four neighbors' tables installed | 680 MB | + +A session is processed on one thread, so one neighbor's table arrives at +the rate of one CPU core, and several neighbors' tables arrive in +parallel. Installing routes in the kernel takes longer than receiving +them, and continues after the last route has arrived. diff --git a/scripts/profile/full-table.sh b/scripts/profile/full-table.sh new file mode 100755 index 0000000..930e297 --- /dev/null +++ b/scripts/profile/full-table.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Profiles the daemon as it takes in a full Internet routing table, on a +# Linux machine: a flame graph and a flat profile of where the CPU time +# goes, and how long the table takes to arrive and to reach the kernel. +# +# scripts/profile/full-table.sh one peer sends the table +# PEERS=4 scripts/profile/full-table.sh four peers send it at once +# KERNEL=0 scripts/profile/full-table.sh the routes stay in the table +# SCALE_DOWN=10 scripts/profile/full-table.sh a tenth of the table +# +# From a machine that is not Linux: scripts/linux-vm.sh scripts/profile/full-table.sh +# +# The daemon runs in a network namespace of its own, so that nothing it +# installs touches the machine's own routing, joined by a veth pair to a +# second namespace where the load generator (crates/bench/examples/feed.rs) +# plays the peers: neighbours on the link, as real peers would be. `perf` +# samples the daemon's threads meanwhile. Needs perf, inferno (cargo +# install inferno), curl and jq, and sudo for the namespaces and the +# kernel's routing table. +# +# Results go to $OUT (default ~/profile): flame.svg, top-self.txt (functions +# by their own time), top-children.txt (by the time of everything they +# call), perf.data, and the daemon's and the load generator's logs. +set -eu + +peers=${PEERS:-1} +scale=${SCALE_DOWN:-1} +kernel=${KERNEL:-1} +hold=${HOLD_SECS:-30} +freq=${FREQ:-1999} +out=${OUT:-$HOME/profile} +ns=ferrix-profile +ns_peer=ferrix-profile-peer + +repository="$(cd "$(dirname "$0")/../.." && pwd)" +target="${CARGO_TARGET_DIR:-$repository/target}" +daemon_bin="$target/profiling/bgp" +feed_bin="$target/profiling/examples/feed" + +fail() { echo "error: $*" >&2; exit 1; } +for tool in perf inferno-collapse-perf inferno-flamegraph curl jq; do + command -v "$tool" >/dev/null || fail "$tool not found" +done + +# Release code with line tables, and frame pointers for perf to walk the +# stacks by (cheaper and more complete than DWARF unwinding here; the +# allocator is built with them too). +cd "$repository" +export RUSTFLAGS="-C force-frame-pointers=yes" CFLAGS="-fno-omit-frame-pointer" +cargo build --profile profiling --locked -p bgp --bin bgp +cargo build --profile profiling --locked -p bgp-bench --example feed + +mkdir -p "$out" +run() { sudo ip netns exec "$ns" "$@"; } +peer() { sudo ip netns exec "$ns_peer" "$@"; } +cleanup() { + sudo pkill -x bgp 2>/dev/null || true + sudo pkill -x feed 2>/dev/null || true + sudo ip netns del "$ns" 2>/dev/null || true + sudo ip netns del "$ns_peer" 2>/dev/null || true +} +trap cleanup EXIT +cleanup +sudo ip netns add "$ns" +sudo ip netns add "$ns_peer" +sudo ip link add bench0 netns "$ns" type veth peer name peer0 netns "$ns_peer" +run ip link set lo up +run ip addr add 192.0.2.254/24 dev bench0 +run ip -6 addr add 2001:db8::ffff/64 dev bench0 +run ip link set bench0 up +peer ip link set lo up +for n in $(seq "$peers"); do + peer ip addr add "192.0.2.$n/24" dev peer0 + peer ip -6 addr add "2001:db8::$n/64" dev peer0 +done +peer ip link set peer0 up +peer ping -c 1 -W 2 192.0.2.254 > /dev/null || fail "the namespaces do not reach each other" + +# An AS number that is in none of the synthetic paths: a route with the +# daemon's own AS in its path is refused as a loop. +cat > "$out/daemon.textpb" <> "$out/daemon.textpb" +fi + +run "$daemon_bin" "$out/daemon.textpb" --level warn > "$out/daemon.log" 2>&1 & +for _ in $(seq 100); do + run curl -sf http://127.0.0.1:8080/api/status > /dev/null 2>&1 && break + sleep 0.1 +done +daemon_pid=$(pgrep -nx bgp) || fail "the daemon did not start; see $out/daemon.log" +status() { run curl -sf "http://127.0.0.1:8080/api/$1" | jq -r "$2"; } +now() { date +%s.%N; } +since() { echo "$(now) - $1" | bc; } + +prefixes=$(( 1000000 / scale + 230000 / scale )) +routes=$(( prefixes * peers )) + +sudo perf record -F "$freq" -g -o "$out/perf.data" -p "$daemon_pid" > "$out/perf.log" 2>&1 & +perf_pid=$! +sleep 1 + +peer env FEED_TO=192.0.2.254:1790 FEED_PEERS="$peers" FEED_SCALE_DOWN="$scale" \ + FEED_FROM=192.0.2 FEED_HOLD_SECS="$hold" "$feed_bin" > "$out/feed.log" 2>&1 & +feed_pid=$! +feeding() { kill -0 "$feed_pid" 2>/dev/null || fail "the load generator stopped; see $out/feed.log"; } + +# The clock starts at the first route in, not when the load generator +# starts: making the table takes it a while. +while [ "$(status status .received_routes)" = 0 ]; do feeding; sleep 0.02; done +start=$(now) +# Routes the policy refuses (the synthetic table has a few bogons in it) +# count as dealt with. +dealt_with() { + status status .received_routes + status peers '[.[] | .rejected_routes // 0] | add // 0' +} +while [ "$(dealt_with | paste -sd+ | bc)" -lt "$routes" ]; do feeding; sleep 0.05; done +arrived=$(since "$start") +received=$(status status .received_routes) +refused=$(status peers '[.[] | .rejected_routes // 0] | add // 0') +echo "$routes routes dealt with after $arrived s ($(echo "$routes / $arrived / 1000000" | bc -l | cut -c1-4) M routes/s): $received taken, $refused refused" +[ "$refused" = 0 ] || run curl -sf http://127.0.0.1:8080/api/rejections | jq -c '.' | cut -c1-400 +if [ "$kernel" = 1 ]; then + prefixes=$(status status .received_prefixes) + while [ "$(status kernel '.synced and .installed >= '"$prefixes")" != true ]; do feeding; sleep 0.05; done + installed=$(since "$start") + echo "$prefixes routes in the kernel after $installed s ($(echo "$prefixes / $installed / 1000" | bc -l | cut -c1-3) k routes/s)" +fi +rss=$(awk '/VmRSS/ { printf "%.0f", $2 / 1024 }' "/proc/$daemon_pid/status") +echo "daemon: $rss MiB resident with the table in" +sleep 1 +sudo kill -INT "$perf_pid" +wait "$perf_pid" || true +cat "$out/feed.log" + +sudo chown "$(id -u)" "$out/perf.data" +# Without the period field, so that each sample counts one whatever event +# perf sampled by (cpu-clock on a machine without performance counters). +perf script -i "$out/perf.data" -F comm,pid,tid,time,event,ip,sym,dso 2> /dev/null \ + | inferno-collapse-perf > "$out/stacks.folded" +title="ferrix: $peers peer(s) send $prefixes prefixes each" +[ "$kernel" = 1 ] && title="$title, into the kernel" +inferno-flamegraph --title "$title" --countname samples --width 1600 "$out/stacks.folded" > "$out/flame.svg" +perf report -i "$out/perf.data" --no-children --stdio --sort symbol --percent-limit 0.5 -g none 2> /dev/null \ + | grep -v '^#' | grep -v '^$' | sed 's/ *$//' | head -80 > "$out/top-self.txt" +perf report -i "$out/perf.data" --children --stdio --sort symbol --percent-limit 1 -g none 2> /dev/null \ + | grep -v '^#' | grep -v '^$' | sed 's/ *$//' | head -80 > "$out/top-children.txt" +echo "samples: $(awk '{ s += $NF } END { print s }' "$out/stacks.folded")" +echo "results in $out: flame.svg top-self.txt top-children.txt"