ferrix/crates/bench/benches/full_table.rs

//! What it costs to take in the Internet routing table.
//!
//! Run with `cargo bench -p bgp-bench`. Set `BENCH_PEERS` (default 4) and
//! `BENCH_SCALE_DOWN` (default 1; 10 makes the table ten times smaller) to
//! change the size.
//!
//! This is a stopwatch and an allocation counter around whole operations at
//! full size, not a statistical micro-benchmark: the questions are "how
//! long until a full table is in" and "how much memory does it take", and
//! their answers are seconds and gigabytes.

use std::alloc::System;
use std::hint::black_box;
use std::net::{Ipv4Addr, SocketAddr};
use std::time::{Duration, Instant};

use bgp_bench::{Prefixes, Shape, announcements, encoded, wire_context};
use bgp_integration_tests::test_peer::TestPeer;
use bgp_integration_tests::test_server::TestServer;
use bgp_packet::message::{Message, PathAttribute, UpdateMessage};
use bgp_primitives::ip_prefix::IpPrefix;
use bgp_rib::{AdjRibIn, PeerId, ReceivedUpdate, RibEntry, SnapshotEntry};
use bytes::BytesMut;
use stats_alloc::{INSTRUMENTED_SYSTEM, Region, StatsAlloc};

#[global_allocator]
static GLOBAL: &StatsAlloc<System> = &INSTRUMENTED_SYSTEM;

fn env(name: &str, default: usize) -> usize {
    std::env::var(name)
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(default)
}

/// Bytes allocated and not freed since `region` began.
fn retained(region: &Region<'_, System>) -> i64 {
    let stats = region.change();
    stats.bytes_allocated as i64 - stats.bytes_deallocated as i64
}

fn mib(bytes: i64) -> f64 {
    bytes as f64 / (1024.0 * 1024.0)
}

fn per_second(count: usize, elapsed: Duration) -> String {
    let rate = count as f64 / elapsed.as_secs_f64();
    if rate >= 1e6 {
        format!("{:.2} M/s", rate / 1e6)
    } else {
        format!("{:.0} k/s", rate / 1e3)
    }
}

fn row(what: &str, elapsed: Duration, detail: String) {
    println!("{what:<44} {:>9.3} s   {detail}", elapsed.as_secs_f64());
}

fn peer_id(peer: u32) -> PeerId {
    PeerId {
        addr: SocketAddr::new(Ipv4Addr::new(192, 0, 2, peer as u8).into(), 179),
        asn: 64512 + peer,
    }
}

fn main() {
    let peers = env("BENCH_PEERS", 4) as u32;
    let shape = Shape::INTERNET.scaled_down(env("BENCH_SCALE_DOWN", 1));

    println!("== Sizes of the types a route is made of");
    for (name, size) in [
        ("IpPrefix", size_of::<IpPrefix>()),
        ("RibEntry", size_of::<RibEntry>()),
        (
            "(IpPrefix, RibEntry), a table slot",
            size_of::<(IpPrefix, RibEntry)>(),
        ),
        ("PathAttribute", size_of::<PathAttribute>()),
        ("SnapshotEntry", size_of::<SnapshotEntry>()),
    ] {
        println!("{name:<44} {size:>5} bytes");
    }

    println!(
        "\n== Table: {} IPv4 + {} IPv6 prefixes, from each of {peers} peers",
        shape.ipv4_prefixes, shape.ipv6_prefixes
    );
    let prefixes = Prefixes::generate(shape);
    let per_peer: Vec<Vec<UpdateMessage>> = (1..=peers)
        .map(|peer| announcements(&prefixes, shape, peer))
        .collect();
    let wire: Vec<Vec<BytesMut>> = per_peer.iter().map(|updates| encoded(updates)).collect();
    let updates: usize = per_peer.iter().map(Vec::len).sum();
    let routes = shape.prefixes() * peers as usize;
    let wire_bytes: usize = wire.iter().flatten().map(BytesMut::len).sum();
    println!(
        "{updates} UPDATE messages, {:.1} prefixes each, {:.1} MiB on the wire, {routes} routes",
        routes as f64 / updates as f64,
        mib(wire_bytes as i64),
    );

    println!("\n== Reading the messages (one thread)");
    let ctx = wire_context();
    let started = Instant::now();
    let decoded: Vec<Vec<Message>> = wire
        .iter()
        .map(|messages| {
            messages
                .iter()
                .map(|m| Message::decode(&ctx, m).expect("what we wrote is well formed"))
                .collect()
        })
        .collect();
    let elapsed = started.elapsed();
    row(
        "decode UPDATEs",
        elapsed,
        format!(
            "{} routes, {} messages, {:.0} MiB/s",
            per_second(routes, elapsed),
            per_second(updates, elapsed),
            mib(wire_bytes as i64) / elapsed.as_secs_f64()
        ),
    );
    drop(decoded);

    floor(&prefixes, &per_peer, routes);

    println!("\n== The same, as the daemon does it today (one thread)");
    // From here to the filled table: what the routes cost to hold.
    let table_region = Region::new(GLOBAL);
    let region = Region::new(GLOBAL);
    let started = Instant::now();
    let received: Vec<Vec<ReceivedUpdate>> = per_peer
        .iter()
        .map(|updates| updates.iter().map(ReceivedUpdate::from_message).collect())
        .collect();
    let elapsed = started.elapsed();
    row(
        "UPDATE -> routes (ReceivedUpdate)",
        elapsed,
        format!(
            "{} routes, holding {:.0} MiB",
            per_second(routes, elapsed),
            mib(retained(&region))
        ),
    );

    let runtime = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .build()
        .expect("a runtime");

    println!("\n== Adj-RIB-In (the actor, fed from one task)");
    runtime.block_on(async {
        let rib = AdjRibIn::spawn();
        let started = Instant::now();
        for (peer, updates) in received.into_iter().enumerate() {
            let peer = peer_id(peer as u32 + 1);
            for update in updates {
                rib.apply(peer, update).await.expect("the actor runs");
            }
        }
        let stored = rib.len().await.expect("the actor runs");
        let elapsed = started.elapsed();
        assert_eq!(stored, routes);
        let held = retained(&table_region);
        row(
            "insert every route",
            elapsed,
            format!(
                "{} routes; table holds {:.0} MiB = {:.0} bytes a route",
                per_second(routes, elapsed),
                mib(held),
                held as f64 / routes as f64
            ),
        );

        let started = Instant::now();
        black_box(rib.len().await.expect("the actor runs"));
        row("count routes", started.elapsed(), String::new());

        let before = Region::new(GLOBAL);
        let started = Instant::now();
        let snapshot = rib.snapshot().await.expect("the actor runs");
        let elapsed = started.elapsed();
        row(
            "snapshot (what /api/routes/received does)",
            elapsed,
            format!(
                "copies {:.0} MiB; nothing else is applied meanwhile",
                mib(retained(&before))
            ),
        );
        drop(snapshot);

        // A second announcement of everything by one peer: every route is
        // replaced, as after a route refresh.
        let again: Vec<ReceivedUpdate> = per_peer[0]
            .iter()
            .map(ReceivedUpdate::from_message)
            .collect();
        let started = Instant::now();
        for update in again {
            rib.apply(peer_id(1), update).await.expect("the actor runs");
        }
        rib.len().await.expect("the actor runs");
        let elapsed = started.elapsed();
        row(
            "replace one peer's routes",
            elapsed,
            per_second(shape.prefixes(), elapsed) + " routes",
        );

        let started = Instant::now();
        let removed = rib.remove_peer(peer_id(1)).await.expect("the actor runs");
        let elapsed = started.elapsed();
        assert_eq!(removed, shape.prefixes());
        row(
            "remove one peer (session down)",
            elapsed,
            per_second(removed, elapsed) + " routes",
        );
    });

    println!("\n== End to end: {peers} peers send the table over TCP at once");
    for logging in [false, true] {
        // The daemon logs a line for every UPDATE at INFO. With no subscriber
        // installed that costs almost nothing; this shows what it costs
        // when the lines are formatted, as they are in production (here to
        // nowhere, so without the cost of the terminal or the disk).
        if logging {
            tracing::subscriber::set_global_default(
                tracing_subscriber::fmt()
                    .with_writer(std::io::sink)
                    .with_max_level(tracing::Level::INFO)
                    .finish(),
            )
            .expect("no subscriber was installed before");
        }
        runtime.block_on(async {
            let server = TestServer::start(65000).await.expect("a server");
            let mut senders = Vec::new();
            for (index, messages) in wire.iter().enumerate() {
                let peer = index as u32 + 1;
                let mut session = TestPeer::connect(server.addr).await.expect("a connection");
                session
                    .handshake(64512 + peer, Ipv4Addr::new(10, 0, 0, peer as u8), 90)
                    .await
                    .expect("a session");
                let stream: Vec<u8> = messages.iter().flat_map(|m| m.iter().copied()).collect();
                senders.push((session, stream));
            }
            let started = Instant::now();
            let sending: Vec<_> = senders
                .into_iter()
                .map(|(mut session, stream)| {
                    tokio::spawn(async move {
                        session.send_raw(&stream).await.expect("the server reads");
                        // Held open until the table is in.
                        session
                    })
                })
                .collect();
            while server.adj_rib_in.len().await.expect("the actor runs") < routes {
                tokio::time::sleep(Duration::from_millis(5)).await;
            }
            let elapsed = started.elapsed();
            row(
                if logging {
                    "with a log line per UPDATE"
                } else {
                    "with logging off"
                },
                elapsed,
                per_second(routes, elapsed) + " routes",
            );
            for session in sending {
                drop(session.await);
            }
        });
    }
}

/// A straw man, not a design: the same routes in the plainest compact form,
/// to show how far the current table is from what a hash table can do.
///
/// A prefix is one integer and a route is the number of its attribute set
/// (here simply the number of the UPDATE that announced it; a real table
/// would intern the sets). One map per peer and address family, filled on
/// one thread with no channel in between.
fn floor(prefixes: &Prefixes, per_peer: &[Vec<UpdateMessage>], routes: usize) {
    use std::collections::HashMap;

    fn key(prefix: &IpPrefix) -> u128 {
        let mut address = [0u8; 16];
        address[..prefix.bytes().len()].copy_from_slice(prefix.bytes());
        // The address in the high bits, the length in the low eight. An IPv6
        // prefix longer than /120 would not fit; the table has none.
        (u128::from_be_bytes(address) & !0xff) | u128::from(prefix.length())
    }

    println!("\n== Straw man: packed prefix -> attribute set number, no actor");
    let region = Region::new(GLOBAL);
    let started = Instant::now();
    let mut tables: Vec<(HashMap<u64, u32>, HashMap<u128, u32>)> = Vec::new();
    for updates in per_peer {
        let mut v4: HashMap<u64, u32> = HashMap::with_capacity(prefixes.ipv4.len());
        let mut v6: HashMap<u128, u32> = HashMap::with_capacity(prefixes.ipv6.len());
        for (number, update) in updates.iter().enumerate() {
            for prefix in &update.announced_routes {
                v4.insert(
                    (key(prefix) >> 64) as u64 | u64::from(prefix.length()),
                    number as u32,
                );
            }
            for attribute in &update.path_attributes {
                if let PathAttribute::MpReachNlri(mp) = attribute {
                    for prefix in &mp.prefixes {
                        v6.insert(key(prefix), number as u32);
                    }
                }
            }
        }
        tables.push((v4, v6));
    }
    let elapsed = started.elapsed();
    let stored: usize = tables.iter().map(|(v4, v6)| v4.len() + v6.len()).sum();
    assert_eq!(stored, routes);
    let held = retained(&region);
    row(
        "insert every route",
        elapsed,
        format!(
            "{} routes; {:.0} MiB = {:.0} bytes a route (attribute sets not counted)",
            per_second(routes, elapsed),
            mib(held),
            held as f64 / routes as f64
        ),
    );
    black_box(tables);
}