diff --git a/crates/bench/benches/full_table.rs b/crates/bench/benches/full_table.rs index 206fc2c..5148813 100644 --- a/crates/bench/benches/full_table.rs +++ b/crates/bench/benches/full_table.rs @@ -3,7 +3,8 @@ //! //! Run with `cargo bench -p bgp-bench --bench full_table`. Set //! `BENCH_PEERS` (default 4) and `BENCH_SCALE_DOWN` (default 1; 10 makes the -//! table ten times smaller) to change the size. The table is synthetic (see +//! table ten times smaller) to change the size. `BENCH_PRESIZE=0` leaves +//! the table to grow as it fills, as it does without `expected_routes`. The table is synthetic (see //! the crate documentation); `real_table` runs on a real one. //! //! This is a stopwatch around whole operations at full size, not a @@ -148,6 +149,9 @@ fn main() { fn into_table(wire: &[Vec], routes: usize, threads: usize) { let region = Region::begin(); let table = RibTable::new(threads * 16); + if env("BENCH_PRESIZE", 1) == 1 { + table.expect_prefixes(routes / wire.len()); + } let peers: Vec = (0..wire.len()) .map(|peer| { table diff --git a/crates/rib/src/table/mod.rs b/crates/rib/src/table/mod.rs index 6f74c8b..0b3d5aa 100644 --- a/crates/rib/src/table/mod.rs +++ b/crates/rib/src/table/mod.rs @@ -670,6 +670,23 @@ impl RibTable { self.len() == 0 } + /// Makes room for `prefixes` prefixes, spread over the shards, so that + /// a table of that size goes in without the shards growing as it + /// does. Growing a shard moves every row it has into a bigger table; + /// over a full table that was a seventh of what a session spent + /// taking it in. Room that is already there is kept. + pub fn expect_prefixes(&self, prefixes: usize) { + let each = prefixes.div_ceil(self.shards.len()); + for shard in &self.shards { + let mut shard = write(shard); + let hasher = shard.hasher.clone(); + let additional = each.saturating_sub(shard.rows.len()); + shard + .rows + .reserve(additional, |row| hasher.hash_one(row.prefix)); + } + } + /// How many prefixes have at least one route. pub fn prefix_count(&self) -> usize { self.shards.iter().map(|shard| read(shard).rows.len()).sum() diff --git a/crates/rib/src/table/tests.rs b/crates/rib/src/table/tests.rs index f856d02..54500db 100644 --- a/crates/rib/src/table/tests.rs +++ b/crates/rib/src/table/tests.rs @@ -1002,3 +1002,31 @@ fn routes_can_be_picked_out_of_the_table() { ); assert!(table.routes_where(0, |_| true).is_empty()); } + +#[test] +fn a_table_told_what_to_expect_has_the_room_before_the_routes_come() { + let table = RibTable::new(8); + table.expect_prefixes(100_000); + let room: usize = table + .shards + .iter() + .map(|shard| read(shard).rows.capacity()) + .sum(); + assert!(room >= 100_000, "room for {room} prefixes"); + // Asking again for less, or for the same, takes nothing away. + table.expect_prefixes(10); + let still: usize = table + .shards + .iter() + .map(|shard| read(shard).rows.capacity()) + .sum(); + assert_eq!(still, room); + // And it is a table like any other. + let peer = table.add_peer(external(1)).unwrap(); + let attrs = table + .attrs() + .intern(&Route::via(&[64512]).attributes(), None) + .unwrap(); + table.announce(peer, &attrs, &[Nlri::from(prefix("192.0.2.0/24"))]); + assert_eq!(table.prefix_count(), 1); +} diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index acb0d3e..4c655a5 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -407,7 +407,9 @@ message Peer { // the table with it, and shows an established session with fewer than // half as needing attention: a peer that is up but has sent next to // nothing looks, without this, exactly like one that is fine. Zero, the - // default, compares nothing. + // default, compares nothing. The routing table is also made with room + // for the largest of these from the start, so that it need not grow, + // and so use twice its memory for a moment, while the tables come in. uint32 expected_routes = 17; } diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index 4b7f208..e0f5518 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -118,7 +118,7 @@ impl BgpServer { for route in &config.originate { local_rib.announce(route.clone()).await?; } - let configured = config + let configured: Vec = config .peers .iter() .map(|p| ConfiguredPeer { @@ -134,6 +134,11 @@ impl BgpServer { // first routes are. let (rpki, rpki_client) = config.rpki.clone().map(Rpki::spawn).unzip(); let rib = Arc::new(RibTable::new(table_shards())); + // Sized for the largest table a peer is expected to send: the + // prefixes of the others are mostly the same ones. + if let Some(expected) = configured.iter().filter_map(|p| p.expected_routes).max() { + rib.expect_prefixes(expected as usize); + } // Started now, so that a machine on which it cannot be done stops // the daemon from starting. There is nothing to install yet. let (kernel, kernel_task) = match config.kernel { diff --git a/docs/content/docs/development/testing.md b/docs/content/docs/development/testing.md index 3e94f48..67f2967 100644 --- a/docs/content/docs/development/testing.md +++ b/docs/content/docs/development/testing.md @@ -240,13 +240,14 @@ prefixes. | Measurement | Result | |---|---| -| One neighbor; routes in the routing table | 0.7 seconds, 1.7 million routes per second | +| One neighbor; routes in the routing table | 0.8 seconds, 1.5 million routes per second | | One neighbor; the best routes installed in the kernel | 2.0 seconds after the first route | | One neighbor, without the kernel; routes in the routing table | 0.6 seconds, 2.0 million routes per second | -| Four neighbors; 4.92 million routes in the routing table | 1.1 seconds, 4.5 million routes per second | -| Four neighbors; the best routes installed in the kernel | 3.0 seconds after the first route | -| Resident memory with one neighbor's table installed | 520 MB | -| Resident memory with four neighbors' tables installed | 680 MB | +| Four neighbors; 4.92 million routes in the routing table | 1.1 seconds, 4.4 million routes per second | +| Four neighbors; the best routes installed in the kernel | 2.2 seconds after the first route | +| Four neighbors; routes the kernel was asked to install | 1.25 million, for 1.23 million prefixes | +| Resident memory with one neighbor's table installed | 440 MB | +| Resident memory with four neighbors' tables installed | 630 MB | A session is processed on one thread, so one neighbor's table arrives at the rate of one CPU core, and several neighbors' tables arrive in diff --git a/docs/content/docs/guides/peers.md b/docs/content/docs/guides/peers.md index 0faf7f0..0f1cad5 100644 --- a/docs/content/docs/guides/peers.md +++ b/docs/content/docs/guides/peers.md @@ -259,8 +259,11 @@ with this value. An established session with fewer than half the expected routes is listed under **Needing attention**, with the expected number beside the actual one. The comparison starts one minute after the session is established, so that a neighbor that is still sending its table is not -reported. The value does not affect route processing; `max_prefixes` is -the limit that Ferrix enforces. +reported. The value does not affect which routes are accepted; +`max_prefixes` is the limit that Ferrix enforces. Ferrix also sizes the +routing table at startup for the largest `expected_routes` value among +the neighbors, so that the table does not grow while the tables are +received. ## Verifying BGP Sessions diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md index daaac17..e4f7ce1 100644 --- a/docs/content/docs/reference/configuration.md +++ b/docs/content/docs/reference/configuration.md @@ -77,7 +77,7 @@ Statements for a BGP neighbor. | `role` [Role](#role) | None | Specifies the BGP role of the neighbor (RFC 9234). Not permitted for IBGP neighbors. Required for every EBGP neighbor if any `export` statement enables `learned_routes`. | | `require_role` bool | `false` | Rejects the session if the neighbor does not advertise a BGP role. Requires `role`. | | `passive` bool | `false` | Prevents Ferrix from initiating connections to the neighbor. Ferrix accepts connections from the address of `endpoint`; the port of `endpoint` is not used. See [Passive Neighbors](@/docs/guides/peers.md#passive-neighbors). | -| `expected_routes` uint32 | 0 | Specifies the approximate number of routes that the neighbor advertises in normal operation. The status page marks an established session with fewer than half this number of routes as needing attention. The value does not affect route processing. 0 disables the comparison. See [Expected Route Counts](@/docs/guides/peers.md#expected-route-counts). | +| `expected_routes` uint32 | 0 | Specifies the approximate number of routes that the neighbor advertises in normal operation. The status page marks an established session with fewer than half this number of routes as needing attention. Ferrix also allocates the routing table with room for the largest `expected_routes` value among the neighbors at startup, which avoids the table growing, and briefly using twice its memory, while the tables are received. The value does not affect which routes are accepted. 0 disables the comparison. See [Expected Route Counts](@/docs/guides/peers.md#expected-route-counts). | | `export` [ExportPolicy](#exportpolicy) | Top-level `export` | Specifies the export policy for the neighbor. Replaces the top-level `export` statement. | | `graceful_restart` [GracefulRestart](#gracefulrestart) | Top-level `graceful_restart` | Specifies graceful restart for the neighbor. Each statement that is set replaces the corresponding top-level statement. | | `route_reflector_client` bool | `false` | Makes the IBGP neighbor a route reflector client: Ferrix advertises to it the routes learned from other IBGP neighbors, and advertises its routes to them. Permitted only for IBGP neighbors. See [Route Reflection](@/docs/guides/passing-routes-on.md#route-reflection). | diff --git a/scripts/profile/full-table.sh b/scripts/profile/full-table.sh index e2b1935..b776584 100755 --- a/scripts/profile/full-table.sh +++ b/scripts/profile/full-table.sh @@ -87,8 +87,8 @@ listen_endpoints { address: "192.0.2.254" port: 1790 } http_endpoint { address: "127.0.0.1" port: 8080 } CONFIG # The peers are configured as a router's would be, with how many routes -# to expect of each; the synthetic table has a few bogons, which the -# default import policy refuses. +# to expect of each. They have private AS numbers and the synthetic table +# has a few bogon prefixes, so bogons are taken. for n in $(seq "$peers"); do cat >> "$out/daemon.textpb" <