ferrix/crates/rib/src/table/journal_tests.rs

use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};

use bgp_primitives::constants::AddressFamilyId;
use bgp_primitives::ip_prefix::IpPrefix;
use proptest::prelude::*;

use super::super::tests::{Fixture, Route, external, internal, prefix, route};
use super::super::{AttrSet, PeerIndex, PeerInfo, RibTable};
use super::{Event, Limits, Reader};

/// What a follower would keep: its own copy of the best routes, brought
/// up to date from the events it is given.
#[derive(Default)]
struct Mirror {
    best: HashMap<IpPrefix, (PeerIndex, Arc<AttrSet>)>,
    /// What was held before a reset and has not been heard of since.
    stale: HashMap<IpPrefix, (PeerIndex, Arc<AttrSet>)>,
    resets: usize,
    synced: usize,
    best_events: usize,
    withdrawn_events: usize,
}

impl Mirror {
    fn apply(&mut self, event: Event<'_>) {
        match event {
            Event::Reset => {
                self.resets += 1;
                self.stale = std::mem::take(&mut self.best);
            }
            Event::Best {
                prefix,
                peer,
                attrs,
            } => {
                self.best_events += 1;
                self.stale.remove(&prefix);
                self.best.insert(prefix, (peer, Arc::clone(attrs)));
            }
            Event::Withdrawn { prefix } => {
                self.withdrawn_events += 1;
                self.best.remove(&prefix);
            }
            Event::Synced => {
                self.synced += 1;
                self.stale.clear();
            }
        }
    }

    /// Reads until there is nothing more, in steps of `budget`.
    fn catch_up(&mut self, reader: &mut Reader, budget: usize) {
        while !reader.poll(budget, |event| self.apply(event)).caught_up {}
    }

    #[track_caller]
    fn assert_matches(&self, table: &RibTable) {
        let mut expected = HashMap::new();
        table.for_each_best(|prefix, peer, attrs| {
            expected.insert(*prefix, (peer, Arc::clone(attrs)));
        });
        assert_eq!(
            self.best.len(),
            expected.len(),
            "the mirror has {} prefixes, the table {}",
            self.best.len(),
            expected.len()
        );
        for (prefix, (peer, attrs)) in &expected {
            let Some((mirrored_peer, mirrored_attrs)) = self.best.get(prefix) else {
                panic!("{prefix} is in the table and not in the mirror");
            };
            assert_eq!(mirrored_peer, peer, "{prefix}: from the wrong peer");
            assert!(
                Arc::ptr_eq(mirrored_attrs, attrs),
                "{prefix}: wrong attributes"
            );
        }
    }
}

fn nth_prefix(n: u32) -> IpPrefix {
    IpPrefix::new(AddressFamilyId::Ipv4, (n << 8).to_be_bytes(), 24).unwrap()
}

#[test]
fn a_new_reader_is_given_the_table_and_then_the_changes() {
    let fixture = Fixture::new(&[external(1), external(2)]);
    let (a, b) = (prefix("10.0.0.0/8"), prefix("2001:db8::/32"));
    fixture.announce(0, &a, &Route::via(&[1, 2]));
    fixture.announce(0, &b, &Route::via(&[1, 2]));

    let mut reader = fixture.table.reader();
    let mut mirror = Mirror::default();
    let mut events = Vec::new();
    let progress = reader.poll(100, |event| {
        events.push(
            format!("{event:?}")
                .split([' ', '{'])
                .next()
                .unwrap()
                .to_string(),
        );
        mirror.apply(event);
    });
    assert!(progress.caught_up);
    assert_eq!(events, ["Reset", "Best", "Best", "Synced"]);
    mirror.assert_matches(&fixture.table);

    // Nothing has happened: nothing to read.
    assert_eq!(reader.poll(100, |_| panic!("no event expected")).events, 0);

    // A change that does not change the best route is not an event.
    fixture.announce(1, &a, &Route::via(&[3, 4, 5]));
    assert_eq!(reader.poll(100, |_| panic!("no event expected")).events, 0);

    fixture.announce(1, &a, &Route::via(&[3]));
    fixture.withdraw(0, &b);
    mirror.catch_up(&mut reader, 100);
    assert_eq!((mirror.best_events, mirror.withdrawn_events), (3, 1));
    assert_eq!((mirror.resets, mirror.synced), (1, 1));
    mirror.assert_matches(&fixture.table);
}

/// A follower wants what the table has now, not how it got there.
#[test]
fn many_changes_to_a_prefix_between_reads_are_one_event() {
    let fixture = Fixture::new(&[external(1), external(2)]);
    let p = prefix("10.0.0.0/8");
    let mut reader = fixture.table.reader();
    let mut mirror = Mirror::default();
    mirror.catch_up(&mut reader, 100);

    for round in 0..50 {
        fixture.announce(0, &p, &Route::via(&[1, 2, 3]));
        fixture.announce(1, &p, &Route::via(&[4, 5]));
        fixture.withdraw(1, &p);
        if round % 2 == 0 {
            fixture.withdraw(0, &p);
        }
    }
    // It ended with peer 0's route in place.
    mirror.catch_up(&mut reader, 100);
    assert_eq!((mirror.best_events, mirror.withdrawn_events), (1, 0));
    mirror.assert_matches(&fixture.table);

    // And a prefix that came and went again is, at most, a withdrawal.
    let q = prefix("10.1.0.0/16");
    for _ in 0..50 {
        fixture.announce(0, &q, &Route::via(&[1]));
        fixture.withdraw(0, &q);
    }
    mirror.catch_up(&mut reader, 100);
    assert_eq!((mirror.best_events, mirror.withdrawn_events), (1, 1));
    mirror.assert_matches(&fixture.table);
}

#[test]
fn reading_in_small_steps_gives_the_same_as_reading_at_once() {
    let fixture = Fixture::new(&[external(1)]);
    for n in 0..500 {
        fixture.announce(0, &nth_prefix(n), &Route::via(&[1, n % 7 + 1]));
    }
    let mut reader = fixture.table.reader();
    let mut mirror = Mirror::default();
    mirror.catch_up(&mut reader, 7);
    mirror.assert_matches(&fixture.table);

    for n in 0..500 {
        if n % 3 == 0 {
            fixture.withdraw(0, &nth_prefix(n));
        } else {
            fixture.announce(0, &nth_prefix(n), &Route::via(&[2, n % 5 + 1]));
        }
    }
    let mut steps = 0;
    while !reader.poll(7, |event| mirror.apply(event)).caught_up {
        steps += 1;
    }
    assert!(steps > 50, "the budget was kept to: {steps} steps");
    mirror.assert_matches(&fixture.table);
    assert_eq!(mirror.resets, 1);
}

#[test]
fn with_no_reader_nothing_is_kept() {
    let fixture = Fixture::new(&[external(1)]);
    for n in 0..5000 {
        fixture.announce(0, &nth_prefix(n), &Route::via(&[1]));
        fixture.withdraw(0, &nth_prefix(n));
    }
    assert_eq!(fixture.table.journal_len(), 0);

    // And what a reader was holding back goes when the reader does.
    let reader = fixture.table.reader();
    for n in 0..100 {
        fixture.announce(0, &nth_prefix(n), &Route::via(&[1]));
    }
    assert_eq!(fixture.table.journal_len(), 100);
    drop(reader);
    assert_eq!(fixture.table.journal_len(), 0);
}

/// A reader that stops reading must not make the table keep a growing
/// journal for it. It is the one that pays: it is given the table again.
#[test]
fn a_reader_that_stops_reading_is_reset_and_holds_nothing_up() {
    // One shard, so that the journal's length is that shard's.
    let fixture = Fixture::on(RibTable::new(1), &[external(1)]);
    fixture.announce(0, &prefix("192.0.2.0/24"), &Route::via(&[1]));

    let mut stuck = fixture.table.reader();
    let mut stuck_mirror = Mirror::default();
    stuck_mirror.catch_up(&mut stuck, 100);
    let mut diligent = fixture.table.reader();
    let mut diligent_mirror = Mirror::default();

    // Prefixes come and go, each a different one, so no entry is made
    // dead by a later one for the same prefix.
    let limits = Limits::default();
    let mut longest = 0;
    for n in 0..20_000 {
        fixture.announce(0, &nth_prefix(n), &Route::via(&[1]));
        fixture.withdraw(0, &nth_prefix(n));
        longest = longest.max(fixture.table.journal_len());
        if n % 100 == 0 {
            diligent_mirror.catch_up(&mut diligent, 1000);
        }
    }
    assert!(
        longest <= limits.tidy_at.max(2 * (1 + limits.slack)) + 2,
        "the journal reached {longest} entries"
    );

    // The one that kept up never noticed.
    diligent_mirror.catch_up(&mut diligent, 1000);
    assert_eq!(
        diligent_mirror.resets, 1,
        "only the reset every reader starts with"
    );
    diligent_mirror.assert_matches(&fixture.table);

    // The one that did not is told, and ends up right all the same.
    stuck_mirror.catch_up(&mut stuck, 1000);
    assert_eq!(stuck_mirror.resets, 2);
    assert_eq!(stuck_mirror.synced, 2);
    stuck_mirror.assert_matches(&fixture.table);
}

#[tokio::test]
async fn a_waiting_reader_is_woken_by_a_change_of_best_route() {
    let fixture = Fixture::new(&[external(1), external(2)]);
    let p = prefix("10.0.0.0/8");
    let mut reader = fixture.table.reader();
    let mut mirror = Mirror::default();

    // Not caught up yet: no waiting.
    reader.changed().await;
    mirror.catch_up(&mut reader, 100);

    let waiting = tokio::time::timeout(std::time::Duration::from_millis(50), reader.changed());
    assert!(waiting.await.is_err(), "nothing changed, so it waits");

    let table = Arc::clone(&fixture.table);
    let attrs = table
        .attrs()
        .intern(&Route::via(&[1]).attributes(), None)
        .unwrap();
    let peer = fixture.peers[0];
    let writer = tokio::spawn(async move {
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
        table.announce(peer, &attrs, &[p]);
    });
    tokio::time::timeout(std::time::Duration::from_secs(5), reader.changed())
        .await
        .expect("woken by the announcement");
    writer.await.unwrap();
    mirror.catch_up(&mut reader, 100);
    mirror.assert_matches(&fixture.table);

    // A change made before the wait began is not missed either.
    fixture.withdraw(0, &p);
    tokio::time::timeout(std::time::Duration::from_secs(5), reader.changed())
        .await
        .expect("the change was already there");
}

/// Writers on several threads, a reader on another, reading while they
/// write. When the writers are done and the reader has caught up, it has
/// what the table has.
#[test]
fn a_reader_keeps_up_with_writers_on_other_threads() {
    const WRITERS: usize = 4;
    const PREFIXES: u32 = 300;
    let infos: Vec<PeerInfo> = (0..WRITERS).map(|i| external(i as u8 + 1)).collect();
    // Small journals, so that tidying and falling behind happen too.
    let fixture = Fixture::on(
        RibTable::with_journal_limits(
            8,
            Limits {
                tidy_at: 16,
                slack: 4,
            },
        ),
        &infos,
    );
    let table = &fixture.table;
    let done = AtomicBool::new(false);
    let mut mirror = Mirror::default();

    std::thread::scope(|scope| {
        let writers: Vec<_> = (0..WRITERS)
            .map(|writer| {
                let peer = fixture.peers[writer];
                scope.spawn(move || {
                    // A simple generator; each writer its own sequence.
                    let mut state = 0x9e37_79b9_7f4a_7c15u64.wrapping_mul(writer as u64 + 1);
                    let mut next = || {
                        state ^= state << 13;
                        state ^= state >> 7;
                        state ^= state << 17;
                        state
                    };
                    for _ in 0..30_000 {
                        let p = nth_prefix((next() % u64::from(PREFIXES)) as u32);
                        if next() % 3 == 0 {
                            table.withdraw(peer, &[p]);
                        } else {
                            let route = Route {
                                med: Some((next() % 3) as u32),
                                ..Route::via(
                                    &[(next() % 3) as u32 + 1, 9][..(next() % 2 + 1) as usize],
                                )
                            };
                            let attrs = table.attrs().intern(&route.attributes(), None).unwrap();
                            table.announce(peer, &attrs, &[p]);
                        }
                    }
                })
            })
            .collect();

        let mirror = &mut mirror;
        let done = &done;
        let reading = scope.spawn(move || {
            let mut reader = table.reader();
            while !done.load(Ordering::Acquire) {
                reader.poll(50, |event| mirror.apply(event));
            }
            mirror.catch_up(&mut reader, 50);
        });
        for writer in writers {
            writer.join().unwrap();
        }
        done.store(true, Ordering::Release);
        reading.join().unwrap();
    });

    mirror.assert_matches(table);
    assert!(mirror.best_events > 0);
}

// ---------------------------------------------------------------------------
// Readers against the table, under any interleaving
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
enum Op {
    Announce {
        peer: usize,
        prefix: usize,
        route: Route,
    },
    Withdraw {
        peer: usize,
        prefix: usize,
    },
    Flap {
        peer: usize,
    },
    Poll {
        reader: usize,
        budget: usize,
    },
    /// The reader is dropped and a new one takes its place.
    Replace {
        reader: usize,
    },
}

const PEERS: usize = 4;
const PREFIXES: usize = 12;
const READERS: usize = 3;

fn op() -> impl Strategy<Value = Op> {
    prop_oneof![
        8 => (0..PEERS, 0..PREFIXES, route())
            .prop_map(|(peer, prefix, route)| Op::Announce { peer, prefix, route }),
        5 => (0..PEERS, 0..PREFIXES).prop_map(|(peer, prefix)| Op::Withdraw { peer, prefix }),
        1 => (0..PEERS).prop_map(|peer| Op::Flap { peer }),
        // Reader 0 is polled rarely: it is the one that falls behind.
        4 => (1..READERS, 1usize..6).prop_map(|(reader, budget)| Op::Poll { reader, budget }),
        1 => (0..READERS, 1usize..6).prop_map(|(reader, budget)| Op::Poll { reader, budget }),
        1 => (1..READERS).prop_map(|reader| Op::Replace { reader }),
    ]
}

proptest! {
    #![proptest_config(ProptestConfig::with_cases(1500))]

    /// However changes and reads are interleaved, and however small the
    /// journals: a reader that has caught up holds exactly the table's best
    /// routes, and every reader gets there in the end.
    #[test]
    fn a_reader_that_has_caught_up_has_what_the_table_has(
        ops in proptest::collection::vec(op(), 1..150),
        shards in 1usize..4,
    ) {
        let infos: Vec<PeerInfo> = (0..PEERS)
            .map(|i| if i == 0 { internal(9) } else { external(i as u8) })
            .collect();
        // Journals that are tidied, and found too long, almost at once.
        let limits = Limits { tidy_at: 3, slack: 1 };
        let mut fixture = Fixture::on(RibTable::with_journal_limits(shards, limits), &infos);
        let prefixes: Vec<IpPrefix> = (0..PREFIXES as u32).map(nth_prefix).collect();
        let mut readers: Vec<(Reader, Mirror)> = (0..READERS)
            .map(|_| (fixture.table.reader(), Mirror::default()))
            .collect();

        for op in ops {
            match op {
                Op::Announce { peer, prefix, route } => {
                    fixture.announce(peer, &prefixes[prefix], &route);
                }
                Op::Withdraw { peer, prefix } => {
                    fixture.withdraw(peer, &prefixes[prefix]);
                }
                Op::Flap { peer } => {
                    fixture.table.remove_peer(fixture.peers[peer]);
                    fixture.peers[peer] = fixture.table.add_peer(infos[peer]).unwrap();
                }
                Op::Poll { reader, budget } => {
                    let (reader, mirror) = &mut readers[reader];
                    let progress = reader.poll(budget, |event| mirror.apply(event));
                    if progress.caught_up {
                        mirror.assert_matches(&fixture.table);
                    }
                }
                Op::Replace { reader } => {
                    readers[reader] = (fixture.table.reader(), Mirror::default());
                }
            }
            // Never much longer than the table, whatever the readers do.
            let bound = shards * (2 * (PREFIXES + limits.slack) + limits.tidy_at + PREFIXES);
            prop_assert!(fixture.table.journal_len() <= bound);
        }

        for (reader, mirror) in &mut readers {
            mirror.catch_up(reader, 5);
            mirror.assert_matches(&fixture.table);
            prop_assert_eq!(mirror.resets, mirror.synced);
        }
    }
}