Rayhaan JaufeerallyandClaude Fable 5.1
321af51e4b
scripts/interop/lab.sh runs the daemon against FRR, BIRD and Routinator, each in a network namespace on one exchange LAN, and checks what every side ends up with: sessions and routes over IPv4 and IPv6 with their attributes, four-octet AS numbers, ADD-PATH, RFC 9234 roles and Only to Customer, RTR version 2 and invalid routes, route refresh, session resets, killed peers, a cut wire, the daemon stopped and killed, a table of a million routes and a restart under it, role mismatch and the prefix limit. All of it passes against FRR 10.5.1, BIRD 2.18 and Routinator 0.15.2. scripts/interop/real-rpki.sh checks the RTR client against a Routinator with the real RPKI: 713,258 IPv4 and 187,479 IPv6 VRPs and 3,379 ASPAs, the same numbers on both sides. What the lab found, fixed here: - A daemon restarted with its routes in the kernel removed them all at once, its table being empty until the peers had sent theirs again, and then put them back: no forwarding meanwhile. Routes of an earlier run are now kept for `stale_seconds` (120 by default) and only those nobody announced again are removed. A restart under a million routes takes 1.1 s to be in step, with nothing removed. - FRR ends a session it loses to a collision with a state machine error without a subcode, which was shown as "error code 5 subcode 0". CI gets a job that runs the lab. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
674 lines
30 KiB
Bash
Executable File
674 lines
30 KiB
Bash
Executable File
#!/bin/bash
|
|
# An interoperability lab: this daemon against FRR, BIRD and Routinator,
|
|
# each in a network namespace of its own on one exchange LAN.
|
|
#
|
|
# 10.9.0.0/24, 2001:db8:9::/64
|
|
# ┌───────────────────┬──────────────────────┐
|
|
# dut (.1) AS 64601 frr (.2) AS 64602 bird (.3) AS 4200000003
|
|
# lan 10.1.0.1/24 lan 10.2.0.1/24 lan 10.3.0.1/24
|
|
# 2001:db8:1::1/48 2001:db8:2::1/48 2001:db8:3::1/48
|
|
# + Routinator on 127.0.0.1:3323
|
|
#
|
|
# "dut" is the daemon under test. FRR is its peer, BIRD its provider; both
|
|
# say so in their OPEN (RFC 9234). Routinator serves validated ROA payloads
|
|
# made up for the lab from a local exceptions file, so nothing is fetched
|
|
# from the Internet.
|
|
#
|
|
# scripts/interop/lab.sh run every phase, then take the lab down
|
|
# scripts/interop/lab.sh up bring the lab up and leave it up
|
|
# scripts/interop/lab.sh down take it down
|
|
# PHASES="basics scale" scripts/interop/lab.sh only these phases
|
|
# SCALE_ROUTES=1000000 ... the size of the table BIRD sends
|
|
#
|
|
# Needs Linux, sudo, and: frr, bird2, jq, routinator, a built `bgp`
|
|
# (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS;
|
|
# scripts/interop/install.sh installs what is missing on Ubuntu.
|
|
set -euo pipefail
|
|
|
|
cd "$(dirname "$0")/../.."
|
|
bgp="${CARGO_TARGET_DIR:-target}/release/bgp"
|
|
work=/tmp/ferrix-lab
|
|
pathspace=ferrixlab
|
|
all_phases="basics roles rpki refresh flaps restart scale limits"
|
|
phases="${PHASES:-$all_phases}"
|
|
scale_routes="${SCALE_ROUTES:-200000}"
|
|
|
|
# --- plumbing ---------------------------------------------------------------
|
|
|
|
in_ns() { local name=$1; shift; sudo ip netns exec "lab-$name" "$@"; }
|
|
api() { in_ns dut curl -sf "http://127.0.0.1:8080$1"; }
|
|
vty() { in_ns frr vtysh -N "$pathspace" "$@" 2>/dev/null; }
|
|
bird_ctl() { sudo birdc -s "$work/bird.ctl" "$@"; }
|
|
say() { printf '%s\n' "$*"; }
|
|
ok() { say " ok: $*"; }
|
|
fail() { say "FAILED: $*" >&2; exit 1; }
|
|
|
|
# wait_for "what" command...: until the command succeeds.
|
|
wait_for() {
|
|
local what=$1; shift
|
|
local tries=${WAIT_TRIES:-150}
|
|
for _ in $(seq "$tries"); do
|
|
if "$@" >/dev/null 2>&1; then return 0; fi
|
|
sleep 0.2
|
|
done
|
|
say "timed out waiting for: $what" >&2
|
|
return 1
|
|
}
|
|
|
|
down() {
|
|
for pidfile in "$work"/dut.pid "$work"/routinator.pid; do
|
|
[ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
[ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.3
|
|
for name in dut frr bird ix; do sudo ip netns del "lab-$name" 2>/dev/null || true; done
|
|
sudo rm -rf "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
}
|
|
|
|
# --- the network ------------------------------------------------------------
|
|
|
|
network_up() {
|
|
sudo modprobe dummy 2>/dev/null || true
|
|
sudo ip netns add lab-ix
|
|
in_ns ix ip link add ix type bridge
|
|
in_ns ix ip link set ix up
|
|
local n=0
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
sudo ip netns add "lab-$name"
|
|
sudo ip link add ix0 netns "lab-$name" type veth peer name "to-$name" netns lab-ix
|
|
in_ns ix ip link set "to-$name" master ix up
|
|
in_ns "$name" ip link set lo up
|
|
in_ns "$name" ip addr add "10.9.0.$n/24" dev ix0
|
|
in_ns "$name" ip -6 addr add "2001:db8:9::$n/64" dev ix0 nodad
|
|
in_ns "$name" ip link set ix0 up
|
|
in_ns "$name" ip link add lan type dummy
|
|
in_ns "$name" ip addr add "10.$n.0.1/24" dev lan
|
|
in_ns "$name" ip -6 addr add "2001:db8:$n::1/48" dev lan nodad
|
|
in_ns "$name" ip link set lan up
|
|
done
|
|
}
|
|
|
|
# --- Routinator ---------------------------------------------------------------
|
|
|
|
# routinator_up [asn]: with `asn` the AS that may originate 10.66.0.0/24.
|
|
routinator_up() {
|
|
local owner=${1:-64999}
|
|
mkdir -p "$work/rpki/repository" "$work/rpki/tals"
|
|
# 10.2.0.0/24 is FRR's to originate. 10.66.0.0/24 is someone else's,
|
|
# and FRR announces it all the same. 10.3.0.0/16 is BIRD's, as far as
|
|
# /24.
|
|
cat > "$work/rpki/slurm.json" <<END
|
|
{
|
|
"slurmVersion": 1,
|
|
"validationOutputFilters": { "prefixFilters": [], "bgpsecFilters": [] },
|
|
"locallyAddedAssertions": {
|
|
"prefixAssertions": [
|
|
{ "asn": 64602, "prefix": "10.2.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": $owner, "prefix": "10.66.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": 4200000003, "prefix": "10.3.0.0/16", "maxPrefixLength": 24 },
|
|
{ "asn": 64602, "prefix": "2001:db8:2::/48", "maxPrefixLength": 48 }
|
|
],
|
|
"bgpsecAssertions": []
|
|
}
|
|
}
|
|
END
|
|
# By its full name: root's PATH does not have a user's ~/.cargo/bin.
|
|
in_ns dut "$(command -v routinator)" --no-rir-tals --extra-tals-dir "$work/rpki/tals" \
|
|
-r "$work/rpki/repository" -x "$work/rpki/slurm.json" \
|
|
--logfile "$work/routinator.log" \
|
|
server --rtr 127.0.0.1:3323 --http 127.0.0.1:8323 --refresh 3600 \
|
|
> /dev/null 2>&1 &
|
|
echo $! | sudo tee "$work/routinator.pid" > /dev/null
|
|
}
|
|
|
|
# --- the daemon under test ------------------------------------------------------
|
|
|
|
# dut_config [frr-role] [frr-max-prefixes]
|
|
dut_config() {
|
|
local frr_role=${1:-PEER} frr_limit=${2:-2000000}
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "10.9.0.1" port: 179 }
|
|
listen_endpoints { address: "2001:db8:9::1" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr4" endpoint { address: "10.9.0.2" } remote_asn: 64602
|
|
address_families: IPV4_UNICAST
|
|
role: $frr_role
|
|
import { accept_bogons: true max_prefixes: $frr_limit hold_down_seconds: 5 }
|
|
}
|
|
peers {
|
|
name: "frr6" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
|
|
address_families: IPV6_UNICAST
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
peers {
|
|
name: "bird4" endpoint { address: "10.9.0.3" } remote_asn: 4200000003
|
|
address_families: IPV4_UNICAST
|
|
role: PROVIDER require_role: true
|
|
add_path: ADD_PATH_RECEIVE
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
peers {
|
|
name: "bird6" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
|
|
address_families: IPV6_UNICAST
|
|
role: PROVIDER require_role: true
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" communities: "64601:100" }
|
|
originate { prefix: "2001:db8:1::/48" large_communities: "64601:1:2" }
|
|
kernel { stale_seconds: 3 }
|
|
rpki { caches { address: "127.0.0.1" port: 3323 } retry_seconds: 1 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
END
|
|
}
|
|
|
|
dut_up() {
|
|
sudo ip netns exec lab-dut \
|
|
setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups \
|
|
--inh-caps +net_admin,+net_bind_service --ambient-caps +net_admin,+net_bind_service \
|
|
"$bgp" "$work/dut.textpb" >> "$work/dut.log" 2>&1 &
|
|
echo $! > "$work/dut.pid"
|
|
# Not the shell's to report on when it is killed, which the tests do.
|
|
disown
|
|
wait_for "the daemon's status interface" api /api/status
|
|
}
|
|
|
|
# dut_down [signal]: stops the daemon and waits until it has gone.
|
|
dut_down() {
|
|
local signal=${1:-TERM}
|
|
# The pid recorded is sudo's; the daemon is the one in the namespace
|
|
# that is not Routinator.
|
|
local pid
|
|
for pid in $(sudo ip netns pids lab-dut); do
|
|
if [ "$(cat "/proc/$pid/comm" 2>/dev/null)" = bgp ]; then
|
|
sudo kill "-$signal" "$pid"
|
|
while [ -e "/proc/$pid" ]; do sleep 0.05; done
|
|
fi
|
|
done
|
|
}
|
|
|
|
# --- FRR ------------------------------------------------------------------------
|
|
|
|
frr_up() {
|
|
sudo mkdir -p "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor 10.9.0.1 remote-as 64601
|
|
neighbor 10.9.0.1 local-role peer
|
|
neighbor 10.9.0.1 timers connect 1
|
|
neighbor 2001:db8:9::1 remote-as 64601
|
|
neighbor 2001:db8:9::1 local-role peer
|
|
neighbor 2001:db8:9::1 timers connect 1
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.66.0.0/24
|
|
neighbor 10.9.0.1 activate
|
|
no neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
sudo touch "/etc/frr/$pathspace/vtysh.conf" "$work/frr.log"
|
|
sudo chown -R frr:frr "/etc/frr/$pathspace" "/var/run/frr/$pathspace" "$work/frr.log"
|
|
for daemon in zebra bgpd; do
|
|
# Run from a copy: the distribution's AppArmor profile for the
|
|
# daemon in its usual place does not allow it a directory of its
|
|
# own, and the machine's profiles are not the lab's to change.
|
|
sudo cp "/usr/lib/frr/$daemon" "$work/$daemon"
|
|
in_ns frr "$work/$daemon" -d -N "$pathspace" \
|
|
-f "/etc/frr/$pathspace/frr.conf" > "$work/$daemon.out" 2>&1 \
|
|
|| fail "$daemon did not start: $(cat "$work/$daemon.out")"
|
|
# zebra makes the directory its sockets are in root's.
|
|
sudo chown -R frr:frr "/var/run/frr/$pathspace"
|
|
done
|
|
}
|
|
|
|
frr_down() {
|
|
local signal=${1:-TERM} daemon
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "-$signal" "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.5
|
|
}
|
|
|
|
# --- BIRD -----------------------------------------------------------------------
|
|
|
|
# bird_config [extra-static-routes-file]
|
|
bird_config() {
|
|
local extra=${1:-}
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 {
|
|
ipv4;
|
|
route 10.3.0.0/24 blackhole;
|
|
route 10.3.7.0/24 blackhole;
|
|
${extra:+include \"$extra\";}
|
|
}
|
|
protocol static static6 {
|
|
ipv6;
|
|
route 2001:db8:3::/48 blackhole;
|
|
}
|
|
template bgp dut {
|
|
local as 4200000003;
|
|
local role provider;
|
|
connect retry time 1;
|
|
hold time 9;
|
|
graceful restart off;
|
|
}
|
|
protocol bgp dut4 from dut {
|
|
neighbor 10.9.0.1 as 64601;
|
|
source address 10.9.0.3;
|
|
ipv4 { import all; export where source = RTS_STATIC; add paths tx; };
|
|
}
|
|
protocol bgp dut6 from dut {
|
|
neighbor 2001:db8:9::1 as 64601;
|
|
source address 2001:db8:9::3;
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
bird_up() {
|
|
sudo ip netns exec lab-bird bird -c "$work/bird.conf" -s "$work/bird.ctl" -P "$work/bird.pid"
|
|
}
|
|
|
|
# --- what is looked at ------------------------------------------------------------
|
|
|
|
established() { # how many of the daemon's sessions are up
|
|
api /api/peers | jq -e --argjson n "$1" '[.[] | select(.state == "Established")] | length == $n'
|
|
}
|
|
dut_has() { # dut_has prefix peer-name: the daemon has the route, from that peer
|
|
api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer)'
|
|
}
|
|
dut_lacks() { ! api "/api/routes/received?prefix=$1" | jq -e '.routes | length > 0' >/dev/null; }
|
|
dut_kernel() { # dut_kernel family "route as ip shows it, without nhid"
|
|
# All of it is read: with a table's worth of routes, a grep that
|
|
# stopped at the first match would leave `ip` writing to nobody.
|
|
in_ns dut ip "$1" route show proto bgp | sed 's/ */ /g; s/ nhid [0-9]*//; s/ pref medium//; s/ *$//' | grep -xF "$2" >/dev/null
|
|
}
|
|
dut_kernel_lacks() { ! in_ns dut ip "$1" route show proto bgp | grep "^$2 " >/dev/null; }
|
|
frr_has() { # frr_has family prefix: FRR has a route for it by way of the daemon
|
|
vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | any(.aspath.string == "64601")'
|
|
}
|
|
frr_lacks() { ! vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | length > 0' >/dev/null; }
|
|
bird_has() { bird_ctl show route for "$1" all | grep -q "BGP.as_path: 64601"; }
|
|
|
|
lab_up() {
|
|
down
|
|
sudo rm -rf "$work"; mkdir -p "$work"; chmod 777 "$work"
|
|
[ -x "$bgp" ] || fail "$bgp not found: cargo build --release -p bgp"
|
|
network_up
|
|
routinator_up
|
|
frr_up
|
|
bird_config; bird_up
|
|
dut_config; dut_up
|
|
}
|
|
|
|
converged() {
|
|
wait_for "four sessions" established 4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.0.0/24 frr4
|
|
wait_for "BIRD's routes at the daemon" dut_has 10.3.0.0/24 bird4
|
|
wait_for "FRR's IPv6 route" dut_has 2001:db8:2::/48 frr6
|
|
wait_for "BIRD's IPv6 route" dut_has 2001:db8:3::/48 bird6
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20"
|
|
wait_for "the daemon's route at FRR" frr_has ipv4 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
wait_for "the daemon's route at BIRD" bird_has 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at BIRD" bird_has 2001:db8:1::/48
|
|
}
|
|
|
|
# --- the phases ---------------------------------------------------------------------
|
|
|
|
frr_neighbor() { # frr_neighbor address jq-filter
|
|
vty -c "show bgp neighbors $1 json" | jq -e --arg n "$1" ".[\$n] | $2"
|
|
}
|
|
frr_configure() { # each argument a line of configuration
|
|
local args=(-c "configure terminal")
|
|
for line in "$@"; do args+=(-c "$line"); done
|
|
vty "${args[@]}"
|
|
}
|
|
peer_is() { # peer_is name jq-filter: of the daemon's view of a peer
|
|
api /api/peers | jq -e --arg n "$1" ".[] | select(.name == \$n) | $2"
|
|
}
|
|
|
|
# Sessions come up, routes go both ways with their attributes intact, and
|
|
# packets follow them.
|
|
phase_basics() {
|
|
# What FRR has of ours: the path, the next hop, the communities.
|
|
local route
|
|
route=$(vty -c "show bgp ipv4 unicast 10.1.0.0/24 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and .nexthops[0].ip == "10.9.0.1"
|
|
and .community.string == "64601:100" and .origin == "IGP"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(vty -c "show bgp ipv6 unicast 2001:db8:1::/48 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and (.nexthops | any(.ip == "2001:db8:9::1"))
|
|
and .largeCommunity.string == "64601:1:2"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "FRR has our routes, with path, next hop and communities"
|
|
|
|
route=$(bird_ctl show route for 10.1.0.0/24 all)
|
|
grep -q "BGP.next_hop: 10.9.0.1" <<<"$route" && grep -q "BGP.community: (64601,100)" <<<"$route" \
|
|
|| fail "BIRD's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(bird_ctl show route for 2001:db8:1::/48 all)
|
|
grep -q "BGP.next_hop: 2001:db8:9::1" <<<"$route" && grep -q "BGP.large_community: (64601, 1, 2)" <<<"$route" \
|
|
|| fail "BIRD's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "BIRD has our routes, with path, next hop and communities"
|
|
|
|
# What we have of theirs. BIRD's AS number needs four octets, and it
|
|
# sends path identifiers (ADD-PATH).
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [4200000003] and .summary.next_hop == "10.9.0.3" and .path_id != 0' >/dev/null \
|
|
|| fail "our copy of BIRD's 10.3.0.0/24 is not as sent"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [64602] and .summary.next_hop == "10.9.0.2" and .best' >/dev/null \
|
|
|| fail "our copy of FRR's 10.2.0.0/24 is not as sent"
|
|
peer_is bird4 '.four_octet_asn and (.add_path_receive == ["IPv4 unicast"])' >/dev/null \
|
|
|| fail "the session with BIRD did not agree on four-octet AS numbers and ADD-PATH"
|
|
ok "we have their routes; four-octet AS numbers and ADD-PATH with BIRD"
|
|
|
|
# Packets, there and back by the routes BGP put in three kernels.
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no ping to BIRD's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
|
|
ok "packets get through to both, over IPv4 and IPv6"
|
|
}
|
|
|
|
# RFC 9234: each end says what it is to the other, and routes are marked
|
|
# Only to Customer where they should be.
|
|
phase_roles() {
|
|
api /api/peers | jq -e 'all(.[]; .role_confirmed)' >/dev/null \
|
|
|| fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')"
|
|
frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \
|
|
|| fail "FRR does not see us as its peer"
|
|
bird_ctl show protocols all dut4 | grep -A12 "Neighbor capabilities" | grep -q "Role: customer" \
|
|
|| fail "BIRD does not see us as its customer"
|
|
ok "roles agree with FRR (peer) and BIRD (provider)"
|
|
|
|
# To a peer our routes are marked; to a provider they are not.
|
|
vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].otc == 64601' >/dev/null \
|
|
|| fail "FRR's copy of our route is not marked Only to Customer by us"
|
|
if bird_ctl show route for 10.1.0.0/24 all | grep -qi "otc\|only_to_customer"; then
|
|
fail "our route to our provider is marked Only to Customer"
|
|
fi
|
|
# What comes from a peer or a provider is marked by them, or by us.
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 64602)' >/dev/null \
|
|
|| fail "FRR's route is not marked Only to Customer"
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 4200000003)' >/dev/null \
|
|
|| fail "BIRD's route is not marked Only to Customer"
|
|
ok "Only to Customer is where it should be, in both directions"
|
|
}
|
|
|
|
# Routinator's data over RTR: an invalid route is refused, and taken when
|
|
# the data comes to allow it.
|
|
phase_rpki() {
|
|
wait_for "the daemon to have Routinator's data" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/rpki | jq -e '.state == \"synchronised\" and .vrps_ipv4 == 3 and .vrps_ipv6 == 1'"
|
|
ok "synchronised with Routinator over RTR version $(api /api/rpki | jq .protocol_version): 3 IPv4 and 1 IPv6 VRPs"
|
|
|
|
# FRR announces 10.66.0.0/24, which is AS 64999's to announce.
|
|
wait_for "the invalid route to be refused" dut_lacks 10.66.0.0/24
|
|
wait_for "the invalid route to leave the kernel" dut_kernel_lacks -4 10.66.0.0/24
|
|
api "/api/routes/rejected?peer=frr4" | jq -e '.[0].by_reason["RPKI invalid"] == 1' >/dev/null \
|
|
|| fail "the refusal is not on record: $(api /api/routes/rejected)"
|
|
peer_is frr4 '.held_routes == 1' >/dev/null || fail "the invalid route is not held"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "FRR's own route is not valid"
|
|
api "/api/routes/received?prefix=10.3.7.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "BIRD's more specific route is not valid"
|
|
ok "FRR's invalid route is refused and held; the others are valid"
|
|
|
|
# The data changes, by way of a Routinator that is restarted: the
|
|
# route is authorised now.
|
|
sudo kill "$(cat "$work/routinator.pid")"; sleep 0.5
|
|
routinator_up 64602
|
|
wait_for "the held route to be taken" dut_has 10.66.0.0/24 frr4
|
|
wait_for "the route in the kernel" dut_kernel -4 "10.66.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
peer_is frr4 '.held_routes == 0' >/dev/null || fail "the route is still held"
|
|
ok "after Routinator restarts with new data, the held route is taken"
|
|
}
|
|
|
|
# Changes after the first exchange: new routes, withdrawals, and being
|
|
# asked to send everything again.
|
|
phase_refresh() {
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.9.0/24"
|
|
wait_for "FRR's new route" dut_has 10.2.9.0/24 frr4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.2.9.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.9.0/24"
|
|
wait_for "FRR's withdrawal" dut_lacks 10.2.9.0/24
|
|
wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.9.0/24
|
|
ok "a route FRR adds and withdraws comes and goes, in the table and the kernel"
|
|
|
|
# FRR asks for our routes again (ROUTE-REFRESH), and gets them.
|
|
local before
|
|
before=$(frr_neighbor 10.9.0.1 '.messageStats.updatesRecv')
|
|
vty -c "clear bgp ipv4 unicast 10.9.0.1 soft in"
|
|
wait_for "our routes to be sent again" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq '.[\"10.9.0.1\"].messageStats.updatesRecv')\" -gt $before ]"
|
|
frr_neighbor 10.9.0.1 '.bgpState == "Established"' >/dev/null || fail "the session did not survive a route refresh"
|
|
wait_for "our route still at FRR" frr_has ipv4 10.1.0.0/24
|
|
ok "FRR's route refresh is answered with our routes"
|
|
|
|
# BIRD sends its routes again unasked; nothing changes.
|
|
bird_ctl reload out dut4 >/dev/null
|
|
sleep 0.5
|
|
converged
|
|
ok "BIRD re-sending its routes changes nothing"
|
|
}
|
|
|
|
# Sessions that end in every way they can, and come back.
|
|
phase_flaps() {
|
|
for round in 1 2 3 4 5; do
|
|
vty -c "clear bgp *"
|
|
wait_for "FRR's routes to go (round $round)" dut_lacks 10.2.0.0/24
|
|
converged
|
|
done
|
|
ok "five resets by FRR: routes go and come back each time"
|
|
|
|
bird_ctl restart dut4 >/dev/null; bird_ctl restart dut6 >/dev/null
|
|
converged
|
|
ok "BIRD restarting its sessions"
|
|
|
|
# FRR dies without a word.
|
|
frr_down KILL
|
|
wait_for "FRR's routes to go" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
dut_has 10.3.0.0/24 bird4 >/dev/null || fail "BIRD's routes went with FRR's"
|
|
frr_up
|
|
converged
|
|
ok "FRR killed and started again"
|
|
|
|
# The wire to FRR is cut: nothing says so but the silence. The hold
|
|
# timer (nine seconds here) has to notice.
|
|
in_ns ix ip link set to-frr down
|
|
WAIT_TRIES=100 wait_for "the hold timer to end the session" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
in_ns ix ip link set to-frr up
|
|
converged
|
|
ok "a cut wire is noticed by the hold timer, and mended"
|
|
|
|
sudo kill -KILL "$(sudo cat "$work/bird.pid")"
|
|
wait_for "BIRD's routes to go" dut_lacks 10.3.0.0/24
|
|
sudo rm -f "$work/bird.ctl"
|
|
bird_up
|
|
converged
|
|
ok "BIRD killed and started again"
|
|
}
|
|
|
|
# The daemon itself stopping, in good order and not.
|
|
phase_restart() {
|
|
dut_down TERM
|
|
[ -z "$(in_ns dut ip route show proto bgp)" ] && [ -z "$(in_ns dut ip -6 route show proto bgp)" ] \
|
|
|| fail "routes left in the kernel after a clean stop: $(in_ns dut ip route show proto bgp)"
|
|
wait_for "FRR to drop our route" frr_lacks ipv4 10.1.0.0/24
|
|
frr_neighbor 10.9.0.1 '.lastNotificationReason // .lastResetDueTo | test("Cease|Notification"; "i")' >/dev/null \
|
|
|| fail "FRR was not told we were going: $(frr_neighbor 10.9.0.1 '{lastResetDueTo, lastNotificationReason}')"
|
|
dut_up
|
|
converged
|
|
ok "stopped in good order: kernel emptied, peers told; and back"
|
|
|
|
# Killed: its routes stay in the kernel. While it is dead BIRD stops
|
|
# announcing one of them; the daemon that starts next has to clear
|
|
# that one out and keep the rest.
|
|
dut_down KILL
|
|
dut_kernel -4 "10.3.7.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "the kernel's routes went with the killed daemon"
|
|
sed -i '/route 10.3.7.0\/24 blackhole;/d' "$work/bird.conf"
|
|
bird_ctl configure >/dev/null
|
|
dut_up
|
|
converged
|
|
wait_for "the stale route to be cleared" dut_kernel_lacks -4 10.3.7.0/24
|
|
ok "killed: routes stay in the kernel, and the next start clears the stale one"
|
|
}
|
|
|
|
# A full table's worth of routes from BIRD, into the table and the kernel.
|
|
phase_scale() {
|
|
# Out of 100.0.0.0/6 and up: as many /24s as asked for.
|
|
awk -v n="$scale_routes" 'BEGIN { for (i = 0; i < n; i++)
|
|
printf "route %d.%d.%d.0/24 blackhole;\n", 100 + int(i / 65536), int(i / 256) % 256, i % 256 }' \
|
|
> "$work/bird-table.conf"
|
|
bird_config "$work/bird-table.conf"
|
|
local base start
|
|
base=$(api /api/status | jq .received_prefixes)
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to arrive" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -ge $((base + scale_routes)) ]"
|
|
local arrived
|
|
arrived=$(date +%s.%N)
|
|
# By the daemon's own count first: listing a million routes with `ip`
|
|
# takes longer than installing them.
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.installed >= $scale_routes'"
|
|
local installed
|
|
installed=$(date +%s.%N)
|
|
local listed
|
|
listed=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed" -ge "$scale_routes" ] || fail "the kernel lists $listed routes, not $scale_routes"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "$scale_routes routes from BIRD: in the table after $(echo "$arrived - $start" | bc)s, in the kernel after $(echo "$installed - $start" | bc)s"
|
|
peer_is frr4 '.state == "Established"' >/dev/null || fail "the session with FRR did not survive the table"
|
|
|
|
# The daemon is killed with the table in the kernel, and started
|
|
# again: it has to find its routes there, take the table from BIRD
|
|
# anew, and end up with the same, none refused and none left over.
|
|
dut_down KILL
|
|
start=$(date +%s.%N)
|
|
dut_up
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel again" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.synced and .installed >= $scale_routes'"
|
|
installed=$(date +%s.%N)
|
|
converged
|
|
local listed_again
|
|
listed_again=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed_again" -eq "$listed" ] || fail "the kernel lists $listed_again routes after a restart; it listed $listed before"
|
|
# They were kept, not taken out and put back: next to nothing was
|
|
# removed on the way.
|
|
local removed
|
|
removed=$(api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }')
|
|
[ "${removed:-0}" -lt 100 ] || fail "$removed routes were removed from the kernel on the way"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "killed and restarted under the table: in step again after $(echo "$installed - $start" | bc)s, the kernel's routes as before"
|
|
|
|
bird_config
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to be withdrawn" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -le $((base + 1)) ]"
|
|
WAIT_TRIES=1500 wait_for "the table out of the kernel" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut ip -4 route show proto bgp | wc -l)\" -le $((base + 1)) ]"
|
|
ok "withdrawn again after $(echo "$(date +%s.%N) - $start" | bc)s"
|
|
converged
|
|
}
|
|
|
|
# What the daemon refuses: a peer that is not what it is configured as,
|
|
# and one that sends more than it may.
|
|
phase_limits() {
|
|
# FRR says it is our peer; here it is configured as our customer.
|
|
dut_down TERM
|
|
dut_config CUSTOMER
|
|
dut_up
|
|
wait_for "the sessions that should come up" established 3
|
|
wait_for "the role mismatch to be on record" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | .state != \"Established\" and (.last_error // \"\" | test(\"role mismatch\"; \"i\"))'"
|
|
sleep 1
|
|
peer_is frr4 '.state != "Established"' >/dev/null || fail "a session came up across a role mismatch"
|
|
frr_neighbor 10.9.0.1 '.bgpState != "Established"' >/dev/null || fail "FRR thinks the session is up"
|
|
ok "a peer whose role does not fit ours gets no session: $(peer_is frr4 '.last_error')"
|
|
|
|
# FRR sends at least two routes that pass; it may have one.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config PEER 1
|
|
dut_up
|
|
wait_for "the limit to end the session" \
|
|
bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq -e '.[\"10.9.0.1\"] | (.lastNotificationReason // \"\") | test(\"Maximum Number of Prefixes\"; \"i\")'"
|
|
wait_for "the peer to be held down" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | (.last_error // \"\") | test(\"held down|prefix\"; \"i\")'"
|
|
dut_lacks 10.2.0.0/24 || fail "routes of a peer over its limit are still there"
|
|
ok "a peer over its prefix limit is told so (Cease / Maximum Number of Prefixes) and held down"
|
|
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# --- main -----------------------------------------------------------------------
|
|
|
|
case "${1:-test}" in
|
|
down) down; exit 0 ;;
|
|
up) lab_up; converged; say "the lab is up; work files in $work"; exit 0 ;;
|
|
test) ;;
|
|
*) fail "usage: $0 [test|up|down]" ;;
|
|
esac
|
|
|
|
on_exit() {
|
|
status=$?
|
|
if [ "$status" -ne 0 ]; then
|
|
say "--- the daemon's last words:" >&2
|
|
tail -n 25 "$work/dut.log" >&2 || true
|
|
fi
|
|
[ -n "${KEEP:-}" ] || down
|
|
exit "$status"
|
|
}
|
|
trap on_exit EXIT
|
|
lab_up
|
|
say "lab: converging"
|
|
converged
|
|
for phase in $phases; do
|
|
say "phase: $phase"
|
|
"phase_$phase"
|
|
done
|
|
say "all phases passed: $phases"
|