Rayhaan JaufeerallyandClaude Fable 5.1
d7d7d39bb9
The kernel's view of a session's TCP connection, which the status page shows on request since the last change, could not be looked at after the fact: a feed that crawled overnight left no record of whether the peer's window or our own side held it up. This change samples it at every scrape of `/metricz`: for each established session, one `NETLINK_SOCK_DIAG` round trip, and twenty gauges labelled by peer, `bgp_peer_tcp_*`: smoothed, least and variance of the round-trip time; the congestion window and segment size; the peer's receive window and ours; the send queue, the part of it not yet sent, the receive queue and the segments in flight; retransmitted and lost segments; the busy time and the time held up by the peer's window and by our send buffer, cumulative, so that their rates give the share of time a session is held up; octets sent and received; and the delivery and pacing rates. A session the kernel has no connection for loses its series, as does one that ends. `SessionHandle::metric_label` is the one place a peer's metric label is made (its name, or its address), used by the advertiser's gauge and here. Documentation: the metrics reference lists them; the monitoring guide gives the query for the share of time a session is held up by the neighbor's receive window, which is the question the overnight feed raised. Testing: a daemon in the Linux VM with one session shows the series and drops them when the session ends; the lab's basics phase checks the scrape for the FRR session's round-trip time; the workspace tests pass. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1321 lines
62 KiB
Bash
Executable File
1321 lines
62 KiB
Bash
Executable File
#!/bin/bash
|
|
# An interoperability lab: this daemon against FRR, BIRD and Routinator,
|
|
# each in a network namespace of its own on one exchange LAN.
|
|
#
|
|
# 10.9.0.0/24, 2001:db8:9::/64
|
|
# ┌───────────────────┬──────────────────────┐
|
|
# dut (.1) AS 64601 frr (.2) AS 64602 bird (.3) AS 4200000003
|
|
# lan 10.1.0.1/24 lan 10.2.0.1/24 lan 10.3.0.1/24
|
|
# 2001:db8:1::1/48 2001:db8:2::1/48 2001:db8:3::1/48
|
|
# + Routinator on 127.0.0.1:3323
|
|
#
|
|
# "dut" is the daemon under test. FRR is its peer, BIRD its provider; both
|
|
# say so in their OPEN (RFC 9234). Routinator serves validated ROA payloads
|
|
# made up for the lab from a local exceptions file, so nothing is fetched
|
|
# from the Internet.
|
|
#
|
|
# scripts/interop/lab.sh run every phase, then take the lab down
|
|
# scripts/interop/lab.sh up bring the lab up and leave it up
|
|
# scripts/interop/lab.sh down take it down
|
|
# PHASES="basics scale" scripts/interop/lab.sh only these phases
|
|
# SCALE_ROUTES=1000000 ... the size of the table BIRD sends
|
|
#
|
|
# Needs Linux, sudo, and: frr, bird2, radvd, jq, routinator, a built `bgp`
|
|
# (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS;
|
|
# scripts/interop/install.sh installs what is missing on Ubuntu.
|
|
set -euo pipefail
|
|
|
|
cd "$(dirname "$0")/../.."
|
|
bgp="${CARGO_TARGET_DIR:-target}/release/bgp"
|
|
work=/tmp/ferrix-lab
|
|
pathspace=ferrixlab
|
|
all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit collector leaks v6only linklocal"
|
|
phases="${PHASES:-$all_phases}"
|
|
scale_routes="${SCALE_ROUTES:-200000}"
|
|
|
|
# --- plumbing ---------------------------------------------------------------
|
|
|
|
in_ns() { local name=$1; shift; sudo ip netns exec "lab-$name" "$@"; }
|
|
api() { in_ns dut curl -sf "http://127.0.0.1:8080$1"; }
|
|
vty() { in_ns frr vtysh -N "$pathspace" "$@" 2>/dev/null; }
|
|
bird_ctl() { sudo birdc -s "$work/bird.ctl" "$@"; }
|
|
say() { printf '%s\n' "$*"; }
|
|
ok() { say " ok: $*"; }
|
|
fail() { say "FAILED: $*" >&2; exit 1; }
|
|
|
|
# wait_for "what" command...: until the command succeeds.
|
|
wait_for() {
|
|
local what=$1; shift
|
|
local tries=${WAIT_TRIES:-150}
|
|
for _ in $(seq "$tries"); do
|
|
if "$@" >/dev/null 2>&1; then return 0; fi
|
|
sleep 0.2
|
|
done
|
|
say "timed out waiting for: $what" >&2
|
|
return 1
|
|
}
|
|
|
|
down() {
|
|
for pidfile in "$work"/dut.pid "$work"/routinator.pid; do
|
|
[ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
[ -f "$work/radvd.pid" ] && sudo kill "$(sudo cat "$work/radvd.pid")" 2>/dev/null || true
|
|
[ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.3
|
|
for name in dut frr bird ix; do sudo ip netns del "lab-$name" 2>/dev/null || true; done
|
|
sudo rm -rf "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
}
|
|
|
|
# --- the network ------------------------------------------------------------
|
|
|
|
network_up() {
|
|
sudo modprobe dummy 2>/dev/null || true
|
|
sudo ip netns add lab-ix
|
|
in_ns ix ip link add ix type bridge
|
|
in_ns ix ip link set ix up
|
|
local n=0
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
sudo ip netns add "lab-$name"
|
|
sudo ip link add ix0 netns "lab-$name" type veth peer name "to-$name" netns lab-ix
|
|
in_ns ix ip link set "to-$name" master ix up
|
|
in_ns "$name" ip link set lo up
|
|
in_ns "$name" ip addr add "10.9.0.$n/24" dev ix0
|
|
in_ns "$name" ip -6 addr add "2001:db8:9::$n/64" dev ix0 nodad
|
|
in_ns "$name" ip link set ix0 up
|
|
in_ns "$name" ip link add lan type dummy
|
|
in_ns "$name" ip addr add "10.$n.0.1/24" dev lan
|
|
in_ns "$name" ip -6 addr add "2001:db8:$n::1/48" dev lan nodad
|
|
in_ns "$name" ip link set lan up
|
|
done
|
|
}
|
|
|
|
# --- Routinator ---------------------------------------------------------------
|
|
|
|
# routinator_up [asn]: with `asn` the AS that may originate 10.66.0.0/24.
|
|
routinator_up() {
|
|
local owner=${1:-64999}
|
|
mkdir -p "$work/rpki/repository" "$work/rpki/tals"
|
|
# 10.2.0.0/24 is FRR's to originate. 10.66.0.0/24 is someone else's,
|
|
# and FRR announces it all the same. 10.3.0.0/16 is BIRD's, as far as
|
|
# /24.
|
|
cat > "$work/rpki/slurm.json" <<END
|
|
{
|
|
"slurmVersion": 1,
|
|
"validationOutputFilters": { "prefixFilters": [], "bgpsecFilters": [] },
|
|
"locallyAddedAssertions": {
|
|
"prefixAssertions": [
|
|
{ "asn": 64602, "prefix": "10.2.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": $owner, "prefix": "10.66.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": 4200000003, "prefix": "10.3.0.0/16", "maxPrefixLength": 24 },
|
|
{ "asn": 64602, "prefix": "2001:db8:2::/48", "maxPrefixLength": 48 }
|
|
],
|
|
"bgpsecAssertions": []
|
|
}
|
|
}
|
|
END
|
|
# By its full name: root's PATH does not have a user's ~/.cargo/bin.
|
|
in_ns dut "$(command -v routinator)" --no-rir-tals --extra-tals-dir "$work/rpki/tals" \
|
|
-r "$work/rpki/repository" -x "$work/rpki/slurm.json" \
|
|
--logfile "$work/routinator.log" \
|
|
server --rtr 127.0.0.1:3323 --http 127.0.0.1:8323 --refresh 3600 \
|
|
> /dev/null 2>&1 &
|
|
echo $! | sudo tee "$work/routinator.pid" > /dev/null
|
|
}
|
|
|
|
# --- the daemon under test ------------------------------------------------------
|
|
|
|
# dut_config [frr-role] [frr-max-prefixes] [more of frr4's import policy]
|
|
# [more of frr4] [more of bird4] [more of the configuration]
|
|
dut_config() {
|
|
local frr_role=${1:-PEER} frr_limit=${2:-2000000} frr_import=${3:-}
|
|
local frr_more=${4:-} bird_more=${5:-} more=${6:-}
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "10.9.0.1" port: 179 }
|
|
listen_endpoints { address: "2001:db8:9::1" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr4" endpoint { address: "10.9.0.2" } remote_asn: 64602
|
|
address_families: IPV4_UNICAST
|
|
role: $frr_role
|
|
import { accept_bogons: true max_prefixes: $frr_limit hold_down_seconds: 5 $frr_import }
|
|
$frr_more
|
|
}
|
|
peers {
|
|
name: "frr6" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
|
|
address_families: IPV6_UNICAST
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
peers {
|
|
name: "bird4" endpoint { address: "10.9.0.3" } remote_asn: 4200000003
|
|
address_families: IPV4_UNICAST
|
|
role: PROVIDER require_role: true
|
|
add_path: ADD_PATH_RECEIVE
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
$bird_more
|
|
}
|
|
peers {
|
|
name: "bird6" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
|
|
address_families: IPV6_UNICAST
|
|
role: PROVIDER require_role: true
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" communities: "64601:100" }
|
|
originate { prefix: "2001:db8:1::/48" large_communities: "64601:1:2" }
|
|
kernel { stale_seconds: 3 }
|
|
rpki { caches { address: "127.0.0.1" port: 3323 } retry_seconds: 1 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
$more
|
|
END
|
|
}
|
|
|
|
dut_config_v6only() {
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "2001:db8:9::1" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
peers {
|
|
name: "bird" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
|
|
role: PROVIDER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" }
|
|
originate { prefix: "2001:db8:1::/48" }
|
|
kernel { stale_seconds: 3 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
END
|
|
}
|
|
|
|
dut_up() {
|
|
sudo ip netns exec lab-dut \
|
|
setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups \
|
|
--inh-caps +net_admin,+net_bind_service --ambient-caps +net_admin,+net_bind_service \
|
|
"$bgp" "$work/dut.textpb" >> "$work/dut.log" 2>&1 &
|
|
echo $! > "$work/dut.pid"
|
|
# Not the shell's to report on when it is killed, which the tests do.
|
|
disown
|
|
wait_for "the daemon's status interface" api /api/status
|
|
}
|
|
|
|
# dut_down [signal]: stops the daemon and waits until it has gone.
|
|
dut_down() {
|
|
local signal=${1:-TERM}
|
|
# The pid recorded is sudo's; the daemon is the one in the namespace
|
|
# that is not Routinator.
|
|
local pid
|
|
for pid in $(sudo ip netns pids lab-dut); do
|
|
if [ "$(cat "/proc/$pid/comm" 2>/dev/null)" = bgp ]; then
|
|
sudo kill "-$signal" "$pid"
|
|
while [ -e "/proc/$pid" ]; do sleep 0.05; done
|
|
fi
|
|
done
|
|
}
|
|
|
|
# --- FRR ------------------------------------------------------------------------
|
|
|
|
frr_config() {
|
|
sudo mkdir -p "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor 10.9.0.1 remote-as 64601
|
|
neighbor 10.9.0.1 local-role peer
|
|
neighbor 10.9.0.1 timers connect 1
|
|
neighbor 2001:db8:9::1 remote-as 64601
|
|
neighbor 2001:db8:9::1 local-role peer
|
|
neighbor 2001:db8:9::1 timers connect 1
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.66.0.0/24
|
|
neighbor 10.9.0.1 activate
|
|
no neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
# One session, over IPv6, for both families: IPv4 routes go with IPv6 next
|
|
# hops (RFC 8950).
|
|
frr_config_v6only() {
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor 2001:db8:9::1 remote-as 64601
|
|
neighbor 2001:db8:9::1 local-role peer
|
|
neighbor 2001:db8:9::1 timers connect 1
|
|
neighbor 2001:db8:9::1 capability extended-nexthop
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.2.5.0/24
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
frr_up() {
|
|
sudo touch "/etc/frr/$pathspace/vtysh.conf" "$work/frr.log"
|
|
sudo chown -R frr:frr "/etc/frr/$pathspace" "/var/run/frr/$pathspace" "$work/frr.log"
|
|
for daemon in zebra bgpd; do
|
|
# Run from a copy: the distribution's AppArmor profile for the
|
|
# daemon in its usual place does not allow it a directory of its
|
|
# own, and the machine's profiles are not the lab's to change.
|
|
sudo cp "/usr/lib/frr/$daemon" "$work/$daemon"
|
|
in_ns frr "$work/$daemon" -d -N "$pathspace" \
|
|
-f "/etc/frr/$pathspace/frr.conf" > "$work/$daemon.out" 2>&1 \
|
|
|| fail "$daemon did not start: $(cat "$work/$daemon.out")"
|
|
# zebra makes the directory its sockets are in root's.
|
|
sudo chown -R frr:frr "/var/run/frr/$pathspace"
|
|
done
|
|
}
|
|
|
|
frr_down() {
|
|
local signal=${1:-TERM} daemon
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "-$signal" "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.5
|
|
}
|
|
|
|
# --- BIRD -----------------------------------------------------------------------
|
|
|
|
# bird_config [extra-static-routes-file] [more-configuration]
|
|
bird_config() {
|
|
local extra=${1:-} more=${2:-}
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 {
|
|
ipv4;
|
|
route 10.3.0.0/24 blackhole;
|
|
route 10.3.7.0/24 blackhole;
|
|
${extra:+include \"$extra\";}
|
|
}
|
|
protocol static static6 {
|
|
ipv6;
|
|
route 2001:db8:3::/48 blackhole;
|
|
}
|
|
template bgp dut {
|
|
local as 4200000003;
|
|
local role provider;
|
|
connect retry time 1;
|
|
hold time 9;
|
|
graceful restart off;
|
|
}
|
|
protocol bgp dut4 from dut {
|
|
neighbor 10.9.0.1 as 64601;
|
|
source address 10.9.0.3;
|
|
ipv4 { import all; export where source = RTS_STATIC; add paths tx; };
|
|
}
|
|
protocol bgp dut6 from dut {
|
|
neighbor 2001:db8:9::1 as 64601;
|
|
source address 2001:db8:9::3;
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
$more
|
|
END
|
|
}
|
|
|
|
bird_config_v6only() {
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
|
|
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
|
|
protocol bgp dut {
|
|
local as 4200000003;
|
|
neighbor 2001:db8:9::1 as 64601;
|
|
source address 2001:db8:9::3;
|
|
local role provider;
|
|
connect retry time 1;
|
|
hold time 9;
|
|
graceful restart off;
|
|
ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
bird_up() {
|
|
sudo ip netns exec lab-bird bird -c "$work/bird.conf" -s "$work/bird.ctl" -P "$work/bird.pid"
|
|
}
|
|
|
|
# --- what is looked at ------------------------------------------------------------
|
|
|
|
established() { # how many of the daemon's sessions are up
|
|
api /api/peers | jq -e --argjson n "$1" '[.[] | select(.state == "Established")] | length == $n'
|
|
}
|
|
dut_has() { # dut_has prefix peer-name: the daemon has the route, from that peer
|
|
api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer)'
|
|
}
|
|
dut_lacks() { ! api "/api/routes/received?prefix=$1" | jq -e '.routes | length > 0' >/dev/null; }
|
|
dut_kernel() { # dut_kernel family "route as ip shows it, without nhid"
|
|
# All of it is read: with a table's worth of routes, a grep that
|
|
# stopped at the first match would leave `ip` writing to nobody.
|
|
in_ns dut ip "$1" route show proto bgp | sed 's/ */ /g; s/ nhid [0-9]*//; s/ pref medium//; s/ *$//' | grep -xF "$2" >/dev/null
|
|
}
|
|
dut_kernel_lacks() { ! in_ns dut ip "$1" route show proto bgp | grep "^$2 " >/dev/null; }
|
|
frr_has() { # frr_has family prefix: FRR has a route for it by way of the daemon
|
|
vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | any(.aspath.string == "64601")'
|
|
}
|
|
frr_lacks() { ! vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | length > 0' >/dev/null; }
|
|
bird_has() { bird_ctl show route for "$1" all | grep -q "BGP.as_path: 64601"; }
|
|
|
|
lab_up() {
|
|
down
|
|
sudo rm -rf "$work"; mkdir -p "$work"; chmod 777 "$work"
|
|
[ -x "$bgp" ] || fail "$bgp not found: cargo build --release -p bgp"
|
|
network_up
|
|
routinator_up
|
|
frr_config; frr_up
|
|
bird_config; bird_up
|
|
dut_config; dut_up
|
|
}
|
|
|
|
converged() {
|
|
wait_for "four sessions" established 4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.0.0/24 frr4
|
|
wait_for "BIRD's routes at the daemon" dut_has 10.3.0.0/24 bird4
|
|
wait_for "FRR's IPv6 route" dut_has 2001:db8:2::/48 frr6
|
|
wait_for "BIRD's IPv6 route" dut_has 2001:db8:3::/48 bird6
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20"
|
|
wait_for "the daemon's route at FRR" frr_has ipv4 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
wait_for "the daemon's route at BIRD" bird_has 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at BIRD" bird_has 2001:db8:1::/48
|
|
}
|
|
|
|
# --- the phases ---------------------------------------------------------------------
|
|
|
|
frr_neighbor() { # frr_neighbor address jq-filter
|
|
vty -c "show bgp neighbors $1 json" | jq -e --arg n "$1" ".[\$n] | $2"
|
|
}
|
|
frr_configure() { # each argument a line of configuration
|
|
local args=(-c "configure terminal")
|
|
for line in "$@"; do args+=(-c "$line"); done
|
|
vty "${args[@]}"
|
|
}
|
|
peer_is() { # peer_is name jq-filter: of the daemon's view of a peer
|
|
api /api/peers | jq -e --arg n "$1" ".[] | select(.name == \$n) | $2"
|
|
}
|
|
|
|
# Sessions come up, routes go both ways with their attributes intact, and
|
|
# packets follow them.
|
|
phase_basics() {
|
|
# What FRR has of ours: the path, the next hop, the communities.
|
|
local route
|
|
route=$(vty -c "show bgp ipv4 unicast 10.1.0.0/24 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and .nexthops[0].ip == "10.9.0.1"
|
|
and .community.string == "64601:100" and .origin == "IGP"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(vty -c "show bgp ipv6 unicast 2001:db8:1::/48 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and (.nexthops | any(.ip == "2001:db8:9::1"))
|
|
and .largeCommunity.string == "64601:1:2"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "FRR has our routes, with path, next hop and communities"
|
|
|
|
route=$(bird_ctl show route for 10.1.0.0/24 all)
|
|
grep -q "BGP.next_hop: 10.9.0.1" <<<"$route" && grep -q "BGP.community: (64601,100)" <<<"$route" \
|
|
|| fail "BIRD's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(bird_ctl show route for 2001:db8:1::/48 all)
|
|
grep -q "BGP.next_hop: 2001:db8:9::1" <<<"$route" && grep -q "BGP.large_community: (64601, 1, 2)" <<<"$route" \
|
|
|| fail "BIRD's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "BIRD has our routes, with path, next hop and communities"
|
|
|
|
# What we have of theirs. BIRD's AS number needs four octets, and it
|
|
# sends path identifiers (ADD-PATH).
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [4200000003] and .summary.next_hop == "10.9.0.3" and .path_id != 0' >/dev/null \
|
|
|| fail "our copy of BIRD's 10.3.0.0/24 is not as sent"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [64602] and .summary.next_hop == "10.9.0.2" and .best' >/dev/null \
|
|
|| fail "our copy of FRR's 10.2.0.0/24 is not as sent"
|
|
peer_is bird4 '.four_octet_asn and (.add_path_receive == ["IPv4 unicast"])' >/dev/null \
|
|
|| fail "the session with BIRD did not agree on four-octet AS numbers and ADD-PATH"
|
|
ok "we have their routes; four-octet AS numbers and ADD-PATH with BIRD"
|
|
|
|
# Packets, there and back by the routes BGP put in three kernels.
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no ping to BIRD's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
|
|
ok "packets get through to both, over IPv4 and IPv6"
|
|
|
|
# The kernel's view of a session's connection, as the page asks for it.
|
|
frr4_addr=$(api /api/peers | jq -r '.[] | select(.name == "frr4") | .peer_addr')
|
|
api "/api/tcp?peer=$frr4_addr" | jq -e '.available and .connection.state == "established" and .connection.mss > 0' >/dev/null \
|
|
|| fail "no TCP details for frr4: $(api "/api/tcp?peer=$frr4_addr")"
|
|
api /metricz | grep -q "^bgp_peer_tcp_rtt_seconds{peer=\"frr4\"}" || fail "no TCP metrics for frr4"
|
|
ok "the kernel tells of a session's TCP connection"
|
|
}
|
|
|
|
# RFC 9234: each end says what it is to the other, and routes are marked
|
|
# Only to Customer where they should be.
|
|
phase_roles() {
|
|
api /api/peers | jq -e 'all(.[]; .role_confirmed)' >/dev/null \
|
|
|| fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')"
|
|
frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \
|
|
|| fail "FRR does not see us as its peer"
|
|
bird_ctl show protocols all dut4 | grep -A40 "Neighbor capabilities" | grep -q "Role: customer" \
|
|
|| fail "BIRD does not see us as its customer"
|
|
ok "roles agree with FRR (peer) and BIRD (provider)"
|
|
|
|
# To a peer our routes are marked; to a provider they are not.
|
|
vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].otc == 64601' >/dev/null \
|
|
|| fail "FRR's copy of our route is not marked Only to Customer by us"
|
|
if bird_ctl show route for 10.1.0.0/24 all | grep -qi "otc\|only_to_customer"; then
|
|
fail "our route to our provider is marked Only to Customer"
|
|
fi
|
|
# What comes from a peer or a provider is marked by them, or by us.
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 64602)' >/dev/null \
|
|
|| fail "FRR's route is not marked Only to Customer"
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 4200000003)' >/dev/null \
|
|
|| fail "BIRD's route is not marked Only to Customer"
|
|
ok "Only to Customer is where it should be, in both directions"
|
|
}
|
|
|
|
# Routinator's data over RTR: an invalid route is refused, and taken when
|
|
# the data comes to allow it.
|
|
phase_rpki() {
|
|
wait_for "the daemon to have Routinator's data" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/rpki | jq -e '.state == \"synchronised\" and .vrps_ipv4 == 3 and .vrps_ipv6 == 1'"
|
|
ok "synchronised with Routinator over RTR version $(api /api/rpki | jq .protocol_version): 3 IPv4 and 1 IPv6 VRPs"
|
|
|
|
# FRR announces 10.66.0.0/24, which is AS 64999's to announce.
|
|
wait_for "the invalid route to be refused" dut_lacks 10.66.0.0/24
|
|
wait_for "the invalid route to leave the kernel" dut_kernel_lacks -4 10.66.0.0/24
|
|
api "/api/routes/rejected?peer=frr4" | jq -e '.[0].by_reason["RPKI invalid"] == 1' >/dev/null \
|
|
|| fail "the refusal is not on record: $(api /api/routes/rejected)"
|
|
peer_is frr4 '.held_routes == 1' >/dev/null || fail "the invalid route is not held"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "FRR's own route is not valid"
|
|
api "/api/routes/received?prefix=10.3.7.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "BIRD's more specific route is not valid"
|
|
ok "FRR's invalid route is refused and held; the others are valid"
|
|
|
|
# The data changes, by way of a Routinator that is restarted: the
|
|
# route is authorised now.
|
|
sudo kill "$(cat "$work/routinator.pid")"; sleep 0.5
|
|
routinator_up 64602
|
|
wait_for "the held route to be taken" dut_has 10.66.0.0/24 frr4
|
|
wait_for "the route in the kernel" dut_kernel -4 "10.66.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
peer_is frr4 '.held_routes == 0' >/dev/null || fail "the route is still held"
|
|
ok "after Routinator restarts with new data, the held route is taken"
|
|
}
|
|
|
|
# Changes after the first exchange: new routes, withdrawals, and being
|
|
# asked to send everything again.
|
|
phase_refresh() {
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.9.0/24"
|
|
wait_for "FRR's new route" dut_has 10.2.9.0/24 frr4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.2.9.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.9.0/24"
|
|
wait_for "FRR's withdrawal" dut_lacks 10.2.9.0/24
|
|
wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.9.0/24
|
|
ok "a route FRR adds and withdraws comes and goes, in the table and the kernel"
|
|
|
|
# FRR asks for our routes again (ROUTE-REFRESH), and gets them.
|
|
local before
|
|
before=$(frr_neighbor 10.9.0.1 '.messageStats.updatesRecv')
|
|
vty -c "clear bgp ipv4 unicast 10.9.0.1 soft in"
|
|
wait_for "our routes to be sent again" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq '.[\"10.9.0.1\"].messageStats.updatesRecv')\" -gt $before ]"
|
|
frr_neighbor 10.9.0.1 '.bgpState == "Established"' >/dev/null || fail "the session did not survive a route refresh"
|
|
wait_for "our route still at FRR" frr_has ipv4 10.1.0.0/24
|
|
ok "FRR's route refresh is answered with our routes"
|
|
|
|
# BIRD sends its routes again unasked; nothing changes.
|
|
bird_ctl reload out dut4 >/dev/null
|
|
sleep 0.5
|
|
converged
|
|
ok "BIRD re-sending its routes changes nothing"
|
|
}
|
|
|
|
# Sessions that end in every way they can, and come back.
|
|
phase_flaps() {
|
|
for round in 1 2 3 4 5; do
|
|
vty -c "clear bgp *"
|
|
wait_for "FRR's routes to go (round $round)" dut_lacks 10.2.0.0/24
|
|
converged
|
|
done
|
|
ok "five resets by FRR: routes go and come back each time"
|
|
|
|
bird_ctl restart dut4 >/dev/null; bird_ctl restart dut6 >/dev/null
|
|
converged
|
|
ok "BIRD restarting its sessions"
|
|
|
|
# FRR dies without a word.
|
|
frr_down KILL
|
|
wait_for "FRR's routes to go" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
dut_has 10.3.0.0/24 bird4 >/dev/null || fail "BIRD's routes went with FRR's"
|
|
frr_up
|
|
converged
|
|
ok "FRR killed and started again"
|
|
|
|
# The wire to FRR is cut: nothing says so but the silence. The hold
|
|
# timer (nine seconds here) has to notice.
|
|
in_ns ix ip link set to-frr down
|
|
WAIT_TRIES=100 wait_for "the hold timer to end the session" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
in_ns ix ip link set to-frr up
|
|
converged
|
|
ok "a cut wire is noticed by the hold timer, and mended"
|
|
|
|
sudo kill -KILL "$(sudo cat "$work/bird.pid")"
|
|
wait_for "BIRD's routes to go" dut_lacks 10.3.0.0/24
|
|
sudo rm -f "$work/bird.ctl"
|
|
bird_up
|
|
converged
|
|
ok "BIRD killed and started again"
|
|
}
|
|
|
|
# The daemon itself stopping, in good order and not.
|
|
phase_restart() {
|
|
dut_down TERM
|
|
[ -z "$(in_ns dut ip route show proto bgp)" ] && [ -z "$(in_ns dut ip -6 route show proto bgp)" ] \
|
|
|| fail "routes left in the kernel after a clean stop: $(in_ns dut ip route show proto bgp)"
|
|
wait_for "FRR to drop our route" frr_lacks ipv4 10.1.0.0/24
|
|
frr_neighbor 10.9.0.1 '.lastNotificationReason // .lastResetDueTo | test("Cease|Notification"; "i")' >/dev/null \
|
|
|| fail "FRR was not told we were going: $(frr_neighbor 10.9.0.1 '{lastResetDueTo, lastNotificationReason}')"
|
|
dut_up
|
|
converged
|
|
ok "stopped in good order: kernel emptied, peers told; and back"
|
|
|
|
# Killed: its routes stay in the kernel. While it is dead BIRD stops
|
|
# announcing one of them; the daemon that starts next has to clear
|
|
# that one out and keep the rest.
|
|
dut_down KILL
|
|
dut_kernel -4 "10.3.7.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "the kernel's routes went with the killed daemon"
|
|
sed -i '/route 10.3.7.0\/24 blackhole;/d' "$work/bird.conf"
|
|
bird_ctl configure >/dev/null
|
|
dut_up
|
|
converged
|
|
wait_for "the stale route to be cleared" dut_kernel_lacks -4 10.3.7.0/24
|
|
ok "killed: routes stay in the kernel, and the next start clears the stale one"
|
|
}
|
|
|
|
# Graceful restart (RFC 4724): the daemon restarts for an upgrade without
|
|
# its routes going anywhere, and keeps BIRD's while BIRD restarts.
|
|
frr_stale() { # frr_stale prefix: FRR keeps our route, as stale
|
|
vty -c "show bgp ipv4 unicast $1 json" | jq -e '.paths | any(.aspath.string == "64601" and .stale == true)'
|
|
}
|
|
frr_fresh() { frr_has ipv4 "$1" && ! frr_stale "$1"; }
|
|
dut_stale() { # dut_stale prefix peer-name: the daemon keeps the peer's route, as stale
|
|
api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer and .stale)'
|
|
}
|
|
kernel_removals() {
|
|
api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }'
|
|
}
|
|
phase_graceful() {
|
|
sed -i 's/graceful restart off;/graceful restart on;/' "$work/bird.conf"
|
|
bird_ctl configure >/dev/null
|
|
wait_for "BIRD to come back with graceful restart" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '[.[] | select(.name == \"bird4\" or .name == \"bird6\") | select(.graceful_restart != null)] | length == 2'"
|
|
converged
|
|
|
|
# The daemon restarts: everything stays where it is meanwhile.
|
|
dut_down USR1
|
|
grep -q "Leaving to be started again" "$work/dut.log" || fail "the daemon did not leave for a restart"
|
|
dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20" || fail "FRR's route left the kernel"
|
|
dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
|
|
dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20" || fail "BIRD's IPv6 route left the kernel"
|
|
wait_for "FRR to keep our route as stale" frr_stale 10.1.0.0/24
|
|
bird_has 10.1.0.0/24 || fail "BIRD dropped our route"
|
|
bird_has 2001:db8:1::/48 || fail "BIRD dropped our IPv6 route"
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "packets stopped while the daemon was away"
|
|
dut_up
|
|
converged
|
|
wait_for "FRR to take our route afresh" frr_fresh 10.1.0.0/24
|
|
wait_for "nothing kept in the kernel any more" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.kept == 0'"
|
|
local removed
|
|
removed=$(kernel_removals)
|
|
[ "${removed:-0}" -eq 0 ] || fail "$removed routes were taken out of the kernel over the restart"
|
|
ok "restarted on SIGUSR1: kernel routes kept, FRR and BIRD kept ours, nothing withdrawn"
|
|
|
|
# BIRD restarts: the daemon keeps its routes meanwhile, in the table
|
|
# and in the kernel, and takes them afresh when it is back.
|
|
bird_ctl graceful restart >/dev/null || true
|
|
while [ -e "/proc/$(sudo cat "$work/bird.pid" 2>/dev/null || echo 0)" ]; do sleep 0.05; done
|
|
wait_for "the daemon to keep BIRD's route as stale" dut_stale 10.3.0.0/24 bird4
|
|
dut_stale 2001:db8:3::/48 bird6 >/dev/null || fail "BIRD's IPv6 route was not kept"
|
|
dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
|
|
peer_is bird4 '.stale_routes > 0' >/dev/null || fail "the daemon does not say it keeps BIRD's routes: $(peer_is bird4 .)"
|
|
sudo rm -f "$work/bird.ctl"
|
|
bird_up
|
|
converged
|
|
wait_for "BIRD's routes to be fresh" \
|
|
bash -c "! sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?peer=bird4' | jq -e '.routes | any(.stale)' >/dev/null"
|
|
ok "BIRD restarted gracefully: its routes were kept, and taken afresh"
|
|
|
|
sed -i 's/graceful restart on;/graceful restart off;/' "$work/bird.conf"
|
|
bird_ctl configure >/dev/null
|
|
converged
|
|
}
|
|
|
|
# A full table's worth of routes from BIRD, into the table and the kernel.
|
|
phase_scale() {
|
|
# Out of 100.0.0.0/6 and up: as many /24s as asked for.
|
|
awk -v n="$scale_routes" 'BEGIN { for (i = 0; i < n; i++)
|
|
printf "route %d.%d.%d.0/24 blackhole;\n", 100 + int(i / 65536), int(i / 256) % 256, i % 256 }' \
|
|
> "$work/bird-table.conf"
|
|
bird_config "$work/bird-table.conf"
|
|
local base start
|
|
base=$(api /api/status | jq .received_prefixes)
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to arrive" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -ge $((base + scale_routes)) ]"
|
|
local arrived
|
|
arrived=$(date +%s.%N)
|
|
# By the daemon's own count first: listing a million routes with `ip`
|
|
# takes longer than installing them.
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.installed >= $scale_routes'"
|
|
local installed
|
|
installed=$(date +%s.%N)
|
|
local listed
|
|
listed=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed" -ge "$scale_routes" ] || fail "the kernel lists $listed routes, not $scale_routes"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "$scale_routes routes from BIRD: in the table after $(echo "$arrived - $start" | bc)s, in the kernel after $(echo "$installed - $start" | bc)s"
|
|
peer_is frr4 '.state == "Established"' >/dev/null || fail "the session with FRR did not survive the table"
|
|
|
|
# The daemon is killed with the table in the kernel, and started
|
|
# again: it has to find its routes there, take the table from BIRD
|
|
# anew, and end up with the same, none refused and none left over.
|
|
dut_down KILL
|
|
start=$(date +%s.%N)
|
|
dut_up
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel again" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.synced and .installed >= $scale_routes'"
|
|
installed=$(date +%s.%N)
|
|
converged
|
|
local listed_again
|
|
listed_again=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed_again" -eq "$listed" ] || fail "the kernel lists $listed_again routes after a restart; it listed $listed before"
|
|
# They were kept, not taken out and put back: next to nothing was
|
|
# removed on the way.
|
|
local removed
|
|
removed=$(api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }')
|
|
[ "${removed:-0}" -lt 100 ] || fail "$removed routes were removed from the kernel on the way"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "killed and restarted under the table: in step again after $(echo "$installed - $start" | bc)s, the kernel's routes as before"
|
|
|
|
bird_config
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to be withdrawn" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -le $((base + 1)) ]"
|
|
WAIT_TRIES=1500 wait_for "the table out of the kernel" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut ip -4 route show proto bgp | wc -l)\" -le $((base + 1)) ]"
|
|
ok "withdrawn again after $(echo "$(date +%s.%N) - $start" | bc)s"
|
|
converged
|
|
}
|
|
|
|
# What the daemon refuses: a peer that is not what it is configured as,
|
|
# and one that sends more than it may.
|
|
phase_limits() {
|
|
# FRR says it is our peer; here it is configured as our customer.
|
|
dut_down TERM
|
|
dut_config CUSTOMER
|
|
dut_up
|
|
wait_for "the sessions that should come up" established 3
|
|
wait_for "the role mismatch to be on record" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | .state != \"Established\" and (.last_error // \"\" | test(\"role mismatch\"; \"i\"))'"
|
|
sleep 1
|
|
peer_is frr4 '.state != "Established"' >/dev/null || fail "a session came up across a role mismatch"
|
|
frr_neighbor 10.9.0.1 '.bgpState != "Established"' >/dev/null || fail "FRR thinks the session is up"
|
|
ok "a peer whose role does not fit ours gets no session: $(peer_is frr4 '.last_error')"
|
|
|
|
# FRR sends at least two routes that pass; it may have one.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config PEER 1
|
|
dut_up
|
|
wait_for "the limit to end the session" \
|
|
bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq -e '.[\"10.9.0.1\"] | (.lastNotificationReason // \"\") | test(\"Maximum Number of Prefixes\"; \"i\")'"
|
|
wait_for "the peer to be held down" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | (.last_error // \"\") | test(\"held down|prefix\"; \"i\")'"
|
|
dut_lacks 10.2.0.0/24 || fail "routes of a peer over its limit are still there"
|
|
ok "a peer over its prefix limit is told so (Cease / Maximum Number of Prefixes) and held down"
|
|
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# FRR tries what a peer must not get away with: announcing our own
|
|
# prefix, announcing the exchange LAN itself (which would take the
|
|
# sessions with everyone on it), and sending our traffic to someone else.
|
|
refused_for() { # refused_for reason: how many of frr4's routes were refused for it
|
|
api "/api/routes/rejected?peer=frr4" | jq -e --arg why "$1" '.[0].by_reason[$why] // 0 | . >= 1'
|
|
}
|
|
phase_hijacks() {
|
|
frr_configure \
|
|
"ip prefix-list ELSEWHERE seq 5 permit 10.2.7.0/24" \
|
|
"route-map TO-DUT permit 10" "match ip address prefix-list ELSEWHERE" "set ip next-hop 10.9.0.3" \
|
|
"route-map TO-DUT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT out" \
|
|
"network 10.1.0.0/24" "network 10.9.0.0/24" "network 10.2.7.0/24"
|
|
|
|
# Our own prefix, from someone else.
|
|
wait_for "our own prefix to be refused" refused_for "prefix is our own"
|
|
dut_lacks 10.1.0.0/24 || fail "our own prefix was taken from FRR"
|
|
ok "our own prefix is not taken from a peer"
|
|
|
|
# A route whose next hop is BIRD, from FRR.
|
|
wait_for "the third-party next hop to be refused" refused_for "next hop is not the peer"
|
|
dut_lacks 10.2.7.0/24 || fail "a route by way of a third party was taken"
|
|
ok "a route whose next hop is someone else is not taken"
|
|
|
|
# The LAN's own prefix. The lab takes bogons, so the policy lets it
|
|
# by; it is still not a route the kernel could be given, and so not
|
|
# one to take: it would compete with the connected route.
|
|
wait_for "the LAN's prefix to be held" refused_for "prefix is a network of this router"
|
|
dut_lacks 10.9.0.0/24 || fail "the LAN's own prefix was taken from FRR"
|
|
in_ns dut ip -4 route show proto bgp | grep -q "^10.9.0.0/24 " && fail "the LAN's own prefix was installed by way of a peer"
|
|
in_ns dut ping -q -c 1 -W 2 10.9.0.3 >/dev/null || fail "the LAN is not reachable any more"
|
|
ok "a route for the exchange LAN itself is not taken, and stays out of the kernel"
|
|
|
|
frr_configure \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"no network 10.1.0.0/24" "no network 10.9.0.0/24" "no network 10.2.7.0/24" \
|
|
"no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# FRR and BIRD both have a route for one prefix, and FRR's is the better.
|
|
# But its next hop is nowhere the machine can reach: then BIRD's is the
|
|
# one chosen and installed (RFC 4271 §9.1.2.1), until the next hop is
|
|
# somewhere, and again when it no longer is.
|
|
best_is() { # best_is prefix peer-name
|
|
api "/api/routes/received?prefix=$1&best=true" | jq -e --arg peer "$2" '.routes | length == 1 and .[0].peer_name == $peer'
|
|
}
|
|
phase_nexthops() {
|
|
dut_down TERM
|
|
dut_config PEER 2000000 "accept_third_party_next_hop: true"
|
|
dut_up
|
|
converged
|
|
# A prefix nobody has said anything about in the RPKI, from both.
|
|
echo "route 10.77.0.0/24 blackhole;" > "$work/bird-both.conf"
|
|
bird_config "$work/bird-both.conf"
|
|
bird_ctl configure >/dev/null
|
|
wait_for "BIRD's route for it" dut_has 10.77.0.0/24 bird4
|
|
frr_configure \
|
|
"ip prefix-list FAR seq 5 permit 10.77.0.0/24" \
|
|
"route-map TO-DUT permit 10" "match ip address prefix-list FAR" "set ip next-hop 10.250.0.1" \
|
|
"route-map TO-DUT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT out" "network 10.77.0.0/24"
|
|
|
|
wait_for "FRR's route to be held" refused_for "next hop unreachable"
|
|
peer_is frr4 '.held_routes >= 1' >/dev/null || fail "the route is not held: $(peer_is frr4 .)"
|
|
best_is 10.77.0.0/24 bird4 >/dev/null || fail "BIRD's route is not the one chosen"
|
|
dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route is not in the kernel"
|
|
ok "a better route whose next hop is nowhere is held; the one that works is chosen and installed"
|
|
|
|
# The next hop becomes a neighbor.
|
|
in_ns dut ip addr add 10.250.0.9/24 dev lan
|
|
wait_for "FRR's route to be taken" best_is 10.77.0.0/24 frr4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.250.0.1 dev lan metric 20"
|
|
ok "when its next hop can be reached the better route is taken, chosen and installed"
|
|
|
|
# And stops being one.
|
|
in_ns dut ip addr del 10.250.0.9/24 dev lan
|
|
wait_for "BIRD's route to be chosen again" best_is 10.77.0.0/24 bird4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20"
|
|
peer_is frr4 '.held_routes >= 1' >/dev/null || fail "FRR's route is not held again"
|
|
ok "and when it no longer can, the other route is back"
|
|
|
|
frr_configure \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"no network 10.77.0.0/24" "no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
|
|
bird_config
|
|
bird_ctl configure >/dev/null
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# Routes are passed on: FRR becomes our customer and BIRD stays our
|
|
# provider, so what FRR announces goes up to BIRD with our AS on the path,
|
|
# what BIRD announces goes down to FRR marked "only to customer", and
|
|
# packets go between their networks through us. FRR, as a customer, steers
|
|
# its routes with our control communities: one keeps a route from BIRD,
|
|
# another has us prepend on the way to BIRD's session group. BIRD, as a
|
|
# provider, may not: its "not to FRR" is taken off as its route comes in.
|
|
phase_transit() {
|
|
dut_down TERM
|
|
dut_config CUSTOMER 2000000 "accept_control_communities: true" "" "groups: 7" \
|
|
'export { learned_routes: true }
|
|
control_communities: true'
|
|
echo "route 10.3.8.0/24 blackhole { bgp_large_community.add((64601, 0, 64602)); };" > "$work/bird-transit.conf"
|
|
bird_config "$work/bird-transit.conf"
|
|
bird_ctl configure >/dev/null
|
|
in_ns dut sysctl -qw net.ipv4.ip_forward=1
|
|
dut_up
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \
|
|
"ip prefix-list KEPT seq 5 permit 10.2.9.0/24" \
|
|
"ip prefix-list FAR seq 5 permit 10.2.10.0/24" \
|
|
"route-map TO-DUT-TRANSIT permit 10" "match ip address prefix-list KEPT" "set large-community 64601:0:4200000003" \
|
|
"route-map TO-DUT-TRANSIT permit 15" "match ip address prefix-list FAR" "set large-community 64601:31:7" \
|
|
"route-map TO-DUT-TRANSIT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "network 10.2.9.0/24" "network 10.2.10.0/24"
|
|
wait_for "four sessions" established 4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.9.0/24 frr4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.10.0/24 frr4
|
|
|
|
# Up to the provider, with our AS on the path; not the one that says
|
|
# not to.
|
|
wait_for "FRR's route at BIRD, by way of us" bash -c "sudo birdc -s $work/bird.ctl show route for 10.2.0.0/24 all | grep -q 'BGP.as_path: 64601 64602'"
|
|
sleep 1
|
|
bird_ctl show route for 10.2.9.0/24 all | grep -q "64601 64602" && fail "BIRD was told the route whose community says not to"
|
|
ok "the customer's routes reach the provider with our AS on the path; one kept back by its community"
|
|
|
|
# Prepended once more towards BIRD's group, at the customer's word,
|
|
# and the community that asked for it gone.
|
|
wait_for "the prepended route at BIRD" bash -c "sudo birdc -s $work/bird.ctl show route for 10.2.10.0/24 all | grep -q 'BGP.as_path: 64601 64601 64602'"
|
|
bird_ctl show route for 10.2.10.0/24 all | grep -q "(64601, 31, 7)" && fail "the control community went on to BIRD"
|
|
ok "a customer's control community has us prepend towards a session group, and is taken off"
|
|
|
|
# The provider's "not to FRR" is not ours to act on: taken off as the
|
|
# route comes in, and the route goes down to FRR all the same.
|
|
wait_for "BIRD's route with our community at FRR" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.8.0/24 json' 2>/dev/null | jq -e '.paths[0].aspath.string == \"64601 4200000003\"'"
|
|
api "/api/routes/received?prefix=10.3.8.0/24" | jq -e '.routes | length > 0 and all(.summary.large_communities | index("64601:0:64602") | not)' >/dev/null \
|
|
|| fail "the provider's control community was kept: $(api "/api/routes/received?prefix=10.3.8.0/24")"
|
|
ok "a provider's control community is taken off as its route comes in"
|
|
|
|
# Down to the customer, marked.
|
|
wait_for "BIRD's route at FRR, by way of us" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths[0].aspath.string == \"64601 4200000003\"'"
|
|
# Marked on the way in from the provider, with the provider's AS, and
|
|
# the mark kept as it goes down (RFC 9234 §5).
|
|
vty -c "show bgp ipv4 unicast 10.3.0.0/24" | grep -q "otc 4200000003" || fail "the route to the customer is not marked only to customer"
|
|
ok "the provider's routes reach the customer, marked only to customer"
|
|
|
|
# And packets follow them, through us.
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.3.0.1 >/dev/null || fail "no ping from the customer's network to the provider's through us"
|
|
in_ns bird ping -q -c 2 -W 2 -I 10.3.0.1 10.2.0.1 >/dev/null || fail "no ping from the provider's network to the customer's through us"
|
|
ok "packets go between the customer's and the provider's networks through us"
|
|
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" \
|
|
"address-family ipv4 unicast" "no network 10.2.9.0/24" "no network 10.2.10.0/24" "no neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "no route-map TO-DUT-TRANSIT"
|
|
bird_config
|
|
bird_ctl configure >/dev/null
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# FRR is a route collector of ours, told every route with its path
|
|
# identifier (ADD-PATH). BIRD talks to us from a second address too, so
|
|
# that there are two routes for each of its prefixes; FRR is told both,
|
|
# and holds them apart. One of BIRD's sessions goes, and so does one of
|
|
# the two.
|
|
phase_collector() {
|
|
dut_down TERM
|
|
in_ns bird ip addr add 10.9.0.4/24 dev ix0
|
|
dut_config CUSTOMER 2000000 "" 'add_path: ADD_PATH_SEND
|
|
export { learned_routes: true all_paths: true }' "" 'peers {
|
|
name: "bird4b" endpoint { address: "10.9.0.4" } remote_asn: 4200000003
|
|
address_families: IPV4_UNICAST
|
|
role: PROVIDER
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}'
|
|
bird_config "" 'protocol bgp dut4b from dut {
|
|
neighbor 10.9.0.1 as 64601;
|
|
source address 10.9.0.4;
|
|
ipv4 { import all; export where source = RTS_STATIC; };
|
|
}'
|
|
bird_ctl configure >/dev/null
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer"
|
|
dut_up
|
|
wait_for "five sessions" established 5
|
|
wait_for "BIRD's route from both its sessions" bash -c "sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?prefix=10.3.0.0/24' | jq -e '.routes | length == 2'"
|
|
peer_is frr4 '.add_path_send | index("IPv4 unicast") != null' >/dev/null \
|
|
|| fail "ADD-PATH was not agreed with FRR: $(peer_is frr4 .add_path_send)"
|
|
wait_for "both routes at FRR, told apart" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths | length == 2 and all(.aspath.string == \"64601 4200000003\")'"
|
|
# Its two prefixes twice, and our own route: what the page and the
|
|
# metrics say it holds from us.
|
|
wait_for "the count of routes the collector holds" peer_is frr4 '.advertised_routes == 5'
|
|
ok "the collector is told both of BIRD's routes for a prefix, with path identifiers"
|
|
|
|
bird_ctl disable dut4b >/dev/null
|
|
wait_for "one route left at FRR" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths | length == 1'"
|
|
wait_for "the count of routes the collector holds" peer_is frr4 '.advertised_routes == 3'
|
|
ok "a route that goes is withdrawn from the collector by its identifier, the other stays"
|
|
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer"
|
|
bird_config
|
|
bird_ctl configure >/dev/null
|
|
in_ns bird ip addr del 10.9.0.4/24 dev ix0
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# A customer leaks, and the export limits hold. FRR, now our customer,
|
|
# announces a route outside its address space and then more routes than
|
|
# BIRD, our provider, is to be told of. A customer's routes go to
|
|
# everyone as far as the roles go; the export limits are the check on
|
|
# that. The route outside our and the customer's space never goes up to
|
|
# BIRD; when more were to go than the limit allows, none of them does,
|
|
# and the session with BIRD ends with an Administrative Shutdown that
|
|
# says why (RFC 9003).
|
|
phase_leaks() {
|
|
dut_down TERM
|
|
dut_config CUSTOMER 2000000 "" "" "" 'export { learned_routes: true }
|
|
export_limits {
|
|
prefixes { prefix: "10.1.0.0/16" max_length: 24 }
|
|
prefixes { prefix: "10.2.0.0/16" max_length: 24 }
|
|
prefixes { prefix: "2001:db8:1::/48" }
|
|
prefixes { prefix: "2001:db8:2::/48" }
|
|
max_prefixes: 5
|
|
}'
|
|
dut_up
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \
|
|
"address-family ipv4 unicast" "network 10.77.0.0/24"
|
|
wait_for "four sessions" established 4
|
|
wait_for "the route outside the customer's space at the daemon" dut_has 10.77.0.0/24 frr4
|
|
|
|
# The customer's route goes up to the provider; the one outside the
|
|
# export limits does not.
|
|
wait_for "FRR's route at BIRD" bird_has 10.2.0.0/24
|
|
sleep 1
|
|
bird_has 10.77.0.0/24 && fail "BIRD was told a route outside the export limits"
|
|
ok "a customer's route outside the export limits is not passed on to the provider"
|
|
|
|
# More routes than the limit: ours and the customer's two, and four
|
|
# more of the customer's, would be six of five.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" \
|
|
"network 10.2.11.0/24" "network 10.2.12.0/24" "network 10.2.13.0/24" "network 10.2.14.0/24"
|
|
wait_for "the export limit to end BIRD's session" \
|
|
peer_is bird4 '.state != "Established" and (.last_error // "" | test("export limit"; "i"))'
|
|
wait_for "BIRD to be told why" sudo grep -q "export limit of 5 routes reached" "$work/bird.log"
|
|
bird_has 10.2.11.0/24 && fail "BIRD was told a route past the export limit"
|
|
ok "past its export limit the provider is told nothing more, and why (Cease / Administrative Shutdown): $(peer_is bird4 '.last_error')"
|
|
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" "address-family ipv4 unicast" \
|
|
"no network 10.77.0.0/24" "no network 10.2.11.0/24" "no network 10.2.12.0/24" \
|
|
"no network 10.2.13.0/24" "no network 10.2.14.0/24"
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# The exchange LAN loses its IPv4 addresses. One session over IPv6 with
|
|
# each router carries both families, and IPv4 routes have IPv6 next hops
|
|
# (RFC 8950), in BGP and in the kernel. Run after the rest: the lab stays
|
|
# that way.
|
|
phase_v6only() {
|
|
dut_down TERM
|
|
frr_down TERM
|
|
sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
|
|
local n=0 name
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0
|
|
done
|
|
frr_config_v6only; frr_up
|
|
bird_config_v6only; bird_up
|
|
dut_config_v6only; dut_up
|
|
|
|
wait_for "two sessions over IPv6" established 2
|
|
api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
|
|
|| fail "the sessions did not agree on IPv4 with IPv6 next hops: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
|
|
ok "both sessions carry IPv4 and IPv6, with IPv6 next hops for IPv4 (RFC 8950)"
|
|
|
|
# What they send us: IPv4 routes whose next hop is an IPv6 address,
|
|
# and the same in the kernel.
|
|
wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
|
|
wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].summary.next_hop | startswith("2001:db8:9::2")' >/dev/null \
|
|
|| fail "FRR's IPv4 route has the next hop $(api "/api/routes/received?prefix=10.2.0.0/24" | jq '.routes[0].summary.next_hop')"
|
|
wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 2001:db8:9::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 2001:db8:9::3 dev ix0 metric 20"
|
|
wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
|
|
ok "their IPv4 routes arrive with IPv6 next hops and are in the kernel that way"
|
|
|
|
# What we send them.
|
|
wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
|
|
vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].nexthops | any(.ip == "2001:db8:9::1" and .afi == "ipv6")' >/dev/null \
|
|
|| fail "FRR's copy of our IPv4 route has the next hops $(vty -c 'show bgp ipv4 unicast 10.1.0.0/24 json' | jq -c '.paths[0].nexthops')"
|
|
wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
|
|
bird_ctl show route for 10.1.0.0/24 all | grep -q "BGP.next_hop: 2001:db8:9::1" \
|
|
|| fail "BIRD's copy of our IPv4 route: $(bird_ctl show route for 10.1.0.0/24 all)"
|
|
wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
ok "FRR and BIRD have our IPv4 route with our IPv6 address as next hop"
|
|
|
|
# IPv4 packets across a link that has no IPv4 addresses.
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
ok "IPv4 packets cross the IPv6-only link, both ways"
|
|
|
|
# A withdrawal by FRR (in MP_UNREACH_NLRI) takes the route out.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.5.0/24"
|
|
wait_for "FRR's withdrawal" dut_lacks 10.2.5.0/24
|
|
wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.5.0/24
|
|
dut_has 10.2.0.0/24 frr >/dev/null || fail "the withdrawal took the other route along"
|
|
ok "an IPv4 route withdrawn over the IPv6 session goes"
|
|
}
|
|
|
|
# The exchange LAN loses its global IPv6 addresses too: all that is left on
|
|
# it are link-local ones, the same on every link a router has, which mean
|
|
# something only together with the interface. The sessions are between
|
|
# those, as are the next hops, for both families. Run after v6only.
|
|
dut_config_linklocal() {
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "::" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr" endpoint { address: "fe80::2%ix0" } remote_asn: 64602
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
peers {
|
|
name: "bird" endpoint { address: "fe80::3%ix0" } remote_asn: 4200000003
|
|
role: PROVIDER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" }
|
|
originate { prefix: "2001:db8:1::/48" }
|
|
kernel { stale_seconds: 3 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
END
|
|
}
|
|
|
|
# FRR is told the interface and not the address ("BGP unnumbered"), which
|
|
# is how it is mostly set up for this. It learns who is on the link from
|
|
# router advertisements, which the daemon does not send: radvd does, as it
|
|
# would have to beside the daemon on a real router with such a neighbor.
|
|
radvd_up() {
|
|
cat > "$work/radvd.conf" <<END
|
|
interface ix0 {
|
|
AdvSendAdvert on;
|
|
MinRtrAdvInterval 3;
|
|
MaxRtrAdvInterval 4;
|
|
AdvDefaultLifetime 0;
|
|
};
|
|
END
|
|
in_ns dut "$(command -v radvd || echo /usr/sbin/radvd)" -C "$work/radvd.conf" -p "$work/radvd.pid" -m logfile -l "$work/radvd.log"
|
|
}
|
|
|
|
frr_config_linklocal() {
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor ix0 interface remote-as 64601
|
|
neighbor ix0 local-role peer
|
|
neighbor ix0 timers connect 1
|
|
neighbor ix0 capability extended-nexthop
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.2.5.0/24
|
|
neighbor ix0 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor ix0 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
bird_config_linklocal() {
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
|
|
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
|
|
protocol bgp dut {
|
|
local as 4200000003;
|
|
neighbor fe80::1%ix0 as 64601;
|
|
local role provider;
|
|
connect retry time 1;
|
|
# Back soon after the hold timer ran out, which the phase makes it do.
|
|
error wait time 1, 2;
|
|
hold time 9;
|
|
graceful restart off;
|
|
ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
phase_linklocal() {
|
|
dut_down TERM
|
|
frr_down TERM
|
|
sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
|
|
local n=0 name
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0 2>/dev/null || true
|
|
in_ns "$name" ip -6 addr flush dev ix0
|
|
in_ns "$name" ip -6 addr add "fe80::$n/64" dev ix0 nodad
|
|
# The same address on another link: it must not be mistaken for
|
|
# the one the session is on.
|
|
in_ns "$name" ip -6 addr flush dev lan scope link
|
|
in_ns "$name" ip -6 addr add "fe80::$n/64" dev lan nodad
|
|
done
|
|
in_ns dut sysctl -qw net.ipv6.conf.ix0.forwarding=1
|
|
radvd_up
|
|
frr_config_linklocal; frr_up
|
|
bird_config_linklocal; bird_up
|
|
dut_config_linklocal; dut_up
|
|
|
|
wait_for "two sessions between link-local addresses" established 2
|
|
peer_is frr '.peer_addr | startswith("[fe80::2%ix0]:")' >/dev/null \
|
|
&& peer_is bird '.peer_addr | startswith("[fe80::3%ix0]:")' >/dev/null \
|
|
&& peer_is frr '.local_addr | startswith("[fe80::1%ix0]:")' >/dev/null \
|
|
|| fail "the peers are at $(api /api/peers | jq -c '[.[] | {name, peer_addr, local_addr}]')"
|
|
api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
|
|
|| fail "the sessions did not agree on both families: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
|
|
ok "sessions with FRR and BIRD at fe80::2%ix0 and fe80::3%ix0, for both families"
|
|
|
|
# Their routes: link-local next hops, on the session's interface.
|
|
wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
|
|
wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
|
|
wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 fe80::3 dev ix0 metric 20"
|
|
wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via fe80::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv6 route in the kernel" dut_kernel -6 "2001:db8:3::/48 via fe80::3 dev ix0 metric 20"
|
|
api /api/kernel | jq -e '.unresolved == {}' >/dev/null || fail "some next hops did not resolve: $(api /api/kernel)"
|
|
ok "their routes of both families are in the kernel by way of fe80::… on ix0"
|
|
|
|
# Ours, at them.
|
|
wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
|
|
wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
|
|
wait_for "our IPv6 route at BIRD" bird_has 2001:db8:1::/48
|
|
wait_for "our IPv4 route in FRR's kernel" in_ns frr ip -4 route get 10.1.0.1
|
|
wait_for "our IPv6 route in BIRD's kernel" in_ns bird ip -6 route get 2001:db8:1::1 from 2001:db8:3::1
|
|
ok "FRR and BIRD have our routes of both families"
|
|
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
|
|
in_ns bird ping -q -c 2 -W 2 -I 2001:db8:3::1 2001:db8:1::1 >/dev/null || fail "no IPv6 ping from BIRD's lan"
|
|
ok "IPv4 and IPv6 packets cross a link with only link-local addresses, both ways"
|
|
|
|
# The interface goes and comes back, as a tunnel does when it is
|
|
# restarted: the sessions come back by themselves.
|
|
in_ns dut ip link set ix0 down
|
|
wait_for "the sessions to go" established 0
|
|
in_ns dut ip link set ix0 up
|
|
in_ns dut ip -6 addr flush dev ix0
|
|
in_ns dut ip -6 addr add fe80::1/64 dev ix0 nodad
|
|
wait_for "the sessions to come back" established 2
|
|
wait_for "FRR's IPv4 route in the kernel again" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
|
|
ok "the sessions and the routes come back after the interface was down"
|
|
}
|
|
|
|
# --- main -----------------------------------------------------------------------
|
|
|
|
case "${1:-test}" in
|
|
down) down; exit 0 ;;
|
|
up) lab_up; converged; say "the lab is up; work files in $work"; exit 0 ;;
|
|
test) ;;
|
|
*) fail "usage: $0 [test|up|down]" ;;
|
|
esac
|
|
|
|
on_exit() {
|
|
status=$?
|
|
if [ "$status" -ne 0 ]; then
|
|
say "--- the daemon's last words:" >&2
|
|
tail -n 25 "$work/dut.log" >&2 || true
|
|
fi
|
|
[ -n "${KEEP:-}" ] || down
|
|
exit "$status"
|
|
}
|
|
trap on_exit EXIT
|
|
lab_up
|
|
say "lab: converging"
|
|
converged
|
|
for phase in $phases; do
|
|
say "phase: $phase"
|
|
"phase_$phase"
|
|
done
|
|
say "all phases passed: $phases"
|