Rayhaan JaufeerallyandClaude Fable 5.1
1e2e8f2d86
What was to be scripted is configuration, done natively:
- import { add_communities, add_large_communities, remove_communities,
local_pref }: every route taken from a peer is marked (where it was
learned, say), with nobody else's word for our marks taken, and given
its preference, which selection goes by.
- export { add_communities, add_large_communities, remove_communities,
as_path_prepend, learned_routes }, for a peer or for all that have none
of their own.
- control_communities: the communities by which routes say which peers
they are passed on to, as exchanges' route servers have them (0:P, L:P,
0:L, and the large forms L:0:P, L:1:P, L:0:0), taken off as the route
goes on.
And what the last of these is for: with export.learned_routes a peer is
told the best route of every prefix the others announced. The rules are
BGP's and cannot be turned off: not back where it came from, not to a
peer whose AS is on the path, not between two internal peers, not past
NO_EXPORT or NO_ADVERTISE; our AS on the path and ourselves as next hop
to another network, LOCAL_PREF to our own; to a route server's client
the path, MED and next hop as they came (RFC 7947). Which routes go where
follows from the roles of RFC 9234: a route that came down or across goes
on down only, so that with roles on every peer this makes us transit for
our customers and nothing else. A peer without a role is told everything,
and the daemon says so at startup.
Each session follows the table's best routes with a reader and keeps the
prefixes it has announced, so that it can withdraw them. Routes that go
out alike go in one UPDATE: changes are gathered for 50 ms first, and a
table that is given out anew is gathered whole (at most four sessions at
a time; it is tens of megabytes). For that to work the table's journal
had to give out together what was written together: every change now
carries a time on a table-wide clock, a reader's pass reads every shard
up to one time, no further than the changes that are complete in every
shard, and a resync takes its places in the journals as of one time.
One peer sending 1.23 M routes over a session, another told of them:
in our table after 0.47 s, all heard after 2.1 s, four routes an UPDATE.
The lab has a phase, transit: FRR as our customer and BIRD as our
provider, routes each way with the right paths and marks, one kept back
by a community, and pings between their networks through us.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1131 lines
51 KiB
Bash
Executable File
1131 lines
51 KiB
Bash
Executable File
#!/bin/bash
|
|
# An interoperability lab: this daemon against FRR, BIRD and Routinator,
|
|
# each in a network namespace of its own on one exchange LAN.
|
|
#
|
|
# 10.9.0.0/24, 2001:db8:9::/64
|
|
# ┌───────────────────┬──────────────────────┐
|
|
# dut (.1) AS 64601 frr (.2) AS 64602 bird (.3) AS 4200000003
|
|
# lan 10.1.0.1/24 lan 10.2.0.1/24 lan 10.3.0.1/24
|
|
# 2001:db8:1::1/48 2001:db8:2::1/48 2001:db8:3::1/48
|
|
# + Routinator on 127.0.0.1:3323
|
|
#
|
|
# "dut" is the daemon under test. FRR is its peer, BIRD its provider; both
|
|
# say so in their OPEN (RFC 9234). Routinator serves validated ROA payloads
|
|
# made up for the lab from a local exceptions file, so nothing is fetched
|
|
# from the Internet.
|
|
#
|
|
# scripts/interop/lab.sh run every phase, then take the lab down
|
|
# scripts/interop/lab.sh up bring the lab up and leave it up
|
|
# scripts/interop/lab.sh down take it down
|
|
# PHASES="basics scale" scripts/interop/lab.sh only these phases
|
|
# SCALE_ROUTES=1000000 ... the size of the table BIRD sends
|
|
#
|
|
# Needs Linux, sudo, and: frr, bird2, radvd, jq, routinator, a built `bgp`
|
|
# (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS;
|
|
# scripts/interop/install.sh installs what is missing on Ubuntu.
|
|
set -euo pipefail
|
|
|
|
cd "$(dirname "$0")/../.."
|
|
bgp="${CARGO_TARGET_DIR:-target}/release/bgp"
|
|
work=/tmp/ferrix-lab
|
|
pathspace=ferrixlab
|
|
all_phases="basics roles rpki refresh flaps restart scale limits hijacks nexthops transit v6only linklocal"
|
|
phases="${PHASES:-$all_phases}"
|
|
scale_routes="${SCALE_ROUTES:-200000}"
|
|
|
|
# --- plumbing ---------------------------------------------------------------
|
|
|
|
in_ns() { local name=$1; shift; sudo ip netns exec "lab-$name" "$@"; }
|
|
api() { in_ns dut curl -sf "http://127.0.0.1:8080$1"; }
|
|
vty() { in_ns frr vtysh -N "$pathspace" "$@" 2>/dev/null; }
|
|
bird_ctl() { sudo birdc -s "$work/bird.ctl" "$@"; }
|
|
say() { printf '%s\n' "$*"; }
|
|
ok() { say " ok: $*"; }
|
|
fail() { say "FAILED: $*" >&2; exit 1; }
|
|
|
|
# wait_for "what" command...: until the command succeeds.
|
|
wait_for() {
|
|
local what=$1; shift
|
|
local tries=${WAIT_TRIES:-150}
|
|
for _ in $(seq "$tries"); do
|
|
if "$@" >/dev/null 2>&1; then return 0; fi
|
|
sleep 0.2
|
|
done
|
|
say "timed out waiting for: $what" >&2
|
|
return 1
|
|
}
|
|
|
|
down() {
|
|
for pidfile in "$work"/dut.pid "$work"/routinator.pid; do
|
|
[ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
[ -f "$work/radvd.pid" ] && sudo kill "$(sudo cat "$work/radvd.pid")" 2>/dev/null || true
|
|
[ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.3
|
|
for name in dut frr bird ix; do sudo ip netns del "lab-$name" 2>/dev/null || true; done
|
|
sudo rm -rf "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
}
|
|
|
|
# --- the network ------------------------------------------------------------
|
|
|
|
network_up() {
|
|
sudo modprobe dummy 2>/dev/null || true
|
|
sudo ip netns add lab-ix
|
|
in_ns ix ip link add ix type bridge
|
|
in_ns ix ip link set ix up
|
|
local n=0
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
sudo ip netns add "lab-$name"
|
|
sudo ip link add ix0 netns "lab-$name" type veth peer name "to-$name" netns lab-ix
|
|
in_ns ix ip link set "to-$name" master ix up
|
|
in_ns "$name" ip link set lo up
|
|
in_ns "$name" ip addr add "10.9.0.$n/24" dev ix0
|
|
in_ns "$name" ip -6 addr add "2001:db8:9::$n/64" dev ix0 nodad
|
|
in_ns "$name" ip link set ix0 up
|
|
in_ns "$name" ip link add lan type dummy
|
|
in_ns "$name" ip addr add "10.$n.0.1/24" dev lan
|
|
in_ns "$name" ip -6 addr add "2001:db8:$n::1/48" dev lan nodad
|
|
in_ns "$name" ip link set lan up
|
|
done
|
|
}
|
|
|
|
# --- Routinator ---------------------------------------------------------------
|
|
|
|
# routinator_up [asn]: with `asn` the AS that may originate 10.66.0.0/24.
|
|
routinator_up() {
|
|
local owner=${1:-64999}
|
|
mkdir -p "$work/rpki/repository" "$work/rpki/tals"
|
|
# 10.2.0.0/24 is FRR's to originate. 10.66.0.0/24 is someone else's,
|
|
# and FRR announces it all the same. 10.3.0.0/16 is BIRD's, as far as
|
|
# /24.
|
|
cat > "$work/rpki/slurm.json" <<END
|
|
{
|
|
"slurmVersion": 1,
|
|
"validationOutputFilters": { "prefixFilters": [], "bgpsecFilters": [] },
|
|
"locallyAddedAssertions": {
|
|
"prefixAssertions": [
|
|
{ "asn": 64602, "prefix": "10.2.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": $owner, "prefix": "10.66.0.0/24", "maxPrefixLength": 24 },
|
|
{ "asn": 4200000003, "prefix": "10.3.0.0/16", "maxPrefixLength": 24 },
|
|
{ "asn": 64602, "prefix": "2001:db8:2::/48", "maxPrefixLength": 48 }
|
|
],
|
|
"bgpsecAssertions": []
|
|
}
|
|
}
|
|
END
|
|
# By its full name: root's PATH does not have a user's ~/.cargo/bin.
|
|
in_ns dut "$(command -v routinator)" --no-rir-tals --extra-tals-dir "$work/rpki/tals" \
|
|
-r "$work/rpki/repository" -x "$work/rpki/slurm.json" \
|
|
--logfile "$work/routinator.log" \
|
|
server --rtr 127.0.0.1:3323 --http 127.0.0.1:8323 --refresh 3600 \
|
|
> /dev/null 2>&1 &
|
|
echo $! | sudo tee "$work/routinator.pid" > /dev/null
|
|
}
|
|
|
|
# --- the daemon under test ------------------------------------------------------
|
|
|
|
# dut_config [frr-role] [frr-max-prefixes] [more of frr4's import policy]
|
|
# [more of frr4] [more of bird4] [more of the configuration]
|
|
dut_config() {
|
|
local frr_role=${1:-PEER} frr_limit=${2:-2000000} frr_import=${3:-}
|
|
local frr_more=${4:-} bird_more=${5:-} more=${6:-}
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "10.9.0.1" port: 179 }
|
|
listen_endpoints { address: "2001:db8:9::1" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr4" endpoint { address: "10.9.0.2" } remote_asn: 64602
|
|
address_families: IPV4_UNICAST
|
|
role: $frr_role
|
|
import { accept_bogons: true max_prefixes: $frr_limit hold_down_seconds: 5 $frr_import }
|
|
$frr_more
|
|
}
|
|
peers {
|
|
name: "frr6" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
|
|
address_families: IPV6_UNICAST
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
peers {
|
|
name: "bird4" endpoint { address: "10.9.0.3" } remote_asn: 4200000003
|
|
address_families: IPV4_UNICAST
|
|
role: PROVIDER require_role: true
|
|
add_path: ADD_PATH_RECEIVE
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
$bird_more
|
|
}
|
|
peers {
|
|
name: "bird6" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
|
|
address_families: IPV6_UNICAST
|
|
role: PROVIDER require_role: true
|
|
import { accept_bogons: true max_prefixes: 2000000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" communities: "64601:100" }
|
|
originate { prefix: "2001:db8:1::/48" large_communities: "64601:1:2" }
|
|
kernel { stale_seconds: 3 }
|
|
rpki { caches { address: "127.0.0.1" port: 3323 } retry_seconds: 1 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
$more
|
|
END
|
|
}
|
|
|
|
dut_config_v6only() {
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "2001:db8:9::1" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
peers {
|
|
name: "bird" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
|
|
role: PROVIDER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" }
|
|
originate { prefix: "2001:db8:1::/48" }
|
|
kernel { stale_seconds: 3 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
END
|
|
}
|
|
|
|
dut_up() {
|
|
sudo ip netns exec lab-dut \
|
|
setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups \
|
|
--inh-caps +net_admin,+net_bind_service --ambient-caps +net_admin,+net_bind_service \
|
|
"$bgp" "$work/dut.textpb" >> "$work/dut.log" 2>&1 &
|
|
echo $! > "$work/dut.pid"
|
|
# Not the shell's to report on when it is killed, which the tests do.
|
|
disown
|
|
wait_for "the daemon's status interface" api /api/status
|
|
}
|
|
|
|
# dut_down [signal]: stops the daemon and waits until it has gone.
|
|
dut_down() {
|
|
local signal=${1:-TERM}
|
|
# The pid recorded is sudo's; the daemon is the one in the namespace
|
|
# that is not Routinator.
|
|
local pid
|
|
for pid in $(sudo ip netns pids lab-dut); do
|
|
if [ "$(cat "/proc/$pid/comm" 2>/dev/null)" = bgp ]; then
|
|
sudo kill "-$signal" "$pid"
|
|
while [ -e "/proc/$pid" ]; do sleep 0.05; done
|
|
fi
|
|
done
|
|
}
|
|
|
|
# --- FRR ------------------------------------------------------------------------
|
|
|
|
frr_config() {
|
|
sudo mkdir -p "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor 10.9.0.1 remote-as 64601
|
|
neighbor 10.9.0.1 local-role peer
|
|
neighbor 10.9.0.1 timers connect 1
|
|
neighbor 2001:db8:9::1 remote-as 64601
|
|
neighbor 2001:db8:9::1 local-role peer
|
|
neighbor 2001:db8:9::1 timers connect 1
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.66.0.0/24
|
|
neighbor 10.9.0.1 activate
|
|
no neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
# One session, over IPv6, for both families: IPv4 routes go with IPv6 next
|
|
# hops (RFC 8950).
|
|
frr_config_v6only() {
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor 2001:db8:9::1 remote-as 64601
|
|
neighbor 2001:db8:9::1 local-role peer
|
|
neighbor 2001:db8:9::1 timers connect 1
|
|
neighbor 2001:db8:9::1 capability extended-nexthop
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.2.5.0/24
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor 2001:db8:9::1 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
frr_up() {
|
|
sudo touch "/etc/frr/$pathspace/vtysh.conf" "$work/frr.log"
|
|
sudo chown -R frr:frr "/etc/frr/$pathspace" "/var/run/frr/$pathspace" "$work/frr.log"
|
|
for daemon in zebra bgpd; do
|
|
# Run from a copy: the distribution's AppArmor profile for the
|
|
# daemon in its usual place does not allow it a directory of its
|
|
# own, and the machine's profiles are not the lab's to change.
|
|
sudo cp "/usr/lib/frr/$daemon" "$work/$daemon"
|
|
in_ns frr "$work/$daemon" -d -N "$pathspace" \
|
|
-f "/etc/frr/$pathspace/frr.conf" > "$work/$daemon.out" 2>&1 \
|
|
|| fail "$daemon did not start: $(cat "$work/$daemon.out")"
|
|
# zebra makes the directory its sockets are in root's.
|
|
sudo chown -R frr:frr "/var/run/frr/$pathspace"
|
|
done
|
|
}
|
|
|
|
frr_down() {
|
|
local signal=${1:-TERM} daemon
|
|
for daemon in bgpd zebra; do
|
|
local pidfile="/var/run/frr/$pathspace/$daemon.pid"
|
|
sudo test -f "$pidfile" && sudo kill "-$signal" "$(sudo cat "$pidfile")" 2>/dev/null || true
|
|
done
|
|
sleep 0.5
|
|
}
|
|
|
|
# --- BIRD -----------------------------------------------------------------------
|
|
|
|
# bird_config [extra-static-routes-file]
|
|
bird_config() {
|
|
local extra=${1:-}
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 {
|
|
ipv4;
|
|
route 10.3.0.0/24 blackhole;
|
|
route 10.3.7.0/24 blackhole;
|
|
${extra:+include \"$extra\";}
|
|
}
|
|
protocol static static6 {
|
|
ipv6;
|
|
route 2001:db8:3::/48 blackhole;
|
|
}
|
|
template bgp dut {
|
|
local as 4200000003;
|
|
local role provider;
|
|
connect retry time 1;
|
|
hold time 9;
|
|
graceful restart off;
|
|
}
|
|
protocol bgp dut4 from dut {
|
|
neighbor 10.9.0.1 as 64601;
|
|
source address 10.9.0.3;
|
|
ipv4 { import all; export where source = RTS_STATIC; add paths tx; };
|
|
}
|
|
protocol bgp dut6 from dut {
|
|
neighbor 2001:db8:9::1 as 64601;
|
|
source address 2001:db8:9::3;
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
bird_config_v6only() {
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
|
|
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
|
|
protocol bgp dut {
|
|
local as 4200000003;
|
|
neighbor 2001:db8:9::1 as 64601;
|
|
source address 2001:db8:9::3;
|
|
local role provider;
|
|
connect retry time 1;
|
|
hold time 9;
|
|
graceful restart off;
|
|
ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
bird_up() {
|
|
sudo ip netns exec lab-bird bird -c "$work/bird.conf" -s "$work/bird.ctl" -P "$work/bird.pid"
|
|
}
|
|
|
|
# --- what is looked at ------------------------------------------------------------
|
|
|
|
established() { # how many of the daemon's sessions are up
|
|
api /api/peers | jq -e --argjson n "$1" '[.[] | select(.state == "Established")] | length == $n'
|
|
}
|
|
dut_has() { # dut_has prefix peer-name: the daemon has the route, from that peer
|
|
api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer)'
|
|
}
|
|
dut_lacks() { ! api "/api/routes/received?prefix=$1" | jq -e '.routes | length > 0' >/dev/null; }
|
|
dut_kernel() { # dut_kernel family "route as ip shows it, without nhid"
|
|
# All of it is read: with a table's worth of routes, a grep that
|
|
# stopped at the first match would leave `ip` writing to nobody.
|
|
in_ns dut ip "$1" route show proto bgp | sed 's/ */ /g; s/ nhid [0-9]*//; s/ pref medium//; s/ *$//' | grep -xF "$2" >/dev/null
|
|
}
|
|
dut_kernel_lacks() { ! in_ns dut ip "$1" route show proto bgp | grep "^$2 " >/dev/null; }
|
|
frr_has() { # frr_has family prefix: FRR has a route for it by way of the daemon
|
|
vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | any(.aspath.string == "64601")'
|
|
}
|
|
frr_lacks() { ! vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | length > 0' >/dev/null; }
|
|
bird_has() { bird_ctl show route for "$1" all | grep -q "BGP.as_path: 64601"; }
|
|
|
|
lab_up() {
|
|
down
|
|
sudo rm -rf "$work"; mkdir -p "$work"; chmod 777 "$work"
|
|
[ -x "$bgp" ] || fail "$bgp not found: cargo build --release -p bgp"
|
|
network_up
|
|
routinator_up
|
|
frr_config; frr_up
|
|
bird_config; bird_up
|
|
dut_config; dut_up
|
|
}
|
|
|
|
converged() {
|
|
wait_for "four sessions" established 4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.0.0/24 frr4
|
|
wait_for "BIRD's routes at the daemon" dut_has 10.3.0.0/24 bird4
|
|
wait_for "FRR's IPv6 route" dut_has 2001:db8:2::/48 frr6
|
|
wait_for "BIRD's IPv6 route" dut_has 2001:db8:3::/48 bird6
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
|
|
wait_for "routes in the kernel" dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20"
|
|
wait_for "the daemon's route at FRR" frr_has ipv4 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
wait_for "the daemon's route at BIRD" bird_has 10.1.0.0/24
|
|
wait_for "the daemon's IPv6 route at BIRD" bird_has 2001:db8:1::/48
|
|
}
|
|
|
|
# --- the phases ---------------------------------------------------------------------
|
|
|
|
frr_neighbor() { # frr_neighbor address jq-filter
|
|
vty -c "show bgp neighbors $1 json" | jq -e --arg n "$1" ".[\$n] | $2"
|
|
}
|
|
frr_configure() { # each argument a line of configuration
|
|
local args=(-c "configure terminal")
|
|
for line in "$@"; do args+=(-c "$line"); done
|
|
vty "${args[@]}"
|
|
}
|
|
peer_is() { # peer_is name jq-filter: of the daemon's view of a peer
|
|
api /api/peers | jq -e --arg n "$1" ".[] | select(.name == \$n) | $2"
|
|
}
|
|
|
|
# Sessions come up, routes go both ways with their attributes intact, and
|
|
# packets follow them.
|
|
phase_basics() {
|
|
# What FRR has of ours: the path, the next hop, the communities.
|
|
local route
|
|
route=$(vty -c "show bgp ipv4 unicast 10.1.0.0/24 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and .nexthops[0].ip == "10.9.0.1"
|
|
and .community.string == "64601:100" and .origin == "IGP"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(vty -c "show bgp ipv6 unicast 2001:db8:1::/48 json")
|
|
jq -e '.paths[0] | .aspath.string == "64601" and (.nexthops | any(.ip == "2001:db8:9::1"))
|
|
and .largeCommunity.string == "64601:1:2"' <<<"$route" >/dev/null \
|
|
|| fail "FRR's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "FRR has our routes, with path, next hop and communities"
|
|
|
|
route=$(bird_ctl show route for 10.1.0.0/24 all)
|
|
grep -q "BGP.next_hop: 10.9.0.1" <<<"$route" && grep -q "BGP.community: (64601,100)" <<<"$route" \
|
|
|| fail "BIRD's copy of 10.1.0.0/24 is not as sent: $route"
|
|
route=$(bird_ctl show route for 2001:db8:1::/48 all)
|
|
grep -q "BGP.next_hop: 2001:db8:9::1" <<<"$route" && grep -q "BGP.large_community: (64601, 1, 2)" <<<"$route" \
|
|
|| fail "BIRD's copy of 2001:db8:1::/48 is not as sent: $route"
|
|
ok "BIRD has our routes, with path, next hop and communities"
|
|
|
|
# What we have of theirs. BIRD's AS number needs four octets, and it
|
|
# sends path identifiers (ADD-PATH).
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [4200000003] and .summary.next_hop == "10.9.0.3" and .path_id != 0' >/dev/null \
|
|
|| fail "our copy of BIRD's 10.3.0.0/24 is not as sent"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0]
|
|
| .summary.as_path == [64602] and .summary.next_hop == "10.9.0.2" and .best' >/dev/null \
|
|
|| fail "our copy of FRR's 10.2.0.0/24 is not as sent"
|
|
peer_is bird4 '.four_octet_asn and (.add_path_receive == ["IPv4 unicast"])' >/dev/null \
|
|
|| fail "the session with BIRD did not agree on four-octet AS numbers and ADD-PATH"
|
|
ok "we have their routes; four-octet AS numbers and ADD-PATH with BIRD"
|
|
|
|
# Packets, there and back by the routes BGP put in three kernels.
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no ping to BIRD's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
|
|
ok "packets get through to both, over IPv4 and IPv6"
|
|
}
|
|
|
|
# RFC 9234: each end says what it is to the other, and routes are marked
|
|
# Only to Customer where they should be.
|
|
phase_roles() {
|
|
api /api/peers | jq -e 'all(.[]; .role_confirmed)' >/dev/null \
|
|
|| fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')"
|
|
frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \
|
|
|| fail "FRR does not see us as its peer"
|
|
bird_ctl show protocols all dut4 | grep -A12 "Neighbor capabilities" | grep -q "Role: customer" \
|
|
|| fail "BIRD does not see us as its customer"
|
|
ok "roles agree with FRR (peer) and BIRD (provider)"
|
|
|
|
# To a peer our routes are marked; to a provider they are not.
|
|
vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].otc == 64601' >/dev/null \
|
|
|| fail "FRR's copy of our route is not marked Only to Customer by us"
|
|
if bird_ctl show route for 10.1.0.0/24 all | grep -qi "otc\|only_to_customer"; then
|
|
fail "our route to our provider is marked Only to Customer"
|
|
fi
|
|
# What comes from a peer or a provider is marked by them, or by us.
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 64602)' >/dev/null \
|
|
|| fail "FRR's route is not marked Only to Customer"
|
|
api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 4200000003)' >/dev/null \
|
|
|| fail "BIRD's route is not marked Only to Customer"
|
|
ok "Only to Customer is where it should be, in both directions"
|
|
}
|
|
|
|
# Routinator's data over RTR: an invalid route is refused, and taken when
|
|
# the data comes to allow it.
|
|
phase_rpki() {
|
|
wait_for "the daemon to have Routinator's data" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/rpki | jq -e '.state == \"synchronised\" and .vrps_ipv4 == 3 and .vrps_ipv6 == 1'"
|
|
ok "synchronised with Routinator over RTR version $(api /api/rpki | jq .protocol_version): 3 IPv4 and 1 IPv6 VRPs"
|
|
|
|
# FRR announces 10.66.0.0/24, which is AS 64999's to announce.
|
|
wait_for "the invalid route to be refused" dut_lacks 10.66.0.0/24
|
|
wait_for "the invalid route to leave the kernel" dut_kernel_lacks -4 10.66.0.0/24
|
|
api "/api/routes/rejected?peer=frr4" | jq -e '.[0].by_reason["RPKI invalid"] == 1' >/dev/null \
|
|
|| fail "the refusal is not on record: $(api /api/routes/rejected)"
|
|
peer_is frr4 '.held_routes == 1' >/dev/null || fail "the invalid route is not held"
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "FRR's own route is not valid"
|
|
api "/api/routes/received?prefix=10.3.7.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
|
|
|| fail "BIRD's more specific route is not valid"
|
|
ok "FRR's invalid route is refused and held; the others are valid"
|
|
|
|
# The data changes, by way of a Routinator that is restarted: the
|
|
# route is authorised now.
|
|
sudo kill "$(cat "$work/routinator.pid")"; sleep 0.5
|
|
routinator_up 64602
|
|
wait_for "the held route to be taken" dut_has 10.66.0.0/24 frr4
|
|
wait_for "the route in the kernel" dut_kernel -4 "10.66.0.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
peer_is frr4 '.held_routes == 0' >/dev/null || fail "the route is still held"
|
|
ok "after Routinator restarts with new data, the held route is taken"
|
|
}
|
|
|
|
# Changes after the first exchange: new routes, withdrawals, and being
|
|
# asked to send everything again.
|
|
phase_refresh() {
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.9.0/24"
|
|
wait_for "FRR's new route" dut_has 10.2.9.0/24 frr4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.2.9.0/24 via 10.9.0.2 dev ix0 metric 20"
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.9.0/24"
|
|
wait_for "FRR's withdrawal" dut_lacks 10.2.9.0/24
|
|
wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.9.0/24
|
|
ok "a route FRR adds and withdraws comes and goes, in the table and the kernel"
|
|
|
|
# FRR asks for our routes again (ROUTE-REFRESH), and gets them.
|
|
local before
|
|
before=$(frr_neighbor 10.9.0.1 '.messageStats.updatesRecv')
|
|
vty -c "clear bgp ipv4 unicast 10.9.0.1 soft in"
|
|
wait_for "our routes to be sent again" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq '.[\"10.9.0.1\"].messageStats.updatesRecv')\" -gt $before ]"
|
|
frr_neighbor 10.9.0.1 '.bgpState == "Established"' >/dev/null || fail "the session did not survive a route refresh"
|
|
wait_for "our route still at FRR" frr_has ipv4 10.1.0.0/24
|
|
ok "FRR's route refresh is answered with our routes"
|
|
|
|
# BIRD sends its routes again unasked; nothing changes.
|
|
bird_ctl reload out dut4 >/dev/null
|
|
sleep 0.5
|
|
converged
|
|
ok "BIRD re-sending its routes changes nothing"
|
|
}
|
|
|
|
# Sessions that end in every way they can, and come back.
|
|
phase_flaps() {
|
|
for round in 1 2 3 4 5; do
|
|
vty -c "clear bgp *"
|
|
wait_for "FRR's routes to go (round $round)" dut_lacks 10.2.0.0/24
|
|
converged
|
|
done
|
|
ok "five resets by FRR: routes go and come back each time"
|
|
|
|
bird_ctl restart dut4 >/dev/null; bird_ctl restart dut6 >/dev/null
|
|
converged
|
|
ok "BIRD restarting its sessions"
|
|
|
|
# FRR dies without a word.
|
|
frr_down KILL
|
|
wait_for "FRR's routes to go" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
dut_has 10.3.0.0/24 bird4 >/dev/null || fail "BIRD's routes went with FRR's"
|
|
frr_up
|
|
converged
|
|
ok "FRR killed and started again"
|
|
|
|
# The wire to FRR is cut: nothing says so but the silence. The hold
|
|
# timer (nine seconds here) has to notice.
|
|
in_ns ix ip link set to-frr down
|
|
WAIT_TRIES=100 wait_for "the hold timer to end the session" dut_lacks 10.2.0.0/24
|
|
wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
|
|
in_ns ix ip link set to-frr up
|
|
converged
|
|
ok "a cut wire is noticed by the hold timer, and mended"
|
|
|
|
sudo kill -KILL "$(sudo cat "$work/bird.pid")"
|
|
wait_for "BIRD's routes to go" dut_lacks 10.3.0.0/24
|
|
sudo rm -f "$work/bird.ctl"
|
|
bird_up
|
|
converged
|
|
ok "BIRD killed and started again"
|
|
}
|
|
|
|
# The daemon itself stopping, in good order and not.
|
|
phase_restart() {
|
|
dut_down TERM
|
|
[ -z "$(in_ns dut ip route show proto bgp)" ] && [ -z "$(in_ns dut ip -6 route show proto bgp)" ] \
|
|
|| fail "routes left in the kernel after a clean stop: $(in_ns dut ip route show proto bgp)"
|
|
wait_for "FRR to drop our route" frr_lacks ipv4 10.1.0.0/24
|
|
frr_neighbor 10.9.0.1 '.lastNotificationReason // .lastResetDueTo | test("Cease|Notification"; "i")' >/dev/null \
|
|
|| fail "FRR was not told we were going: $(frr_neighbor 10.9.0.1 '{lastResetDueTo, lastNotificationReason}')"
|
|
dut_up
|
|
converged
|
|
ok "stopped in good order: kernel emptied, peers told; and back"
|
|
|
|
# Killed: its routes stay in the kernel. While it is dead BIRD stops
|
|
# announcing one of them; the daemon that starts next has to clear
|
|
# that one out and keep the rest.
|
|
dut_down KILL
|
|
dut_kernel -4 "10.3.7.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "the kernel's routes went with the killed daemon"
|
|
sed -i '/route 10.3.7.0\/24 blackhole;/d' "$work/bird.conf"
|
|
bird_ctl configure >/dev/null
|
|
dut_up
|
|
converged
|
|
wait_for "the stale route to be cleared" dut_kernel_lacks -4 10.3.7.0/24
|
|
ok "killed: routes stay in the kernel, and the next start clears the stale one"
|
|
}
|
|
|
|
# A full table's worth of routes from BIRD, into the table and the kernel.
|
|
phase_scale() {
|
|
# Out of 100.0.0.0/6 and up: as many /24s as asked for.
|
|
awk -v n="$scale_routes" 'BEGIN { for (i = 0; i < n; i++)
|
|
printf "route %d.%d.%d.0/24 blackhole;\n", 100 + int(i / 65536), int(i / 256) % 256, i % 256 }' \
|
|
> "$work/bird-table.conf"
|
|
bird_config "$work/bird-table.conf"
|
|
local base start
|
|
base=$(api /api/status | jq .received_prefixes)
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to arrive" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -ge $((base + scale_routes)) ]"
|
|
local arrived
|
|
arrived=$(date +%s.%N)
|
|
# By the daemon's own count first: listing a million routes with `ip`
|
|
# takes longer than installing them.
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.installed >= $scale_routes'"
|
|
local installed
|
|
installed=$(date +%s.%N)
|
|
local listed
|
|
listed=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed" -ge "$scale_routes" ] || fail "the kernel lists $listed routes, not $scale_routes"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "$scale_routes routes from BIRD: in the table after $(echo "$arrived - $start" | bc)s, in the kernel after $(echo "$installed - $start" | bc)s"
|
|
peer_is frr4 '.state == "Established"' >/dev/null || fail "the session with FRR did not survive the table"
|
|
|
|
# The daemon is killed with the table in the kernel, and started
|
|
# again: it has to find its routes there, take the table from BIRD
|
|
# anew, and end up with the same, none refused and none left over.
|
|
dut_down KILL
|
|
start=$(date +%s.%N)
|
|
dut_up
|
|
WAIT_TRIES=1500 wait_for "the table in the kernel again" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.synced and .installed >= $scale_routes'"
|
|
installed=$(date +%s.%N)
|
|
converged
|
|
local listed_again
|
|
listed_again=$(in_ns dut ip -4 route show proto bgp | wc -l)
|
|
[ "$listed_again" -eq "$listed" ] || fail "the kernel lists $listed_again routes after a restart; it listed $listed before"
|
|
# They were kept, not taken out and put back: next to nothing was
|
|
# removed on the way.
|
|
local removed
|
|
removed=$(api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }')
|
|
[ "${removed:-0}" -lt 100 ] || fail "$removed routes were removed from the kernel on the way"
|
|
api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
|
|
ok "killed and restarted under the table: in step again after $(echo "$installed - $start" | bc)s, the kernel's routes as before"
|
|
|
|
bird_config
|
|
start=$(date +%s.%N)
|
|
bird_ctl configure >/dev/null
|
|
WAIT_TRIES=1500 wait_for "the table to be withdrawn" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -le $((base + 1)) ]"
|
|
WAIT_TRIES=1500 wait_for "the table out of the kernel" \
|
|
bash -c "[ \"\$(sudo ip netns exec lab-dut ip -4 route show proto bgp | wc -l)\" -le $((base + 1)) ]"
|
|
ok "withdrawn again after $(echo "$(date +%s.%N) - $start" | bc)s"
|
|
converged
|
|
}
|
|
|
|
# What the daemon refuses: a peer that is not what it is configured as,
|
|
# and one that sends more than it may.
|
|
phase_limits() {
|
|
# FRR says it is our peer; here it is configured as our customer.
|
|
dut_down TERM
|
|
dut_config CUSTOMER
|
|
dut_up
|
|
wait_for "the sessions that should come up" established 3
|
|
wait_for "the role mismatch to be on record" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | .state != \"Established\" and (.last_error // \"\" | test(\"role mismatch\"; \"i\"))'"
|
|
sleep 1
|
|
peer_is frr4 '.state != "Established"' >/dev/null || fail "a session came up across a role mismatch"
|
|
frr_neighbor 10.9.0.1 '.bgpState != "Established"' >/dev/null || fail "FRR thinks the session is up"
|
|
ok "a peer whose role does not fit ours gets no session: $(peer_is frr4 '.last_error')"
|
|
|
|
# FRR sends at least two routes that pass; it may have one.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config PEER 1
|
|
dut_up
|
|
wait_for "the limit to end the session" \
|
|
bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq -e '.[\"10.9.0.1\"] | (.lastNotificationReason // \"\") | test(\"Maximum Number of Prefixes\"; \"i\")'"
|
|
wait_for "the peer to be held down" \
|
|
bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | (.last_error // \"\") | test(\"held down|prefix\"; \"i\")'"
|
|
dut_lacks 10.2.0.0/24 || fail "routes of a peer over its limit are still there"
|
|
ok "a peer over its prefix limit is told so (Cease / Maximum Number of Prefixes) and held down"
|
|
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.8.0/24"
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# FRR tries what a peer must not get away with: announcing our own
|
|
# prefix, announcing the exchange LAN itself (which would take the
|
|
# sessions with everyone on it), and sending our traffic to someone else.
|
|
refused_for() { # refused_for reason: how many of frr4's routes were refused for it
|
|
api "/api/routes/rejected?peer=frr4" | jq -e --arg why "$1" '.[0].by_reason[$why] // 0 | . >= 1'
|
|
}
|
|
phase_hijacks() {
|
|
frr_configure \
|
|
"ip prefix-list ELSEWHERE seq 5 permit 10.2.7.0/24" \
|
|
"route-map TO-DUT permit 10" "match ip address prefix-list ELSEWHERE" "set ip next-hop 10.9.0.3" \
|
|
"route-map TO-DUT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT out" \
|
|
"network 10.1.0.0/24" "network 10.9.0.0/24" "network 10.2.7.0/24"
|
|
|
|
# Our own prefix, from someone else.
|
|
wait_for "our own prefix to be refused" refused_for "prefix is our own"
|
|
dut_lacks 10.1.0.0/24 || fail "our own prefix was taken from FRR"
|
|
ok "our own prefix is not taken from a peer"
|
|
|
|
# A route whose next hop is BIRD, from FRR.
|
|
wait_for "the third-party next hop to be refused" refused_for "next hop is not the peer"
|
|
dut_lacks 10.2.7.0/24 || fail "a route by way of a third party was taken"
|
|
ok "a route whose next hop is someone else is not taken"
|
|
|
|
# The LAN's own prefix. The lab takes bogons, so the policy lets it
|
|
# by; it is still not a route the kernel could be given, and so not
|
|
# one to take: it would compete with the connected route.
|
|
wait_for "the LAN's prefix to be held" refused_for "next hop unreachable"
|
|
dut_lacks 10.9.0.0/24 || fail "the LAN's own prefix was taken from FRR"
|
|
in_ns dut ip -4 route show proto bgp | grep -q "^10.9.0.0/24 " && fail "the LAN's own prefix was installed by way of a peer"
|
|
in_ns dut ping -q -c 1 -W 2 10.9.0.3 >/dev/null || fail "the LAN is not reachable any more"
|
|
ok "a route for the exchange LAN itself is not taken, and stays out of the kernel"
|
|
|
|
frr_configure \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"no network 10.1.0.0/24" "no network 10.9.0.0/24" "no network 10.2.7.0/24" \
|
|
"no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# FRR and BIRD both have a route for one prefix, and FRR's is the better.
|
|
# But its next hop is nowhere the machine can reach: then BIRD's is the
|
|
# one chosen and installed (RFC 4271 §9.1.2.1), until the next hop is
|
|
# somewhere, and again when it no longer is.
|
|
best_is() { # best_is prefix peer-name
|
|
api "/api/routes/received?prefix=$1&best=true" | jq -e --arg peer "$2" '.routes | length == 1 and .[0].peer_name == $peer'
|
|
}
|
|
phase_nexthops() {
|
|
dut_down TERM
|
|
dut_config PEER 2000000 "accept_third_party_next_hop: true"
|
|
dut_up
|
|
converged
|
|
# A prefix nobody has said anything about in the RPKI, from both.
|
|
echo "route 10.77.0.0/24 blackhole;" > "$work/bird-both.conf"
|
|
bird_config "$work/bird-both.conf"
|
|
bird_ctl configure >/dev/null
|
|
wait_for "BIRD's route for it" dut_has 10.77.0.0/24 bird4
|
|
frr_configure \
|
|
"ip prefix-list FAR seq 5 permit 10.77.0.0/24" \
|
|
"route-map TO-DUT permit 10" "match ip address prefix-list FAR" "set ip next-hop 10.250.0.1" \
|
|
"route-map TO-DUT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT out" "network 10.77.0.0/24"
|
|
|
|
wait_for "FRR's route to be held" refused_for "next hop unreachable"
|
|
peer_is frr4 '.held_routes >= 1' >/dev/null || fail "the route is not held: $(peer_is frr4 .)"
|
|
best_is 10.77.0.0/24 bird4 >/dev/null || fail "BIRD's route is not the one chosen"
|
|
dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route is not in the kernel"
|
|
ok "a better route whose next hop is nowhere is held; the one that works is chosen and installed"
|
|
|
|
# The next hop becomes a neighbor.
|
|
in_ns dut ip addr add 10.250.0.9/24 dev lan
|
|
wait_for "FRR's route to be taken" best_is 10.77.0.0/24 frr4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.250.0.1 dev lan metric 20"
|
|
ok "when its next hop can be reached the better route is taken, chosen and installed"
|
|
|
|
# And stops being one.
|
|
in_ns dut ip addr del 10.250.0.9/24 dev lan
|
|
wait_for "BIRD's route to be chosen again" best_is 10.77.0.0/24 bird4
|
|
wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20"
|
|
peer_is frr4 '.held_routes >= 1' >/dev/null || fail "FRR's route is not held again"
|
|
ok "and when it no longer can, the other route is back"
|
|
|
|
frr_configure \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"no network 10.77.0.0/24" "no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
|
|
bird_config
|
|
bird_ctl configure >/dev/null
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# Routes are passed on: FRR becomes our customer and BIRD stays our
|
|
# provider, so what FRR announces goes up to BIRD with our AS on the path,
|
|
# what BIRD announces goes down to FRR marked "only to customer", and
|
|
# packets go between their networks through us. A community on one of
|
|
# FRR's routes keeps it from BIRD.
|
|
phase_transit() {
|
|
dut_down TERM
|
|
dut_config CUSTOMER 2000000 "" "" "" \
|
|
'export { learned_routes: true }
|
|
control_communities: true'
|
|
in_ns dut sysctl -qw net.ipv4.ip_forward=1
|
|
dut_up
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \
|
|
"ip prefix-list KEPT seq 5 permit 10.2.9.0/24" \
|
|
"route-map TO-DUT-TRANSIT permit 10" "match ip address prefix-list KEPT" "set large-community 64601:0:4200000003" \
|
|
"route-map TO-DUT-TRANSIT permit 20" \
|
|
"router bgp 64602" "address-family ipv4 unicast" \
|
|
"neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "network 10.2.9.0/24"
|
|
wait_for "four sessions" established 4
|
|
wait_for "FRR's routes at the daemon" dut_has 10.2.9.0/24 frr4
|
|
|
|
# Up to the provider, with our AS on the path; not the one that says
|
|
# not to.
|
|
wait_for "FRR's route at BIRD, by way of us" bash -c "sudo birdc -s $work/bird.ctl show route for 10.2.0.0/24 all | grep -q 'BGP.as_path: 64601 64602'"
|
|
sleep 1
|
|
bird_ctl show route for 10.2.9.0/24 all | grep -q "64601 64602" && fail "BIRD was told the route whose community says not to"
|
|
ok "the customer's routes reach the provider with our AS on the path; one kept back by its community"
|
|
|
|
# Down to the customer, marked.
|
|
wait_for "BIRD's route at FRR, by way of us" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths[0].aspath.string == \"64601 4200000003\"'"
|
|
# Marked on the way in from the provider, with the provider's AS, and
|
|
# the mark kept as it goes down (RFC 9234 §5).
|
|
vty -c "show bgp ipv4 unicast 10.3.0.0/24" | grep -q "otc 4200000003" || fail "the route to the customer is not marked only to customer"
|
|
ok "the provider's routes reach the customer, marked only to customer"
|
|
|
|
# And packets follow them, through us.
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.3.0.1 >/dev/null || fail "no ping from the customer's network to the provider's through us"
|
|
in_ns bird ping -q -c 2 -W 2 -I 10.3.0.1 10.2.0.1 >/dev/null || fail "no ping from the provider's network to the customer's through us"
|
|
ok "packets go between the customer's and the provider's networks through us"
|
|
|
|
frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" \
|
|
"address-family ipv4 unicast" "no network 10.2.9.0/24" "no neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "no route-map TO-DUT-TRANSIT"
|
|
dut_down TERM
|
|
dut_config
|
|
dut_up
|
|
converged
|
|
ok "back to normal"
|
|
}
|
|
|
|
# The exchange LAN loses its IPv4 addresses. One session over IPv6 with
|
|
# each router carries both families, and IPv4 routes have IPv6 next hops
|
|
# (RFC 8950), in BGP and in the kernel. Run after the rest: the lab stays
|
|
# that way.
|
|
phase_v6only() {
|
|
dut_down TERM
|
|
frr_down TERM
|
|
sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
|
|
local n=0 name
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0
|
|
done
|
|
frr_config_v6only; frr_up
|
|
bird_config_v6only; bird_up
|
|
dut_config_v6only; dut_up
|
|
|
|
wait_for "two sessions over IPv6" established 2
|
|
api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
|
|
|| fail "the sessions did not agree on IPv4 with IPv6 next hops: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
|
|
ok "both sessions carry IPv4 and IPv6, with IPv6 next hops for IPv4 (RFC 8950)"
|
|
|
|
# What they send us: IPv4 routes whose next hop is an IPv6 address,
|
|
# and the same in the kernel.
|
|
wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
|
|
wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
|
|
api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].summary.next_hop | startswith("2001:db8:9::2")' >/dev/null \
|
|
|| fail "FRR's IPv4 route has the next hop $(api "/api/routes/received?prefix=10.2.0.0/24" | jq '.routes[0].summary.next_hop')"
|
|
wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 2001:db8:9::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 2001:db8:9::3 dev ix0 metric 20"
|
|
wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
|
|
ok "their IPv4 routes arrive with IPv6 next hops and are in the kernel that way"
|
|
|
|
# What we send them.
|
|
wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
|
|
vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].nexthops | any(.ip == "2001:db8:9::1" and .afi == "ipv6")' >/dev/null \
|
|
|| fail "FRR's copy of our IPv4 route has the next hops $(vty -c 'show bgp ipv4 unicast 10.1.0.0/24 json' | jq -c '.paths[0].nexthops')"
|
|
wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
|
|
bird_ctl show route for 10.1.0.0/24 all | grep -q "BGP.next_hop: 2001:db8:9::1" \
|
|
|| fail "BIRD's copy of our IPv4 route: $(bird_ctl show route for 10.1.0.0/24 all)"
|
|
wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
ok "FRR and BIRD have our IPv4 route with our IPv6 address as next hop"
|
|
|
|
# IPv4 packets across a link that has no IPv4 addresses.
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
ok "IPv4 packets cross the IPv6-only link, both ways"
|
|
|
|
# A withdrawal by FRR (in MP_UNREACH_NLRI) takes the route out.
|
|
frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.5.0/24"
|
|
wait_for "FRR's withdrawal" dut_lacks 10.2.5.0/24
|
|
wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.5.0/24
|
|
dut_has 10.2.0.0/24 frr >/dev/null || fail "the withdrawal took the other route along"
|
|
ok "an IPv4 route withdrawn over the IPv6 session goes"
|
|
}
|
|
|
|
# The exchange LAN loses its global IPv6 addresses too: all that is left on
|
|
# it are link-local ones, the same on every link a router has, which mean
|
|
# something only together with the interface. The sessions are between
|
|
# those, as are the next hops, for both families. Run after v6only.
|
|
dut_config_linklocal() {
|
|
cat > "$work/dut.textpb" <<END
|
|
router_id: "10.9.0.1"
|
|
local_asn: 64601
|
|
listen_endpoints { address: "::" port: 179 }
|
|
connect_retry_seconds: 1
|
|
peers {
|
|
name: "frr" endpoint { address: "fe80::2%ix0" } remote_asn: 64602
|
|
role: PEER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
peers {
|
|
name: "bird" endpoint { address: "fe80::3%ix0" } remote_asn: 4200000003
|
|
role: PROVIDER
|
|
import { accept_bogons: true max_prefixes: 1000 }
|
|
}
|
|
originate { prefix: "10.1.0.0/24" }
|
|
originate { prefix: "2001:db8:1::/48" }
|
|
kernel { stale_seconds: 3 }
|
|
http_endpoint { address: "127.0.0.1" port: 8080 }
|
|
END
|
|
}
|
|
|
|
# FRR is told the interface and not the address ("BGP unnumbered"), which
|
|
# is how it is mostly set up for this. It learns who is on the link from
|
|
# router advertisements, which the daemon does not send: radvd does, as it
|
|
# would have to beside the daemon on a real router with such a neighbor.
|
|
radvd_up() {
|
|
cat > "$work/radvd.conf" <<END
|
|
interface ix0 {
|
|
AdvSendAdvert on;
|
|
MinRtrAdvInterval 3;
|
|
MaxRtrAdvInterval 4;
|
|
AdvDefaultLifetime 0;
|
|
};
|
|
END
|
|
in_ns dut "$(command -v radvd || echo /usr/sbin/radvd)" -C "$work/radvd.conf" -p "$work/radvd.pid" -m logfile -l "$work/radvd.log"
|
|
}
|
|
|
|
frr_config_linklocal() {
|
|
sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
|
|
frr defaults traditional
|
|
hostname frr
|
|
log file $work/frr.log
|
|
!
|
|
router bgp 64602
|
|
bgp router-id 10.9.0.2
|
|
no bgp ebgp-requires-policy
|
|
no bgp network import-check
|
|
timers bgp 3 9
|
|
neighbor ix0 interface remote-as 64601
|
|
neighbor ix0 local-role peer
|
|
neighbor ix0 timers connect 1
|
|
neighbor ix0 capability extended-nexthop
|
|
!
|
|
address-family ipv4 unicast
|
|
network 10.2.0.0/24
|
|
network 10.2.5.0/24
|
|
neighbor ix0 activate
|
|
exit-address-family
|
|
!
|
|
address-family ipv6 unicast
|
|
network 2001:db8:2::/48
|
|
neighbor ix0 activate
|
|
exit-address-family
|
|
!
|
|
END
|
|
}
|
|
|
|
bird_config_linklocal() {
|
|
cat > "$work/bird.conf" <<END
|
|
log "$work/bird.log" all;
|
|
router id 10.9.0.3;
|
|
protocol device {}
|
|
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
|
|
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
|
|
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
|
|
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
|
|
protocol bgp dut {
|
|
local as 4200000003;
|
|
neighbor fe80::1%ix0 as 64601;
|
|
local role provider;
|
|
connect retry time 1;
|
|
# Back soon after the hold timer ran out, which the phase makes it do.
|
|
error wait time 1, 2;
|
|
hold time 9;
|
|
graceful restart off;
|
|
ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
|
|
ipv6 { import all; export where source = RTS_STATIC; };
|
|
}
|
|
END
|
|
}
|
|
|
|
phase_linklocal() {
|
|
dut_down TERM
|
|
frr_down TERM
|
|
sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
|
|
local n=0 name
|
|
for name in dut frr bird; do
|
|
n=$((n + 1))
|
|
in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0 2>/dev/null || true
|
|
in_ns "$name" ip -6 addr flush dev ix0
|
|
in_ns "$name" ip -6 addr add "fe80::$n/64" dev ix0 nodad
|
|
# The same address on another link: it must not be mistaken for
|
|
# the one the session is on.
|
|
in_ns "$name" ip -6 addr flush dev lan scope link
|
|
in_ns "$name" ip -6 addr add "fe80::$n/64" dev lan nodad
|
|
done
|
|
in_ns dut sysctl -qw net.ipv6.conf.ix0.forwarding=1
|
|
radvd_up
|
|
frr_config_linklocal; frr_up
|
|
bird_config_linklocal; bird_up
|
|
dut_config_linklocal; dut_up
|
|
|
|
wait_for "two sessions between link-local addresses" established 2
|
|
peer_is frr '.peer_addr | startswith("[fe80::2%ix0]:")' >/dev/null \
|
|
&& peer_is bird '.peer_addr | startswith("[fe80::3%ix0]:")' >/dev/null \
|
|
&& peer_is frr '.local_addr | startswith("[fe80::1%ix0]:")' >/dev/null \
|
|
|| fail "the peers are at $(api /api/peers | jq -c '[.[] | {name, peer_addr, local_addr}]')"
|
|
api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
|
|
|| fail "the sessions did not agree on both families: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
|
|
ok "sessions with FRR and BIRD at fe80::2%ix0 and fe80::3%ix0, for both families"
|
|
|
|
# Their routes: link-local next hops, on the session's interface.
|
|
wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
|
|
wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
|
|
wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 fe80::3 dev ix0 metric 20"
|
|
wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via fe80::2 dev ix0 metric 20"
|
|
wait_for "BIRD's IPv6 route in the kernel" dut_kernel -6 "2001:db8:3::/48 via fe80::3 dev ix0 metric 20"
|
|
api /api/kernel | jq -e '.unresolved == {}' >/dev/null || fail "some next hops did not resolve: $(api /api/kernel)"
|
|
ok "their routes of both families are in the kernel by way of fe80::… on ix0"
|
|
|
|
# Ours, at them.
|
|
wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
|
|
wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
|
|
wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
|
|
wait_for "our IPv6 route at BIRD" bird_has 2001:db8:1::/48
|
|
wait_for "our IPv4 route in FRR's kernel" in_ns frr ip -4 route get 10.1.0.1
|
|
wait_for "our IPv6 route in BIRD's kernel" in_ns bird ip -6 route get 2001:db8:1::1 from 2001:db8:3::1
|
|
ok "FRR and BIRD have our routes of both families"
|
|
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
|
|
in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
|
|
in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
|
|
in_ns bird ping -q -c 2 -W 2 -I 2001:db8:3::1 2001:db8:1::1 >/dev/null || fail "no IPv6 ping from BIRD's lan"
|
|
ok "IPv4 and IPv6 packets cross a link with only link-local addresses, both ways"
|
|
|
|
# The interface goes and comes back, as a tunnel does when it is
|
|
# restarted: the sessions come back by themselves.
|
|
in_ns dut ip link set ix0 down
|
|
wait_for "the sessions to go" established 0
|
|
in_ns dut ip link set ix0 up
|
|
in_ns dut ip -6 addr flush dev ix0
|
|
in_ns dut ip -6 addr add fe80::1/64 dev ix0 nodad
|
|
wait_for "the sessions to come back" established 2
|
|
wait_for "FRR's IPv4 route in the kernel again" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
|
|
ok "the sessions and the routes come back after the interface was down"
|
|
}
|
|
|
|
# --- main -----------------------------------------------------------------------
|
|
|
|
case "${1:-test}" in
|
|
down) down; exit 0 ;;
|
|
up) lab_up; converged; say "the lab is up; work files in $work"; exit 0 ;;
|
|
test) ;;
|
|
*) fail "usage: $0 [test|up|down]" ;;
|
|
esac
|
|
|
|
on_exit() {
|
|
status=$?
|
|
if [ "$status" -ne 0 ]; then
|
|
say "--- the daemon's last words:" >&2
|
|
tail -n 25 "$work/dut.log" >&2 || true
|
|
fi
|
|
[ -n "${KEEP:-}" ] || down
|
|
exit "$status"
|
|
}
|
|
trap on_exit EXIT
|
|
lab_up
|
|
say "lab: converging"
|
|
converged
|
|
for phase in $phases; do
|
|
say "phase: $phase"
|
|
"phase_$phase"
|
|
done
|
|
say "all phases passed: $phases"
|