#!/bin/bash
# An interoperability lab: this daemon against FRR, BIRD and Routinator,
# each in a network namespace of its own on one exchange LAN.
#
#                      10.9.0.0/24, 2001:db8:9::/64
#        ┌───────────────────┬──────────────────────┐
#   dut (.1) AS 64601     frr (.2) AS 64602     bird (.3) AS 4200000003
#   lan 10.1.0.1/24       lan 10.2.0.1/24       lan 10.3.0.1/24
#       2001:db8:1::1/48      2001:db8:2::1/48      2001:db8:3::1/48
#   + Routinator on 127.0.0.1:3323
#
# "dut" is the daemon under test. FRR is its peer, BIRD its provider; both
# say so in their OPEN (RFC 9234). Routinator serves validated ROA payloads
# made up for the lab from a local exceptions file, so nothing is fetched
# from the Internet.
#
#   scripts/interop/lab.sh            run every phase, then take the lab down
#   scripts/interop/lab.sh up         bring the lab up and leave it up
#   scripts/interop/lab.sh down       take it down
#   PHASES="basics scale" scripts/interop/lab.sh     only these phases
#   SCALE_ROUTES=1000000 ...          the size of the table BIRD sends
#
# Needs Linux, sudo, and: frr, bird2, radvd, jq, routinator, a built `bgp`
# (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS;
# scripts/interop/install.sh installs what is missing on Ubuntu.
set -euo pipefail

cd "$(dirname "$0")/../.."
bgp="${CARGO_TARGET_DIR:-target}/release/bgp"
work=/tmp/ferrix-lab
pathspace=ferrixlab
all_phases="basics roles rpki refresh flaps restart graceful scale limits hijacks nexthops transit collector leaks v6only linklocal"
phases="${PHASES:-$all_phases}"
scale_routes="${SCALE_ROUTES:-200000}"

# --- plumbing ---------------------------------------------------------------

in_ns() { local name=$1; shift; sudo ip netns exec "lab-$name" "$@"; }
api() { in_ns dut curl -sf "http://127.0.0.1:8080$1"; }
vty() { in_ns frr vtysh -N "$pathspace" "$@" 2>/dev/null; }
bird_ctl() { sudo birdc -s "$work/bird.ctl" "$@"; }
say() { printf '%s\n' "$*"; }
ok() { say "  ok: $*"; }
fail() { say "FAILED: $*" >&2; exit 1; }

# wait_for "what" command...: until the command succeeds.
wait_for() {
    local what=$1; shift
    local tries=${WAIT_TRIES:-150}
    for _ in $(seq "$tries"); do
        if "$@" >/dev/null 2>&1; then return 0; fi
        sleep 0.2
    done
    say "timed out waiting for: $what" >&2
    return 1
}

down() {
    for pidfile in "$work"/dut.pid "$work"/routinator.pid; do
        [ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true
    done
    [ -f "$work/radvd.pid" ] && sudo kill "$(sudo cat "$work/radvd.pid")" 2>/dev/null || true
    [ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true
    for daemon in bgpd zebra; do
        local pidfile="/var/run/frr/$pathspace/$daemon.pid"
        sudo test -f "$pidfile" && sudo kill "$(sudo cat "$pidfile")" 2>/dev/null || true
    done
    sleep 0.3
    for name in dut frr bird ix; do sudo ip netns del "lab-$name" 2>/dev/null || true; done
    sudo rm -rf "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
}

# --- the network ------------------------------------------------------------

network_up() {
    sudo modprobe dummy 2>/dev/null || true
    sudo ip netns add lab-ix
    in_ns ix ip link add ix type bridge
    in_ns ix ip link set ix up
    local n=0
    for name in dut frr bird; do
        n=$((n + 1))
        sudo ip netns add "lab-$name"
        sudo ip link add ix0 netns "lab-$name" type veth peer name "to-$name" netns lab-ix
        in_ns ix ip link set "to-$name" master ix up
        in_ns "$name" ip link set lo up
        in_ns "$name" ip addr add "10.9.0.$n/24" dev ix0
        in_ns "$name" ip -6 addr add "2001:db8:9::$n/64" dev ix0 nodad
        in_ns "$name" ip link set ix0 up
        in_ns "$name" ip link add lan type dummy
        in_ns "$name" ip addr add "10.$n.0.1/24" dev lan
        in_ns "$name" ip -6 addr add "2001:db8:$n::1/48" dev lan nodad
        in_ns "$name" ip link set lan up
    done
}

# --- Routinator ---------------------------------------------------------------

# routinator_up [asn]: with `asn` the AS that may originate 10.66.0.0/24.
routinator_up() {
    local owner=${1:-64999}
    mkdir -p "$work/rpki/repository" "$work/rpki/tals"
    # 10.2.0.0/24 is FRR's to originate. 10.66.0.0/24 is someone else's,
    # and FRR announces it all the same. 10.3.0.0/16 is BIRD's, as far as
    # /24.
    cat > "$work/rpki/slurm.json" <<END
{
  "slurmVersion": 1,
  "validationOutputFilters": { "prefixFilters": [], "bgpsecFilters": [] },
  "locallyAddedAssertions": {
    "prefixAssertions": [
      { "asn": 64602, "prefix": "10.2.0.0/24", "maxPrefixLength": 24 },
      { "asn": $owner, "prefix": "10.66.0.0/24", "maxPrefixLength": 24 },
      { "asn": 4200000003, "prefix": "10.3.0.0/16", "maxPrefixLength": 24 },
      { "asn": 64602, "prefix": "2001:db8:2::/48", "maxPrefixLength": 48 }
    ],
    "bgpsecAssertions": []
  }
}
END
    # By its full name: root's PATH does not have a user's ~/.cargo/bin.
    in_ns dut "$(command -v routinator)" --no-rir-tals --extra-tals-dir "$work/rpki/tals" \
        -r "$work/rpki/repository" -x "$work/rpki/slurm.json" \
        --logfile "$work/routinator.log" \
        server --rtr 127.0.0.1:3323 --http 127.0.0.1:8323 --refresh 3600 \
        > /dev/null 2>&1 &
    echo $! | sudo tee "$work/routinator.pid" > /dev/null
}

# --- the daemon under test ------------------------------------------------------

# dut_config [frr-role] [frr-max-prefixes] [more of frr4's import policy]
#            [more of frr4] [more of bird4] [more of the configuration]
dut_config() {
    local frr_role=${1:-PEER} frr_limit=${2:-2000000} frr_import=${3:-}
    local frr_more=${4:-} bird_more=${5:-} more=${6:-}
    cat > "$work/dut.textpb" <<END
router_id: "10.9.0.1"
local_asn: 64601
listen_endpoints { address: "10.9.0.1" port: 179 }
listen_endpoints { address: "2001:db8:9::1" port: 179 }
connect_retry_seconds: 1
peers {
  name: "frr4" endpoint { address: "10.9.0.2" } remote_asn: 64602
  address_families: IPV4_UNICAST
  role: $frr_role
  import { accept_bogons: true max_prefixes: $frr_limit hold_down_seconds: 5 $frr_import }
  $frr_more
}
peers {
  name: "frr6" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
  address_families: IPV6_UNICAST
  role: PEER
  import { accept_bogons: true max_prefixes: 2000000 }
}
peers {
  name: "bird4" endpoint { address: "10.9.0.3" } remote_asn: 4200000003
  address_families: IPV4_UNICAST
  role: PROVIDER require_role: true
  add_path: ADD_PATH_RECEIVE
  import { accept_bogons: true max_prefixes: 2000000 }
  $bird_more
}
peers {
  name: "bird6" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
  address_families: IPV6_UNICAST
  role: PROVIDER require_role: true
  import { accept_bogons: true max_prefixes: 2000000 }
}
originate { prefix: "10.1.0.0/24" communities: "64601:100" }
originate { prefix: "2001:db8:1::/48" large_communities: "64601:1:2" }
kernel { stale_seconds: 3 }
rpki { caches { address: "127.0.0.1" port: 3323 } retry_seconds: 1 }
http_endpoint { address: "127.0.0.1" port: 8080 }
$more
END
}

dut_config_v6only() {
    cat > "$work/dut.textpb" <<END
router_id: "10.9.0.1"
local_asn: 64601
listen_endpoints { address: "2001:db8:9::1" port: 179 }
connect_retry_seconds: 1
peers {
  name: "frr" endpoint { address: "2001:db8:9::2" } remote_asn: 64602
  role: PEER
  import { accept_bogons: true max_prefixes: 1000 }
}
peers {
  name: "bird" endpoint { address: "2001:db8:9::3" } remote_asn: 4200000003
  role: PROVIDER
  import { accept_bogons: true max_prefixes: 1000 }
}
originate { prefix: "10.1.0.0/24" }
originate { prefix: "2001:db8:1::/48" }
kernel { stale_seconds: 3 }
http_endpoint { address: "127.0.0.1" port: 8080 }
END
}

dut_up() {
    sudo ip netns exec lab-dut \
        setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups \
        --inh-caps +net_admin,+net_bind_service --ambient-caps +net_admin,+net_bind_service \
        "$bgp" "$work/dut.textpb" >> "$work/dut.log" 2>&1 &
    echo $! > "$work/dut.pid"
    # Not the shell's to report on when it is killed, which the tests do.
    disown
    wait_for "the daemon's status interface" api /api/status
}

# dut_down [signal]: stops the daemon and waits until it has gone.
dut_down() {
    local signal=${1:-TERM}
    # The pid recorded is sudo's; the daemon is the one in the namespace
    # that is not Routinator.
    local pid
    for pid in $(sudo ip netns pids lab-dut); do
        if [ "$(cat "/proc/$pid/comm" 2>/dev/null)" = bgp ]; then
            sudo kill "-$signal" "$pid"
            while [ -e "/proc/$pid" ]; do sleep 0.05; done
        fi
    done
}

# --- FRR ------------------------------------------------------------------------

frr_config() {
    sudo mkdir -p "/etc/frr/$pathspace" "/var/run/frr/$pathspace"
    sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
frr defaults traditional
hostname frr
log file $work/frr.log
!
router bgp 64602
 bgp router-id 10.9.0.2
 no bgp ebgp-requires-policy
 no bgp network import-check
 timers bgp 3 9
 neighbor 10.9.0.1 remote-as 64601
 neighbor 10.9.0.1 local-role peer
 neighbor 10.9.0.1 timers connect 1
 neighbor 2001:db8:9::1 remote-as 64601
 neighbor 2001:db8:9::1 local-role peer
 neighbor 2001:db8:9::1 timers connect 1
 !
 address-family ipv4 unicast
  network 10.2.0.0/24
  network 10.66.0.0/24
  neighbor 10.9.0.1 activate
  no neighbor 2001:db8:9::1 activate
 exit-address-family
 !
 address-family ipv6 unicast
  network 2001:db8:2::/48
  neighbor 2001:db8:9::1 activate
 exit-address-family
!
END
}

# One session, over IPv6, for both families: IPv4 routes go with IPv6 next
# hops (RFC 8950).
frr_config_v6only() {
    sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
frr defaults traditional
hostname frr
log file $work/frr.log
!
router bgp 64602
 bgp router-id 10.9.0.2
 no bgp ebgp-requires-policy
 no bgp network import-check
 timers bgp 3 9
 neighbor 2001:db8:9::1 remote-as 64601
 neighbor 2001:db8:9::1 local-role peer
 neighbor 2001:db8:9::1 timers connect 1
 neighbor 2001:db8:9::1 capability extended-nexthop
 !
 address-family ipv4 unicast
  network 10.2.0.0/24
  network 10.2.5.0/24
  neighbor 2001:db8:9::1 activate
 exit-address-family
 !
 address-family ipv6 unicast
  network 2001:db8:2::/48
  neighbor 2001:db8:9::1 activate
 exit-address-family
!
END
}

frr_up() {
    sudo touch "/etc/frr/$pathspace/vtysh.conf" "$work/frr.log"
    sudo chown -R frr:frr "/etc/frr/$pathspace" "/var/run/frr/$pathspace" "$work/frr.log"
    for daemon in zebra bgpd; do
        # Run from a copy: the distribution's AppArmor profile for the
        # daemon in its usual place does not allow it a directory of its
        # own, and the machine's profiles are not the lab's to change.
        sudo cp "/usr/lib/frr/$daemon" "$work/$daemon"
        in_ns frr "$work/$daemon" -d -N "$pathspace" \
            -f "/etc/frr/$pathspace/frr.conf" > "$work/$daemon.out" 2>&1 \
            || fail "$daemon did not start: $(cat "$work/$daemon.out")"
        # zebra makes the directory its sockets are in root's.
        sudo chown -R frr:frr "/var/run/frr/$pathspace"
    done
}

frr_down() {
    local signal=${1:-TERM} daemon
    for daemon in bgpd zebra; do
        local pidfile="/var/run/frr/$pathspace/$daemon.pid"
        sudo test -f "$pidfile" && sudo kill "-$signal" "$(sudo cat "$pidfile")" 2>/dev/null || true
    done
    sleep 0.5
}

# --- BIRD -----------------------------------------------------------------------

# bird_config [extra-static-routes-file] [more-configuration]
bird_config() {
    local extra=${1:-} more=${2:-}
    cat > "$work/bird.conf" <<END
log "$work/bird.log" all;
router id 10.9.0.3;
protocol device {}
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
protocol static static4 {
  ipv4;
  route 10.3.0.0/24 blackhole;
  route 10.3.7.0/24 blackhole;
  ${extra:+include \"$extra\";}
}
protocol static static6 {
  ipv6;
  route 2001:db8:3::/48 blackhole;
}
template bgp dut {
  local as 4200000003;
  local role provider;
  connect retry time 1;
  hold time 9;
  graceful restart off;
}
protocol bgp dut4 from dut {
  neighbor 10.9.0.1 as 64601;
  source address 10.9.0.3;
  ipv4 { import all; export where source = RTS_STATIC; add paths tx; };
}
protocol bgp dut6 from dut {
  neighbor 2001:db8:9::1 as 64601;
  source address 2001:db8:9::3;
  ipv6 { import all; export where source = RTS_STATIC; };
}
$more
END
}

bird_config_v6only() {
    cat > "$work/bird.conf" <<END
log "$work/bird.log" all;
router id 10.9.0.3;
protocol device {}
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
protocol bgp dut {
  local as 4200000003;
  neighbor 2001:db8:9::1 as 64601;
  source address 2001:db8:9::3;
  local role provider;
  connect retry time 1;
  hold time 9;
  graceful restart off;
  ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
  ipv6 { import all; export where source = RTS_STATIC; };
}
END
}

bird_up() {
    sudo ip netns exec lab-bird bird -c "$work/bird.conf" -s "$work/bird.ctl" -P "$work/bird.pid"
}

# --- what is looked at ------------------------------------------------------------

established() {  # how many of the daemon's sessions are up
    api /api/peers | jq -e --argjson n "$1" '[.[] | select(.state == "Established")] | length == $n'
}
dut_has() {  # dut_has prefix peer-name: the daemon has the route, from that peer
    api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer)'
}
dut_lacks() { ! api "/api/routes/received?prefix=$1" | jq -e '.routes | length > 0' >/dev/null; }
dut_kernel() {  # dut_kernel family "route as ip shows it, without nhid"
    # All of it is read: with a table's worth of routes, a grep that
    # stopped at the first match would leave `ip` writing to nobody.
    in_ns dut ip "$1" route show proto bgp | sed 's/  */ /g; s/ nhid [0-9]*//; s/ pref medium//; s/ *$//' | grep -xF "$2" >/dev/null
}
dut_kernel_lacks() { ! in_ns dut ip "$1" route show proto bgp | grep "^$2 " >/dev/null; }
frr_has() {  # frr_has family prefix: FRR has a route for it by way of the daemon
    vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | any(.aspath.string == "64601")'
}
frr_lacks() { ! vty -c "show bgp $1 unicast $2 json" | jq -e '.paths | length > 0' >/dev/null; }
bird_has() { bird_ctl show route for "$1" all | grep -q "BGP.as_path: 64601"; }

lab_up() {
    down
    sudo rm -rf "$work"; mkdir -p "$work"; chmod 777 "$work"
    [ -x "$bgp" ] || fail "$bgp not found: cargo build --release -p bgp"
    network_up
    routinator_up
    frr_config; frr_up
    bird_config; bird_up
    dut_config; dut_up
}

converged() {
    wait_for "four sessions" established 4
    wait_for "FRR's routes at the daemon" dut_has 10.2.0.0/24 frr4
    wait_for "BIRD's routes at the daemon" dut_has 10.3.0.0/24 bird4
    wait_for "FRR's IPv6 route" dut_has 2001:db8:2::/48 frr6
    wait_for "BIRD's IPv6 route" dut_has 2001:db8:3::/48 bird6
    wait_for "routes in the kernel" dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20"
    wait_for "routes in the kernel" dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20"
    wait_for "routes in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
    wait_for "routes in the kernel" dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20"
    wait_for "the daemon's route at FRR" frr_has ipv4 10.1.0.0/24
    wait_for "the daemon's IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
    wait_for "the daemon's route at BIRD" bird_has 10.1.0.0/24
    wait_for "the daemon's IPv6 route at BIRD" bird_has 2001:db8:1::/48
}

# --- the phases ---------------------------------------------------------------------

frr_neighbor() {  # frr_neighbor address jq-filter
    vty -c "show bgp neighbors $1 json" | jq -e --arg n "$1" ".[\$n] | $2"
}
frr_configure() {  # each argument a line of configuration
    local args=(-c "configure terminal")
    for line in "$@"; do args+=(-c "$line"); done
    vty "${args[@]}"
}
peer_is() {  # peer_is name jq-filter: of the daemon's view of a peer
    api /api/peers | jq -e --arg n "$1" ".[] | select(.name == \$n) | $2"
}

# Sessions come up, routes go both ways with their attributes intact, and
# packets follow them.
phase_basics() {
    # What FRR has of ours: the path, the next hop, the communities.
    local route
    route=$(vty -c "show bgp ipv4 unicast 10.1.0.0/24 json")
    jq -e '.paths[0] | .aspath.string == "64601" and .nexthops[0].ip == "10.9.0.1"
           and .community.string == "64601:100" and .origin == "IGP"' <<<"$route" >/dev/null \
        || fail "FRR's copy of 10.1.0.0/24 is not as sent: $route"
    route=$(vty -c "show bgp ipv6 unicast 2001:db8:1::/48 json")
    jq -e '.paths[0] | .aspath.string == "64601" and (.nexthops | any(.ip == "2001:db8:9::1"))
           and .largeCommunity.string == "64601:1:2"' <<<"$route" >/dev/null \
        || fail "FRR's copy of 2001:db8:1::/48 is not as sent: $route"
    ok "FRR has our routes, with path, next hop and communities"

    route=$(bird_ctl show route for 10.1.0.0/24 all)
    grep -q "BGP.next_hop: 10.9.0.1" <<<"$route" && grep -q "BGP.community: (64601,100)" <<<"$route" \
        || fail "BIRD's copy of 10.1.0.0/24 is not as sent: $route"
    route=$(bird_ctl show route for 2001:db8:1::/48 all)
    grep -q "BGP.next_hop: 2001:db8:9::1" <<<"$route" && grep -q "BGP.large_community: (64601, 1, 2)" <<<"$route" \
        || fail "BIRD's copy of 2001:db8:1::/48 is not as sent: $route"
    ok "BIRD has our routes, with path, next hop and communities"

    # What we have of theirs. BIRD's AS number needs four octets, and it
    # sends path identifiers (ADD-PATH).
    api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0]
        | .summary.as_path == [4200000003] and .summary.next_hop == "10.9.0.3" and .path_id != 0' >/dev/null \
        || fail "our copy of BIRD's 10.3.0.0/24 is not as sent"
    api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0]
        | .summary.as_path == [64602] and .summary.next_hop == "10.9.0.2" and .best' >/dev/null \
        || fail "our copy of FRR's 10.2.0.0/24 is not as sent"
    peer_is bird4 '.four_octet_asn and (.add_path_receive == ["IPv4 unicast"])' >/dev/null \
        || fail "the session with BIRD did not agree on four-octet AS numbers and ADD-PATH"
    ok "we have their routes; four-octet AS numbers and ADD-PATH with BIRD"

    # Packets, there and back by the routes BGP put in three kernels.
    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no ping to FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no ping to BIRD's lan"
    in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
    ok "packets get through to both, over IPv4 and IPv6"

    # The kernel's view of a session's connection, as the page asks for it.
    frr4_addr=$(api /api/peers | jq -r '.[] | select(.name == "frr4") | .peer_addr')
    api "/api/tcp?peer=$frr4_addr" | jq -e '.available and .connection.state == "established" and .connection.mss > 0' >/dev/null \
        || fail "no TCP details for frr4: $(api "/api/tcp?peer=$frr4_addr")"
    ok "the kernel tells of a session's TCP connection"
}

# RFC 9234: each end says what it is to the other, and routes are marked
# Only to Customer where they should be.
phase_roles() {
    api /api/peers | jq -e 'all(.[]; .role_confirmed)' >/dev/null \
        || fail "not every peer confirmed its role: $(api /api/peers | jq -c '[.[] | {name, role, role_confirmed}]')"
    frr_neighbor 10.9.0.1 '.localRole == "peer" and .remoteRole == "peer"' >/dev/null \
        || fail "FRR does not see us as its peer"
    bird_ctl show protocols all dut4 | grep -A40 "Neighbor capabilities" | grep -q "Role: customer" \
        || fail "BIRD does not see us as its customer"
    ok "roles agree with FRR (peer) and BIRD (provider)"

    # To a peer our routes are marked; to a provider they are not.
    vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].otc == 64601' >/dev/null \
        || fail "FRR's copy of our route is not marked Only to Customer by us"
    if bird_ctl show route for 10.1.0.0/24 all | grep -qi "otc\|only_to_customer"; then
        fail "our route to our provider is marked Only to Customer"
    fi
    # What comes from a peer or a provider is marked by them, or by us.
    api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 64602)' >/dev/null \
        || fail "FRR's route is not marked Only to Customer"
    api "/api/routes/received?prefix=10.3.0.0/24" | jq -e '.routes[0].attributes | any(.OnlyToCustomer? == 4200000003)' >/dev/null \
        || fail "BIRD's route is not marked Only to Customer"
    ok "Only to Customer is where it should be, in both directions"
}

# Routinator's data over RTR: an invalid route is refused, and taken when
# the data comes to allow it.
phase_rpki() {
    wait_for "the daemon to have Routinator's data" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/rpki | jq -e '.state == \"synchronised\" and .vrps_ipv4 == 3 and .vrps_ipv6 == 1'"
    ok "synchronised with Routinator over RTR version $(api /api/rpki | jq .protocol_version): 3 IPv4 and 1 IPv6 VRPs"

    # FRR announces 10.66.0.0/24, which is AS 64999's to announce.
    wait_for "the invalid route to be refused" dut_lacks 10.66.0.0/24
    wait_for "the invalid route to leave the kernel" dut_kernel_lacks -4 10.66.0.0/24
    api "/api/routes/rejected?peer=frr4" | jq -e '.[0].by_reason["RPKI invalid"] == 1' >/dev/null \
        || fail "the refusal is not on record: $(api /api/routes/rejected)"
    peer_is frr4 '.held_routes == 1' >/dev/null || fail "the invalid route is not held"
    api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
        || fail "FRR's own route is not valid"
    api "/api/routes/received?prefix=10.3.7.0/24" | jq -e '.routes[0].rpki == "valid"' >/dev/null \
        || fail "BIRD's more specific route is not valid"
    ok "FRR's invalid route is refused and held; the others are valid"

    # The data changes, by way of a Routinator that is restarted: the
    # route is authorised now.
    sudo kill "$(cat "$work/routinator.pid")"; sleep 0.5
    routinator_up 64602
    wait_for "the held route to be taken" dut_has 10.66.0.0/24 frr4
    wait_for "the route in the kernel" dut_kernel -4 "10.66.0.0/24 via 10.9.0.2 dev ix0 metric 20"
    peer_is frr4 '.held_routes == 0' >/dev/null || fail "the route is still held"
    ok "after Routinator restarts with new data, the held route is taken"
}

# Changes after the first exchange: new routes, withdrawals, and being
# asked to send everything again.
phase_refresh() {
    frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.9.0/24"
    wait_for "FRR's new route" dut_has 10.2.9.0/24 frr4
    wait_for "it in the kernel" dut_kernel -4 "10.2.9.0/24 via 10.9.0.2 dev ix0 metric 20"
    frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.9.0/24"
    wait_for "FRR's withdrawal" dut_lacks 10.2.9.0/24
    wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.9.0/24
    ok "a route FRR adds and withdraws comes and goes, in the table and the kernel"

    # FRR asks for our routes again (ROUTE-REFRESH), and gets them.
    local before
    before=$(frr_neighbor 10.9.0.1 '.messageStats.updatesRecv')
    vty -c "clear bgp ipv4 unicast 10.9.0.1 soft in"
    wait_for "our routes to be sent again" \
        bash -c "[ \"\$(sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq '.[\"10.9.0.1\"].messageStats.updatesRecv')\" -gt $before ]"
    frr_neighbor 10.9.0.1 '.bgpState == "Established"' >/dev/null || fail "the session did not survive a route refresh"
    wait_for "our route still at FRR" frr_has ipv4 10.1.0.0/24
    ok "FRR's route refresh is answered with our routes"

    # BIRD sends its routes again unasked; nothing changes.
    bird_ctl reload out dut4 >/dev/null
    sleep 0.5
    converged
    ok "BIRD re-sending its routes changes nothing"
}

# Sessions that end in every way they can, and come back.
phase_flaps() {
    for round in 1 2 3 4 5; do
        vty -c "clear bgp *"
        wait_for "FRR's routes to go (round $round)" dut_lacks 10.2.0.0/24
        converged
    done
    ok "five resets by FRR: routes go and come back each time"

    bird_ctl restart dut4 >/dev/null; bird_ctl restart dut6 >/dev/null
    converged
    ok "BIRD restarting its sessions"

    # FRR dies without a word.
    frr_down KILL
    wait_for "FRR's routes to go" dut_lacks 10.2.0.0/24
    wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
    dut_has 10.3.0.0/24 bird4 >/dev/null || fail "BIRD's routes went with FRR's"
    frr_up
    converged
    ok "FRR killed and started again"

    # The wire to FRR is cut: nothing says so but the silence. The hold
    # timer (nine seconds here) has to notice.
    in_ns ix ip link set to-frr down
    WAIT_TRIES=100 wait_for "the hold timer to end the session" dut_lacks 10.2.0.0/24
    wait_for "them out of the kernel" dut_kernel_lacks -4 10.2.0.0/24
    in_ns ix ip link set to-frr up
    converged
    ok "a cut wire is noticed by the hold timer, and mended"

    sudo kill -KILL "$(sudo cat "$work/bird.pid")"
    wait_for "BIRD's routes to go" dut_lacks 10.3.0.0/24
    sudo rm -f "$work/bird.ctl"
    bird_up
    converged
    ok "BIRD killed and started again"
}

# The daemon itself stopping, in good order and not.
phase_restart() {
    dut_down TERM
    [ -z "$(in_ns dut ip route show proto bgp)" ] && [ -z "$(in_ns dut ip -6 route show proto bgp)" ] \
        || fail "routes left in the kernel after a clean stop: $(in_ns dut ip route show proto bgp)"
    wait_for "FRR to drop our route" frr_lacks ipv4 10.1.0.0/24
    frr_neighbor 10.9.0.1 '.lastNotificationReason // .lastResetDueTo | test("Cease|Notification"; "i")' >/dev/null \
        || fail "FRR was not told we were going: $(frr_neighbor 10.9.0.1 '{lastResetDueTo, lastNotificationReason}')"
    dut_up
    converged
    ok "stopped in good order: kernel emptied, peers told; and back"

    # Killed: its routes stay in the kernel. While it is dead BIRD stops
    # announcing one of them; the daemon that starts next has to clear
    # that one out and keep the rest.
    dut_down KILL
    dut_kernel -4 "10.3.7.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "the kernel's routes went with the killed daemon"
    sed -i '/route 10.3.7.0\/24 blackhole;/d' "$work/bird.conf"
    bird_ctl configure >/dev/null
    dut_up
    converged
    wait_for "the stale route to be cleared" dut_kernel_lacks -4 10.3.7.0/24
    ok "killed: routes stay in the kernel, and the next start clears the stale one"
}

# Graceful restart (RFC 4724): the daemon restarts for an upgrade without
# its routes going anywhere, and keeps BIRD's while BIRD restarts.
frr_stale() {  # frr_stale prefix: FRR keeps our route, as stale
    vty -c "show bgp ipv4 unicast $1 json" | jq -e '.paths | any(.aspath.string == "64601" and .stale == true)'
}
frr_fresh() { frr_has ipv4 "$1" && ! frr_stale "$1"; }
dut_stale() {  # dut_stale prefix peer-name: the daemon keeps the peer's route, as stale
    api "/api/routes/received?prefix=$1" | jq -e --arg peer "$2" '.routes | any(.peer_name == $peer and .stale)'
}
kernel_removals() {
    api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }'
}
phase_graceful() {
    sed -i 's/graceful restart off;/graceful restart on;/' "$work/bird.conf"
    bird_ctl configure >/dev/null
    wait_for "BIRD to come back with graceful restart" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '[.[] | select(.name == \"bird4\" or .name == \"bird6\") | select(.graceful_restart != null)] | length == 2'"
    converged

    # The daemon restarts: everything stays where it is meanwhile.
    dut_down USR1
    grep -q "Leaving to be started again" "$work/dut.log" || fail "the daemon did not leave for a restart"
    dut_kernel -4 "10.2.0.0/24 via 10.9.0.2 dev ix0 metric 20" || fail "FRR's route left the kernel"
    dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
    dut_kernel -6 "2001:db8:3::/48 via 2001:db8:9::3 dev ix0 metric 20" || fail "BIRD's IPv6 route left the kernel"
    wait_for "FRR to keep our route as stale" frr_stale 10.1.0.0/24
    bird_has 10.1.0.0/24 || fail "BIRD dropped our route"
    bird_has 2001:db8:1::/48 || fail "BIRD dropped our IPv6 route"
    in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "packets stopped while the daemon was away"
    dut_up
    converged
    wait_for "FRR to take our route afresh" frr_fresh 10.1.0.0/24
    wait_for "nothing kept in the kernel any more" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.kept == 0'"
    local removed
    removed=$(kernel_removals)
    [ "${removed:-0}" -eq 0 ] || fail "$removed routes were taken out of the kernel over the restart"
    ok "restarted on SIGUSR1: kernel routes kept, FRR and BIRD kept ours, nothing withdrawn"

    # BIRD restarts: the daemon keeps its routes meanwhile, in the table
    # and in the kernel, and takes them afresh when it is back.
    bird_ctl graceful restart >/dev/null || true
    while [ -e "/proc/$(sudo cat "$work/bird.pid" 2>/dev/null || echo 0)" ]; do sleep 0.05; done
    wait_for "the daemon to keep BIRD's route as stale" dut_stale 10.3.0.0/24 bird4
    dut_stale 2001:db8:3::/48 bird6 >/dev/null || fail "BIRD's IPv6 route was not kept"
    dut_kernel -4 "10.3.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route left the kernel"
    peer_is bird4 '.stale_routes > 0' >/dev/null || fail "the daemon does not say it keeps BIRD's routes: $(peer_is bird4 .)"
    sudo rm -f "$work/bird.ctl"
    bird_up
    converged
    wait_for "BIRD's routes to be fresh" \
        bash -c "! sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?peer=bird4' | jq -e '.routes | any(.stale)' >/dev/null"
    ok "BIRD restarted gracefully: its routes were kept, and taken afresh"

    sed -i 's/graceful restart on;/graceful restart off;/' "$work/bird.conf"
    bird_ctl configure >/dev/null
    converged
}

# A full table's worth of routes from BIRD, into the table and the kernel.
phase_scale() {
    # Out of 100.0.0.0/6 and up: as many /24s as asked for.
    awk -v n="$scale_routes" 'BEGIN { for (i = 0; i < n; i++)
        printf "route %d.%d.%d.0/24 blackhole;\n", 100 + int(i / 65536), int(i / 256) % 256, i % 256 }' \
        > "$work/bird-table.conf"
    bird_config "$work/bird-table.conf"
    local base start
    base=$(api /api/status | jq .received_prefixes)
    start=$(date +%s.%N)
    bird_ctl configure >/dev/null
    WAIT_TRIES=1500 wait_for "the table to arrive" \
        bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -ge $((base + scale_routes)) ]"
    local arrived
    arrived=$(date +%s.%N)
    # By the daemon's own count first: listing a million routes with `ip`
    # takes longer than installing them.
    WAIT_TRIES=1500 wait_for "the table in the kernel" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.installed >= $scale_routes'"
    local installed
    installed=$(date +%s.%N)
    local listed
    listed=$(in_ns dut ip -4 route show proto bgp | wc -l)
    [ "$listed" -ge "$scale_routes" ] || fail "the kernel lists $listed routes, not $scale_routes"
    api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
    ok "$scale_routes routes from BIRD: in the table after $(echo "$arrived - $start" | bc)s, in the kernel after $(echo "$installed - $start" | bc)s"
    peer_is frr4 '.state == "Established"' >/dev/null || fail "the session with FRR did not survive the table"

    # The daemon is killed with the table in the kernel, and started
    # again: it has to find its routes there, take the table from BIRD
    # anew, and end up with the same, none refused and none left over.
    dut_down KILL
    start=$(date +%s.%N)
    dut_up
    WAIT_TRIES=1500 wait_for "the table in the kernel again" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/kernel | jq -e '.synced and .installed >= $scale_routes'"
    installed=$(date +%s.%N)
    converged
    local listed_again
    listed_again=$(in_ns dut ip -4 route show proto bgp | wc -l)
    [ "$listed_again" -eq "$listed" ] || fail "the kernel lists $listed_again routes after a restart; it listed $listed before"
    # They were kept, not taken out and put back: next to nothing was
    # removed on the way.
    local removed
    removed=$(api /metricz | awk '/^bgp_kernel_route_changes_total\{change="remove"\}/ { print $2 }')
    [ "${removed:-0}" -lt 100 ] || fail "$removed routes were removed from the kernel on the way"
    api /api/kernel | jq -e '.errors == 0' >/dev/null || fail "the kernel refused routes: $(api /api/kernel | jq -c '{errors, last_error}')"
    ok "killed and restarted under the table: in step again after $(echo "$installed - $start" | bc)s, the kernel's routes as before"

    bird_config
    start=$(date +%s.%N)
    bird_ctl configure >/dev/null
    WAIT_TRIES=1500 wait_for "the table to be withdrawn" \
        bash -c "[ \"\$(sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/status | jq .received_prefixes)\" -le $((base + 1)) ]"
    WAIT_TRIES=1500 wait_for "the table out of the kernel" \
        bash -c "[ \"\$(sudo ip netns exec lab-dut ip -4 route show proto bgp | wc -l)\" -le $((base + 1)) ]"
    ok "withdrawn again after $(echo "$(date +%s.%N) - $start" | bc)s"
    converged
}

# What the daemon refuses: a peer that is not what it is configured as,
# and one that sends more than it may.
phase_limits() {
    # FRR says it is our peer; here it is configured as our customer.
    dut_down TERM
    dut_config CUSTOMER
    dut_up
    wait_for "the sessions that should come up" established 3
    wait_for "the role mismatch to be on record" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | .state != \"Established\" and (.last_error // \"\" | test(\"role mismatch\"; \"i\"))'"
    sleep 1
    peer_is frr4 '.state != "Established"' >/dev/null || fail "a session came up across a role mismatch"
    frr_neighbor 10.9.0.1 '.bgpState != "Established"' >/dev/null || fail "FRR thinks the session is up"
    ok "a peer whose role does not fit ours gets no session: $(peer_is frr4 '.last_error')"

    # FRR sends at least two routes that pass; it may have one.
    frr_configure "router bgp 64602" "address-family ipv4 unicast" "network 10.2.8.0/24"
    dut_down TERM
    dut_config PEER 1
    dut_up
    wait_for "the limit to end the session" \
        bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp neighbors 10.9.0.1 json' 2>/dev/null | jq -e '.[\"10.9.0.1\"] | (.lastNotificationReason // \"\") | test(\"Maximum Number of Prefixes\"; \"i\")'"
    wait_for "the peer to be held down" \
        bash -c "sudo ip netns exec lab-dut curl -sf http://127.0.0.1:8080/api/peers | jq -e '.[] | select(.name == \"frr4\") | (.last_error // \"\") | test(\"held down|prefix\"; \"i\")'"
    dut_lacks 10.2.0.0/24 || fail "routes of a peer over its limit are still there"
    ok "a peer over its prefix limit is told so (Cease / Maximum Number of Prefixes) and held down"

    frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.8.0/24"
    dut_down TERM
    dut_config
    dut_up
    converged
    ok "back to normal"
}

# FRR tries what a peer must not get away with: announcing our own
# prefix, announcing the exchange LAN itself (which would take the
# sessions with everyone on it), and sending our traffic to someone else.
refused_for() {  # refused_for reason: how many of frr4's routes were refused for it
    api "/api/routes/rejected?peer=frr4" | jq -e --arg why "$1" '.[0].by_reason[$why] // 0 | . >= 1'
}
phase_hijacks() {
    frr_configure \
        "ip prefix-list ELSEWHERE seq 5 permit 10.2.7.0/24" \
        "route-map TO-DUT permit 10" "match ip address prefix-list ELSEWHERE" "set ip next-hop 10.9.0.3" \
        "route-map TO-DUT permit 20" \
        "router bgp 64602" "address-family ipv4 unicast" \
        "neighbor 10.9.0.1 route-map TO-DUT out" \
        "network 10.1.0.0/24" "network 10.9.0.0/24" "network 10.2.7.0/24"

    # Our own prefix, from someone else.
    wait_for "our own prefix to be refused" refused_for "prefix is our own"
    dut_lacks 10.1.0.0/24 || fail "our own prefix was taken from FRR"
    ok "our own prefix is not taken from a peer"

    # A route whose next hop is BIRD, from FRR.
    wait_for "the third-party next hop to be refused" refused_for "next hop is not the peer"
    dut_lacks 10.2.7.0/24 || fail "a route by way of a third party was taken"
    ok "a route whose next hop is someone else is not taken"

    # The LAN's own prefix. The lab takes bogons, so the policy lets it
    # by; it is still not a route the kernel could be given, and so not
    # one to take: it would compete with the connected route.
    wait_for "the LAN's prefix to be held" refused_for "prefix is a network of this router"
    dut_lacks 10.9.0.0/24 || fail "the LAN's own prefix was taken from FRR"
    in_ns dut ip -4 route show proto bgp | grep -q "^10.9.0.0/24 " && fail "the LAN's own prefix was installed by way of a peer"
    in_ns dut ping -q -c 1 -W 2 10.9.0.3 >/dev/null || fail "the LAN is not reachable any more"
    ok "a route for the exchange LAN itself is not taken, and stays out of the kernel"

    frr_configure \
        "router bgp 64602" "address-family ipv4 unicast" \
        "no network 10.1.0.0/24" "no network 10.9.0.0/24" "no network 10.2.7.0/24" \
        "no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
    converged
    ok "back to normal"
}

# FRR and BIRD both have a route for one prefix, and FRR's is the better.
# But its next hop is nowhere the machine can reach: then BIRD's is the
# one chosen and installed (RFC 4271 §9.1.2.1), until the next hop is
# somewhere, and again when it no longer is.
best_is() {  # best_is prefix peer-name
    api "/api/routes/received?prefix=$1&best=true" | jq -e --arg peer "$2" '.routes | length == 1 and .[0].peer_name == $peer'
}
phase_nexthops() {
    dut_down TERM
    dut_config PEER 2000000 "accept_third_party_next_hop: true"
    dut_up
    converged
    # A prefix nobody has said anything about in the RPKI, from both.
    echo "route 10.77.0.0/24 blackhole;" > "$work/bird-both.conf"
    bird_config "$work/bird-both.conf"
    bird_ctl configure >/dev/null
    wait_for "BIRD's route for it" dut_has 10.77.0.0/24 bird4
    frr_configure \
        "ip prefix-list FAR seq 5 permit 10.77.0.0/24" \
        "route-map TO-DUT permit 10" "match ip address prefix-list FAR" "set ip next-hop 10.250.0.1" \
        "route-map TO-DUT permit 20" \
        "router bgp 64602" "address-family ipv4 unicast" \
        "neighbor 10.9.0.1 route-map TO-DUT out" "network 10.77.0.0/24"

    wait_for "FRR's route to be held" refused_for "next hop unreachable"
    peer_is frr4 '.held_routes >= 1' >/dev/null || fail "the route is not held: $(peer_is frr4 .)"
    best_is 10.77.0.0/24 bird4 >/dev/null || fail "BIRD's route is not the one chosen"
    dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20" || fail "BIRD's route is not in the kernel"
    ok "a better route whose next hop is nowhere is held; the one that works is chosen and installed"

    # The next hop becomes a neighbor.
    in_ns dut ip addr add 10.250.0.9/24 dev lan
    wait_for "FRR's route to be taken" best_is 10.77.0.0/24 frr4
    wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.250.0.1 dev lan metric 20"
    ok "when its next hop can be reached the better route is taken, chosen and installed"

    # And stops being one.
    in_ns dut ip addr del 10.250.0.9/24 dev lan
    wait_for "BIRD's route to be chosen again" best_is 10.77.0.0/24 bird4
    wait_for "it in the kernel" dut_kernel -4 "10.77.0.0/24 via 10.9.0.3 dev ix0 metric 20"
    peer_is frr4 '.held_routes >= 1' >/dev/null || fail "FRR's route is not held again"
    ok "and when it no longer can, the other route is back"

    frr_configure \
        "router bgp 64602" "address-family ipv4 unicast" \
        "no network 10.77.0.0/24" "no neighbor 10.9.0.1 route-map TO-DUT out" "no route-map TO-DUT"
    bird_config
    bird_ctl configure >/dev/null
    dut_down TERM
    dut_config
    dut_up
    converged
    ok "back to normal"
}

# Routes are passed on: FRR becomes our customer and BIRD stays our
# provider, so what FRR announces goes up to BIRD with our AS on the path,
# what BIRD announces goes down to FRR marked "only to customer", and
# packets go between their networks through us. FRR, as a customer, steers
# its routes with our control communities: one keeps a route from BIRD,
# another has us prepend on the way to BIRD's session group. BIRD, as a
# provider, may not: its "not to FRR" is taken off as its route comes in.
phase_transit() {
    dut_down TERM
    dut_config CUSTOMER 2000000 "accept_control_communities: true" "" "groups: 7" \
        'export { learned_routes: true }
control_communities: true'
    echo "route 10.3.8.0/24 blackhole { bgp_large_community.add((64601, 0, 64602)); };" > "$work/bird-transit.conf"
    bird_config "$work/bird-transit.conf"
    bird_ctl configure >/dev/null
    in_ns dut sysctl -qw net.ipv4.ip_forward=1
    dut_up
    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \
        "ip prefix-list KEPT seq 5 permit 10.2.9.0/24" \
        "ip prefix-list FAR seq 5 permit 10.2.10.0/24" \
        "route-map TO-DUT-TRANSIT permit 10" "match ip address prefix-list KEPT" "set large-community 64601:0:4200000003" \
        "route-map TO-DUT-TRANSIT permit 15" "match ip address prefix-list FAR" "set large-community 64601:31:7" \
        "route-map TO-DUT-TRANSIT permit 20" \
        "router bgp 64602" "address-family ipv4 unicast" \
        "neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "network 10.2.9.0/24" "network 10.2.10.0/24"
    wait_for "four sessions" established 4
    wait_for "FRR's routes at the daemon" dut_has 10.2.9.0/24 frr4
    wait_for "FRR's routes at the daemon" dut_has 10.2.10.0/24 frr4

    # Up to the provider, with our AS on the path; not the one that says
    # not to.
    wait_for "FRR's route at BIRD, by way of us" bash -c "sudo birdc -s $work/bird.ctl show route for 10.2.0.0/24 all | grep -q 'BGP.as_path: 64601 64602'"
    sleep 1
    bird_ctl show route for 10.2.9.0/24 all | grep -q "64601 64602" && fail "BIRD was told the route whose community says not to"
    ok "the customer's routes reach the provider with our AS on the path; one kept back by its community"

    # Prepended once more towards BIRD's group, at the customer's word,
    # and the community that asked for it gone.
    wait_for "the prepended route at BIRD" bash -c "sudo birdc -s $work/bird.ctl show route for 10.2.10.0/24 all | grep -q 'BGP.as_path: 64601 64601 64602'"
    bird_ctl show route for 10.2.10.0/24 all | grep -q "(64601, 31, 7)" && fail "the control community went on to BIRD"
    ok "a customer's control community has us prepend towards a session group, and is taken off"

    # The provider's "not to FRR" is not ours to act on: taken off as the
    # route comes in, and the route goes down to FRR all the same.
    wait_for "BIRD's route with our community at FRR" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.8.0/24 json' 2>/dev/null | jq -e '.paths[0].aspath.string == \"64601 4200000003\"'"
    api "/api/routes/received?prefix=10.3.8.0/24" | jq -e '.routes | length > 0 and all(.summary.large_communities | index("64601:0:64602") | not)' >/dev/null \
        || fail "the provider's control community was kept: $(api "/api/routes/received?prefix=10.3.8.0/24")"
    ok "a provider's control community is taken off as its route comes in"

    # Down to the customer, marked.
    wait_for "BIRD's route at FRR, by way of us" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths[0].aspath.string == \"64601 4200000003\"'"
    # Marked on the way in from the provider, with the provider's AS, and
    # the mark kept as it goes down (RFC 9234 §5).
    vty -c "show bgp ipv4 unicast 10.3.0.0/24" | grep -q "otc 4200000003" || fail "the route to the customer is not marked only to customer"
    ok "the provider's routes reach the customer, marked only to customer"

    # And packets follow them, through us.
    in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.3.0.1 >/dev/null || fail "no ping from the customer's network to the provider's through us"
    in_ns bird ping -q -c 2 -W 2 -I 10.3.0.1 10.2.0.1 >/dev/null || fail "no ping from the provider's network to the customer's through us"
    ok "packets go between the customer's and the provider's networks through us"

    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" \
        "address-family ipv4 unicast" "no network 10.2.9.0/24" "no network 10.2.10.0/24" "no neighbor 10.9.0.1 route-map TO-DUT-TRANSIT out" "no route-map TO-DUT-TRANSIT"
    bird_config
    bird_ctl configure >/dev/null
    dut_down TERM
    dut_config
    dut_up
    converged
    ok "back to normal"
}

# FRR is a route collector of ours, told every route with its path
# identifier (ADD-PATH). BIRD talks to us from a second address too, so
# that there are two routes for each of its prefixes; FRR is told both,
# and holds them apart. One of BIRD's sessions goes, and so does one of
# the two.
phase_collector() {
    dut_down TERM
    in_ns bird ip addr add 10.9.0.4/24 dev ix0
    dut_config CUSTOMER 2000000 "" 'add_path: ADD_PATH_SEND
  export { learned_routes: true all_paths: true }' "" 'peers {
  name: "bird4b" endpoint { address: "10.9.0.4" } remote_asn: 4200000003
  address_families: IPV4_UNICAST
  role: PROVIDER
  import { accept_bogons: true max_prefixes: 2000000 }
}'
    bird_config "" 'protocol bgp dut4b from dut {
  neighbor 10.9.0.1 as 64601;
  source address 10.9.0.4;
  ipv4 { import all; export where source = RTS_STATIC; };
}'
    bird_ctl configure >/dev/null
    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer"
    dut_up
    wait_for "five sessions" established 5
    wait_for "BIRD's route from both its sessions" bash -c "sudo ip netns exec lab-dut curl -sf 'http://127.0.0.1:8080/api/routes/received?prefix=10.3.0.0/24' | jq -e '.routes | length == 2'"
    peer_is frr4 '.add_path_send | index("IPv4 unicast") != null' >/dev/null \
        || fail "ADD-PATH was not agreed with FRR: $(peer_is frr4 .add_path_send)"
    wait_for "both routes at FRR, told apart" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths | length == 2 and all(.aspath.string == \"64601 4200000003\")'"
    # Its two prefixes twice, and our own route: what the page and the
    # metrics say it holds from us.
    wait_for "the count of routes the collector holds" peer_is frr4 '.advertised_routes == 5'
    ok "the collector is told both of BIRD's routes for a prefix, with path identifiers"

    bird_ctl disable dut4b >/dev/null
    wait_for "one route left at FRR" bash -c "sudo ip netns exec lab-frr vtysh -N $pathspace -c 'show bgp ipv4 unicast 10.3.0.0/24 json' 2>/dev/null | jq -e '.paths | length == 1'"
    wait_for "the count of routes the collector holds" peer_is frr4 '.advertised_routes == 3'
    ok "a route that goes is withdrawn from the collector by its identifier, the other stays"

    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer"
    bird_config
    bird_ctl configure >/dev/null
    in_ns bird ip addr del 10.9.0.4/24 dev ix0
    dut_down TERM
    dut_config
    dut_up
    converged
    ok "back to normal"
}

# A customer leaks, and the export limits hold. FRR, now our customer,
# announces a route outside its address space and then more routes than
# BIRD, our provider, is to be told of. A customer's routes go to
# everyone as far as the roles go; the export limits are the check on
# that. The route outside our and the customer's space never goes up to
# BIRD; when more were to go than the limit allows, none of them does,
# and the session with BIRD ends with an Administrative Shutdown that
# says why (RFC 9003).
phase_leaks() {
    dut_down TERM
    dut_config CUSTOMER 2000000 "" "" "" 'export { learned_routes: true }
export_limits {
  prefixes { prefix: "10.1.0.0/16" max_length: 24 }
  prefixes { prefix: "10.2.0.0/16" max_length: 24 }
  prefixes { prefix: "2001:db8:1::/48" }
  prefixes { prefix: "2001:db8:2::/48" }
  max_prefixes: 5
}'
    dut_up
    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role customer" \
        "address-family ipv4 unicast" "network 10.77.0.0/24"
    wait_for "four sessions" established 4
    wait_for "the route outside the customer's space at the daemon" dut_has 10.77.0.0/24 frr4

    # The customer's route goes up to the provider; the one outside the
    # export limits does not.
    wait_for "FRR's route at BIRD" bird_has 10.2.0.0/24
    sleep 1
    bird_has 10.77.0.0/24 && fail "BIRD was told a route outside the export limits"
    ok "a customer's route outside the export limits is not passed on to the provider"

    # More routes than the limit: ours and the customer's two, and four
    # more of the customer's, would be six of five.
    frr_configure "router bgp 64602" "address-family ipv4 unicast" \
        "network 10.2.11.0/24" "network 10.2.12.0/24" "network 10.2.13.0/24" "network 10.2.14.0/24"
    wait_for "the export limit to end BIRD's session" \
        peer_is bird4 '.state != "Established" and (.last_error // "" | test("export limit"; "i"))'
    wait_for "BIRD to be told why" sudo grep -q "export limit of 5 routes reached" "$work/bird.log"
    bird_has 10.2.11.0/24 && fail "BIRD was told a route past the export limit"
    ok "past its export limit the provider is told nothing more, and why (Cease / Administrative Shutdown): $(peer_is bird4 '.last_error')"

    frr_configure "router bgp 64602" "neighbor 10.9.0.1 local-role peer" "address-family ipv4 unicast" \
        "no network 10.77.0.0/24" "no network 10.2.11.0/24" "no network 10.2.12.0/24" \
        "no network 10.2.13.0/24" "no network 10.2.14.0/24"
    dut_down TERM
    dut_config
    dut_up
    converged
    ok "back to normal"
}

# The exchange LAN loses its IPv4 addresses. One session over IPv6 with
# each router carries both families, and IPv4 routes have IPv6 next hops
# (RFC 8950), in BGP and in the kernel. Run after the rest: the lab stays
# that way.
phase_v6only() {
    dut_down TERM
    frr_down TERM
    sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
    local n=0 name
    for name in dut frr bird; do
        n=$((n + 1))
        in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0
    done
    frr_config_v6only; frr_up
    bird_config_v6only; bird_up
    dut_config_v6only; dut_up

    wait_for "two sessions over IPv6" established 2
    api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
        || fail "the sessions did not agree on IPv4 with IPv6 next hops: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
    ok "both sessions carry IPv4 and IPv6, with IPv6 next hops for IPv4 (RFC 8950)"

    # What they send us: IPv4 routes whose next hop is an IPv6 address,
    # and the same in the kernel.
    wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
    wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
    api "/api/routes/received?prefix=10.2.0.0/24" | jq -e '.routes[0].summary.next_hop | startswith("2001:db8:9::2")' >/dev/null \
        || fail "FRR's IPv4 route has the next hop $(api "/api/routes/received?prefix=10.2.0.0/24" | jq '.routes[0].summary.next_hop')"
    wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 2001:db8:9::2 dev ix0 metric 20"
    wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 2001:db8:9::3 dev ix0 metric 20"
    wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via 2001:db8:9::2 dev ix0 metric 20"
    ok "their IPv4 routes arrive with IPv6 next hops and are in the kernel that way"

    # What we send them.
    wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
    vty -c "show bgp ipv4 unicast 10.1.0.0/24 json" | jq -e '.paths[0].nexthops | any(.ip == "2001:db8:9::1" and .afi == "ipv6")' >/dev/null \
        || fail "FRR's copy of our IPv4 route has the next hops $(vty -c 'show bgp ipv4 unicast 10.1.0.0/24 json' | jq -c '.paths[0].nexthops')"
    wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
    bird_ctl show route for 10.1.0.0/24 all | grep -q "BGP.next_hop: 2001:db8:9::1" \
        || fail "BIRD's copy of our IPv4 route: $(bird_ctl show route for 10.1.0.0/24 all)"
    wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
    ok "FRR and BIRD have our IPv4 route with our IPv6 address as next hop"

    # IPv4 packets across a link that has no IPv4 addresses.
    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
    in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
    ok "IPv4 packets cross the IPv6-only link, both ways"

    # A withdrawal by FRR (in MP_UNREACH_NLRI) takes the route out.
    frr_configure "router bgp 64602" "address-family ipv4 unicast" "no network 10.2.5.0/24"
    wait_for "FRR's withdrawal" dut_lacks 10.2.5.0/24
    wait_for "it out of the kernel" dut_kernel_lacks -4 10.2.5.0/24
    dut_has 10.2.0.0/24 frr >/dev/null || fail "the withdrawal took the other route along"
    ok "an IPv4 route withdrawn over the IPv6 session goes"
}

# The exchange LAN loses its global IPv6 addresses too: all that is left on
# it are link-local ones, the same on every link a router has, which mean
# something only together with the interface. The sessions are between
# those, as are the next hops, for both families. Run after v6only.
dut_config_linklocal() {
    cat > "$work/dut.textpb" <<END
router_id: "10.9.0.1"
local_asn: 64601
listen_endpoints { address: "::" port: 179 }
connect_retry_seconds: 1
peers {
  name: "frr" endpoint { address: "fe80::2%ix0" } remote_asn: 64602
  role: PEER
  import { accept_bogons: true max_prefixes: 1000 }
}
peers {
  name: "bird" endpoint { address: "fe80::3%ix0" } remote_asn: 4200000003
  role: PROVIDER
  import { accept_bogons: true max_prefixes: 1000 }
}
originate { prefix: "10.1.0.0/24" }
originate { prefix: "2001:db8:1::/48" }
kernel { stale_seconds: 3 }
http_endpoint { address: "127.0.0.1" port: 8080 }
END
}

# FRR is told the interface and not the address ("BGP unnumbered"), which
# is how it is mostly set up for this. It learns who is on the link from
# router advertisements, which the daemon does not send: radvd does, as it
# would have to beside the daemon on a real router with such a neighbor.
radvd_up() {
    cat > "$work/radvd.conf" <<END
interface ix0 {
  AdvSendAdvert on;
  MinRtrAdvInterval 3;
  MaxRtrAdvInterval 4;
  AdvDefaultLifetime 0;
};
END
    in_ns dut "$(command -v radvd || echo /usr/sbin/radvd)" -C "$work/radvd.conf" -p "$work/radvd.pid" -m logfile -l "$work/radvd.log"
}

frr_config_linklocal() {
    sudo tee "/etc/frr/$pathspace/frr.conf" > /dev/null <<END
frr defaults traditional
hostname frr
log file $work/frr.log
!
router bgp 64602
 bgp router-id 10.9.0.2
 no bgp ebgp-requires-policy
 no bgp network import-check
 timers bgp 3 9
 neighbor ix0 interface remote-as 64601
 neighbor ix0 local-role peer
 neighbor ix0 timers connect 1
 neighbor ix0 capability extended-nexthop
 !
 address-family ipv4 unicast
  network 10.2.0.0/24
  network 10.2.5.0/24
  neighbor ix0 activate
 exit-address-family
 !
 address-family ipv6 unicast
  network 2001:db8:2::/48
  neighbor ix0 activate
 exit-address-family
!
END
}

bird_config_linklocal() {
    cat > "$work/bird.conf" <<END
log "$work/bird.log" all;
router id 10.9.0.3;
protocol device {}
protocol kernel kernel4 { ipv4 { export where source = RTS_BGP; }; }
protocol kernel kernel6 { ipv6 { export where source = RTS_BGP; }; }
protocol static static4 { ipv4; route 10.3.0.0/24 blackhole; }
protocol static static6 { ipv6; route 2001:db8:3::/48 blackhole; }
protocol bgp dut {
  local as 4200000003;
  neighbor fe80::1%ix0 as 64601;
  local role provider;
  connect retry time 1;
  # Back soon after the hold timer ran out, which the phase makes it do.
  error wait time 1, 2;
  hold time 9;
  graceful restart off;
  ipv4 { import all; export where source = RTS_STATIC; extended next hop on; };
  ipv6 { import all; export where source = RTS_STATIC; };
}
END
}

phase_linklocal() {
    dut_down TERM
    frr_down TERM
    sudo kill "$(sudo cat "$work/bird.pid")"; sleep 0.5; sudo rm -f "$work/bird.ctl"
    local n=0 name
    for name in dut frr bird; do
        n=$((n + 1))
        in_ns "$name" ip addr del "10.9.0.$n/24" dev ix0 2>/dev/null || true
        in_ns "$name" ip -6 addr flush dev ix0
        in_ns "$name" ip -6 addr add "fe80::$n/64" dev ix0 nodad
        # The same address on another link: it must not be mistaken for
        # the one the session is on.
        in_ns "$name" ip -6 addr flush dev lan scope link
        in_ns "$name" ip -6 addr add "fe80::$n/64" dev lan nodad
    done
    in_ns dut sysctl -qw net.ipv6.conf.ix0.forwarding=1
    radvd_up
    frr_config_linklocal; frr_up
    bird_config_linklocal; bird_up
    dut_config_linklocal; dut_up

    wait_for "two sessions between link-local addresses" established 2
    peer_is frr '.peer_addr | startswith("[fe80::2%ix0]:")' >/dev/null \
        && peer_is bird '.peer_addr | startswith("[fe80::3%ix0]:")' >/dev/null \
        && peer_is frr '.local_addr | startswith("[fe80::1%ix0]:")' >/dev/null \
        || fail "the peers are at $(api /api/peers | jq -c '[.[] | {name, peer_addr, local_addr}]')"
    api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \
        || fail "the sessions did not agree on both families: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')"
    ok "sessions with FRR and BIRD at fe80::2%ix0 and fe80::3%ix0, for both families"

    # Their routes: link-local next hops, on the session's interface.
    wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr
    wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird
    wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
    wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 fe80::3 dev ix0 metric 20"
    wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via fe80::2 dev ix0 metric 20"
    wait_for "BIRD's IPv6 route in the kernel" dut_kernel -6 "2001:db8:3::/48 via fe80::3 dev ix0 metric 20"
    api /api/kernel | jq -e '.unresolved == {}' >/dev/null || fail "some next hops did not resolve: $(api /api/kernel)"
    ok "their routes of both families are in the kernel by way of fe80::… on ix0"

    # Ours, at them.
    wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24
    wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48
    wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24
    wait_for "our IPv6 route at BIRD" bird_has 2001:db8:1::/48
    wait_for "our IPv4 route in FRR's kernel" in_ns frr ip -4 route get 10.1.0.1
    wait_for "our IPv6 route in BIRD's kernel" in_ns bird ip -6 route get 2001:db8:1::1 from 2001:db8:3::1
    ok "FRR and BIRD have our routes of both families"

    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan"
    in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan"
    in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan"
    in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan"
    in_ns bird ping -q -c 2 -W 2 -I 2001:db8:3::1 2001:db8:1::1 >/dev/null || fail "no IPv6 ping from BIRD's lan"
    ok "IPv4 and IPv6 packets cross a link with only link-local addresses, both ways"

    # The interface goes and comes back, as a tunnel does when it is
    # restarted: the sessions come back by themselves.
    in_ns dut ip link set ix0 down
    wait_for "the sessions to go" established 0
    in_ns dut ip link set ix0 up
    in_ns dut ip -6 addr flush dev ix0
    in_ns dut ip -6 addr add fe80::1/64 dev ix0 nodad
    wait_for "the sessions to come back" established 2
    wait_for "FRR's IPv4 route in the kernel again" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20"
    ok "the sessions and the routes come back after the interface was down"
}

# --- main -----------------------------------------------------------------------

case "${1:-test}" in
    down) down; exit 0 ;;
    up) lab_up; converged; say "the lab is up; work files in $work"; exit 0 ;;
    test) ;;
    *) fail "usage: $0 [test|up|down]" ;;
esac

on_exit() {
    status=$?
    if [ "$status" -ne 0 ]; then
        say "--- the daemon's last words:" >&2
        tail -n 25 "$work/dut.log" >&2 || true
    fi
    [ -n "${KEEP:-}" ] || down
    exit "$status"
}
trap on_exit EXIT
lab_up
say "lab: converging"
converged
for phase in $phases; do
    say "phase: $phase"
    "phase_$phase"
done
say "all phases passed: $phases"