diff --git a/Cargo.lock b/Cargo.lock index 95b011e..a101496 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,6 +292,7 @@ dependencies = [ "bytes", "clap", "futures", + "nix", "prometheus", "proptest", "protobuf", @@ -381,6 +382,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "clap" version = "4.5.53" @@ -954,6 +961,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -994,6 +1010,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nom" version = "8.0.0" diff --git a/Cargo.toml b/Cargo.toml index 81ed11a..9329b3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ bytes = "1.10.1" eyre = "0.6.12" nom = "8.0.0" serde = { version = "1.0.219", features = ["derive"] } +nix = { version = "0.31", default-features = false, features = ["net"] } socket2 = "0.6" serde_json = "1.0.140" serde_repr = "0.1.20" diff --git a/bin/bgp/README.md b/bin/bgp/README.md index 9147b19..fcfc188 100644 --- a/bin/bgp/README.md +++ b/bin/bgp/README.md @@ -184,9 +184,44 @@ link-local one. In the kernel such a route is an IPv4 route by way of an IPv6 neighbor (`10.2.0.0/24 via inet6 2001:db8::2 dev eth0`, Linux 5.2 and later). -The session's addresses have to be global ones (or unique local): a peer -configured by a link-local address (`fe80::...`), which needs the -interface named with it, is not supported yet. +## Peers at link-local addresses + +A link with no addresses but the link-local ones every interface has +(`fe80::/10`) is enough for a session. Such an address means something +only on its link, so the peer is the address and the interface together: + +``` +peers { + name: "upstream" + endpoint { address: "fe80::1%eno3" } + remote_asn: 64602 +} +``` + +- The interface is looked up by its name each time it is needed: one that + is not there yet, or is made anew with another number (a tunnel that is + restarted), is found when it is. Until then the peer's state says there + is no such interface. +- The same address on two interfaces is two peers. A link-local address + without an interface is refused, as is an interface on any other + address. +- To take the peer's connections the daemon must listen on `::` (as it + does by default), not on one global address. +- Both address families go over the session (IPv4 as above, if the peer + agrees). Our routes go with our link-local address as next hop, in both + places MP_REACH_NLRI has for one, as FRR sends its own. The peer's next + hop is taken in any of the forms in use: the link-local address alone, + twice, or after `::`. In the kernel the routes go by way of that address + on the session's interface (`10.2.0.0/24 via inet6 fe80::1 dev eno3`). +- `local_address`, if set to a link-local address of ours, is taken to be + on the peer's interface. + +With FRR, configure the peer by interface (`neighbor eno3 interface +remote-as ...`, "BGP unnumbered"). FRR then learns who is on the link from +IPv6 router advertisements, which this daemon does not send: run `radvd` +(or the like) on the interface beside it. FRR's older `neighbor fe80::1 +interface eno3` brings the session up but FRR 10.5 then takes every route +from us for unreachable. BIRD needs only `neighbor fe80::1%eno3 as ...`. ## Forwarding: routes in the kernel (Linux) @@ -293,14 +328,15 @@ Routinator 0.15.2. What it covers: | `scale` | A table from BIRD (`SCALE_ROUTES`, a million for the Internet's size) into the table and the kernel; the daemon killed and restarted under it; the table withdrawn | | `limits` | A peer whose role does not fit gets no session; a peer over its prefix limit gets Cease / Maximum Number of Prefixes and is held down | | `v6only` | The exchange LAN without IPv4 addresses: one IPv6 session with each router for both families, IPv4 routes with IPv6 next hops (RFC 8950) in BGP and in the kernel, IPv4 pings across | +| `linklocal` | The exchange LAN with link-local addresses only, and the same ones on a second interface: sessions with FRR (by interface, with radvd beside the daemon) and BIRD at `fe80::…%ix0`, both families in BGP and in the kernel, pings across, the interface taken down and up | `scripts/interop/real-rpki.sh` checks the RTR client against a Routinator that has the real RPKI, ASPAs included: the daemon has to end up with as many VRPs and ASPAs as the validator says it has. Not covered: routers other than FRR and BIRD; IPv6 routes over an IPv4 -session (the daemon does not send them); sessions between link-local -addresses; long runs. +session (the daemon does not send them); an interface that is removed and +made anew under a session; long runs. ## Finding a route diff --git a/crates/bench/benches/full_table.rs b/crates/bench/benches/full_table.rs index 16e3cf8..50af1f3 100644 --- a/crates/bench/benches/full_table.rs +++ b/crates/bench/benches/full_table.rs @@ -162,6 +162,7 @@ fn into_table(wire: &[Vec], routes: usize, threads: usize) { asn: 64512 + peer as u32 + 1, router_id: Ipv4Addr::from(peer as u32 + 1), address: Ipv4Addr::new(192, 0, 2, peer as u8 + 1).into(), + interface: 0, ibgp: false, }) .expect("room for the peer") diff --git a/crates/bench/benches/real_table.rs b/crates/bench/benches/real_table.rs index 5c98fe8..24977ea 100644 --- a/crates/bench/benches/real_table.rs +++ b/crates/bench/benches/real_table.rs @@ -281,6 +281,7 @@ fn add_peers(table: &RibTable, dump: &Dump, peers: &[usize]) -> Vec { asn: dump.peers[peer].asn, router_id: Ipv4Addr::from(peer as u32 + 1), address: dump.peers[peer].address, + interface: 0, ibgp: false, }) .expect("room for the peer") diff --git a/crates/fib/src/kernel.rs b/crates/fib/src/kernel.rs index f36df44..d54bb4f 100644 --- a/crates/fib/src/kernel.rs +++ b/crates/fib/src/kernel.rs @@ -30,7 +30,7 @@ use crate::netlink::{ /// "Input/output error": for what should not happen. const EIO: i32 = 5; -use crate::plan::{Op, Plan, Target, Unresolved, resolve}; +use crate::plan::{Neighbor, Op, Plan, Target, Unresolved, resolve}; use crate::{Fib, FibConfig, FibError, InterfaceStatus}; /// How many best routes are taken from the routing table at a time. @@ -305,7 +305,7 @@ async fn write( } /// What a next hop is resolved for: see [`Task::resolved`]. -type Resolving = (Option, IpVersion, Option); +type Resolving = (Option, IpVersion, Option); /// A change to the best routes, as taken out of the routing table to be /// dealt with once the table is let go of. @@ -689,10 +689,13 @@ impl Task { next_hop, Some(NlriNextHop::Ipv6(_) | NlriNextHop::Ipv6WithLl { .. }) ); - let peer: Option = needs_peer + let peer: Option = needs_peer .then(|| self.rib.peer(peer)) .flatten() - .map(|peer| peer.address); + .map(|peer| Neighbor { + address: peer.address, + interface: peer.interface, + }); let interfaces = &self.interfaces; let target = *self .resolved diff --git a/crates/fib/src/plan.rs b/crates/fib/src/plan.rs index 6edb33a..2fa31b7 100644 --- a/crates/fib/src/plan.rs +++ b/crates/fib/src/plan.rs @@ -60,8 +60,27 @@ fn is_link_local(address: &Ipv6Addr) -> bool { address.segments()[0] & 0xffc0 == 0xfe80 } +/// The peer a route was learned from, as far as its next hop is concerned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Neighbor { + pub address: IpAddr, + /// The interface the session with the peer is on, where the peer's + /// address is a link-local one and says nothing without it; zero + /// otherwise. + pub interface: u32, +} + +impl From for Neighbor { + fn from(address: IpAddr) -> Self { + Self { + address, + interface: 0, + } + } +} + /// Works out where the kernel is to send packets for a route to `prefix` -/// with the next hop `next_hop`, learned from the peer at `peer`. +/// with the next hop `next_hop`, learned from `peer`. /// /// Only a next hop that is a neighbor on one of our interfaces is /// resolved. A link-local next hop means something only on one link, and @@ -70,7 +89,7 @@ pub fn resolve( interfaces: &Interfaces, prefix: &IpPrefix, next_hop: Option<&NlriNextHop>, - peer: Option, + peer: Option, ) -> Result { let next_hop = next_hop.ok_or(Unresolved::Missing)?; let version = match next_hop { @@ -97,12 +116,29 @@ pub fn resolve( }; // The interface the peer is a neighbor on, for a link-local gateway. let on_peers_link = |gateway: Ipv6Addr| { - peer.and_then(|peer| interfaces.connected(peer)) - .map(|connected| Target { - gateway: gateway.into(), - interface: connected.interface, - }) - .ok_or(Unresolved::NotConnected) + let interface = match peer.ok_or(Unresolved::NotConnected)? { + // The session is on a link-local address itself, and so says + // which link: every interface has fe80::/64, and looking the + // peer up there would find any of them. + Neighbor { interface, .. } if interface != 0 => interfaces + .link(interface) + .filter(|link| link.up) + .map(|link| link.index), + Neighbor { address, .. } => interfaces + .connected(address) + .map(|connected| connected.interface), + } + .ok_or(Unresolved::NotConnected)?; + if interfaces + .addresses_of(interface) + .any(|ours| ours.local == IpAddr::V6(gateway)) + { + return Err(Unresolved::Ourselves); + } + Ok(Target { + gateway: gateway.into(), + interface, + }) }; match *next_hop { NlriNextHop::Ipv4(gateway) => direct(gateway.into()), @@ -110,6 +146,13 @@ pub fn resolve( NlriNextHop::Ipv6(gateway) => direct(gateway.into()), // The global address if it is a neighbor; else the link-local one, // which is there for exactly that case (RFC 2545 §3). + // Some put the link-local address, or nothing, where the global + // one goes, having no global one to give. + NlriNextHop::Ipv6WithLl { global, link_local } + if global.is_unspecified() || is_link_local(&global) => + { + on_peers_link(link_local) + } NlriNextHop::Ipv6WithLl { global, link_local } => match direct(global.into()) { Err(Unresolved::NotConnected) => on_peers_link(link_local), resolved => resolved, @@ -405,12 +448,16 @@ mod tests { interfaces } + fn neighbor(address: &str) -> Neighbor { + address.parse::().unwrap().into() + } + #[test] fn a_next_hop_resolves_if_it_is_a_neighbor() { let interfaces = interfaces(); let v4 = prefix("203.0.113.0/24"); let v6 = prefix("2001:db8:100::/48"); - let peer = Some("192.0.2.2".parse().unwrap()); + let peer = Some(neighbor("192.0.2.2")); let v4_hop = |s: &str| NlriNextHop::Ipv4(s.parse().unwrap()); let v6_hop = |s: &str| NlriNextHop::Ipv6(s.parse().unwrap()); @@ -452,7 +499,7 @@ mod tests { global: "2001:db8:ffff::2".parse().unwrap(), link_local: "fe80::2".parse().unwrap(), }), - Some("2001:db8::2".parse().unwrap()), + Some(neighbor("2001:db8::2")), ), Ok(target("fe80::2", 2)) ); @@ -462,7 +509,7 @@ mod tests { fn a_link_local_next_hop_is_on_the_peers_link() { let interfaces = interfaces(); let v6 = prefix("2001:db8:100::/48"); - let peer = Some("2001:db8::2".parse().unwrap()); + let peer = Some(neighbor("2001:db8::2")); let link_local: Ipv6Addr = "fe80::2".parse().unwrap(); let both = |global: &str| NlriNextHop::Ipv6WithLl { global: global.parse().unwrap(), @@ -483,7 +530,7 @@ mod tests { Ok(target("fe80::2", 2)) ); // A peer that is not a neighbor says nothing about the link. - let far = Some("2001:db8:ffff::9".parse().unwrap()); + let far = Some(neighbor("2001:db8:ffff::9")); assert_eq!( resolve(&interfaces, &v6, Some(&NlriNextHop::Ipv6(link_local)), far), Err(Unresolved::NotConnected) @@ -494,6 +541,76 @@ mod tests { ); } + /// A session on link-local addresses: which link is known from the + /// session and from nothing else, every interface having fe80::/64. + #[test] + fn a_link_local_peer_is_on_the_interface_of_its_session() { + let mut interfaces = interfaces(); + for (index, local) in [(2, "fe80::a"), (3, "fe80::b")] { + interfaces.apply(&Message::Address { + present: true, + address: Address { + interface: index, + local: local.parse().unwrap(), + network: prefix("fe80::/64"), + }, + }); + } + let v4 = prefix("203.0.113.0/24"); + let v6 = prefix("2001:db8:100::/48"); + let link_local: Ipv6Addr = "fe80::2".parse().unwrap(); + let on = |interface| { + Some(Neighbor { + address: link_local.into(), + interface, + }) + }; + // However the peer says it: the address alone, or twice, or after + // nothing where a global one would go. + for next_hop in [ + NlriNextHop::Ipv6(link_local), + NlriNextHop::Ipv6WithLl { + global: link_local, + link_local, + }, + NlriNextHop::Ipv6WithLl { + global: Ipv6Addr::UNSPECIFIED, + link_local, + }, + ] { + for interface in [2, 3] { + for route in [&v4, &v6] { + assert_eq!( + resolve(&interfaces, route, Some(&next_hop), on(interface)), + Ok(target("fe80::2", interface)), + "{next_hop:?} on {interface}" + ); + } + } + // An interface that is not there, or is down. + assert_eq!( + resolve(&interfaces, &v6, Some(&next_hop), on(9)), + Err(Unresolved::NotConnected) + ); + } + // Our own address on that link, which on another is a neighbor's. + let ours = NlriNextHop::Ipv6("fe80::a".parse().unwrap()); + assert_eq!( + resolve(&interfaces, &v6, Some(&ours), on(2)), + Err(Unresolved::Ourselves) + ); + assert_eq!( + resolve(&interfaces, &v6, Some(&ours), on(3)), + Ok(target("fe80::a", 3)) + ); + // A global next hop on such a session is looked up as any other. + let global = NlriNextHop::Ipv6("2001:db8::2".parse().unwrap()); + assert_eq!( + resolve(&interfaces, &v6, Some(&global), on(3)), + Ok(target("2001:db8::2", 2)) + ); + } + #[test] fn the_kernel_is_asked_only_for_what_changes() { let mut plan = Plan::default(); diff --git a/crates/fib/tests/kernel.rs b/crates/fib/tests/kernel.rs index b998c9e..ede5a20 100644 --- a/crates/fib/tests/kernel.rs +++ b/crates/fib/tests/kernel.rs @@ -120,6 +120,7 @@ impl Machine { asn: 65000 + u32::from(host), router_id: self.neighbor(host), address: self.neighbor(host).into(), + interface: 0, ibgp: false, }) .unwrap() @@ -616,6 +617,7 @@ async fn an_ipv4_route_goes_by_way_of_an_ipv6_neighbor() { asn: 65002, router_id: machine.neighbor(2), address: neighbor, + interface: 0, ibgp: false, }) .unwrap(); diff --git a/crates/integration-tests/tests/add_path.rs b/crates/integration-tests/tests/add_path.rs index 9033566..d9fdb4b 100644 --- a/crates/integration-tests/tests/add_path.rs +++ b/crates/integration-tests/tests/add_path.rs @@ -41,7 +41,7 @@ async fn server_offering( TestServer::start_with(LOCAL_ASN, |c| { c.peers = vec![PeerConfig { name: "route-server-client".into(), - endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1), + endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/admission.rs b/crates/integration-tests/tests/admission.rs index 8a2226d..ff60c34 100644 --- a/crates/integration-tests/tests/admission.rs +++ b/crates/integration-tests/tests/admission.rs @@ -19,7 +19,7 @@ const TIMEOUT: Duration = Duration::from_secs(5); fn peer_at(endpoint: SocketAddr) -> PeerConfig { PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/aspa.rs b/crates/integration-tests/tests/aspa.rs index 03007bf..77ec20a 100644 --- a/crates/integration-tests/tests/aspa.rs +++ b/crates/integration-tests/tests/aspa.rs @@ -57,7 +57,7 @@ async fn server_with( }); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/collision.rs b/crates/integration-tests/tests/collision.rs index 2c7c8c6..3f1e616 100644 --- a/crates/integration-tests/tests/collision.rs +++ b/crates/integration-tests/tests/collision.rs @@ -37,7 +37,7 @@ async fn server_dialing(listener: &TcpListener) -> Result { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/debugging.rs b/crates/integration-tests/tests/debugging.rs index 333890a..007cdf2 100644 --- a/crates/integration-tests/tests/debugging.rs +++ b/crates/integration-tests/tests/debugging.rs @@ -167,7 +167,7 @@ async fn test_refused_routes_can_be_seen_and_counted() -> Result<()> { c.accept_unconfigured_peers = false; c.peers = vec![PeerConfig { name: "refused-demo".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: 3333, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/dialer.rs b/crates/integration-tests/tests/dialer.rs index 313ce7a..44c1d9a 100644 --- a/crates/integration-tests/tests/dialer.rs +++ b/crates/integration-tests/tests/dialer.rs @@ -28,7 +28,7 @@ async fn server_dialing(listener: &TcpListener, remote_asn: u32) -> Result Result<()> { c.connect_retry = Duration::from_secs(60); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1), + endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, @@ -236,7 +236,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: Some(Ipv4Addr::LOCALHOST.into()), @@ -256,7 +256,7 @@ async fn test_local_address_is_used_as_the_source() -> Result<()> { c.connect_retry = Duration::from_millis(200); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: Some(Ipv4Addr::new(192, 0, 2, 1).into()), diff --git a/crates/integration-tests/tests/import_policy.rs b/crates/integration-tests/tests/import_policy.rs index e721e4d..57483df 100644 --- a/crates/integration-tests/tests/import_policy.rs +++ b/crates/integration-tests/tests/import_policy.rs @@ -41,7 +41,7 @@ async fn server_with(import: ImportPolicy) -> Result<(TestServer, TcpListener)> c.connect_retry = Duration::from_millis(50); c.peers = vec![PeerConfig { name: "r1".into(), - endpoint, + endpoint: endpoint.into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/negotiation.rs b/crates/integration-tests/tests/negotiation.rs index 8730cba..3dfa4ad 100644 --- a/crates/integration-tests/tests/negotiation.rs +++ b/crates/integration-tests/tests/negotiation.rs @@ -184,7 +184,7 @@ async fn test_no_family_in_common_refuses_the_session() -> Result<()> { // Inbound connections from this address inherit the peer's settings. c.peers = vec![PeerConfig { name: "v6-only".into(), - endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1), + endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(), remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None, diff --git a/crates/integration-tests/tests/roles.rs b/crates/integration-tests/tests/roles.rs index 16e79ae..29d5f5e 100644 --- a/crates/integration-tests/tests/roles.rs +++ b/crates/integration-tests/tests/roles.rs @@ -48,7 +48,7 @@ async fn server_with(role: Option, require_role: bool) -> Result PeerInfo { asn: 65000 + u32::from(number), router_id: Ipv4Addr::new(10, 0, 0, number), address: IpAddr::from(Ipv4Addr::new(192, 0, 2, number)), + interface: 0, ibgp: false, } } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 45facd9..86b9690 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -23,6 +23,7 @@ tracing-subscriber.workspace = true tracing.workspace = true protobuf.workspace = true thiserror.workspace = true +nix.workspace = true [build-dependencies] protobuf-codegen.workspace = true diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index 53af4b7..21cff60 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -141,7 +141,8 @@ message OriginatedRoute { // An IP address and port pair. message TcpEndpoint { - // IP address (V4 or V6). + // IP address (V4 or V6). A peer at a link-local IPv6 address has the + // interface it is on after it: "fe80::1%eth0". string address = 1; // TCP port. Defaults to 179 for BGP when unset. uint32 port = 2; diff --git a/crates/server/src/address.rs b/crates/server/src/address.rs new file mode 100644 index 0000000..e4605cf --- /dev/null +++ b/crates/server/src/address.rs @@ -0,0 +1,354 @@ +//! Where a peer is. +//! +//! Most peers are told apart by their IP address alone. A link-local +//! address (`fe80::/10`) means something only on one link, and the same +//! one is often used on every link a router has, so such a peer is the +//! address together with the interface it is reached on: `fe80::1%eth0`. +//! +//! The kernel knows an interface by a number that changes when the +//! interface is made anew, as a tunnel is when it is restarted. So the +//! configuration keeps the name, and the number is looked up whenever it +//! is needed. + +use std::fmt; +use std::io; +use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; + +/// The longest name the kernel gives an interface. +const MAX_INTERFACE_NAME: usize = 15; + +/// A peer's IP address, and for a link-local address the interface it is +/// on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PeerAddress { + pub ip: IpAddr, + /// The number of the interface, for a link-local address; zero for + /// any other. + pub interface: u32, +} + +impl From for PeerAddress { + fn from(address: SocketAddr) -> Self { + let interface = match address { + SocketAddr::V6(v6) if is_link_local(v6.ip()) => v6.scope_id(), + _ => 0, + }; + Self { + ip: address.ip(), + interface, + } + } +} + +impl PeerAddress { + /// Whether `text` is this address as someone might type it: as it is + /// written out, or without the interface. + pub fn is_called(&self, text: &str) -> bool { + text.parse() == Ok(self.ip) || self.to_string() == text + } +} + +impl From for PeerAddress { + fn from(ip: IpAddr) -> Self { + Self { ip, interface: 0 } + } +} + +/// The address, and the interface by name if there is one: `fe80::1%eth0`. +/// An interface that is no longer there goes by its number. +impl fmt::Display for PeerAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.ip)?; + match self.interface { + 0 => Ok(()), + index => match interface_name(index) { + Some(name) => write!(f, "%{name}"), + None => write!(f, "%{index}"), + }, + } + } +} + +/// `address` as text, with an interface by its name: `[fe80::1%eth0]:179`. +pub fn socket_text(address: SocketAddr) -> String { + match PeerAddress::from(address) { + PeerAddress { interface: 0, ip } => SocketAddr::new(ip, address.port()).to_string(), + scoped => format!("[{scoped}]:{}", address.port()), + } +} + +pub fn is_link_local(address: &Ipv6Addr) -> bool { + address.segments()[0] & 0xffc0 == 0xfe80 +} + +/// The number of the interface called `name`, if there is one now. +pub fn interface_index(name: &str) -> Option { + nix::net::if_::if_nametoindex(name).ok() +} + +/// The name of the interface with this number, if there is one now. +pub fn interface_name(index: u32) -> Option { + nix::net::if_::if_indextoname(index) + .ok()? + .into_string() + .ok() +} + +/// A peer's address as the configuration has it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Endpoint { + pub ip: IpAddr, + pub port: u16, + /// The interface a link-local address is on; `None` for any other + /// address. + pub interface: Option, +} + +/// What is wrong with a peer's address. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum EndpointProblem { + #[error("{0:?} is not an IP address")] + NotAnAddress(String), + #[error( + "{0} is a link-local address, which needs the interface it is on: {0}%eth0, for example" + )] + NoInterface(Ipv6Addr), + #[error("{0} is not a link-local address, and takes no interface")] + NotLinkLocal(IpAddr), + #[error("{0:?} is not the name of an interface")] + InterfaceName(String), +} + +impl Endpoint { + /// Reads `text`: an IP address, with `%` and the name of an interface + /// after it if, and only if, it is link-local. + pub fn parse(text: &str, port: u16) -> Result { + let (address, interface) = match text.split_once('%') { + Some((address, interface)) => (address, Some(interface)), + None => (text, None), + }; + let ip: IpAddr = address + .parse() + .map_err(|_| EndpointProblem::NotAnAddress(text.to_string()))?; + let link_local = match ip { + IpAddr::V6(v6) if is_link_local(&v6) => Some(v6), + _ => None, + }; + let interface = match (link_local, interface) { + (None, None) => None, + (Some(v6), None) => return Err(EndpointProblem::NoInterface(v6)), + (None, Some(_)) => return Err(EndpointProblem::NotLinkLocal(ip)), + (Some(_), Some(name)) => { + let fine = !name.is_empty() + && name.len() <= MAX_INTERFACE_NAME + && name + .chars() + .all(|c| c.is_ascii_graphic() && c != '/' && c != '%'); + if !fine { + return Err(EndpointProblem::InterfaceName(name.to_string())); + } + Some(name.to_string()) + } + }; + Ok(Self { + ip, + port, + interface, + }) + } + + /// The number of the peer's interface as things are now: zero if the + /// address needs none, and `None` if the interface is not there. + fn scope(&self) -> Option { + match &self.interface { + None => Some(0), + Some(name) => interface_index(name), + } + } + + /// What a connection from the peer comes from, as things are now. + /// `None` if the peer's interface is not there. + pub fn address(&self) -> Option { + Some(PeerAddress { + ip: self.ip, + interface: self.scope()?, + }) + } + + /// Whether a connection from `address` is one from this peer. + pub fn is_at(&self, address: PeerAddress) -> bool { + // The interface is looked up only for a peer whose address it is. + self.ip == address.ip && self.scope() == Some(address.interface) + } + + /// Where to connect to reach the peer. + pub fn to_dial(&self) -> io::Result { + self.socket(self.ip, self.port) + } + + /// `ip` and `port` as an address on the peer's link: for the peer + /// itself, and for our own address there. + pub fn socket(&self, ip: IpAddr, port: u16) -> io::Result { + let scope = self.scope().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!( + "there is no interface {}", + self.interface.as_deref().unwrap_or_default() + ), + ) + })?; + Ok(match ip { + IpAddr::V6(v6) if is_link_local(&v6) => SocketAddrV6::new(v6, port, 0, scope).into(), + ip => SocketAddr::new(ip, port), + }) + } +} + +/// The address and the interface: `fe80::1%eth0`. Without the port. +impl fmt::Display for Endpoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.interface { + None => write!(f, "{}", self.ip), + Some(interface) => write!(f, "{}%{interface}", self.ip), + } + } +} + +impl Endpoint { + /// The address and the port, as [`SocketAddr`] is written: + /// `[fe80::1%eth0]:179`. + pub fn with_port(&self) -> String { + match self.ip { + IpAddr::V4(v4) => format!("{v4}:{}", self.port), + IpAddr::V6(_) => format!("[{self}]:{}", self.port), + } + } +} + +impl From for Endpoint { + /// An address that needs no interface. A link-local one keeps none + /// and so reaches nothing; those are read with [`Endpoint::parse`]. + fn from(address: SocketAddr) -> Self { + Self { + ip: address.ip(), + port: address.port(), + interface: None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An interface every machine has, and its number. + fn loopback() -> (String, u32) { + ["lo", "lo0"] + .iter() + .find_map(|name| Some((name.to_string(), interface_index(name)?))) + .expect("a loopback interface") + } + + #[test] + fn a_link_local_address_is_read_with_its_interface() { + let peer = Endpoint::parse("fe80:1234::1%eno3", 179).unwrap(); + assert_eq!(peer.ip, "fe80:1234::1".parse::().unwrap()); + assert_eq!(peer.interface.as_deref(), Some("eno3")); + assert_eq!(peer.to_string(), "fe80:1234::1%eno3"); + assert_eq!(peer.with_port(), "[fe80:1234::1%eno3]:179"); + + let plain = Endpoint::parse("2001:db8::1", 1179).unwrap(); + assert_eq!(plain.interface, None); + assert_eq!(plain.with_port(), "[2001:db8::1]:1179"); + assert_eq!( + Endpoint::parse("192.0.2.1", 179).unwrap().with_port(), + "192.0.2.1:179" + ); + } + + #[test] + fn an_address_and_an_interface_that_do_not_go_together_are_refused() { + let problem = |text: &str| Endpoint::parse(text, 179).unwrap_err(); + assert_eq!( + problem("fe80::1"), + EndpointProblem::NoInterface("fe80::1".parse().unwrap()) + ); + assert_eq!( + problem("febf::1"), + EndpointProblem::NoInterface("febf::1".parse().unwrap()) + ); + assert_eq!( + problem("2001:db8::1%eth0"), + EndpointProblem::NotLinkLocal("2001:db8::1".parse().unwrap()) + ); + assert_eq!( + problem("192.0.2.1%eth0"), + EndpointProblem::NotLinkLocal("192.0.2.1".parse().unwrap()) + ); + // fec0::/10 was site-local: not link-local. + assert!(Endpoint::parse("fec0::1", 179).is_ok()); + for name in ["", "a/b", "an interface", "sixteen-letters-", "a%b"] { + assert_eq!( + problem(&format!("fe80::1%{name}")), + EndpointProblem::InterfaceName(name.to_string()), + ); + } + assert_eq!( + problem("example"), + EndpointProblem::NotAnAddress("example".into()) + ); + } + + /// The interface's number is whatever it is when it is asked for. + #[test] + fn the_interface_is_looked_up_when_it_is_needed() { + let (name, index) = loopback(); + let peer = Endpoint::parse(&format!("fe80::1%{name}"), 179).unwrap(); + let ip: IpAddr = "fe80::1".parse().unwrap(); + let there = PeerAddress { + ip, + interface: index, + }; + assert_eq!(peer.address(), Some(there)); + assert!(peer.is_at(there)); + // The same address on another link is another peer. + assert!(!peer.is_at(PeerAddress { + ip, + interface: index + 1000 + })); + assert!(!peer.is_at(ip.into())); + match peer.to_dial().unwrap() { + SocketAddr::V6(v6) => assert_eq!((v6.scope_id(), v6.port()), (index, 179)), + SocketAddr::V4(_) => panic!("an IPv6 peer"), + } + assert_eq!(there.to_string(), format!("fe80::1%{name}")); + assert!(there.is_called(&format!("fe80::1%{name}"))); + assert!(there.is_called("fe80::1")); + assert!(!there.is_called("fe80::1%other")); + assert!(!there.is_called("fe80::2")); + assert_eq!( + socket_text(peer.to_dial().unwrap()), + format!("[fe80::1%{name}]:179") + ); + + let gone = Endpoint::parse("fe80::1%not-there", 179).unwrap(); + assert_eq!(gone.address(), None); + assert!(!gone.is_at(there)); + assert_eq!( + gone.to_dial().unwrap_err().to_string(), + "there is no interface not-there" + ); + } + + /// Only a link-local address keeps the interface it came in on. + #[test] + fn only_a_link_local_address_has_an_interface() { + let global: SocketAddr = + SocketAddrV6::new("2001:db8::1".parse().unwrap(), 179, 0, 7).into(); + assert_eq!(PeerAddress::from(global).interface, 0); + let local: SocketAddr = SocketAddrV6::new("fe80::1".parse().unwrap(), 179, 0, 7).into(); + assert_eq!(PeerAddress::from(local).interface, 7); + assert_eq!(socket_text(global), "[2001:db8::1]:179"); + } +} diff --git a/crates/server/src/advertise.rs b/crates/server/src/advertise.rs index 3346bc3..0be473e 100644 --- a/crates/server/src/advertise.rs +++ b/crates/server/src/advertise.rs @@ -16,7 +16,7 @@ //! and stops it when the session goes. use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use bgp_packet::message::{ @@ -34,6 +34,7 @@ use tokio::sync::broadcast::error::RecvError; use tokio::task::JoinHandle; use tracing::{debug, info, warn}; +use crate::address::is_link_local; use crate::policy::PeerRole; use crate::session::{PeerId, SessionClosed, SessionEvent, SessionHandle, SessionRegistry}; @@ -331,7 +332,7 @@ pub fn build_announce(route: &LocalRoute, towards: &Towards) -> Option { Some(PathAttribute::MpReachNlri(MpReachNlriPathAttribute { family: AfiSafi::IPV6_UNICAST, - next_hop: NlriNextHop::Ipv6(next_hop), + next_hop: ipv6_next_hop(next_hop), prefixes: vec![route.prefix.into()], })) } @@ -341,7 +342,7 @@ pub fn build_announce(route: &LocalRoute, towards: &Towards) -> Option { Some(PathAttribute::MpReachNlri(MpReachNlriPathAttribute { family: AfiSafi::IPV4_UNICAST, - next_hop: NlriNextHop::Ipv6(next_hop), + next_hop: ipv6_next_hop(next_hop), prefixes: vec![route.prefix.into()], })) } @@ -395,6 +396,24 @@ pub fn build_announce(route: &LocalRoute, towards: &Towards) -> Option NlriNextHop { + if is_link_local(&ours) { + NlriNextHop::Ipv6WithLl { + global: ours, + link_local: ours, + } + } else { + NlriNextHop::Ipv6(ours) + } +} + /// Builds the UPDATE withdrawing `prefix`. With `extended_next_hop` the /// session has IPv4 routes announced in MP_REACH_NLRI (RFC 8950), and they /// are withdrawn the way they were announced. @@ -439,6 +458,34 @@ mod tests { IpPrefix::try_from(s).unwrap() } + /// On a session between link-local addresses ours is the next hop, + /// written the way FRR writes its own and takes ours. + #[test] + fn a_link_local_next_hop_is_given_twice() { + let ours: Ipv6Addr = "fe80::1".parse().unwrap(); + let towards = Towards { + extended_next_hop: true, + ..towards(65000, IpAddr::V6(ours), true) + }; + for (route, family) in [ + ("10.1.0.0/16", AfiSafi::IPV4_UNICAST), + ("2001:db8:1::/48", AfiSafi::IPV6_UNICAST), + ] { + let local = LocalRoute::new(prefix(route), LocalRouteAttrs::default()); + let update = build_announce(&local, &towards).unwrap(); + assert!(update.path_attributes.contains(&PathAttribute::MpReachNlri( + MpReachNlriPathAttribute { + family, + next_hop: NlriNextHop::Ipv6WithLl { + global: ours, + link_local: ours, + }, + prefixes: vec![prefix(route).into()], + } + ))); + } + } + /// RFC 8950: over a link that has only IPv6 addresses, an IPv4 route /// goes in MP_REACH_NLRI with our IPv6 address as next hop, and is /// withdrawn in MP_UNREACH_NLRI. @@ -830,6 +877,7 @@ mod tests { asn: negotiated.peer_asn, router_id: negotiated.peer_identifier, address, + interface: 0, ibgp: false, }) .unwrap(); diff --git a/crates/server/src/collision.rs b/crates/server/src/collision.rs index f067391..a15d0d8 100644 --- a/crates/server/src/collision.rs +++ b/crates/server/src/collision.rs @@ -24,12 +24,13 @@ //! by the same side the newer wins: the initiator has abandoned the older. use std::collections::HashMap; -use std::net::{IpAddr, Ipv4Addr}; +use std::net::Ipv4Addr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use tokio::sync::oneshot; +use crate::address::PeerAddress; use crate::metrics::COLLISIONS; use crate::sync::IgnorePoison; @@ -83,7 +84,7 @@ fn new_loses( /// Who a connection is with: the address at the other end, and the BGP /// Identifier announced from it. -type Peer = (IpAddr, Ipv4Addr); +type Peer = (PeerAddress, Ipv4Addr); /// Shared by every connection of one server. #[derive(Clone)] @@ -109,7 +110,7 @@ impl CollisionTable { pub fn admit( &self, local_id: Ipv4Addr, - remote_addr: IpAddr, + remote_addr: PeerAddress, remote_id: Ipv4Addr, direction: Direction, ) -> Option { @@ -263,7 +264,7 @@ mod tests { #[tokio::test] async fn table_signals_loser_and_releases_slots() { let table = CollisionTable::new(false); - let addr: IpAddr = Ipv4Addr::new(192, 0, 2, 1).into(); + let addr = PeerAddress::from(std::net::IpAddr::from(Ipv4Addr::new(192, 0, 2, 1))); // We are LOW: outbound first, then the peer's inbound wins. let mut out = table.admit(LOW, addr, HIGH, Outbound).unwrap(); let inb = table.admit(LOW, addr, HIGH, Inbound).unwrap(); @@ -292,8 +293,8 @@ mod tests { #[tokio::test] async fn the_same_identifier_from_another_address_is_another_peer() { let table = CollisionTable::new(true); - let v4: IpAddr = Ipv4Addr::new(192, 0, 2, 1).into(); - let v6: IpAddr = "2001:db8::1".parse().unwrap(); + let v4 = PeerAddress::from(std::net::IpAddr::from(Ipv4Addr::new(192, 0, 2, 1))); + let v6 = PeerAddress::from("2001:db8::1".parse::().unwrap()); let mut over_v4 = table.admit(LOW, v4, HIGH, Outbound).unwrap(); over_v4.set_established(); diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 9625341..7477c06 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -11,6 +11,7 @@ use bgp_primitives::ip_prefix::IpPrefix; use bgp_rib::{LocalRoute, LocalRouteAttrs}; use bgp_rpki::RpkiConfig; +use crate::address::{Endpoint, PeerAddress}; use crate::error::{ConfigError, ImportProblem, OriginateProblem, PeerProblem}; use crate::family::FamilySet; use crate::policy::{ImportPolicy, PeerRole, PrefixSet}; @@ -54,7 +55,8 @@ pub struct Config { #[derive(Debug, Clone)] pub struct PeerConfig { pub name: String, - pub endpoint: SocketAddr, + /// Where the peer is. A link-local address has its interface with it. + pub endpoint: Endpoint, pub remote_asn: u32, pub hold_time_secs: u16, /// Source address for connections we open to this peer; `None` lets the @@ -76,8 +78,8 @@ pub struct PeerConfig { impl Config { /// The configured peer at `address`, if there is one. - pub fn peer_at(&self, address: IpAddr) -> Option<&PeerConfig> { - self.peers.iter().find(|peer| peer.endpoint.ip() == address) + pub fn peer_at(&self, address: PeerAddress) -> Option<&PeerConfig> { + self.peers.iter().find(|peer| peer.endpoint.is_at(address)) } pub fn from_proto(proto: BgpConfig) -> Result { @@ -124,10 +126,10 @@ impl Config { if peers.iter().any(|earlier| earlier.name == peer.name) { return Err(rejected(PeerProblem::DuplicateName)); } - if peers - .iter() - .any(|earlier| earlier.endpoint.ip() == peer.endpoint.ip()) - { + if peers.iter().any(|earlier| { + earlier.endpoint.ip == peer.endpoint.ip + && earlier.endpoint.interface == peer.endpoint.interface + }) { return Err(rejected(PeerProblem::DuplicateAddress)); } peers.push(peer); @@ -197,8 +199,9 @@ impl Config { fn parse_peer(peer: Peer) -> Result { let ep = peer.endpoint.as_ref().ok_or(PeerProblem::MissingEndpoint)?; - let endpoint = - parse_endpoint(&ep.address, ep.port).map_err(|e| PeerProblem::Endpoint(Box::new(e)))?; + let endpoint = parse_port(ep.port) + .and_then(|port| Ok(Endpoint::parse(&ep.address, port)?)) + .map_err(|e| PeerProblem::Endpoint(Box::new(e)))?; if peer.remote_asn == 0 { return Err(PeerProblem::RemoteAsn); @@ -216,7 +219,7 @@ fn parse_peer(peer: Peer) -> Result { } else { let ip = IpAddr::from_str(&peer.local_address) .map_err(|_| PeerProblem::LocalAddress(peer.local_address.clone()))?; - if ip.is_ipv4() != endpoint.is_ipv4() { + if ip.is_ipv4() != endpoint.ip.is_ipv4() { return Err(PeerProblem::AddressFamilyMismatch); } Some(ip) @@ -266,7 +269,7 @@ fn parse_peer(peer: Peer) -> Result { Ok(PeerConfig { // A peer without a name goes by its address. name: if peer.name.is_empty() { - endpoint.ip().to_string() + endpoint.to_string() } else { peer.name }, @@ -374,11 +377,15 @@ fn parse_import(import: &proto::ImportPolicy) -> Result Result { + match port { + 0 => Ok(DEFAULT_BGP_PORT), + port => u16::try_from(port).map_err(|_| ConfigError::Port(port)), + } +} + fn parse_endpoint(address: &str, port: u32) -> Result { - let port = match port { - 0 => DEFAULT_BGP_PORT, - port => u16::try_from(port).map_err(|_| ConfigError::Port(port))?, - }; + let port = parse_port(port)?; let ip = IpAddr::from_str(address).map_err(|_| ConfigError::Address(address.to_string()))?; Ok(SocketAddr::new(ip, port)) } @@ -624,6 +631,28 @@ mod tests { PeerProblem::DuplicateAddress ); + // A link-local address is one peer on each interface it is on. + assert_eq!( + problem(peer("r1", "fe80::1%eth0") + &peer("r2", "fe80::1%eth0")), + PeerProblem::DuplicateAddress + ); + let config = parse(&format!( + "{BASE}{}{}", + peer("", "fe80::1%eth0"), + peer("", "fe80::1%eth1") + )) + .unwrap(); + assert_eq!( + config.peers.iter().map(|p| &p.name).collect::>(), + ["fe80::1%eth0", "fe80::1%eth1"] + ); + // And nothing without one. + assert_eq!( + problem(peer("r1", "fe80::1")).to_string(), + "endpoint is invalid: fe80::1 is a link-local address, which needs the interface it \ + is on: fe80::1%eth0, for example" + ); + // A peer without a name goes by its address, so two of them differ. let config = parse(&format!( "{BASE}{}{}", @@ -635,11 +664,15 @@ mod tests { assert_eq!(config.peers[1].name, "192.0.2.3"); assert_eq!( config - .peer_at("192.0.2.3".parse().unwrap()) + .peer_at("192.0.2.3".parse::().unwrap().into()) .map(|p| &*p.name), Some("192.0.2.3") ); - assert!(config.peer_at("192.0.2.4".parse().unwrap()).is_none()); + assert!( + config + .peer_at("192.0.2.4".parse::().unwrap().into()) + .is_none() + ); } /// A peer would refuse our OPEN. diff --git a/crates/server/src/connection.rs b/crates/server/src/connection.rs index 9a468a2..abda53a 100644 --- a/crates/server/src/connection.rs +++ b/crates/server/src/connection.rs @@ -33,6 +33,7 @@ use tokio::sync::mpsc; use tokio::time::{Instant, Interval, MissedTickBehavior, Sleep, interval_at, sleep, timeout}; use tracing::{debug, error, info, warn}; +use crate::address::{PeerAddress, socket_text}; use crate::collision::{CollisionTable, ConnectionGuard, Direction}; use crate::config::PeerConfig; use crate::error::SessionError; @@ -177,7 +178,7 @@ pub async fn run_session( session .server .hold_downs - .hold(peer_addr.ip(), session.import.hold_down); + .hold(peer_addr.into(), session.import.hold_down); PREFIX_LIMIT_EXCEEDED .with_label_values(&[session.refusals.label()]) .inc(); @@ -409,9 +410,9 @@ impl Session { let label = peer .name .clone() - .unwrap_or_else(|| peer_addr.ip().to_string()); + .unwrap_or_else(|| PeerAddress::from(peer_addr).to_string()); let session = Session { - refusals: server.refusals.of_peer(peer_addr.ip(), &label), + refusals: server.refusals.of_peer(peer_addr.into(), &label), fsm, frames: FrameReader::new(reader), writer, @@ -443,12 +444,12 @@ impl Session { /// out its hold timer on a connection nobody is reading. async fn close_after_panic(&mut self, panic: String) -> SessionError { let error = SessionError::Panicked(panic); - error!(peer_addr = %self.peer_addr, err = %error, "Closing session after a panic"); + error!(peer_addr = %socket_text(self.peer_addr), err = %error, "Closing session after a panic"); if let Some(notification) = error.notification() { let notification = Message::Notification(notification); // Contained as well: whatever panicked may do so again. if !matches!(contained(self.send(¬ification)).await, Ok(Ok(()))) { - warn!(peer_addr = %self.peer_addr, "Could not send NOTIFICATION after a panic"); + warn!(peer_addr = %socket_text(self.peer_addr), "Could not send NOTIFICATION after a panic"); } } error @@ -514,7 +515,7 @@ impl Session { .with_label_values(&[error.handling.as_str()]) .inc(); warn!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), handling = error.handling.as_str(), detail = %error.detail, "Error in received UPDATE" @@ -533,14 +534,14 @@ impl Session { // The session is ending anyway and the Close that // follows says why; a NOTIFICATION is best effort. Err(e) if matches!(msg, Message::Notification(_)) => { - warn!(peer_addr = %self.peer_addr, err = %e, "Failed to send NOTIFICATION"); + warn!(peer_addr = %socket_text(self.peer_addr), err = %e, "Failed to send NOTIFICATION"); } // A message of our own making that cannot be written. // That is a bug here, and nothing the peer did: the // session is worth more than the message. Err(SessionError::Encode(e)) => { error!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), message = message_type_name(&msg), err = %e, "Dropped a message that cannot be encoded" @@ -571,13 +572,13 @@ impl Session { Action::Established(negotiated) => { if let Err(e) = self.establish(&negotiated) { // What the machine wanted next no longer applies. - error!(peer_addr = %self.peer_addr, err = %e, "Cannot take the peer's routes"); + error!(peer_addr = %socket_text(self.peer_addr), err = %e, "Cannot take the peer's routes"); queue = self.fsm.handle(Event::LocalFailure(e)).into(); } } Action::RoutesIgnored(count) => { warn!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), routes = count, "Ignored routes of an address family that was not negotiated" ); @@ -589,13 +590,13 @@ impl Session { if let Err(e) = self.store(&update, attributes_wire.as_deref()) { match &e { SessionError::PrefixLimitExceeded { .. } => warn!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), err = %e, hold_down = ?self.import.hold_down, "Ending the session and keeping the peer down" ), _ => error!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), err = %e, "Cannot store the peer's routes" ), @@ -632,16 +633,16 @@ impl Session { self.collision = self.server.collisions.admit( self.server.router_id, - self.peer_addr.ip(), + self.peer_addr.into(), negotiated.peer_identifier, self.direction, ); if self.collision.is_none() { - info!(peer_addr = %self.peer_addr, "Closing connection: lost collision resolution"); + info!(peer_addr = %socket_text(self.peer_addr), "Closing connection: lost collision resolution"); return false; } info!( - peer_addr = %self.peer_addr, + peer_addr = %socket_text(self.peer_addr), peer_asn = negotiated.peer_asn, negotiated_hold_time = negotiated.hold_time_secs, four_octet_asn = negotiated.four_octet_asn, @@ -665,9 +666,10 @@ impl Session { asn: negotiated.peer_asn, router_id: negotiated.peer_identifier, address: self.peer_addr.ip(), + interface: PeerAddress::from(self.peer_addr).interface, ibgp: negotiated.ibgp, })?; - info!(peer_addr = %self.peer_addr, peer_asn = negotiated.peer_asn, "Session established"); + info!(peer_addr = %socket_text(self.peer_addr), peer_asn = negotiated.peer_asn, "Session established"); SESSIONS_ESTABLISHED_TOTAL.inc(); if let Some(collision) = &mut self.collision { @@ -820,7 +822,7 @@ impl Session { debug!(peer = %established.peer_id, withdrawn, announced, "Sending UPDATE"); } if let Message::Notification(notification) = msg { - info!(peer_addr = %self.peer_addr, error = %notification.error, "Sending NOTIFICATION, closing session"); + info!(peer_addr = %socket_text(self.peer_addr), error = %notification.error, "Sending NOTIFICATION, closing session"); self.notified = true; } self.write_buf.clear(); @@ -1180,6 +1182,7 @@ mod tests { asn: 65001, router_id: Ipv4Addr::new(10, 0, 0, 2), address: Ipv4Addr::new(192, 0, 2, 1).into(), + interface: 0, ibgp: false, }) .unwrap(); @@ -1203,7 +1206,10 @@ mod tests { }, policy: &ImportPolicy::permissive(), checks: Checks::NONE, - refusals: &RefusalLog::default().of_peer(Ipv4Addr::new(192, 0, 2, 1).into(), "test"), + refusals: &RefusalLog::default().of_peer( + std::net::IpAddr::from(Ipv4Addr::new(192, 0, 2, 1)).into(), + "test", + ), }; routes::apply(&table, &import, &mut Held::default(), &update, None).unwrap(); assert_eq!(table.len(), 1); diff --git a/crates/server/src/dialer.rs b/crates/server/src/dialer.rs index 889399a..f11b2b5 100644 --- a/crates/server/src/dialer.rs +++ b/crates/server/src/dialer.rs @@ -37,20 +37,22 @@ pub fn spawn_dialers( /// Opens the TCP connection, from the configured source address if any. async fn connect(peer: &PeerConfig) -> std::io::Result { + // Worked out for each attempt: an interface that was made anew has a + // new number. + let endpoint = peer.endpoint.to_dial()?; let Some(local) = peer.local_address else { - return TcpStream::connect(peer.endpoint).await; + return TcpStream::connect(endpoint).await; }; - let socket = if peer.endpoint.is_ipv4() { + let socket = if endpoint.is_ipv4() { TcpSocket::new_v4()? } else { TcpSocket::new_v6()? }; - socket - .bind(std::net::SocketAddr::new(local, 0)) - .map_err(|e| { - std::io::Error::new(e.kind(), format!("cannot use local_address {local}: {e}")) - })?; - socket.connect(peer.endpoint).await + // A link-local address of our own is on the peer's link. + socket.bind(peer.endpoint.socket(local, 0)?).map_err(|e| { + std::io::Error::new(e.kind(), format!("cannot use local_address {local}: {e}")) + })?; + socket.connect(endpoint).await } async fn dial_loop(peer: PeerConfig, server: Arc, connect_retry: Duration) { @@ -61,14 +63,16 @@ async fn dial_loop(peer: PeerConfig, server: Arc, connect_retry: if registry.is_shutting_down() { return; } - if registry.find(peer.endpoint.ip(), peer.remote_asn).is_some() { + // Where the peer is now; nowhere if its interface is not there. + let address = peer.endpoint.address(); + if address.is_some_and(|address| registry.find(address, peer.remote_asn).is_some()) { // Peer dialled us; nothing to do until that session goes away. registry.set_dial_state(&peer.name, DialState::Idle); tokio::time::sleep(connect_retry).await; continue; } - if let Some(remaining) = server.hold_downs.remaining(peer.endpoint.ip()) { + if let Some(remaining) = address.and_then(|address| server.hold_downs.remaining(address)) { registry.set_dial_state( &peer.name, DialState::Backoff { @@ -86,25 +90,28 @@ async fn dial_loop(peer: PeerConfig, server: Arc, connect_retry: info!(peer = %peer.name, endpoint = %peer.endpoint, "Connecting to peer"); let ended = match tokio::time::timeout(CONNECT_TIMEOUT, connect(&peer)).await { - Ok(Ok(stream)) => { - info!(peer = %peer.name, endpoint = %peer.endpoint, "TCP connected, starting session"); - let end = run_session( - stream, - peer.endpoint, - Direction::Outbound, - Arc::clone(&server), - PeerSettings::from(&peer), - ) - .await; - // A session that came up is a peer worth dialling again - // soon. One that connects and is then refused, for its AS - // number say, will be refused again: it is retried ever - // more slowly, like one that cannot be reached at all. - if end.established { - backoff = connect_retry; + Ok(Ok(stream)) => match stream.peer_addr() { + Ok(peer_addr) => { + info!(peer = %peer.name, endpoint = %peer.endpoint, "TCP connected, starting session"); + let end = run_session( + stream, + peer_addr, + Direction::Outbound, + Arc::clone(&server), + PeerSettings::from(&peer), + ) + .await; + // A session that came up is a peer worth dialling again + // soon. One that connects and is then refused, for its AS + // number say, will be refused again: it is retried ever + // more slowly, like one that cannot be reached at all. + if end.established { + backoff = connect_retry; + } + end.result.map_err(|e| e.to_string()) } - end.result.map_err(|e| e.to_string()) - } + Err(e) => Err(format!("connection lost at once: {e}")), + }, Ok(Err(e)) => Err(format!("connect failed: {e}")), Err(_) => Err(format!("connect timed out after {CONNECT_TIMEOUT:?}")), }; diff --git a/crates/server/src/error.rs b/crates/server/src/error.rs index 9df3fc1..60dd50d 100644 --- a/crates/server/src/error.rs +++ b/crates/server/src/error.rs @@ -2,6 +2,8 @@ use std::io; use std::net::SocketAddr; + +use crate::address::EndpointProblem; use std::time::Duration; use bgp_packet::errors::{BgpError, CeaseSubcode, DecodeError, FsmSubcode, OpenMsgSubcode}; @@ -166,6 +168,8 @@ pub enum ConfigError { LocalAsn, #[error("invalid IP address {0:?}")] Address(String), + #[error(transparent)] + Endpoint(#[from] EndpointProblem), #[error("port {0} is out of range")] Port(u32), #[error("http_endpoint.port is required")] diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index e3ae646..4782bc4 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -1,3 +1,4 @@ +pub mod address; pub mod advertise; pub mod collision; pub mod config; diff --git a/crates/server/src/policy.rs b/crates/server/src/policy.rs index e0e993e..3787f8a 100644 --- a/crates/server/src/policy.rs +++ b/crates/server/src/policy.rs @@ -25,7 +25,6 @@ //! replaced it, and the replacement is not acceptable. use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; use std::sync::{LazyLock, Mutex}; use std::time::{Duration, Instant}; @@ -34,6 +33,7 @@ use bgp_packet::open::BgpRole; use bgp_primitives::constants::IpVersion; use bgp_primitives::ip_prefix::IpPrefix; +use crate::address::PeerAddress; use crate::sync::IgnorePoison; /// The longest prefixes routed between networks: anything more specific is @@ -613,16 +613,16 @@ impl PrefixSet { /// their limit. Shared by the sessions, which put peers here, and by the /// dialer and the listeners, which leave them alone meanwhile. #[derive(Debug, Default)] -pub struct HoldDowns(Mutex>); +pub struct HoldDowns(Mutex>); impl HoldDowns { /// Keeps the peer at `address` down for `duration` from now. - pub fn hold(&self, address: IpAddr, duration: Duration) { + pub fn hold(&self, address: PeerAddress, duration: Duration) { self.0.locked().insert(address, Instant::now() + duration); } /// How much longer the peer at `address` is kept down, if it is. - pub fn remaining(&self, address: IpAddr) -> Option { + pub fn remaining(&self, address: PeerAddress) -> Option { let mut held = self.0.locked(); let until = *held.get(&address)?; let remaining = until.saturating_duration_since(Instant::now()); @@ -1076,13 +1076,16 @@ mod tests { #[test] fn a_peer_is_held_down_for_as_long_as_was_said() { let held = HoldDowns::default(); - let address: IpAddr = "192.0.2.1".parse().unwrap(); + let address = PeerAddress::from("192.0.2.1".parse::().unwrap()); assert_eq!(held.remaining(address), None); held.hold(address, Duration::from_secs(600)); let remaining = held.remaining(address).expect("held"); assert!(remaining > Duration::from_secs(590), "{remaining:?}"); - assert_eq!(held.remaining("192.0.2.2".parse().unwrap()), None); + assert_eq!( + held.remaining("192.0.2.2".parse::().unwrap().into()), + None + ); held.hold(address, Duration::ZERO); assert_eq!(held.remaining(address), None); diff --git a/crates/server/src/refusals.rs b/crates/server/src/refusals.rs index 2326902..2279400 100644 --- a/crates/server/src/refusals.rs +++ b/crates/server/src/refusals.rs @@ -8,7 +8,6 @@ //! worth looking at. use std::collections::{HashMap, VecDeque}; -use std::net::IpAddr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime}; @@ -17,6 +16,7 @@ use bgp_primitives::ip_prefix::{IpPrefix, Nlri}; use prometheus::IntCounter; use tracing::warn; +use crate::address::PeerAddress; use crate::metrics::ROUTES_REJECTED; use crate::policy::Rejection; use crate::sync::IgnorePoison; @@ -161,12 +161,12 @@ impl PeerRefusals { /// The refusals of every peer, by address. Cheap to clone; clones share /// one record. #[derive(Clone, Default)] -pub struct RefusalLog(Arc>>>); +pub struct RefusalLog(Arc>>>); impl RefusalLog { /// The record for the peer at `address`, made if there was none. /// `label` is what to call the peer; see [`PeerRefusals::label`]. - pub(crate) fn of_peer(&self, address: IpAddr, label: &str) -> Arc { + pub(crate) fn of_peer(&self, address: PeerAddress, label: &str) -> Arc { let mut peers = self.0.locked(); if let Some(existing) = peers.get(&address) { return Arc::clone(existing); @@ -181,7 +181,7 @@ impl RefusalLog { } /// Every peer that has had a route refused, with its record. - pub fn peers(&self) -> Vec<(IpAddr, Arc)> { + pub fn peers(&self) -> Vec<(PeerAddress, Arc)> { self.0 .locked() .iter() @@ -205,7 +205,7 @@ mod tests { #[test] fn refusals_are_counted_by_reason_and_the_latest_are_kept() { let log = RefusalLog::default(); - let address: IpAddr = "192.0.2.1".parse().unwrap(); + let address = PeerAddress::from("192.0.2.1".parse::().unwrap()); let record = log.of_peer(address, "refusals-test-1"); assert!(log.peers().is_empty(), "nothing refused yet"); diff --git a/crates/server/src/server.rs b/crates/server/src/server.rs index fdcc8ef..aa8d56d 100644 --- a/crates/server/src/server.rs +++ b/crates/server/src/server.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::io; -use std::net::{IpAddr, SocketAddr}; +use std::net::SocketAddr; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -18,6 +18,7 @@ use bgp_rib::table::RibTable; use bgp_rib::{LocalRib, RibStopped}; use bgp_rpki::Rpki; +use crate::address::{PeerAddress, socket_text}; use crate::advertise::spawn_advertiser; use crate::collision::{CollisionTable, Direction}; use crate::config::Config; @@ -110,7 +111,7 @@ impl BgpServer { .iter() .map(|p| ConfiguredPeer { name: p.name.clone(), - endpoint: p.endpoint, + endpoint: p.endpoint.clone(), remote_asn: p.remote_asn, role: p.role, }) @@ -236,7 +237,7 @@ impl BgpServer { }; match result { Ok((stream, peer_addr)) => { - match admit(&self.config, &context, peer_addr.ip(), &connections) { + match admit(&self.config, &context, peer_addr.into(), &connections) { Ok((peer, slot)) => { let context = Arc::clone(&context); tokio::spawn(serve(stream, peer_addr, context, peer, slot)); @@ -328,7 +329,7 @@ fn listen(addr: SocketAddr) -> io::Result { fn admit( config: &Config, context: &ServerContext, - address: IpAddr, + address: PeerAddress, connections: &ConnectionCounts, ) -> Result<(PeerSettings, ConnectionSlot), Refusal> { if context.hold_downs.remaining(address).is_some() { @@ -358,7 +359,7 @@ async fn serve( ) { let end = run_session(stream, peer_addr, Direction::Inbound, context, peer).await; if let Err(e) = end.result { - warn!(%peer_addr, err = %e, "BGP session terminated with error"); + warn!(peer_addr = %socket_text(peer_addr), err = %e, "BGP session terminated with error"); } } @@ -404,7 +405,7 @@ impl Refusals { return; } warn!( - %peer_addr, + peer_addr = %socket_text(peer_addr), reason = refusal.as_str(), others_since_last_report = self.unlogged, "Refused a connection" @@ -416,12 +417,12 @@ impl Refusals { /// How many connections each address has open to us. #[derive(Clone, Default)] -struct ConnectionCounts(Arc>>); +struct ConnectionCounts(Arc>>); impl ConnectionCounts { /// A place for one more connection from `address`, unless it has as /// many as it may. The place is given back when the slot is dropped. - fn take(&self, address: IpAddr) -> Option { + fn take(&self, address: PeerAddress) -> Option { let mut counts = self.0.locked(); let count = counts.entry(address).or_insert(0); if *count >= MAX_CONNECTIONS_PER_ADDRESS { @@ -437,7 +438,7 @@ impl ConnectionCounts { struct ConnectionSlot { counts: ConnectionCounts, - address: IpAddr, + address: PeerAddress, } impl Drop for ConnectionSlot { @@ -550,8 +551,8 @@ mod tests { #[test] fn an_address_may_have_only_so_many_connections() { let counts = ConnectionCounts::default(); - let address: IpAddr = "192.0.2.1".parse().unwrap(); - let other: IpAddr = "192.0.2.2".parse().unwrap(); + let address = PeerAddress::from("192.0.2.1".parse::().unwrap()); + let other = PeerAddress::from("192.0.2.2".parse::().unwrap()); let mut slots: Vec<_> = (0..MAX_CONNECTIONS_PER_ADDRESS) .map(|_| counts.take(address).expect("room")) diff --git a/crates/server/src/session.rs b/crates/server/src/session.rs index 252c08c..03e89d5 100644 --- a/crates/server/src/session.rs +++ b/crates/server/src/session.rs @@ -6,7 +6,7 @@ //! the registry to enumerate live sessions and queue outbound messages. use std::collections::HashMap; -use std::net::{IpAddr, SocketAddr}; +use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime}; @@ -19,6 +19,7 @@ use bgp_primitives::ip_prefix::Nlri; use bgp_rib::table::PeerIndex; use tokio::sync::{broadcast, mpsc}; +use crate::address::{Endpoint, PeerAddress, socket_text}; use crate::collision::Direction; use crate::family::FamilySet; use crate::fsm::Negotiated; @@ -38,7 +39,7 @@ pub struct PeerId { impl std::fmt::Display for PeerId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}(AS{})", self.addr, self.asn) + write!(f, "{}(AS{})", socket_text(self.addr), self.asn) } } @@ -60,7 +61,7 @@ pub enum DialState { #[derive(Debug, Clone)] pub struct ConfiguredPeer { pub name: String, - pub endpoint: SocketAddr, + pub endpoint: Endpoint, pub remote_asn: u32, /// What the peer is to us, if that was said. pub role: Option, @@ -289,13 +290,13 @@ impl SessionRegistry { self.dial_states.writing().insert(name.to_string(), state); } - /// An established session with this remote IP and ASN, regardless of - /// which side opened the TCP connection. - pub fn find(&self, ip: IpAddr, asn: u32) -> Option { + /// An established session with the peer at this address and with this + /// ASN, regardless of which side opened the TCP connection. + pub fn find(&self, address: PeerAddress, asn: u32) -> Option { self.inner .reading() .values() - .find(|h| h.peer.addr.ip() == ip && h.peer.asn == asn) + .find(|h| PeerAddress::from(h.peer.addr) == address && h.peer.asn == asn) .cloned() } diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index ac06b2e..afae5c4 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -17,6 +17,7 @@ use bgp_primitives::ip_prefix::IpPrefix; use bgp_rib::table::{PeerIndex, RibTable, Route, RouteRef}; use bgp_rib::{LocalRoute, RibStopped}; use bgp_rpki::{CacheState, Update}; +use bgp_server::address::{PeerAddress, socket_text}; use bgp_server::policy::PeerRole; use bgp_server::rpki::PathCheck; use bgp_server::session::{ConfiguredPeer, DialState, PeerId, SessionHandle, SessionRegistry}; @@ -97,11 +98,11 @@ impl From<&SessionHandle> for PeerView { fn from(s: &SessionHandle) -> Self { Self { name: s.name.clone(), - peer_addr: s.peer.addr.to_string(), + peer_addr: socket_text(s.peer.addr), peer_asn: s.peer.asn, state: "Established".into(), direction: Some(s.direction.as_str().into()), - local_addr: Some(s.local_addr.to_string()), + local_addr: Some(socket_text(s.local_addr)), established_at_unix: Some(unix_secs(s.established_at)), uptime_secs: Some(s.established_at.elapsed().map(|d| d.as_secs()).unwrap_or(0)), four_octet_asn: Some(s.four_octet_asn), @@ -131,7 +132,7 @@ impl PeerView { }; Self { name: Some(peer.name.clone()), - peer_addr: peer.endpoint.to_string(), + peer_addr: peer.endpoint.with_port(), peer_asn: peer.remote_asn, state: state_name.into(), direction: None, @@ -165,7 +166,7 @@ pub fn peer_views(registry: &SessionRegistry) -> Vec { .map(|(peer, state)| { match sessions .iter() - .find(|s| s.peer.addr.ip() == peer.endpoint.ip() && s.peer.asn == peer.remote_asn) + .find(|s| peer.endpoint.is_at(s.peer.addr.into()) && s.peer.asn == peer.remote_asn) { Some(session) => { claimed.insert(session.peer); @@ -332,11 +333,10 @@ impl PeerNames { /// The peers that `wanted` names: by configured name, or by address. fn matching(&self, wanted: &str) -> Vec { - let address: Option = wanted.parse().ok(); self.peers .iter() .filter(|(_, (peer, name, _))| { - name.as_deref() == Some(wanted) || address == Some(peer.addr.ip()) + name.as_deref() == Some(wanted) || PeerAddress::from(peer.addr).is_called(wanted) }) .map(|(index, _)| *index) .collect() @@ -351,7 +351,7 @@ impl PeerNames { let (attributes, next_hop) = route.attrs.attributes().unwrap_or_default(); Some(ReceivedRouteView { peer_name: name.clone(), - peer_addr: peer.addr.to_string(), + peer_addr: socket_text(peer.addr), peer_asn: peer.asn, prefix: route.prefix.to_string(), path_id: route.path_id, @@ -972,7 +972,7 @@ pub async fn rejected_routes( query .peer .as_deref() - .is_none_or(|wanted| record.label() == wanted || wanted.parse() == Ok(*address)) + .is_none_or(|wanted| record.label() == wanted || address.is_called(wanted)) }) .map(|(address, record)| RefusalsView { peer: record.label().to_string(), @@ -1023,6 +1023,7 @@ mod tests { asn: 65000 + u32::from(n), router_id: Ipv4Addr::new(10, 0, 0, n), address: Ipv4Addr::new(192, 0, 2, n).into(), + interface: 0, ibgp: false, }) .unwrap() diff --git a/docs/RFCs.md b/docs/RFCs.md index f84aeee..33b61d5 100644 --- a/docs/RFCs.md +++ b/docs/RFCs.md @@ -125,7 +125,7 @@ https://datatracker.ietf.org/doc/html/rfc8950 | IPv4 unicast routes received with an IPv6 next hop (global, or global and link-local) | ✅ | | Such routes installed in the kernel (IPv4 route by way of an IPv6 neighbor) | ✅ | | Other families than IPv4 unicast (VPN-IPv4, labelled) | ❌ | -| Sessions between link-local addresses | ❌ | +| Sessions between link-local addresses (`fe80::1%eth0`), with link-local next hops | ✅ | --- diff --git a/scripts/interop/install.sh b/scripts/interop/install.sh index 43581f6..933e7ea 100755 --- a/scripts/interop/install.sh +++ b/scripts/interop/install.sh @@ -1,9 +1,9 @@ #!/bin/sh # Installs what scripts/interop/lab.sh needs, on Ubuntu or Debian: FRR and # BIRD from the distribution, Routinator from crates.io (a few minutes to -# build). +# build), and radvd, which tells FRR who is on a link. set -eu -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y frr bird2 jq curl +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y frr bird2 jq curl radvd # They are run by the lab, in namespaces, not as services of this machine. -sudo systemctl disable --now frr bird 2>/dev/null || true +sudo systemctl disable --now frr bird radvd 2>/dev/null || true command -v routinator >/dev/null || cargo install --locked routinator diff --git a/scripts/interop/lab.sh b/scripts/interop/lab.sh index 043767d..7621bbe 100755 --- a/scripts/interop/lab.sh +++ b/scripts/interop/lab.sh @@ -20,7 +20,7 @@ # PHASES="basics scale" scripts/interop/lab.sh only these phases # SCALE_ROUTES=1000000 ... the size of the table BIRD sends # -# Needs Linux, sudo, and: frr, bird2, jq, routinator, a built `bgp` +# Needs Linux, sudo, and: frr, bird2, radvd, jq, routinator, a built `bgp` # (cargo build --release -p bgp). scripts/linux-vm.sh runs it from macOS; # scripts/interop/install.sh installs what is missing on Ubuntu. set -euo pipefail @@ -29,7 +29,7 @@ cd "$(dirname "$0")/../.." bgp="${CARGO_TARGET_DIR:-target}/release/bgp" work=/tmp/ferrix-lab pathspace=ferrixlab -all_phases="basics roles rpki refresh flaps restart scale limits v6only" +all_phases="basics roles rpki refresh flaps restart scale limits v6only linklocal" phases="${PHASES:-$all_phases}" scale_routes="${SCALE_ROUTES:-200000}" @@ -59,6 +59,7 @@ down() { for pidfile in "$work"/dut.pid "$work"/routinator.pid; do [ -f "$pidfile" ] && sudo kill "$(cat "$pidfile")" 2>/dev/null || true done + [ -f "$work/radvd.pid" ] && sudo kill "$(sudo cat "$work/radvd.pid")" 2>/dev/null || true [ -f "$work/bird.pid" ] && sudo kill "$(sudo cat "$work/bird.pid")" 2>/dev/null || true for daemon in bgpd zebra; do local pidfile="/var/run/frr/$pathspace/$daemon.pid" @@ -727,7 +728,8 @@ phase_limits() { # The exchange LAN loses its IPv4 addresses. One session over IPv6 with # each router carries both families, and IPv4 routes have IPv6 next hops -# (RFC 8950), in BGP and in the kernel. Run last: the lab stays that way. +# (RFC 8950), in BGP and in the kernel. Run after the rest: the lab stays +# that way. phase_v6only() { dut_down TERM frr_down TERM @@ -782,6 +784,172 @@ phase_v6only() { ok "an IPv4 route withdrawn over the IPv6 session goes" } +# The exchange LAN loses its global IPv6 addresses too: all that is left on +# it are link-local ones, the same on every link a router has, which mean +# something only together with the interface. The sessions are between +# those, as are the next hops, for both families. Run after v6only. +dut_config_linklocal() { + cat > "$work/dut.textpb" < "$work/radvd.conf" < /dev/null < "$work/bird.conf" </dev/null || true + in_ns "$name" ip -6 addr flush dev ix0 + in_ns "$name" ip -6 addr add "fe80::$n/64" dev ix0 nodad + # The same address on another link: it must not be mistaken for + # the one the session is on. + in_ns "$name" ip -6 addr flush dev lan scope link + in_ns "$name" ip -6 addr add "fe80::$n/64" dev lan nodad + done + in_ns dut sysctl -qw net.ipv6.conf.ix0.forwarding=1 + radvd_up + frr_config_linklocal; frr_up + bird_config_linklocal; bird_up + dut_config_linklocal; dut_up + + wait_for "two sessions between link-local addresses" established 2 + peer_is frr '.peer_addr | startswith("[fe80::2%ix0]:")' >/dev/null \ + && peer_is bird '.peer_addr | startswith("[fe80::3%ix0]:")' >/dev/null \ + && peer_is frr '.local_addr | startswith("[fe80::1%ix0]:")' >/dev/null \ + || fail "the peers are at $(api /api/peers | jq -c '[.[] | {name, peer_addr, local_addr}]')" + api /api/peers | jq -e 'all(.[]; .extended_next_hop and (.address_families | length == 2))' >/dev/null \ + || fail "the sessions did not agree on both families: $(api /api/peers | jq -c '[.[] | {name, address_families, extended_next_hop}]')" + ok "sessions with FRR and BIRD at fe80::2%ix0 and fe80::3%ix0, for both families" + + # Their routes: link-local next hops, on the session's interface. + wait_for "FRR's IPv4 route" dut_has 10.2.0.0/24 frr + wait_for "BIRD's IPv4 route" dut_has 10.3.0.0/24 bird + wait_for "FRR's IPv4 route in the kernel" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20" + wait_for "BIRD's IPv4 route in the kernel" dut_kernel -4 "10.3.0.0/24 via inet6 fe80::3 dev ix0 metric 20" + wait_for "FRR's IPv6 route in the kernel" dut_kernel -6 "2001:db8:2::/48 via fe80::2 dev ix0 metric 20" + wait_for "BIRD's IPv6 route in the kernel" dut_kernel -6 "2001:db8:3::/48 via fe80::3 dev ix0 metric 20" + api /api/kernel | jq -e '.unresolved == {}' >/dev/null || fail "some next hops did not resolve: $(api /api/kernel)" + ok "their routes of both families are in the kernel by way of fe80::… on ix0" + + # Ours, at them. + wait_for "our IPv4 route at FRR" frr_has ipv4 10.1.0.0/24 + wait_for "our IPv6 route at FRR" frr_has ipv6 2001:db8:1::/48 + wait_for "our IPv4 route at BIRD" bird_has 10.1.0.0/24 + wait_for "our IPv6 route at BIRD" bird_has 2001:db8:1::/48 + wait_for "our IPv4 route in FRR's kernel" in_ns frr ip -4 route get 10.1.0.1 + wait_for "our IPv6 route in BIRD's kernel" in_ns bird ip -6 route get 2001:db8:1::1 from 2001:db8:3::1 + ok "FRR and BIRD have our routes of both families" + + in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.2.0.1 >/dev/null || fail "no IPv4 ping to FRR's lan" + in_ns dut ping -q -c 2 -W 2 -I 10.1.0.1 10.3.0.1 >/dev/null || fail "no IPv4 ping to BIRD's lan" + in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:2::1 >/dev/null || fail "no IPv6 ping to FRR's lan" + in_ns dut ping -q -c 2 -W 2 -I 2001:db8:1::1 2001:db8:3::1 >/dev/null || fail "no IPv6 ping to BIRD's lan" + in_ns frr ping -q -c 2 -W 2 -I 10.2.0.1 10.1.0.1 >/dev/null || fail "no IPv4 ping from FRR's lan" + in_ns bird ping -q -c 2 -W 2 -I 2001:db8:3::1 2001:db8:1::1 >/dev/null || fail "no IPv6 ping from BIRD's lan" + ok "IPv4 and IPv6 packets cross a link with only link-local addresses, both ways" + + # The interface goes and comes back, as a tunnel does when it is + # restarted: the sessions come back by themselves. + in_ns dut ip link set ix0 down + wait_for "the sessions to go" established 0 + in_ns dut ip link set ix0 up + in_ns dut ip -6 addr flush dev ix0 + in_ns dut ip -6 addr add fe80::1/64 dev ix0 nodad + wait_for "the sessions to come back" established 2 + wait_for "FRR's IPv4 route in the kernel again" dut_kernel -4 "10.2.0.0/24 via inet6 fe80::2 dev ix0 metric 20" + ok "the sessions and the routes come back after the interface was down" +} + # --- main ----------------------------------------------------------------------- case "${1:-test}" in