From fd496d26bf0100cfe80de87106d2ce4121ac6fe2 Mon Sep 17 00:00:00 2001 From: Rayhaan Jaufeerally Date: Sun, 20 Sep 2026 18:15:09 +0200 Subject: [PATCH] feature: run on the router itself: protoc-free build, per-peer source address Preparing to run ferrix next to BIRD on the production router. - build: the config proto is now parsed in pure Rust, so building needs no `protoc` binary. It previously worked only where one happened to be installed; the Dockerfile in this repo could not have built. - peers gain `local_address`, the source IP for connections we open (and therefore the next hop we advertise). When both speakers share a host the OS would otherwise pick an arbitrary local address and the peer, which identifies us by address, would refuse the session. Address-family mismatches are rejected at config load; an address the host lacks shows up as the peer's backoff reason. - docs/lab/prod/on-router: static musl cross-build instructions, config, hardened systemd unit (unprivileged, SIGTERM gives BIRD a Cease), loopback addresses for networkd, the BIRD stanza for a same-host multihop session, and reverse-proxy examples with TLS and a password, since the API has no authentication. Co-Authored-By: Claude Fable 5.1 --- crates/integration-tests/tests/collision.rs | 2 +- crates/integration-tests/tests/dialer.rs | 49 ++++++++++- crates/server/build.rs | 3 + crates/server/proto/config.proto | 7 ++ crates/server/src/config.rs | 15 ++++ crates/server/src/dialer.rs | 16 +++- docs/lab/prod/on-router/05-lab-lo.network | 17 ++++ docs/lab/prod/on-router/README.md | 82 +++++++++++++++++++ .../lab/prod/on-router/bird-ferrix-local.conf | 18 ++++ docs/lab/prod/on-router/ferrix-bgp.service | 34 ++++++++ docs/lab/prod/on-router/ferrix.textpb | 28 +++++++ 11 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 docs/lab/prod/on-router/05-lab-lo.network create mode 100644 docs/lab/prod/on-router/README.md create mode 100644 docs/lab/prod/on-router/bird-ferrix-local.conf create mode 100644 docs/lab/prod/on-router/ferrix-bgp.service create mode 100644 docs/lab/prod/on-router/ferrix.textpb diff --git a/crates/integration-tests/tests/collision.rs b/crates/integration-tests/tests/collision.rs index 2b45ed0..d19a975 100644 --- a/crates/integration-tests/tests/collision.rs +++ b/crates/integration-tests/tests/collision.rs @@ -33,7 +33,7 @@ async fn server_dialing(listener: &TcpListener) -> Result { let endpoint: SocketAddr = listener.local_addr()?; TestServer::start_with(LOCAL_ASN, |c| { c.connect_retry = Duration::from_millis(200); - c.peers = vec![PeerConfig { name: "lab".into(), endpoint, remote_asn: PEER_ASN, hold_time_secs: 90 }]; + c.peers = vec![PeerConfig { name: "lab".into(), endpoint, remote_asn: PEER_ASN, hold_time_secs: 90, local_address: None }]; }) .await } diff --git a/crates/integration-tests/tests/dialer.rs b/crates/integration-tests/tests/dialer.rs index 2db3289..18ffc96 100644 --- a/crates/integration-tests/tests/dialer.rs +++ b/crates/integration-tests/tests/dialer.rs @@ -24,7 +24,7 @@ async fn server_dialing(listener: &TcpListener, remote_asn: u32) -> Result Result<()> { endpoint: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1), remote_asn: PEER_ASN, hold_time_secs: 90, + local_address: None, }]; }) .await?; @@ -175,3 +176,49 @@ async fn test_inbound_from_configured_address_uses_config() -> Result<()> { assert_eq!(listed.name.as_deref(), Some("lab")); Ok(()) } + +/// With `local_address` set the connection must come from that address (the +/// peer may identify us by it), and an address the host does not have is +/// reported as the reason for backing off rather than silently ignored. +#[tokio::test] +async fn test_local_address_is_used_as_the_source() -> Result<()> { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let endpoint: SocketAddr = listener.local_addr()?; + let _server = TestServer::start_with(LOCAL_ASN, |c| { + c.connect_retry = Duration::from_millis(200); + c.peers = vec![PeerConfig { + name: "lab".into(), + endpoint, + remote_asn: PEER_ASN, + hold_time_secs: 90, + local_address: Some(Ipv4Addr::LOCALHOST.into()), + }]; + }) + .await?; + let (_stream, from) = tokio::time::timeout(TIMEOUT, listener.accept()).await??; + assert_eq!(from.ip(), std::net::IpAddr::from(Ipv4Addr::LOCALHOST)); + + // 192.0.2.1 is documentation space; no host has it. + let server = TestServer::start_with(LOCAL_ASN, |c| { + c.connect_retry = Duration::from_millis(200); + c.peers = vec![PeerConfig { + name: "lab".into(), + endpoint, + remote_asn: PEER_ASN, + hold_time_secs: 90, + local_address: Some(Ipv4Addr::new(192, 0, 2, 1).into()), + }]; + }) + .await?; + let reason = tokio::time::timeout(TIMEOUT, async { + loop { + if let Some((_, DialState::Backoff { last_error, .. })) = server.registry.configured_peers().into_iter().next() { + return last_error; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await?; + assert!(reason.contains("local_address 192.0.2.1"), "{reason}"); + Ok(()) +} diff --git a/crates/server/build.rs b/crates/server/build.rs index f941313..a33c9c6 100644 --- a/crates/server/build.rs +++ b/crates/server/build.rs @@ -2,6 +2,9 @@ use protobuf_codegen::Codegen; fn main() { Codegen::new() + // Parse the .proto in pure Rust so building needs no `protoc` binary + // on the machine (routers and minimal containers rarely have one). + .pure() .input("proto/config.proto") .includes(&["proto"]) .cargo_out_dir("protos") diff --git a/crates/server/proto/config.proto b/crates/server/proto/config.proto index c027efa..32fed00 100644 --- a/crates/server/proto/config.proto +++ b/crates/server/proto/config.proto @@ -72,4 +72,11 @@ message Peer { // Hold time to advertise in the OPEN message, in seconds (RFC 4271 ยง4.2). // Must be 0 or >= 3. Defaults to 90 when unset. uint32 hold_time_seconds = 4; + + // Source IP address for connections we open to this peer. It is also the + // next hop of the routes we send it. Required when the peer identifies us + // by address and the host has several, e.g. when both speakers run on the + // same machine. Must be the same address family as the endpoint. Unset + // lets the operating system choose. + string local_address = 5; } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 9ba9e3b..e8bcfbe 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -39,6 +39,9 @@ pub struct PeerConfig { pub endpoint: SocketAddr, pub remote_asn: u32, pub hold_time_secs: u16, + /// Source address for connections we open to this peer; `None` lets the + /// operating system choose. + pub local_address: Option, } impl Config { @@ -81,11 +84,23 @@ impl Config { p.hold_time_seconds as u16 }; + let local_address = if p.local_address.is_empty() { + None + } else { + let ip = IpAddr::from_str(&p.local_address) + .map_err(|_| eyre!("Peer {:?} has invalid local_address {:?}", p.name, p.local_address))?; + if ip.is_ipv4() != endpoint.is_ipv4() { + bail!("Peer {:?}: local_address {ip} and endpoint {endpoint} are different address families", p.name); + } + Some(ip) + }; + Ok(PeerConfig { name: p.name, endpoint, remote_asn: p.remote_asn, hold_time_secs, + local_address, }) }) .collect::>()?; diff --git a/crates/server/src/dialer.rs b/crates/server/src/dialer.rs index 5a6910b..7a51b6a 100644 --- a/crates/server/src/dialer.rs +++ b/crates/server/src/dialer.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; -use tokio::net::TcpStream; +use tokio::net::{TcpSocket, TcpStream}; use tokio::task::JoinHandle; use tracing::{info, warn}; @@ -44,6 +44,18 @@ pub fn spawn_dialers( .collect() } +/// Opens the TCP connection, from the configured source address if any. +async fn connect(peer: &PeerConfig) -> std::io::Result { + let Some(local) = peer.local_address else { + return TcpStream::connect(peer.endpoint).await; + }; + let socket = if peer.endpoint.is_ipv4() { TcpSocket::new_v4()? } else { TcpSocket::new_v6()? }; + socket + .bind(std::net::SocketAddr::new(local, 0)) + .map_err(|e| std::io::Error::new(e.kind(), format!("cannot use local_address {local}: {e}")))?; + socket.connect(peer.endpoint).await +} + async fn dial_loop(peer: Arc, config: SessionConfig, connect_retry: Duration) { let registry: SessionRegistry = config.registry.clone(); let mut backoff = connect_retry; @@ -62,7 +74,7 @@ async fn dial_loop(peer: Arc, config: SessionConfig, connect_retry: registry.set_dial_state(&peer.name, DialState::Connecting); info!(peer = %peer.name, endpoint = %peer.endpoint, "Connecting to peer"); - let outcome = match tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(peer.endpoint)).await { + let outcome = match tokio::time::timeout(CONNECT_TIMEOUT, connect(&peer)).await { Ok(Ok(stream)) => { info!(peer = %peer.name, endpoint = %peer.endpoint, "TCP connected, starting session"); backoff = connect_retry; diff --git a/docs/lab/prod/on-router/05-lab-lo.network b/docs/lab/prod/on-router/05-lab-lo.network new file mode 100644 index 0000000..1effb3e --- /dev/null +++ b/docs/lab/prod/on-router/05-lab-lo.network @@ -0,0 +1,17 @@ +# /etc/systemd/network/05-lab-lo.network +# If a .network file already matches "lo" (e.g. the AnyIP one), merge these +# sections into it instead: networkd applies only the first match. +[Match] +Name=lo + +# One address each for BIRD and ferrix to peer over. +[Address] +Address=fd00:179::1/128 + +[Address] +Address=fd00:179::2/128 + +# AnyIP: the router answers ping for every address in the lab space. +[Route] +Destination=2a0d:d741::/40 +Type=local diff --git a/docs/lab/prod/on-router/README.md b/docs/lab/prod/on-router/README.md new file mode 100644 index 0000000..171ef24 --- /dev/null +++ b/docs/lab/prod/on-router/README.md @@ -0,0 +1,82 @@ +# Running ferrix_bgp on the production router + +ferrix and BIRD run side by side on pr01 and peer over two loopback +addresses. Students reach the status page through a reverse proxy that adds +TLS and a password. Nothing here has been run on the router; treat each step +as something to check. + +## 1. Build a static Linux binary (on the Mac) + +The `x86_64-unknown-linux-musl` target and Homebrew's musl cross-linker are +enough; the router needs no Rust toolchain and no `protoc`. + + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-linux-musl-gcc \ + cargo build --release -p bgp --target x86_64-unknown-linux-musl + + scp target/x86_64-unknown-linux-musl/release/bgp pr01:/tmp/ferrix-bgp + +Check the router really is x86-64 first (`uname -m`). + +## 2. Install on the router + + install -m 0755 /tmp/ferrix-bgp /usr/local/bin/ferrix-bgp + install -d /etc/ferrix-bgp + install -m 0644 ferrix.textpb /etc/ferrix-bgp/ferrix.textpb + install -m 0644 ferrix-bgp.service /etc/systemd/system/ferrix-bgp.service + +## 3. Loopback addresses + +Merge `05-lab-lo.network` into the networkd file that matches `lo`, then +`networkctl reload` and confirm with `ip -6 addr show dev lo`. + +## 4. BIRD + +Replace the `protocol bgp ferrix` stanza with `bird-ferrix-local.conf`, then +`birdc configure check` and `birdc configure timeout 300` / `confirm`. + +## 5. Start + + systemctl daemon-reload + systemctl enable --now ferrix-bgp + journalctl -u ferrix-bgp -f + birdc show protocols all ferrix + +## 6. Let students in + +The API has no authentication and announces into a real AS, so do not expose +port 8080. Put a reverse proxy in front with TLS and a class password. The +event stream is long-lived, so the proxy must not buffer it. + +Caddy: + + lab.example.net { + basic_auth { + student + } + reverse_proxy 127.0.0.1:8080 { + flush_interval -1 + } + } + +nginx, inside a TLS `server` block: + + location / { + auth_basic "BGP lab"; + auth_basic_user_file /etc/nginx/lab.htpasswd; + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_buffering off; + proxy_read_timeout 1h; + } + +Restricting the proxy to the university's address ranges as well is cheap +and worth doing. + +## Things to know + +- Announced routes live in memory. A restart of the service forgets them and + students have to announce again. +- Stopping the service sends BIRD a Cease first, so restarts reconnect at + once. +- ferrix's own safety net is `local_route_policy`; BIRD's import filter and + import limit are the second one. Keep both. diff --git a/docs/lab/prod/on-router/bird-ferrix-local.conf b/docs/lab/prod/on-router/bird-ferrix-local.conf new file mode 100644 index 0000000..df56d83 --- /dev/null +++ b/docs/lab/prod/on-router/bird-ferrix-local.conf @@ -0,0 +1,18 @@ +# Replaces "protocol bgp ferrix" in bird.conf when ferrix runs on this host. +# Only the addresses and "multihop" differ from the WireGuard version; the +# import filter and everything about export control stay as they are. +protocol bgp ferrix +{ + description "ferrix_bgp teaching lab (announce only, same host)"; + local fd00:179::1 as 210036; + neighbor fd00:179::2 as FERRIX_ASN; + # Both ends are addresses of this machine, so there is no directly + # connected neighbour for "direct" to find. + multihop; + passive on; + ipv6 { + import filter ferrix_import; + import limit 130 action block; + export none; + }; +} diff --git a/docs/lab/prod/on-router/ferrix-bgp.service b/docs/lab/prod/on-router/ferrix-bgp.service new file mode 100644 index 0000000..89e27a2 --- /dev/null +++ b/docs/lab/prod/on-router/ferrix-bgp.service @@ -0,0 +1,34 @@ +[Unit] +Description=ferrix_bgp teaching BGP speaker +After=network-online.target bird.service +Wants=network-online.target + +[Service] +ExecStart=/usr/local/bin/ferrix-bgp /etc/ferrix-bgp/ferrix.textpb --level info +# SIGTERM makes ferrix send Cease / Administrative Shutdown to BIRD first. +KillSignal=SIGTERM +TimeoutStopSec=10 +Restart=on-failure +RestartSec=5 + +# It needs the network and nothing else: no root, no privileged port, no +# write access anywhere. +DynamicUser=yes +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +SystemCallArchitectures=native +CapabilityBoundingSet= +MemoryMax=256M + +[Install] +WantedBy=multi-user.target diff --git a/docs/lab/prod/on-router/ferrix.textpb b/docs/lab/prod/on-router/ferrix.textpb new file mode 100644 index 0000000..fc47608 --- /dev/null +++ b/docs/lab/prod/on-router/ferrix.textpb @@ -0,0 +1,28 @@ +# ferrix_bgp running on the production router itself, next to BIRD. +router_id: "10.255.0.2" +local_asn: 4200000001 + +# Unused in practice (BIRD is passive and we dial it), but a listener is +# required. 179 belongs to BIRD, so take another port on our own address. +listen_endpoints { address: "fd00:179::2" port: 9179 } + +peers { + name: "pr01-bird" + endpoint { address: "fd00:179::1" port: 179 } + # BIRD recognises its neighbour by source address. Both addresses live on + # this host, so say which one is ours. + local_address: "fd00:179::2" + remote_asn: 210036 + hold_time_seconds: 90 +} +connect_retry_seconds: 5 +collision_detect_established_state: true + +local_route_policy { + allowed_prefixes: "2a0d:d741::/40" + max_routes_per_owner: 1 +} + +# Loopback only. Students reach it through the reverse proxy, which adds TLS +# and a password; the API itself has neither. +http_endpoint { address: "127.0.0.1" port: 8080 }