From b6963b65e61ce35a70925697307d13f2f01ee999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Fri, 6 Nov 2026 09:51:54 +0100 Subject: [PATCH] Add more options to restrict outgoing traffic So far only the --egress-deny option existed which allowed to restrict link-local and private networks. However, this might not be enough give that some e.g. LLMs reach out to endpoints like cloud storage to store information which might be unwanted. To cover this more flexible A new --egress-allow option was added and further entry forms for both egress options. Details how to use and combine the options were added to the documentation and man pages. With regards to AI workloads the best practices chapter was updated accordingly. Documentation and examples were produced by AI and reviewed by myself. Assisted-by: Claude:claude-opus-4-6 --- common/src/defaults.rs | 11 +- common/src/network.rs | 127 +++++-- completion/flake-ctl | 2 +- doc/flake-ctl-firecracker-network-init.rst | 143 +++++++- doc/flake-pilot-helper.rst | 8 +- doc/user_guide/best_practices.rst | 10 + doc/user_guide/firecracker_networking.rst | 31 ++ flake-ctl/src/cli.rs | 22 +- flake-ctl/src/main.rs | 3 +- flake-ctl/src/network.rs | 398 ++++++++++++++++++++- flake-pilot-helper/src/cli.rs | 8 + flake-pilot-helper/src/firewall.rs | 381 ++++++++++++++------ flake-pilot-helper/src/main.rs | 5 +- flake-pilot-helper/src/network.rs | 35 +- 14 files changed, 990 insertions(+), 194 deletions(-) diff --git a/common/src/defaults.rs b/common/src/defaults.rs index 0ed0b8a7..aa8f671c 100644 --- a/common/src/defaults.rs +++ b/common/src/defaults.rs @@ -85,6 +85,7 @@ pub const MV_TOOL: &str = "/usr/bin/mv"; pub const RSYNC_TOOL: &str = "/usr/bin/rsync"; pub const SYSTEMCTL_TOOL: &str = "/usr/bin/systemctl"; pub const IPTABLES_TOOL: &str = "/usr/sbin/iptables"; +pub const IPTABLES_RESTORE_TOOL: &str = "/usr/sbin/iptables-restore"; pub const IP_TOOL: &str = "/usr/sbin/ip"; pub const SYSCTL_TOOL: &str = "/usr/sbin/sysctl"; pub const NFS_EXPORTFS_TOOL: &str = "/usr/sbin/exportfs"; @@ -105,7 +106,8 @@ pub const FIREWALL_FORWARD_CHAIN: &str = "FLAKE_PILOT_FORWARD"; pub const FIREWALL_INPUT_CHAIN: &str = "FLAKE_PILOT_INPUT"; // Chain of the egress policy of the VM network. The traffic of the // VMs to the outside world passes it before it is forwarded, see -// NETWORK_EGRESS_PRESETS +// NETWORK_EGRESS_PRESETS. The chain is written as a whole, the +// order of its rules decides which traffic passes pub const FIREWALL_EGRESS_CHAIN: &str = "FLAKE_PILOT_EGRESS"; // Max. size of a network interface name including the // terminating zero byte as defined by IFNAMSIZ in linux/if.h @@ -229,9 +231,10 @@ pub const NETWORK_HOST_PREFIX_LEN: u8 = pub const NETWORK_DNS: &str = "8.8.8.8"; // Named sets of networks the egress policy of the VM network can -// deny, see 'flake-ctl firecracker network init --egress-deny'. -// The traffic of the VMs to them is dropped instead of being -// forwarded to the outgoing interface: +// deny or allow, see 'flake-ctl firecracker network init +// --egress-deny' and '--egress-allow'. The traffic of the VMs to +// denied networks is dropped instead of being forwarded to the +// outgoing interface: // // link-local: the link local network, which provides the metadata // service of the cloud the host runs in at 169.254.169.254. diff --git a/common/src/network.rs b/common/src/network.rs index 744490ef..0e1f1427 100644 --- a/common/src/network.rs +++ b/common/src/network.rs @@ -689,6 +689,13 @@ pub struct NetworkState { /// policy became selectable provides none #[serde(default)] pub egress_deny: Vec, + /// Networks in their ADDRESS/PREFIX_LEN notation the traffic + /// of the VMs to the outside world is limited to. If there are + /// any, the traffic to all other networks is dropped, a network + /// which is denied as well stays denied. A state written before + /// the allow list became selectable provides none + #[serde(default)] + pub egress_allow: Vec, /// The addresses handed out to the VM instances of all users #[serde(default)] pub allocations: Vec @@ -711,49 +718,93 @@ impl NetworkState { pub fn get_egress_deny(&self) -> Option> { /*! - Provide the recorded networks of the egress policy, None - if one of them is invalid + Provide the recorded denied networks of the egress policy, + None if one of them is invalid + !*/ + parse_networks(&self.egress_deny) + } + + pub fn get_egress_allow(&self) -> Option> { + /*! + Provide the recorded allowed networks of the egress policy, + None if one of them is invalid !*/ - self.egress_deny.iter() - .map(|network| Ipv4Network::parse(network)) - .collect() + parse_networks(&self.egress_allow) } } +fn parse_networks(networks: &[String]) -> Option> { + /*! + Read the given networks, None if one of them is invalid + !*/ + networks.iter() + .map(|network| Ipv4Network::parse(network)) + .collect() +} + pub fn get_egress_networks( - egress_deny: &[String] + entries: &[String] ) -> Result, String> { /*! - Provide the networks of the egress policy of the VM network + Provide the networks of the given entries of the egress policy + of the VM network, see get_egress_entry_networks(). Networks + which are given more than once are only listed once, in the + order they are given first + !*/ + let mut networks: Vec = Vec::new(); + for entry in entries { + let entry_networks = get_egress_entry_networks(entry).ok_or( + format!( + "Invalid egress policy entry {entry:?}: expected one of {} \ + or ADDRESS[/PREFIX_LEN]", + get_egress_preset_names().join(", ") + ) + )?; + add_networks(&mut networks, entry_networks); + } + Ok(networks) +} + +pub fn get_egress_entry_networks(entry: &str) -> Option> { + /*! + Provide the networks of one entry of the egress policy - Each entry is the name of one of the sets of networks in + The entry is the name of one of the sets of networks in defaults::NETWORK_EGRESS_PRESETS or a network in its ADDRESS/PREFIX_LEN notation. An address without a prefix length - is a single host. Networks which are given more than once are - only listed once, in the order they are given first + is a single host. None if the entry is neither of them !*/ - let mut networks: Vec = Vec::new(); - for entry in egress_deny { - let preset = defaults::NETWORK_EGRESS_PRESETS.iter() - .find(|(name, _)| name == entry); - let entry_networks = match preset { - Some((_, preset_networks)) => preset_networks.iter() + let preset = defaults::NETWORK_EGRESS_PRESETS.iter() + .find(|(name, _)| *name == entry); + match preset { + Some((_, preset_networks)) => Some( + preset_networks.iter() .filter_map(|network| Ipv4Network::parse(network)) - .collect(), - None => vec![Ipv4Network::parse(entry).ok_or(format!( - "Invalid egress policy entry {entry:?}: expected one of {} \ - or ADDRESS[/PREFIX_LEN]", - defaults::NETWORK_EGRESS_PRESETS.iter() - .map(|(name, _)| *name).collect::>().join(", ") - ))?] - }; - for network in entry_networks { - if ! networks.contains(&network) { - networks.push(network) - } + .collect() + ), + None => Ipv4Network::parse(entry).map(|network| vec![network]) + } +} + +pub fn get_egress_preset_names() -> Vec<&'static str> { + /*! + Provide the names of the sets of networks of the egress policy + !*/ + defaults::NETWORK_EGRESS_PRESETS.iter().map(|(name, _)| *name).collect() +} + +pub fn add_networks( + networks: &mut Vec, new_networks: Vec +) { + /*! + Add the given new networks to the given list of networks, + networks which are listed already are skipped + !*/ + for network in new_networks { + if ! networks.contains(&network) { + networks.push(network) } } - Ok(networks) } // Allocation is an address of the private network handed out to @@ -1233,6 +1284,7 @@ mod tests { host_resolver: true, host_dhcp: false, egress_deny: vec!["169.254.0.0/16".to_string()], + egress_allow: vec!["203.0.113.0/24".to_string()], allocations: vec![ allocation( "172.16.0.2", 1000, "tap-1000_myapp", @@ -1249,6 +1301,9 @@ mod tests { assert_eq!( Some(vec![network("169.254.0.0/16")]), state.get_egress_deny() ); + assert_eq!( + Some(vec![network("203.0.113.0/24")]), state.get_egress_allow() + ); assert_eq!( state, serde_saphyr::from_str::(&yaml_data).unwrap() ); @@ -1262,6 +1317,8 @@ mod tests { host_dhcp: false\n\ egress_deny:\n\ - 169.254.0.0/16\n\ + egress_allow:\n\ + - 203.0.113.0/24\n\ allocations:\n\ - address: 172.16.0.2\n \ uid: 1000\n \ @@ -1273,9 +1330,17 @@ mod tests { // an invalid network of the egress policy is not ignored let state = NetworkState { egress_deny: vec!["10.0.0.0/8".to_string(), "bogus".to_string()], + egress_allow: vec!["bogus".to_string()], ..Default::default() }; assert_eq!(None, state.get_egress_deny()); + assert_eq!(None, state.get_egress_allow()); + // a state written before the allow list existed allows all + let state: NetworkState = serde_saphyr::from_str( + "egress_deny:\n\ + - 169.254.0.0/16\n" + ).unwrap(); + assert_eq!(Some(vec![]), state.get_egress_allow()); } #[test] @@ -1304,6 +1369,10 @@ mod tests { assert!(egress_networks(&["metadata"]).is_err()); assert!(egress_networks(&["10.0.0.0/33"]).is_err()); assert!(egress_networks(&["-j ACCEPT"]).is_err()); + // host names and files are resolved by the caller, the + // helper only takes networks and presets + assert!(egress_networks(&["oss-cn-hangzhou.aliyuncs.com"]).is_err()); + assert!(egress_networks(&["@/etc/shadow"]).is_err()); } #[test] diff --git a/completion/flake-ctl b/completion/flake-ctl index 3ec5dec6..b999c938 100755 --- a/completion/flake-ctl +++ b/completion/flake-ctl @@ -69,7 +69,7 @@ __flake_ctl_firecracker_show() { __flake_ctl_firecracker_network() { if [ "${COMP_WORDS[3]}" = "init" ];then - __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --egress-deny --help" + __comp_reply_unused "--outgoing-interface --host-resolver --host-dhcp --egress-deny --egress-allow --help" return 0 fi if [ "${COMP_WORDS[3]}" = "add" ] || [ "${COMP_WORDS[3]}" = "remove" ];then diff --git a/doc/flake-ctl-firecracker-network-init.rst b/doc/flake-ctl-firecracker-network-init.rst index c988fe21..a329a61a 100644 --- a/doc/flake-ctl-firecracker-network-init.rst +++ b/doc/flake-ctl-firecracker-network-init.rst @@ -12,13 +12,14 @@ SYNOPSIS .. code:: bash USAGE: - flake-ctl firecracker network init [--outgoing-interface ] [--host-resolver] [--host-dhcp] [--egress-deny ]... + flake-ctl firecracker network init [--outgoing-interface ] [--host-resolver] [--host-dhcp] [--egress-deny ]... [--egress-allow ]... OPTIONS: --outgoing-interface --host-resolver --host-dhcp --egress-deny + --egress-allow --help DESCRIPTION @@ -64,7 +65,7 @@ selects from, see NETWORK below, and must not collide with a network of the host, the interface has to exist. Both are recorded in the network state file along with the services of the host given by ``--host-resolver`` and ``--host-dhcp`` and the networks of the egress -policy given by ``--egress-deny``. Connecting an application to +policy given by ``--egress-deny`` and ``--egress-allow``. Connecting an application to the host network with **flake-ctl-firecracker-network-add**\(8) takes them from there: the address of the application, the route to it and its firewall rules are only ever created within the recorded network. @@ -74,7 +75,8 @@ firecracker-pilot reads to check the firewall setup of the host. Chains and rules which already exist are not created again. This allows to call the command more than once without stacking up -duplicates of the same rule. The setup is not persistent, it has to be +duplicates of the same rule. The egress chain is the exception, it +is replaced as a whole, see FIREWALL below. The setup is not persistent, it has to be created again after a reboot of the host. **NOTE:** The setup assumes there is no other firewall software @@ -115,6 +117,11 @@ follows, shown in the notation of ``iptables -S``: -A FLAKE_PILOT_EGRESS -d -j DROP -A FLAKE_PILOT_EGRESS -j ACCEPT + # table filter, with --egress-allow + -A FLAKE_PILOT_EGRESS -d -j DROP + -A FLAKE_PILOT_EGRESS -d -j ACCEPT + -A FLAKE_PILOT_EGRESS -j DROP + -A INPUT -i tap-+ -j FLAKE_PILOT_INPUT -A FLAKE_PILOT_INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FLAKE_PILOT_INPUT -j DROP @@ -128,7 +135,15 @@ The effect of the setup is: * The traffic of the VMs to the outside world passes the egress policy before it is forwarded. Its traffic to a network given by ``--egress-deny`` is dropped, there is one ``DROP`` rule for each - of them. Without the option the egress chain accepts all traffic. + of them. Without ``--egress-allow`` the egress chain accepts all + other traffic. With ``--egress-allow`` it only accepts the traffic + to the allowed networks, there is one ``ACCEPT`` rule for each of + them, and drops everything else. The ``DROP`` rules of the denied + networks come first, a denied network stays denied even if it is + part of an allowed one. The egress chain is replaced as a whole + with **iptables-restore**\(8) in one transaction. While the policy + changes, the traffic of the VMs either passes the former or the + new policy, never a mix of both. * Traffic between two VMs, traffic from the outside world to a VM and traffic from a VM to any other interface of the host is dropped. @@ -238,9 +253,67 @@ OPTIONS Network the traffic of the VMs to the outside world must not go to. Can be specified multiple times, the traffic to all of them is - dropped. Either the name of a set of networks or a network in its - ``ADDRESS[/PREFIX_LEN]`` notation, an address without a prefix - length is a single host: + dropped. Each value is one of the entries described in EGRESS + POLICY ENTRIES below, e.g: + + .. code:: bash + + --egress-deny link-local + --egress-deny 100.100.100.200 + --egress-deny oss-cn-hangzhou.aliyuncs.com + --egress-deny @/etc/flakes/egress-deny.list + + The policy only applies to the traffic which is forwarded to the + outgoing interface. The services of the host a VM may use through + its gateway, see FIREWALL above, are not affected. The networks + are recorded in the network state file and take effect for all + applications at once, including the ones which are already + connected. Calling the command again replaces the policy, without + the option the policy is removed. + **flake-ctl-firecracker-network-add**\(8) refuses to connect an + application if the recorded policy is not active + + The policy is IPv4 only, as is the whole setup. IPv6 is switched + off on the TAP devices of the VMs + +--egress-allow + + Network the traffic of the VMs to the outside world is limited to. + Can be specified multiple times, the traffic to all of them is + allowed and the traffic to every other network is dropped. Each + value is one of the entries described in EGRESS POLICY ENTRIES + below. A network given by ``--egress-deny`` stays denied, even if + it is part of an allowed network. The allow list can therefore + only narrow down what the VMs reach, never extend it. + + Unlike a list of denied networks, an allow list also covers + destinations nobody knows about beforehand, e.g a storage service + an application sends data to without being asked. It is the + recommended policy for applications which are not trusted, e.g an + AI agent, which only need a known set of services: + + .. code:: bash + + --egress-allow api.example.com --egress-allow @/etc/flakes/egress-allow.list + + The VMs need a name server. With ``--host-resolver`` they use the + resolver of the host through their gateway, which is not part of + the egress policy. Without it, the public name server ``8.8.8.8`` + is allowed along with the given networks, unless one of them + covers it already. Please note, an application which may use a + name server can still pass data to the outside world through the + names it looks up. A resolver on the host which only answers the + names the application needs closes that path as well. + + Like ``--egress-deny``, the allow list is recorded in the network + state file, takes effect for all applications at once and is + removed by calling the command again without the option + +EGRESS POLICY ENTRIES +--------------------- + +The values of ``--egress-deny`` and ``--egress-allow`` are given in +one of the following forms: link-local ``169.254.0.0/16``. It covers the metadata service a cloud @@ -258,18 +331,48 @@ OPTIONS e.g a proxy or a name server of the local network, cannot reach it anymore - The policy only applies to the traffic which is forwarded to the - outgoing interface. The services of the host a VM may use through - its gateway, see FIREWALL above, are not affected. The networks - are recorded in the network state file and take effect for all - applications at once, including the ones which are already - connected. Calling the command again replaces the policy, without - the option the policy is removed. - **flake-ctl-firecracker-network-add**\(8) refuses to connect an - application if the recorded policy is not active +ADDRESS[/PREFIX_LEN] + A network, e.g ``47.74.0.0/15``. An address without a prefix + length is a single host, e.g ``100.100.100.200`` + +HOSTNAME + A fully qualified host name, e.g ``oss-cn-hangzhou.aliyuncs.com``. + The name is resolved by the command, with the name service of + the host and the identity of the calling user, and is replaced by + the IPv4 addresses it resolves to. A name which cannot be resolved + fails the command. The addresses are the ones the name resolves to + at the time of the call, the policy does not follow later changes. + Names of services which are spread over many addresses, e.g the + storage service of a cloud provider, which provides one name per + region and bucket, are not covered completely by a single name. + Wildcards like ``*.aliyuncs.com`` cannot be resolved. Such a + service is better covered by its address ranges, given in a file, + or by an allow list which does not include it + +@FILE + The entries of the given file, one per line. Each line is one of + the forms above, a file cannot name another file. Empty lines and + everything following a ``#`` are skipped. An invalid entry is + reported along with its line. This allows to keep long lists, e.g + the published address ranges of a cloud provider, apart from the + command line: - The policy is IPv4 only, as is the whole setup. IPv6 is switched - off on the TAP devices of the VMs + .. code:: bash + + # /etc/flakes/egress-deny.list + link-local + private + # address ranges of a cloud provider + 47.74.0.0/15 + 8.208.0.0/12 + + The file is read by the command with the identity of the calling + user + +The helper, see **flake-pilot-helper**\(8), only receives the +resulting networks. It neither resolves host names nor reads files +on behalf of the caller. A network which is given more than once is +only taken once. FILES ----- @@ -291,6 +394,10 @@ EXAMPLE $ flake-ctl firecracker network init --egress-deny link-local --egress-deny private + $ flake-ctl firecracker network init --egress-deny link-local --egress-deny @/etc/flakes/cloud-provider.list + + $ flake-ctl firecracker network init --host-resolver --egress-deny link-local --egress-deny private --egress-allow api.example.com + SEE ALSO -------- diff --git a/doc/flake-pilot-helper.rst b/doc/flake-pilot-helper.rst index 30b7f426..bc9584c6 100644 --- a/doc/flake-pilot-helper.rst +++ b/doc/flake-pilot-helper.rst @@ -15,7 +15,7 @@ SYNOPSIS sudo /usr/libexec/flake-pilot/flake-pilot-helper [--debug] SUBCOMMANDS: - network-init --network --outgoing-interface [--host-resolver] [--host-dhcp] [--egress-deny ]... + network-init --network --outgoing-interface [--host-resolver] [--host-dhcp] [--egress-deny ]... [--egress-allow ]... network-cleanup network-connect --config-file [--instance ] [--uid ] [--prefer ] [--exclude ]... network-disconnect --config-file [--instance ] [--uid ] @@ -73,7 +73,11 @@ network-init from and must not collide with a network of the host, the interface has to exist. Both are recorded in the network state along with the services of the host the VMs use and the networks - of the egress policy, which only adds drop rules. Called by + of the egress policy. The policy only narrows down the traffic of + the VMs: the denied networks are dropped, and with allowed + networks everything else is dropped as well. The networks are + given as presets or in their ADDRESS[/PREFIX_LEN] notation, the + helper neither resolves host names nor reads files. Called by **flake-ctl-firecracker-network-init**\(8) network-cleanup diff --git a/doc/user_guide/best_practices.rst b/doc/user_guide/best_practices.rst index 9bbfaead..b372a7f3 100644 --- a/doc/user_guide/best_practices.rst +++ b/doc/user_guide/best_practices.rst @@ -150,6 +150,16 @@ your local network, from the private networks as well: flake-ctl firecracker network init \ --egress-deny link-local --egress-deny private +If the tool only needs a known set of services, e.g the API of its +model provider, allow only those. Everything else, including services +the tool contacts without being asked, is dropped then: + +.. code-block:: bash + + flake-ctl firecracker network init \ + --egress-deny link-local --egress-deny private \ + --egress-allow api.example.com + :ref:`firecracker-networking` explains the setup and the policy. Share One Directory With the Host diff --git a/doc/user_guide/firecracker_networking.rst b/doc/user_guide/firecracker_networking.rst index ebe16e2b..7db2014e 100644 --- a/doc/user_guide/firecracker_networking.rst +++ b/doc/user_guide/firecracker_networking.rst @@ -213,6 +213,37 @@ through their gateway are not affected. Unlike the settings above, the policy takes effect for all applications at once. Calling ``init`` again replaces it. +Besides presets and networks, ``--egress-deny`` takes a host name, +which is replaced by the IPv4 addresses it resolves to at the time of +the call, and ``@FILE``, which reads one entry per line, e.g the +published address ranges of a cloud provider: + +.. code-block:: bash + + flake-ctl firecracker network init \ + --egress-deny link-local \ + --egress-deny oss-cn-hangzhou.aliyuncs.com \ + --egress-deny @/etc/flakes/cloud-provider.list + +A list of denied networks only covers the destinations you know +about. If an application must not send data anywhere but to a known +set of services, e.g an AI model which was reported to upload data to +a cloud storage bucket, limit its traffic to those services with +``--egress-allow`` instead. The traffic to every other network is +dropped then, denied networks stay denied: + +.. code-block:: bash + + flake-ctl firecracker network init --host-resolver \ + --egress-deny link-local --egress-deny private \ + --egress-allow api.example.com \ + --egress-allow @/etc/flakes/egress-allow.list + +Without ``--host-resolver`` the public name server of the VMs is +allowed along with the given networks. Host names are only resolved +once, by ``init``, call it again if the addresses of an allowed or +denied service change. + Rules of former versions of Flake Pilot for single TAP devices are deleted. The rules for all traffic former versions created, a ``MASQUERADE`` rule for the outgoing interface and a ``FORWARD`` rule diff --git a/flake-ctl/src/cli.rs b/flake-ctl/src/cli.rs index f4efa85d..91fec003 100644 --- a/flake-ctl/src/cli.rs +++ b/flake-ctl/src/cli.rs @@ -417,13 +417,25 @@ pub enum Network { /// Network the traffic of the VMs to the outside world /// must not go to. Either link-local, which covers the /// metadata service of a cloud at 169.254.169.254, private, - /// which covers the private networks of RFC 1918, or a - /// network as ADDRESS[/PREFIX_LEN]. Can be specified - /// multiple times. Without this option the VMs may reach - /// every network the host reaches through the outgoing - /// interface + /// which covers the private networks of RFC 1918, a + /// network as ADDRESS[/PREFIX_LEN], a HOSTNAME, which + /// denies the IPv4 addresses it resolves to at the time + /// of the call, or @FILE, which reads one such entry per + /// line. Can be specified multiple times. Without this + /// option the VMs may reach every network the host + /// reaches through the outgoing interface #[clap(long, multiple_occurrences = true)] egress_deny: Vec, + + /// Network the traffic of the VMs to the outside world + /// is limited to, given like --egress-deny. Can be + /// specified multiple times. With this option the traffic + /// to every network which is not allowed is dropped, a + /// network given by --egress-deny stays denied. Unless + /// --host-resolver is given, the public name server of + /// the VMs is allowed along with the given networks + #[clap(long, multiple_occurrences = true)] + egress_allow: Vec, }, /// Connect a VM application to the host network Add { diff --git a/flake-ctl/src/main.rs b/flake-ctl/src/main.rs index ab690461..28544ca0 100644 --- a/flake-ctl/src/main.rs +++ b/flake-ctl/src/main.rs @@ -190,7 +190,7 @@ async fn main() -> Result> { // init cli::Network::Init { outgoing_interface, host_resolver, host_dhcp, - egress_deny + egress_deny, egress_allow } => { if ! network::init( outgoing_interface.as_deref(), @@ -199,6 +199,7 @@ async fn main() -> Result> { dhcp: *host_dhcp }, egress_deny, + egress_allow, user ) { return Ok(ExitCode::FAILURE) diff --git a/flake-ctl/src/network.rs b/flake-ctl/src/network.rs index 67db2e5b..c2db2b63 100644 --- a/flake-ctl/src/network.rs +++ b/flake-ctl/src/network.rs @@ -23,7 +23,8 @@ // use flakes::helper::helper_call; use flakes::network::{ - get_default_route_interface, get_egress_networks, get_free_network, + add_networks, get_default_route_interface, get_egress_entry_networks, + get_egress_preset_names, get_free_network, get_nameserver, get_network_config_file, get_preferred_network, get_tap_name, lock_network_state, network_collides, read_network_config, @@ -34,7 +35,7 @@ use flakes::registration; use serde::Serialize; use std::collections::HashMap; use std::fs; -use std::net::Ipv4Addr; +use std::net::{Ipv4Addr, SocketAddr, ToSocketAddrs}; use std::path::Path; use std::process::{Command, Stdio}; use uzers::{get_current_uid, get_user_by_name, get_user_by_uid}; @@ -46,7 +47,7 @@ use crate::defaults; pub fn init( outgoing_interface: Option<&str>, services: HostServices, - egress_deny: &[String], usermode: bool + egress_deny: &[String], egress_allow: &[String], usermode: bool ) -> bool { /*! Prepare the host for NAT based VM networking @@ -63,7 +64,10 @@ pub fn init( itself is created by the helper, see its network-init command. The helper checks the network and the interface again and records them along with the given services of the host and the - networks of the given egress policy in the network state. The + denied and allowed networks of the given egress policy in the + network state. Host names and files of the egress policy are + resolved here, see get_egress_policy_networks(), the helper + only receives networks. The setup of the single flakes, done by add(), uses what is recorded there. @@ -76,8 +80,15 @@ pub fn init( firewall is managed by another tool, the rules have to be created with that tool instead !*/ - let egress_networks = match get_egress_networks(egress_deny) { - Ok(egress_networks) => egress_networks, + let egress_deny = match get_egress_policy_networks(egress_deny) { + Ok(egress_deny) => egress_deny, + Err(error) => { + error!("{error}"); + return false + } + }; + let mut egress_allow = match get_egress_policy_networks(egress_allow) { + Ok(egress_allow) => egress_allow, Err(error) => { error!("{error}"); return false @@ -93,6 +104,11 @@ pub fn init( Some(network) => network, None => return false }; + if let Some(nameserver) = allow_nameserver( + &mut egress_allow, &network, services + ) { + info!("Allowing the name server {nameserver} of the VMs"); + } let mut call = helper_call("network-init"); call.arg("--network").arg(network.to_string()) .arg("--outgoing-interface").arg(&outgoing_interface); @@ -102,9 +118,12 @@ pub fn init( if services.dhcp { call.arg("--host-dhcp"); } - for egress_network in &egress_networks { + for egress_network in &egress_deny { call.arg("--egress-deny").arg(egress_network.to_string()); } + for egress_network in &egress_allow { + call.arg("--egress-allow").arg(egress_network.to_string()); + } if ! run_helper(&mut call, "prepare the host network") { return false } @@ -122,16 +141,209 @@ pub fn init( " DHCP server of the host: {}", if services.dhcp { "yes" } else { "no" } ); - if egress_networks.is_empty() { - info!(" egress denied: none"); - } else { + info!(" egress denied: {}", format_networks(&egress_deny, "none")); + info!(" egress allowed: {}", format_networks(&egress_allow, "all")); + true +} + +pub fn get_egress_policy_networks( + entries: &[String] +) -> Result, String> { + /*! + Provide the networks of the given entries of the egress policy + + In addition to the entries the helper takes, see + flakes::network::get_egress_entry_networks(), an entry can be: + + * HOSTNAME: the IPv4 addresses the name resolves to on the host + at the time of the call. A name which is resolved to other + addresses later on is not followed, the policy has to be + created again for that. Names of a service which is spread + over many addresses, e.g a storage service of a cloud, are + better covered by the networks of that service + + * @FILE: the entries of the given file, one per line. Empty + lines and everything following a '#' are skipped. A file + cannot name another file. This allows to keep long lists of + networks, e.g the published address ranges of a cloud + provider, apart from the command line + + The entries are resolved by the calling user. The helper only + receives the resulting networks, it neither resolves names nor + reads files on behalf of the caller. Networks which are given + more than once are only listed once + !*/ + get_egress_policy_networks_with(entries, &resolve_host_name) +} + +fn get_egress_policy_networks_with( + entries: &[String], resolve: &dyn Fn(&str) -> Result, String> +) -> Result, String> { + /*! + Provide the networks of the given entries of the egress policy, + host names are resolved with the given resolver, see + get_egress_policy_networks() + !*/ + let mut networks: Vec = Vec::new(); + for entry in entries { + match entry.strip_prefix('@') { + Some(file) => { + for (line_number, file_entry) in read_egress_file(file)? { + let file_networks = get_egress_policy_entry_networks( + &file_entry, resolve + ).map_err( + |error| format!("{file}, line {line_number}: {error}") + )?; + add_networks(&mut networks, file_networks); + } + }, + None => add_networks( + &mut networks, get_egress_policy_entry_networks(entry, resolve)? + ) + } + } + Ok(networks) +} + +fn get_egress_policy_entry_networks( + entry: &str, resolve: &dyn Fn(&str) -> Result, String> +) -> Result, String> { + /*! + Provide the networks of one entry of the egress policy which + is not a file, see get_egress_policy_networks() + !*/ + if let Some(networks) = get_egress_entry_networks(entry) { + return Ok(networks) + } + if is_host_name(entry) { + let addresses = resolve(entry)?; info!( - " egress denied: {}", - egress_networks.iter() + "Egress policy: {entry} resolves to {}", + addresses.iter() .map(ToString::to_string).collect::>().join(", ") ); + return Ok( + addresses.into_iter() + .map(|address| Ipv4Network::new(address, 32)) + .collect() + ) + } + Err(format!( + "Invalid egress policy entry {entry:?}: expected one of {}, \ + ADDRESS[/PREFIX_LEN], HOSTNAME or @FILE", + get_egress_preset_names().join(", ") + )) +} + +fn read_egress_file(file: &str) -> Result, String> { + /*! + Read the entries of the given egress policy file along with + their line number, see get_egress_policy_networks() + !*/ + let content = fs::read_to_string(file).map_err( + |error| format!("Failed to read egress policy file {file}: {error}") + )?; + let mut entries = Vec::new(); + for (index, line) in content.lines().enumerate() { + let entry = line.split('#').next().unwrap_or_default().trim(); + if entry.is_empty() { + continue + } + if entry.starts_with('@') { + return Err(format!( + "{file}, line {}: an egress policy file cannot name \ + another file", index + 1 + )) + } + entries.push((index + 1, entry.to_string())); + } + Ok(entries) +} + +pub fn is_host_name(name: &str) -> bool { + /*! + Check if the given name is a fully qualified host name + + The name consists of at least two labels of letters, digits + and hyphens, a label neither starts nor ends with a hyphen. + The last label is not numeric, which tells a name apart from + an IPv4 address. A single label is not taken as a host name, + it is more likely a mistyped name of a set of networks + !*/ + let name = name.strip_suffix('.').unwrap_or(name); + let labels: Vec<&str> = name.split('.').collect(); + name.len() <= 253 + && labels.len() >= 2 + && labels.iter().all(|label| { + ! label.is_empty() && label.len() <= 63 + && ! label.starts_with('-') && ! label.ends_with('-') + && label.chars().all( + |letter| letter.is_ascii_alphanumeric() || letter == '-' + ) + }) + && ! labels.last().unwrap_or(&"").chars() + .all(|letter| letter.is_ascii_digit()) +} + +fn resolve_host_name(name: &str) -> Result, String> { + /*! + Provide the IPv4 addresses the given host name resolves to + !*/ + let resolved = (name, 0).to_socket_addrs().map_err( + |error| format!("Failed to resolve egress policy host {name}: {error}") + )?; + let mut addresses: Vec = resolved + .filter_map(|address| match address { + SocketAddr::V4(address) => Some(*address.ip()), + SocketAddr::V6(_) => None + }) + .collect(); + addresses.sort(); + addresses.dedup(); + if addresses.is_empty() { + return Err(format!("Egress policy host {name} has no IPv4 address")) + } + Ok(addresses) +} + +fn allow_nameserver( + egress_allow: &mut Vec, network: &Ipv4Network, + services: HostServices +) -> Option { + /*! + Add the name server of the VMs to the given allowed networks + + With an allow list the VMs would not reach their name server + anymore unless it is allowed. A resolver on the host is reached + through the gateway and is not part of the egress policy. The + public name server is, it is added unless no network is allowed + at all or it is part of an allowed network. The added name + server is returned + !*/ + if egress_allow.is_empty() || services.resolver { + return None + } + let nameserver = Ipv4Network::parse(&get_nameserver(network, services))?; + if egress_allow.iter().any( + |allowed| allowed.contains(&nameserver.address) + ) { + return None + } + egress_allow.push(nameserver); + Some(nameserver) +} + +fn format_networks(networks: &[Ipv4Network], empty: &str) -> String { + /*! + Provide the given networks for the output, long lists are only + counted + !*/ + match networks.len() { + 0 => empty.to_string(), + 1..=8 => networks.iter() + .map(ToString::to_string).collect::>().join(", "), + count => format!("{count} networks") } - true } pub fn add( @@ -1182,12 +1394,164 @@ mod tests { use tempfile::tempdir; use super::{ - configure_flake, get_effective_boot_args, get_flake_taps, - get_network_info, get_remove_command, read_flake_config, - select_outgoing_interface, unconfigure_flake, FlakeNetwork, - NetworkConfig + allow_nameserver, configure_flake, format_networks, + get_effective_boot_args, get_egress_policy_networks_with, + get_flake_taps, get_network_info, get_remove_command, is_host_name, + read_flake_config, select_outgoing_interface, unconfigure_flake, + FlakeNetwork, NetworkConfig }; + fn network(network: &str) -> Ipv4Network { + Ipv4Network::parse(network).unwrap() + } + + fn fake_resolver(name: &str) -> Result, String> { + match name { + "oss-cn-hangzhou.aliyuncs.com" | "oss-cn-hangzhou.aliyuncs.com." => { + Ok(vec![address("118.31.219.1"), address("118.31.219.2")]) + }, + "mirror.example.org" => Ok(vec![address("203.0.113.10")]), + _ => Err(format!("Failed to resolve egress policy host {name}")) + } + } + + fn policy_networks(entries: &[&str]) -> Result, String> { + get_egress_policy_networks_with( + &entries.iter().map(ToString::to_string).collect::>(), + &fake_resolver + ) + } + + #[test] + fn test_is_host_name() { + assert!(is_host_name("oss-cn-hangzhou.aliyuncs.com")); + assert!(is_host_name("bucket.oss-cn-hangzhou.aliyuncs.com.")); + assert!(is_host_name("xn--80ak6aa92e.com")); + // a single label is more likely a mistyped preset + assert!(! is_host_name("metadata")); + assert!(! is_host_name("localhost")); + // addresses and networks are no host names + assert!(! is_host_name("10.0.0.1")); + assert!(! is_host_name("10.0.0.256")); + assert!(! is_host_name("10.0.0.0/8")); + assert!(! is_host_name("*.aliyuncs.com")); + assert!(! is_host_name("-bad.example.com")); + assert!(! is_host_name("bad-.example.com")); + assert!(! is_host_name("a..example.com")); + assert!(! is_host_name("-j ACCEPT")); + assert!(! is_host_name(&format!("{}.com", "a".repeat(64)))); + } + + #[test] + fn test_get_egress_policy_networks() { + assert_eq!(Ok(vec![]), policy_networks(&[])); + // presets and networks are taken as the helper takes them + assert_eq!( + Ok(vec![network("169.254.0.0/16"), network("100.100.100.200")]), + policy_networks(&["link-local", "100.100.100.200"]) + ); + // a host name is replaced by the addresses it resolves to, + // addresses which are given more than once are listed once + assert_eq!( + Ok(vec![ + network("118.31.219.1"), network("118.31.219.2"), + network("203.0.113.10") + ]), + policy_networks(&[ + "oss-cn-hangzhou.aliyuncs.com", "118.31.219.2", + "mirror.example.org" + ]) + ); + // a name which cannot be resolved fails the policy + assert!(policy_networks(&["unknown.example.org"]).is_err()); + // invalid entries are refused + let error = policy_networks(&["metadata"]).unwrap_err(); + assert!(error.contains("HOSTNAME or @FILE"), "{}", error); + assert!(policy_networks(&["*.aliyuncs.com"]).is_err()); + assert!(policy_networks(&["10.0.0.0/33"]).is_err()); + } + + #[test] + fn test_get_egress_policy_networks_from_file() { + let tmp_dir = tempdir().unwrap(); + let list = tmp_dir.path().join("alibaba-cloud.list"); + fs::write( + &list, + "# address ranges of a cloud provider\n\ + 47.74.0.0/15\n\ + \n\ + 8.208.0.0/12 # Europe\n\ + oss-cn-hangzhou.aliyuncs.com\n\ + private\n" + ).unwrap(); + let list_entry = format!("@{}", list.display()); + assert_eq!( + Ok(vec![ + network("169.254.0.0/16"), + network("47.74.0.0/15"), network("8.208.0.0/12"), + network("118.31.219.1"), network("118.31.219.2"), + network("10.0.0.0/8"), network("172.16.0.0/12"), + network("192.168.0.0/16") + ]), + policy_networks(&["link-local", &list_entry]) + ); + // an invalid entry is reported along with its line + let invalid = tmp_dir.path().join("invalid.list"); + fs::write(&invalid, "47.74.0.0/15\n47.74.0.0/40\n").unwrap(); + let error = policy_networks( + &[&format!("@{}", invalid.display())] + ).unwrap_err(); + assert!(error.contains("line 2"), "{}", error); + // a file cannot name another file + let nested = tmp_dir.path().join("nested.list"); + fs::write(&nested, format!("{list_entry}\n")).unwrap(); + assert!(policy_networks(&[&format!("@{}", nested.display())]).is_err()); + // a missing file fails the policy + assert!(policy_networks(&["@/does/not/exist"]).is_err()); + } + + #[test] + fn test_allow_nameserver() { + let private = network("172.16.0.0/24"); + let public_dns = HostServices::default(); + let host_resolver = HostServices { resolver: true, dhcp: false }; + // without an allow list everything is allowed already + let mut allowed = vec![]; + assert_eq!(None, allow_nameserver(&mut allowed, &private, public_dns)); + assert!(allowed.is_empty()); + // the public name server is added to the allow list + let mut allowed = vec![network("203.0.113.0/24")]; + assert_eq!( + Some(network("8.8.8.8")), + allow_nameserver(&mut allowed, &private, public_dns) + ); + assert_eq!( + vec![network("203.0.113.0/24"), network("8.8.8.8")], allowed + ); + // unless an allowed network covers it already + let mut allowed = vec![network("8.8.0.0/16")]; + assert_eq!(None, allow_nameserver(&mut allowed, &private, public_dns)); + assert_eq!(vec![network("8.8.0.0/16")], allowed); + // a resolver on the host is not reached through the policy + let mut allowed = vec![network("203.0.113.0/24")]; + assert_eq!( + None, allow_nameserver(&mut allowed, &private, host_resolver) + ); + assert_eq!(vec![network("203.0.113.0/24")], allowed); + } + + #[test] + fn test_format_networks() { + assert_eq!("none", format_networks(&[], "none")); + assert_eq!( + "10.0.0.0/8, 8.8.8.8/32", + format_networks(&[network("10.0.0.0/8"), network("8.8.8.8")], "") + ); + let many: Vec = (0..9) + .map(|index| network(&format!("10.0.{index}.0/24"))).collect(); + assert_eq!("9 networks", format_networks(&many, "")); + } + fn address(address: &str) -> Ipv4Addr { address.parse().unwrap() } diff --git a/flake-pilot-helper/src/cli.rs b/flake-pilot-helper/src/cli.rs index 1cebd98b..09810eab 100644 --- a/flake-pilot-helper/src/cli.rs +++ b/flake-pilot-helper/src/cli.rs @@ -70,6 +70,14 @@ pub enum Commands { /// Given more than once, all of them are denied #[clap(long, multiple_occurrences = true)] egress_deny: Vec, + + /// Network the traffic of the VMs to the outside world is + /// limited to: link-local, private or ADDRESS[/PREFIX_LEN]. + /// Given more than once, all of them are allowed. Given at + /// all, the traffic to every other network is dropped. A + /// network given by --egress-deny stays denied + #[clap(long, multiple_occurrences = true)] + egress_allow: Vec, }, /// Delete the network setup of the host NetworkCleanup, diff --git a/flake-pilot-helper/src/firewall.rs b/flake-pilot-helper/src/firewall.rs index aa43cb00..df3eab47 100644 --- a/flake-pilot-helper/src/firewall.rs +++ b/flake-pilot-helper/src/firewall.rs @@ -29,6 +29,7 @@ //! state and from the address allocations, never from a rule //! specification of the caller. //! +use std::io::Write; use std::net::Ipv4Addr; use std::process::{Command, Stdio}; @@ -46,9 +47,8 @@ use crate::run::run_ok; // Position a rule is created at in its chain. Accepting rules are // inserted at the top, dropping rules are appended. This keeps // every accepting rule in front of the drop rules, no matter in -// which order the rules are created. The egress chain is the other -// way round: the drop rules of the egress policy are inserted at -// the top, in front of the rule which accepts everything else +// which order the rules are created. The egress chain is not built +// this way, it is written as a whole, see write_egress_chain() pub enum RulePosition { Top, Bottom @@ -72,7 +72,7 @@ pub fn get_firewall_chains() -> [(&'static str, &'static str); 4] { } pub fn get_firewall_rules( - network: &str, outgoing_interface: &str, egress_deny: &[Ipv4Network] + network: &str, outgoing_interface: &str ) -> Vec<(FirewallRule, RulePosition)> { /*! Provide the rules of the host firewall setup and the position @@ -88,20 +88,11 @@ pub fn get_firewall_rules( dropped as well, except for answers to traffic of the host and the services connect() allows for each instance - The rules of the egress chain are listed first, the chain is - complete before traffic is passed to it + The rules of the egress chain are not part of them, the chain + is written before these rules are created, see setup_firewall() !*/ let tap_match = get_tap_match(); - let mut rules: Vec<(FirewallRule, RulePosition)> = get_egress_rules( - egress_deny - ).into_iter().map(|rule| (rule, RulePosition::Top)).collect(); - rules.extend([ - ( - FirewallRule::new("filter", FIREWALL_EGRESS_CHAIN, &[ - "-j", "ACCEPT" - ]), - RulePosition::Bottom - ), + vec![ (get_masquerade_rule(network, outgoing_interface), RulePosition::Bottom), (get_egress_jump_rule(network, outgoing_interface), RulePosition::Top), ( @@ -134,8 +125,7 @@ pub fn get_firewall_rules( FirewallRule::new("filter", FIREWALL_INPUT_CHAIN, &["-j", "DROP"]), RulePosition::Bottom ) - ]); - rules + ] } pub fn get_egress_jump_rule( @@ -152,20 +142,150 @@ pub fn get_egress_jump_rule( ]) } -pub fn get_egress_rules(egress_deny: &[Ipv4Network]) -> Vec { +pub fn get_egress_rules( + egress_deny: &[Ipv4Network], egress_allow: &[Ipv4Network] +) -> Vec { + /*! + Provide the rules of the egress chain, in the order they have + to be in + + The traffic of the VMs to the denied networks is dropped first. + Without allowed networks all other traffic is accepted by the + last rule of the chain. With allowed networks, only the traffic + to them is accepted and the last rule drops everything else. A + network which is denied therefore stays denied, even if it is + part of an allowed one. The allow list can only narrow down what + the VMs reach, never extend it. + + The rules are given in the notation iptables lists them with, + see egress_chain_matches() + !*/ + let rule = |network: &Ipv4Network, target: &str| FirewallRule::new( + "filter", FIREWALL_EGRESS_CHAIN, + &["-d", &network.to_string(), "-j", target] + ); + let mut rules: Vec = egress_deny.iter() + .map(|network| rule(network, "DROP")) + .collect(); + rules.extend(egress_allow.iter().map(|network| rule(network, "ACCEPT"))); + rules.push(FirewallRule::new("filter", FIREWALL_EGRESS_CHAIN, &[ + "-j", if egress_allow.is_empty() { "ACCEPT" } else { "DROP" } + ])); + rules +} + +pub fn get_egress_restore_input(rules: &[FirewallRule]) -> String { /*! - Provide the rules of the egress policy + Provide the input of iptables-restore which replaces the rules + of the egress chain by the given ones - The traffic of the VMs to the given networks is dropped, all - other traffic which enters the egress chain is accepted by its - last rule. The rules are given in the notation iptables lists - them with, see delete_stale_egress_rules() + With --noflush only the chains which are declared in the input + are flushed, the egress chain is the only one. The rules come + from validated networks only, see get_egress_rules() !*/ - egress_deny.iter() - .map(|network| FirewallRule::new("filter", FIREWALL_EGRESS_CHAIN, &[ - "-d", &network.to_string(), "-j", "DROP" - ])) - .collect() + let mut input = format!("*filter\n:{FIREWALL_EGRESS_CHAIN} - [0:0]\n"); + for rule in rules { + input.push_str( + &format!("-A {} {}\n", rule.chain, rule.spec.join(" ")) + ); + } + input.push_str("COMMIT\n"); + input +} + +pub fn write_egress_chain( + egress_deny: &[Ipv4Network], egress_allow: &[Ipv4Network] +) -> bool { + /*! + Replace the rules of the egress chain by the ones of the given + egress policy, see get_egress_rules() + + The rules decide by their order which traffic passes. Adding and + deleting them one by one would let traffic pass, or drop it, by + a mix of the former and the new policy while the chain is + changed. iptables-restore replaces the chain in one transaction + instead, the traffic passes either the former or the new policy + !*/ + let input = get_egress_restore_input( + &get_egress_rules(egress_deny, egress_allow) + ); + info!( + "Writing {FIREWALL_EGRESS_CHAIN} chain: {} denied, {} allowed \ + network(s)...", egress_deny.len(), egress_allow.len() + ); + let mut call = Command::new(defaults::IPTABLES_RESTORE_TOOL); + call.arg("--wait").arg("--noflush") + .stdin(Stdio::piped()).stdout(Stdio::null()); + let mut child = match call.spawn() { + Ok(child) => child, + Err(error) => { + error!("Failed to write {FIREWALL_EGRESS_CHAIN} chain: {error:?}"); + return false + } + }; + let written = match child.stdin.take() { + Some(mut stdin) => stdin.write_all(input.as_bytes()), + None => Err(std::io::Error::other("no stdin")) + }; + // stdin is closed at this point, iptables-restore reads to its end + match (written, child.wait()) { + (Ok(_), Ok(status)) if status.success() => true, + (Err(error), _) | (_, Err(error)) => { + error!("Failed to write {FIREWALL_EGRESS_CHAIN} chain: {error:?}"); + false + }, + (_, Ok(status)) => { + error!("Failed to write {FIREWALL_EGRESS_CHAIN} chain: {status}"); + false + } + } +} + +pub fn egress_chain_matches( + listed: &[Vec], egress_deny: &[Ipv4Network], + egress_allow: &[Ipv4Network] +) -> bool { + /*! + Check if the listed rules of the egress chain are the ones of + the given egress policy + + The drop rules of the denied networks come first, followed by + the accept rules of the allowed networks and the last rule of + the chain, see get_egress_rules(). Within the drop rules and + within the accept rules the order does not matter. Former + versions inserted the drop rules one by one at the top, which + listed them in reverse order + !*/ + let wanted: Vec> = get_egress_rules(egress_deny, egress_allow) + .into_iter().map(|rule| rule.spec).collect(); + if listed.len() != wanted.len() { + return false + } + let same_rules = |listed: &[Vec], wanted: &[Vec]| { + wanted.iter().all(|spec| listed.contains(spec)) + }; + let allow_start = egress_deny.len(); + let allow_end = allow_start + egress_allow.len(); + same_rules(&listed[..allow_start], &wanted[..allow_start]) + && same_rules( + &listed[allow_start..allow_end], &wanted[allow_start..allow_end] + ) + && listed[allow_end..] == wanted[allow_end..] +} + +pub fn get_chain_rules(table: &str, chain: &str) -> Option>> { + /*! + Provide the rules of the given chain in the notation iptables + lists them with, see get_listed_rules() + !*/ + let mut call = iptables(table); + call.arg("-S").arg(chain).stderr(Stdio::null()); + match call.output() { + Ok(output) if output.status.success() => Some(get_listed_rules( + &String::from_utf8_lossy(&output.stdout), chain + )), + _ => None + } } pub fn get_guest_service_rules( @@ -217,33 +337,35 @@ pub fn get_guest_service_rules( pub fn setup_firewall( network: &Ipv4Network, outgoing_interface: &str, - egress_deny: &[Ipv4Network] + egress_deny: &[Ipv4Network], egress_allow: &[Ipv4Network] ) -> bool { /*! Create the chains and rules of the host firewall setup The chains are created first and are entered from the builtin chains last, traffic is therefore only passed to them once they - are complete. Chains and rules which already exist are kept, - this allows to call the setup more than once. The drop rules of - a former egress policy are deleted once the rules of the given - one are in place + are complete. The egress chain is written as a whole before + traffic is passed to it, a former egress policy is replaced in + one step, see write_egress_chain(). All other chains and rules + which already exist are kept, this allows to call the setup + more than once !*/ for (table, chain) in get_firewall_chains() { if ! create_chain(table, chain) { return false } } + if ! write_egress_chain(egress_deny, egress_allow) { + return false + } for (rule, position) in get_firewall_rules( - &network.to_string(), outgoing_interface, egress_deny + &network.to_string(), outgoing_interface ) { if ! add_rule(&rule, position) { return false } } - if ! delete_stale_egress_rules(egress_deny) - || ! delete_unfiltered_forward_rules() - { + if ! delete_unfiltered_forward_rules() { return false } for rule in get_jump_rules() { @@ -273,16 +395,18 @@ pub fn delete_firewall() -> bool { pub fn firewall_is_active( network: &Ipv4Network, outgoing_interface: &str, - egress_deny: &[Ipv4Network] + egress_deny: &[Ipv4Network], egress_allow: &[Ipv4Network] ) -> bool { /*! Check if the host firewall setup is complete, the chains are present and are entered from the builtin chains With an egress policy, the traffic of the VMs has to be passed - to the egress chain and the drop rules of the policy have to be - present. A setup created before the egress policy existed has - no egress chain, it is complete without a policy + to the egress chain and the chain has to hold the rules of the + policy, see egress_chain_matches(). The chain is read in one + call, a policy may consist of many networks. A setup created + before the egress policy existed has no egress chain, it is + complete without a policy !*/ let chains_exist = get_firewall_chains().iter() .filter(|(_, chain)| *chain != FIREWALL_EGRESS_CHAIN) @@ -290,27 +414,16 @@ pub fn firewall_is_active( if ! chains_exist || ! get_jump_rules().iter().all(rule_exists) { return false } - egress_deny.is_empty() || ( - rule_exists( - &get_egress_jump_rule(&network.to_string(), outgoing_interface) - ) && get_egress_rules(egress_deny).iter().all(rule_exists) + if egress_deny.is_empty() && egress_allow.is_empty() { + return true + } + rule_exists( + &get_egress_jump_rule(&network.to_string(), outgoing_interface) + ) && get_chain_rules("filter", FIREWALL_EGRESS_CHAIN).is_some_and( + |listed| egress_chain_matches(&listed, egress_deny, egress_allow) ) } -pub fn delete_stale_egress_rules(egress_deny: &[Ipv4Network]) -> bool { - /*! - Delete the drop rules of the egress chain which are not part of - the given egress policy - !*/ - let wanted: Vec> = get_egress_rules(egress_deny).into_iter() - .map(|rule| rule.spec) - .collect(); - delete_listed_rules("filter", FIREWALL_EGRESS_CHAIN, |spec| { - spec.last().is_some_and(|target| target == "DROP") - && ! wanted.iter().any(|wanted_spec| wanted_spec == spec) - }) -} - pub fn delete_unfiltered_forward_rules() -> bool { /*! Delete the rules which forward the traffic of the VMs without @@ -524,13 +637,9 @@ pub fn delete_listed_rules( each rule is matched by its specification without the leading '-A CHAIN' !*/ - let mut call = iptables(table); - call.arg("-S").arg(chain).stderr(Stdio::null()); - let rules = match call.output() { - Ok(output) if output.status.success() => { - get_listed_rules(&String::from_utf8_lossy(&output.stdout), chain) - }, - _ => { + let rules = match get_chain_rules(table, chain) { + Some(rules) => rules, + None => { warn!("Failed to read the rules of the {chain} chain"); return false } @@ -572,8 +681,9 @@ mod tests { use flakes::network::{HostServices, Ipv4Network}; use super::{ - get_egress_rules, get_firewall_rules, get_guest_service_rules, - get_listed_rules, RulePosition + egress_chain_matches, get_egress_restore_input, get_egress_rules, + get_firewall_rules, get_guest_service_rules, get_listed_rules, + RulePosition }; fn network(network: &str) -> Ipv4Network { @@ -586,10 +696,7 @@ mod tests { #[test] fn test_firewall_rules() { - let rules = get_firewall_rules( - "172.16.0.0/24", "eth0", - &[network("169.254.0.0/16"), network("10.0.0.0/8")] - ); + let rules = get_firewall_rules("172.16.0.0/24", "eth0"); let rule_args = |chain: &str| -> Vec<(String, bool)> { rules.iter() .filter(|(rule, _)| rule.chain == chain) @@ -607,24 +714,9 @@ mod tests { )], rule_args("FLAKE_PILOT_NAT") ); - // the egress chain drops the traffic to the denied networks - // in front of the rule which accepts everything else - assert_eq!( - vec![ - ("-d 169.254.0.0/16 -j DROP".to_string(), true), - ("-d 10.0.0.0/8 -j DROP".to_string(), true), - ("-j ACCEPT".to_string(), false) - ], - rule_args("FLAKE_PILOT_EGRESS") - ); - // the egress chain is complete before traffic is passed to it - let last_egress_rule = rules.iter() - .rposition(|(rule, _)| rule.chain == "FLAKE_PILOT_EGRESS"); - let egress_jump_rule = rules.iter() - .position(|(rule, _)| rule.spec.ends_with(&[ - "-j".to_string(), "FLAKE_PILOT_EGRESS".to_string() - ])); - assert!(last_egress_rule.unwrap() < egress_jump_rule.unwrap()); + // the egress chain is written as a whole, apart from these + // rules + assert!(rule_args("FLAKE_PILOT_EGRESS").is_empty()); // accepting rules are created at the top, the drop rules // for everything else to and from a VM at the bottom. The // traffic to the outside world is only accepted after it @@ -657,31 +749,106 @@ mod tests { ], rule_args("FLAKE_PILOT_INPUT") ); + } + + #[test] + fn test_egress_rules() { + let specs = |deny: &[Ipv4Network], allow: &[Ipv4Network]| { + get_egress_rules(deny, allow).iter() + .inspect(|rule| assert_eq!("FLAKE_PILOT_EGRESS", rule.chain)) + .map(|rule| rule.spec.join(" ")) + .collect::>() + }; // without an egress policy everything passes the egress chain - let rules = get_firewall_rules("172.16.0.0/24", "eth0", &[]); + assert_eq!(vec!["-j ACCEPT"], specs(&[], &[])); + // the traffic to the denied networks is dropped in front of + // the rule which accepts everything else assert_eq!( - vec!["-j ACCEPT"], - rules.iter() - .filter(|(rule, _)| rule.chain == "FLAKE_PILOT_EGRESS") - .map(|(rule, _)| rule.spec.join(" ")) - .collect::>() + vec![ + "-d 169.254.0.0/16 -j DROP", + "-d 10.0.0.0/8 -j DROP", + "-j ACCEPT" + ], + specs(&[network("169.254.0.0/16"), network("10.0.0.0/8")], &[]) + ); + // with an allow list only the traffic to the allowed + // networks passes, the denied networks stay denied + assert_eq!( + vec![ + "-d 47.0.0.0/8 -j DROP", + "-d 8.8.8.8/32 -j ACCEPT", + "-d 0.0.0.0/0 -j ACCEPT", + "-j DROP" + ], + specs( + &[network("47.0.0.0/8")], + &[network("8.8.8.8"), network("0.0.0.0/0")] + ) ); } #[test] - fn test_egress_rules_are_listed_as_created() { - // stale drop rules are recognized by comparing the listed - // rules of the egress chain with the created ones - let rule_list = "-N FLAKE_PILOT_EGRESS\n\ + fn test_egress_restore_input() { + // only the egress chain is declared and therefore replaced, + // its rules are written in their order + assert_eq!( + "*filter\n\ + :FLAKE_PILOT_EGRESS - [0:0]\n\ -A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 203.0.113.0/24 -j ACCEPT\n\ + -A FLAKE_PILOT_EGRESS -j DROP\n\ + COMMIT\n", + get_egress_restore_input(&get_egress_rules( + &[network("169.254.0.0/16")], &[network("203.0.113.0/24")] + )) + ); + } + + #[test] + fn test_egress_chain_matches() { + let listed = |rule_list: &str| get_listed_rules( + &format!("-N FLAKE_PILOT_EGRESS\n{rule_list}"), + "FLAKE_PILOT_EGRESS" + ); + let deny = [network("169.254.0.0/16"), network("100.100.100.200")]; + let allow = [network("203.0.113.0/24")]; + // the rules are listed in the notation they are created with + let denied = listed( + "-A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ -A FLAKE_PILOT_EGRESS -d 100.100.100.200/32 -j DROP\n\ - -A FLAKE_PILOT_EGRESS -j ACCEPT\n"; - assert_eq!( - get_listed_rules(rule_list, "FLAKE_PILOT_EGRESS")[..2].to_vec(), - get_egress_rules(&[ - network("169.254.0.0/16"), network("100.100.100.200") - ]).into_iter().map(|rule| rule.spec).collect::>() + -A FLAKE_PILOT_EGRESS -j ACCEPT\n" + ); + assert!(egress_chain_matches(&denied, &deny, &[])); + // former versions listed the drop rules in reverse order + let reversed = listed( + "-A FLAKE_PILOT_EGRESS -d 100.100.100.200/32 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -j ACCEPT\n" + ); + assert!(egress_chain_matches(&reversed, &deny, &[])); + // a missing, stale or additional rule does not match + assert!(! egress_chain_matches(&denied, &deny[..1], &[])); + assert!(! egress_chain_matches(&denied, &deny, &allow)); + assert!(! egress_chain_matches( + &listed("-A FLAKE_PILOT_EGRESS -j ACCEPT\n"), &deny, &[] + )); + let allowed = listed( + "-A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 100.100.100.200/32 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 203.0.113.0/24 -j ACCEPT\n\ + -A FLAKE_PILOT_EGRESS -j DROP\n" + ); + assert!(egress_chain_matches(&allowed, &deny, &allow)); + assert!(! egress_chain_matches(&allowed, &deny, &[])); + // an allow rule in front of a drop rule would let the + // traffic to the denied network pass + let allowed_first = listed( + "-A FLAKE_PILOT_EGRESS -d 203.0.113.0/24 -j ACCEPT\n\ + -A FLAKE_PILOT_EGRESS -d 169.254.0.0/16 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -d 100.100.100.200/32 -j DROP\n\ + -A FLAKE_PILOT_EGRESS -j DROP\n" ); + assert!(! egress_chain_matches(&allowed_first, &deny, &allow)); } #[test] diff --git a/flake-pilot-helper/src/main.rs b/flake-pilot-helper/src/main.rs index f1cc701c..72442520 100644 --- a/flake-pilot-helper/src/main.rs +++ b/flake-pilot-helper/src/main.rs @@ -86,10 +86,11 @@ fn main() -> ExitCode { setup_logger(debug); let ok = match &args.command { cli::Commands::NetworkInit { - network, outgoing_interface, host_resolver, host_dhcp, egress_deny + network, outgoing_interface, host_resolver, host_dhcp, + egress_deny, egress_allow } => network::init( &caller, network, outgoing_interface, *host_resolver, *host_dhcp, - egress_deny + egress_deny, egress_allow ), cli::Commands::NetworkCleanup => network::cleanup(&caller), cli::Commands::NetworkConnect { diff --git a/flake-pilot-helper/src/network.rs b/flake-pilot-helper/src/network.rs index 6fef378b..274e7e94 100644 --- a/flake-pilot-helper/src/network.rs +++ b/flake-pilot-helper/src/network.rs @@ -69,7 +69,8 @@ pub struct VmInstance<'a> { pub fn init( caller: &Caller, network: &str, outgoing_interface: &str, - host_resolver: bool, host_dhcp: bool, egress_deny: &[String] + host_resolver: bool, host_dhcp: bool, egress_deny: &[String], + egress_allow: &[String] ) -> bool { /*! Prepare the host for NAT based VM networking @@ -85,8 +86,10 @@ pub fn init( selects from, and it must not collide with a network the host is connected to. The outgoing interface has to exist. Both are recorded in the network state along with the services of the - host the VMs use and the networks of the egress policy, see - flakes::network::get_egress_networks(). connect() takes them + host the VMs use and the denied and allowed networks of the + egress policy, see flakes::network::get_egress_networks() and + firewall::get_egress_rules(). The allowed networks only narrow + down what the VMs reach, never extend it. connect() takes them from there and nowhere else: the address of a VM, the route to it and its firewall rules are only ever created within the recorded network. @@ -112,6 +115,13 @@ pub fn init( return false } }; + let egress_allow = match get_egress_networks(egress_allow) { + Ok(egress_allow) => egress_allow, + Err(error) => { + error!("{error}"); + return false + } + }; let _lock = match lock_network_state(true) { Some(lock) => lock, None => return false @@ -144,10 +154,15 @@ pub fn init( state.egress_deny = egress_deny.iter() .map(ToString::to_string) .collect(); + state.egress_allow = egress_allow.iter() + .map(ToString::to_string) + .collect(); if ! write_network_state(&state) { return false } - if ! setup_firewall(&network, outgoing_interface, &egress_deny) { + if ! setup_firewall( + &network, outgoing_interface, &egress_deny, &egress_allow + ) { return false } delete_legacy_rules(outgoing_interface); @@ -250,9 +265,11 @@ pub fn connect( } }; let services = state.get_services(); - let egress_deny = match state.get_egress_deny() { - Some(egress_deny) => egress_deny, - None => { + let (egress_deny, egress_allow) = match ( + state.get_egress_deny(), state.get_egress_allow() + ) { + (Some(egress_deny), Some(egress_allow)) => (egress_deny, egress_allow), + _ => { error!("The recorded egress policy is invalid"); error!("Please run 'flake-ctl firecracker network init' again"); return false @@ -260,7 +277,9 @@ pub fn connect( }; let outgoing_interface = state.outgoing_interface.as_deref() .unwrap_or_default(); - if ! firewall_is_active(&network, outgoing_interface, &egress_deny) { + if ! firewall_is_active( + &network, outgoing_interface, &egress_deny, &egress_allow + ) { error!("The firewall setup of the host is not active"); error!( "Please run 'flake-ctl firecracker network init' first, \