From cabd2ad10753e288b28adb2490bd6486feec2661 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 01/13] add Proxmox VE LXC runtime backend --- .github/workflows/test.yml | 49 + Cargo.lock | 3 +- application/Cargo.toml | 7 +- application/build.rs | 2 + application/src/config.rs | 161 + application/src/main.rs | 48 +- application/src/server/configuration/mod.rs | 2 +- .../server/executor/{docker => }/cgroup.rs | 0 application/src/server/executor/docker/mod.rs | 10 +- application/src/server/executor/mod.rs | 132 +- .../src/server/executor/pve_lxc/cli.rs | 3072 +++++++++++ .../src/server/executor/pve_lxc/firewall.rs | 651 +++ .../src/server/executor/pve_lxc/mod.rs | 4595 +++++++++++++++++ application/src/server/firewall/iptables.rs | 1 + application/src/server/firewall/mod.rs | 102 + application/src/server/firewall/nftables.rs | 2 + application/src/server/firewall/sets.rs | 39 + application/src/server/mod.rs | 7 + application/src/setup/mod.rs | 6 +- application/src/stats.rs | 28 +- application/src/tundra/daemon.rs | 159 +- application/src/tundra/mod.rs | 52 +- docs/pve-lxc-runtime.md | 512 ++ patches/tundra-pve-lxc-process-ref.patch | 169 + scripts/pve-lxc-smoke-test.sh | 513 ++ 25 files changed, 10254 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/test.yml rename application/src/server/executor/{docker => }/cgroup.rs (100%) create mode 100644 application/src/server/executor/pve_lxc/cli.rs create mode 100644 application/src/server/executor/pve_lxc/firewall.rs create mode 100644 application/src/server/executor/pve_lxc/mod.rs create mode 100644 docs/pve-lxc-runtime.md create mode 100644 patches/tundra-pve-lxc-process-ref.patch create mode 100755 scripts/pve-lxc-smoke-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..3555a7f8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: Test + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + linux: + name: Linux tests + runs-on: ubuntu-24.04 + + steps: + - name: Checkout sources + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v2 + with: + toolchain: 1.97.0 + target: x86_64-unknown-linux-gnu + components: rustfmt + cache-key: x86_64-unknown-linux-gnu-test + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Check Tundra process-reference patch + env: + TUNDRA_BASE: ccb05e1406232f8c39f05a984133277026cd79d6 + run: | + git init "$RUNNER_TEMP/tundra" + git -C "$RUNNER_TEMP/tundra" remote add origin https://github.com/calagopus/tundra.git + git -C "$RUNNER_TEMP/tundra" fetch --depth=1 origin "$TUNDRA_BASE" + git -C "$RUNNER_TEMP/tundra" checkout --detach FETCH_HEAD + git -C "$RUNNER_TEMP/tundra" apply --check "$GITHUB_WORKSPACE/patches/tundra-pve-lxc-process-ref.patch" + + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross --rev 8c1a8aa4b661711f4b7b6ac07c2e8929ce2f7d27 --locked + + - name: Run Linux tests + run: cross test --release --target x86_64-unknown-linux-gnu + env: + JEMALLOC_SYS_WITH_LG_PAGE: 12 diff --git a/Cargo.lock b/Cargo.lock index 29df9d38..6164bef3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5416,7 +5416,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tundra-common" version = "1.0.1" -source = "git+https://github.com/calagopus/tundra.git#ccb05e1406232f8c39f05a984133277026cd79d6" +source = "git+https://github.com/calagopus/tundra.git?rev=ccb05e1406232f8c39f05a984133277026cd79d6#ccb05e1406232f8c39f05a984133277026cd79d6" dependencies = [ "base64 0.23.1", "bytes", @@ -6204,6 +6204,7 @@ dependencies = [ "sevenz-rust2", "sha1 0.11.0", "sha2", + "shell-words", "sntpc", "sntpc-net-tokio", "sysinfo", diff --git a/application/Cargo.toml b/application/Cargo.toml index f079f240..6284adc2 100644 --- a/application/Cargo.toml +++ b/application/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" tempfile = "3.27.0" unrar = { path = "../unrar-rs" } pbs-client = { path = "../pbs-client" } -tundra-common = { git = "https://github.com/calagopus/tundra.git" } +tundra-common = { git = "https://github.com/calagopus/tundra.git", rev = "ccb05e1406232f8c39f05a984133277026cd79d6" } tracing = { version = "0.1.41", features = ["max_level_debug"] } tracing-subscriber = { version = "0.3.19", features = ["chrono"] } tracing-appender = "0.2.3" @@ -24,7 +24,7 @@ utoipa = { version = "5.4.0", features = ["axum_extras", "preserve_order", "chro utoipa-axum = "0.2.0" reqwest = { workspace = true } url = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["io-util"] } chrono = { version = "0.4.40", features = ["serde"] } colored = "3.0.0" jsonwebtoken = { version = "11.0.0", features = ["aws_lc_rs"] } @@ -59,7 +59,7 @@ dialoguer = { version = "0.12.0", default-features = false } sysinfo = { version = "0.39.0", default-features = false, features = ["disk", "network", "system", "user"] } crc32fast = { workspace = true } cap-std = "4.0.0" -rustix = { version = "1.0.7", features = ["fs", "process", "net", "thread"] } +rustix = { version = "1.0.7", features = ["fs", "process", "net", "thread", "pty"] } cidr = { version = "0.3.1", features = ["serde"] } hickory-resolver = { workspace = true } constant_time_eq = "0.6.0" @@ -73,6 +73,7 @@ zstd = { workspace = true } base64 = "0.23.0" chunked_vec = "=0.5.1" async-channel = "2.5.0" +shell-words = "1.1.0" lzzzz = "2.0.0" xmltree = { version = "0.12.0", features = ["attribute-order"] } xml = "1.3.0" diff --git a/application/build.rs b/application/build.rs index 75b00df1..7bd6b35b 100644 --- a/application/build.rs +++ b/application/build.rs @@ -191,6 +191,8 @@ fn handle_git_info() { { git_hash = String::from_utf8_lossy(&output.stdout).trim().to_string(); } + } else { + println!("cargo:rustc-env=CARGO_GIT_BRANCH=unknown"); } println!("cargo:rustc-env=CARGO_GIT_COMMIT={git_hash}"); } diff --git a/application/src/config.rs b/application/src/config.rs index 7a2bae45..3658cbcc 100644 --- a/application/src/config.rs +++ b/application/src/config.rs @@ -25,6 +25,9 @@ use utoipa::ToSchema; fn app_name() -> String { "Calagopus".to_string() } +fn default_true() -> bool { + true +} fn api_host() -> String { "0.0.0.0".to_string() } @@ -726,6 +729,84 @@ impl From for SystemPath { } } +#[derive(ToSchema, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Debug, Default)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeBackend { + #[default] + Auto, + Docker, + PveLxc, +} + +#[derive(ToSchema, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Debug, Default)] +#[serde(rename_all = "snake_case")] +pub enum PveLxcFirewallBackend { + #[default] + Auto, + Proxmox, + Nftables, + Iptables, + Disabled, +} + +fn pve_lxc_pct_path() -> String { + "/usr/sbin/pct".to_string() +} + +fn pve_lxc_pvesh_path() -> String { + "/usr/bin/pvesh".to_string() +} + +fn pve_lxc_pveversion_path() -> String { + "/usr/bin/pveversion".to_string() +} +fn pve_lxc_pvesm_path() -> String { + "/usr/sbin/pvesm".to_string() +} + +fn pve_lxc_lxc_attach_path() -> String { + "/usr/bin/lxc-attach".to_string() +} + +fn pve_lxc_lxc_stop_path() -> String { + "/usr/bin/lxc-stop".to_string() +} + +fn pve_lxc_perl_path() -> String { + "/usr/bin/perl".to_string() +} + +fn pve_lxc_template_storage() -> String { + "local".to_string() +} + +fn pve_lxc_rootfs_storage() -> String { + "auto".to_string() +} + +fn pve_lxc_bridge() -> String { + "vmbr0".to_string() +} + +fn pve_lxc_edge_wireguard_interface() -> String { + "wg-calagopus".to_string() +} + +fn pve_lxc_rootfs_size_gib() -> u64 { + 8 +} +fn pve_lxc_console_log_max_bytes() -> u64 { + 5 * 1024 * 1024 +} + +fn pve_lxc_tag_prefix() -> String { + "calagopus".to_string() +} + +fn pve_lxc_managed_file_directory() -> SystemPath { + SystemPath::new("/var/lib/lxc/calagopus-wings-managed") +} + nestify::nest! { #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] pub struct InnerConfig { @@ -741,6 +822,86 @@ nestify::nest! { #[serde(default)] pub token: String, + #[serde(default)] + #[schema(inline)] + pub runtime: #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct Runtime { + #[serde(default)] + pub backend: RuntimeBackend, + + #[serde(default)] + #[schema(inline)] + pub pve_lxc: #[derive(Clone, ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct PveLxcRuntime { + #[serde(default = "pve_lxc_pct_path")] + pub pct_path: String, + #[serde(default = "pve_lxc_pvesh_path")] + pub pvesh_path: String, + #[serde(default = "pve_lxc_pveversion_path")] + pub pveversion_path: String, + #[serde(default = "pve_lxc_pvesm_path")] + pub pvesm_path: String, + #[serde(default = "pve_lxc_lxc_attach_path")] + pub lxc_attach_path: String, + #[serde(default = "pve_lxc_lxc_stop_path")] + pub lxc_stop_path: String, + #[serde(default = "pve_lxc_perl_path")] + pub perl_path: String, + + /// Empty means discover the local Proxmox node name at runtime. + #[serde(default)] + pub node: String, + #[serde(default = "pve_lxc_template_storage")] + pub template_storage: String, + #[serde(default = "pve_lxc_rootfs_storage")] + pub rootfs_storage: String, + #[serde(default = "pve_lxc_bridge")] + pub bridge: String, + /// Optional VLAN tag applied to each LXC veth on `bridge`. + #[serde(default)] + pub vlan_tag: Option, + /// IPv4 CIDR prefix used by static panel allocations (for example, 24). + #[serde(default)] + pub network_prefix: Option, + /// IPv4 gateway used by static panel allocations on the LXC network. + #[serde(default)] + pub gateway: Option, + /// WireGuard interface whose peers publish private LXC allocations. + /// The panel allocation IP is matched against each peer endpoint. + #[serde(default = "pve_lxc_edge_wireguard_interface")] + pub edge_wireguard_interface: String, + /// Restricted SSH key used only to synchronize edge port forwards. + #[serde(default)] + pub edge_ssh_identity_path: String, + /// Pinned SSH known-hosts file for the edge host. + #[serde(default)] + pub edge_known_hosts_path: String, + #[serde(default = "pve_lxc_rootfs_size_gib")] + pub rootfs_size_gib: u64, + /// Maximum retained console bytes for each LXC server. When the + /// limit is reached Wings starts a fresh bounded log. + #[serde(default = "pve_lxc_console_log_max_bytes")] + pub console_log_max_bytes: u64, + #[serde(default = "pve_lxc_tag_prefix")] + pub tag_prefix: String, + /// Host-visible staging directory used for read-only managed file binds. + /// Its parent path must be traversable by unprivileged LXC mount setup. + #[serde(default = "pve_lxc_managed_file_directory")] + pub managed_file_directory: SystemPath, + #[serde(default = "default_true")] + pub unprivileged: bool, + + #[serde(default)] + #[schema(inline)] + pub firewall: #[derive(Clone, ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct PveLxcFirewall { + #[serde(default)] + pub backend: PveLxcFirewallBackend, + #[serde(default = "docker_firewall_source_file_max_entries")] + pub source_file_max_entries: u64, + #[serde(default = "docker_firewall_source_file_max_bytes")] + pub source_file_max_bytes: u64, + }, + }, + }, + #[serde(default)] #[schema(inline)] pub api: #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct Api { diff --git a/application/src/main.rs b/application/src/main.rs index 3bfbd046..f82b1153 100644 --- a/application/src/main.rs +++ b/application/src/main.rs @@ -1,4 +1,4 @@ -use crate::{response::ApiResponse, routes::GetState, server::executor::ServerExecutor}; +use crate::{response::ApiResponse, routes::GetState}; use anyhow::Context; use axum::{ body::Body, @@ -493,44 +493,12 @@ async fn main_rt() { Ok::<_, anyhow::Error>(()) }); - tracing::info!("connecting to docker"); - let (executor, docker) = { - let config_ref = config.load(); - let docker = Arc::new( - match if config_ref.docker.socket.starts_with("http://") - || config_ref.docker.socket.starts_with("tcp://") - { - bollard::Docker::connect_with_http( - &config_ref.docker.socket, - 120, - bollard::API_DEFAULT_VERSION, - ) - } else { - bollard::Docker::connect_with_local( - &config_ref.docker.socket, - 120, - bollard::API_DEFAULT_VERSION, - ) - } { - Ok(docker) => docker, - Err(err) => exit_error!("failed to connect to docker: {:?}", err), - }, - ); - - let own_container = - crate::server::executor::docker::DockerExecutor::own_container(&docker).await; - let firewall = - crate::server::firewall::create(&config, &docker, own_container.as_ref()).await; - - ( - Arc::new(crate::server::executor::docker::DockerExecutor::new( - Arc::clone(&docker), - config.clone(), - firewall, - )), - docker, - ) + let runtime = match crate::server::executor::create_runtime(Arc::clone(&config)).await { + Ok(runtime) => runtime, + Err(err) => exit_error!("failed to initialize server runtime: {:#}", err), }; + let executor = runtime.executor; + let docker = runtime.docker; tracing::info!("running server executor boot tasks"); if let Err(err) = executor.boot().await { @@ -579,7 +547,7 @@ async fn main_rt() { #[cfg(unix)] let tundra = if config.load().tundra.enabled { - match crate::tundra::TundraManager::create(&config, Arc::clone(&docker)) { + match crate::tundra::TundraManager::create(&config, docker.as_ref().map(Arc::clone)) { Ok(tundra) => Some(tundra), Err(err) => exit_error!("failed to set up the tundra control plane: {:?}", err), } @@ -603,7 +571,7 @@ async fn main_rt() { config: Arc::clone(&config), executor, - stats_manager: Arc::new(crate::stats::StatsManager::default()), + stats_manager: Arc::new(crate::stats::StatsManager::new(Arc::clone(&config))), server_manager: Arc::new(crate::server::manager::ServerManager::new(&servers)), backup_manager: Arc::new(crate::server::backup::manager::BackupManager::default()), inotify_manager: Arc::new(crate::server::filesystem::inotify::InotifyManager::new()), diff --git a/application/src/server/configuration/mod.rs b/application/src/server/configuration/mod.rs index 41735976..7b76eb0e 100644 --- a/application/src/server/configuration/mod.rs +++ b/application/src/server/configuration/mod.rs @@ -746,7 +746,7 @@ impl ServerConfiguration { ), }; - if self.build.oom_disabled && crate::server::executor::docker::cgroup::is_unified() { + if self.build.oom_disabled && crate::server::executor::cgroup::is_unified() { tracing::warn!( server = %self.uuid, "oom_disabled is set, but the container engine discards it on cgroup v2 hosts" diff --git a/application/src/server/executor/docker/cgroup.rs b/application/src/server/executor/cgroup.rs similarity index 100% rename from application/src/server/executor/docker/cgroup.rs rename to application/src/server/executor/cgroup.rs diff --git a/application/src/server/executor/docker/mod.rs b/application/src/server/executor/docker/mod.rs index 142c6560..05fb0433 100644 --- a/application/src/server/executor/docker/mod.rs +++ b/application/src/server/executor/docker/mod.rs @@ -17,7 +17,7 @@ use std::{ }; use tokio::io::{AsyncWriteExt, ReadBuf}; -pub mod cgroup; +use super::cgroup; pub mod host_mounts; #[cfg(unix)] pub mod lxcfs; @@ -488,6 +488,7 @@ impl DockerServerConfigurationExt for crate::server::configuration::ServerConfig crate::server::firewall::FirewallServerSpec { server: self.uuid, + target: None, bindings: bindings.into_iter().collect(), container_ports: container_ports.into_iter().collect(), container_ips: Vec::new(), @@ -2918,6 +2919,13 @@ impl super::ServerExecutor for DockerExecutor { Ok(()) } + async fn reconcile_firewall( + &self, + servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + DockerExecutor::reconcile_firewall(self, servers).await + } + async fn setup_server_process( &self, server: &super::super::Server, diff --git a/application/src/server/executor/mod.rs b/application/src/server/executor/mod.rs index 3e52145f..37d14be4 100644 --- a/application/src/server/executor/mod.rs +++ b/application/src/server/executor/mod.rs @@ -1,9 +1,132 @@ +use anyhow::Context; use serde::Serialize; -use std::{collections::HashMap, net::IpAddr, sync::Arc}; +use std::{collections::HashMap, net::IpAddr, path::Path, process::Stdio, sync::Arc}; use utoipa::ToSchema; +pub mod cgroup; pub mod docker; pub mod noop; +pub mod pve_lxc; + +pub struct Runtime { + pub executor: Arc, + /// Transitional escape hatch for Docker-only subsystems such as Tundra. + /// New runtime-neutral code should use `executor` instead. + pub docker: Option>, +} + +pub async fn create_runtime(config: Arc) -> Result { + let backend = resolve_backend(&config).await?; + + match backend { + crate::config::RuntimeBackend::Auto => Err(anyhow::anyhow!( + "automatic runtime selection was not resolved" + )), + crate::config::RuntimeBackend::Docker => { + tracing::info!("connecting to docker"); + let docker = Arc::new(docker_client(&config.load().docker.socket)?); + + let own_container = docker::DockerExecutor::own_container(&docker).await; + let firewall = + crate::server::firewall::create(&config, &docker, own_container.as_ref()).await; + let executor: Arc = Arc::new(docker::DockerExecutor::new( + Arc::clone(&docker), + config, + firewall, + )); + + Ok(Runtime { + executor, + docker: Some(docker), + }) + } + crate::config::RuntimeBackend::PveLxc => { + tracing::info!("initializing Proxmox VE LXC runtime"); + let cli = { + let snapshot = config.load(); + pve_lxc::PveCli::from_config(&snapshot.runtime.pve_lxc) + }; + let firewall = pve_lxc::create_firewall(&config, &cli).await?; + let executor: Arc = + Arc::new(pve_lxc::PveLxcExecutor::new(config, cli, firewall)); + + Ok(Runtime { + executor, + docker: None, + }) + } + } +} + +fn docker_client(socket: &str) -> Result { + if socket.starts_with("http://") || socket.starts_with("tcp://") { + bollard::Docker::connect_with_http(socket, 120, bollard::API_DEFAULT_VERSION) + } else { + bollard::Docker::connect_with_local(socket, 120, bollard::API_DEFAULT_VERSION) + } + .context("failed to connect to docker") +} + +async fn docker_available(socket: &str) -> bool { + match docker_client(socket) { + Ok(docker) => tokio::time::timeout(std::time::Duration::from_secs(3), docker.version()) + .await + .is_ok_and(|result| result.is_ok()), + Err(_) => false, + } +} + +async fn pve_lxc_available(runtime: &crate::config::PveLxcRuntime) -> bool { + if !Path::new(&runtime.pct_path).is_file() || !Path::new(&runtime.pveversion_path).is_file() { + return false; + } + + tokio::time::timeout( + std::time::Duration::from_secs(3), + tokio::process::Command::new(&runtime.pveversion_path) + .arg("--verbose") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .status(), + ) + .await + .ok() + .and_then(Result::ok) + .is_some_and(|status| status.success()) +} + +async fn resolve_backend( + config: &Arc, +) -> Result { + let snapshot = config.load(); + if snapshot.runtime.backend != crate::config::RuntimeBackend::Auto { + return Ok(snapshot.runtime.backend); + } + + let docker_socket = snapshot.docker.socket.clone(); + let pve_lxc = snapshot.runtime.pve_lxc.clone(); + drop(snapshot); + + let selected = if docker_available(&docker_socket).await { + crate::config::RuntimeBackend::Docker + } else if pve_lxc_available(&pve_lxc).await { + crate::config::RuntimeBackend::PveLxc + } else { + anyhow::bail!( + "runtime.backend is auto but neither Docker nor a local Proxmox VE LXC runtime is available" + ); + }; + + let mut persisted: crate::config::InnerConfig = + serde_json::from_value(serde_json::to_value(&**config.load())?)?; + persisted.runtime.backend = selected; + config.replace(persisted)?; + tracing::info!(runtime_backend = ?selected, "automatically selected and persisted runtime backend"); + + Ok(selected) +} type StatusReceiver = tokio::sync::mpsc::Receiver; @@ -46,6 +169,13 @@ pub trait ProcessHandle: Send + Sync { pub trait ServerExecutor: Send + Sync { async fn boot(&self) -> Result<(), anyhow::Error>; + async fn reconcile_firewall( + &self, + _servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + Ok(()) + } + async fn setup_server_process( &self, server: &super::Server, diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs new file mode 100644 index 00000000..8ce922c4 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -0,0 +1,3072 @@ +use anyhow::Context; +use sha2::{Digest, Sha256}; +use std::{ + net::IpAddr, + path::{Component, Path, PathBuf}, + process::Output, + process::Stdio, + time::Duration, +}; +use tokio::io::AsyncWriteExt; + +const TASK_POLL_INTERVAL: Duration = Duration::from_millis(500); +const TASK_TIMEOUT: Duration = Duration::from_secs(15 * 60); +const CONFIG_HELPER_TIMEOUT: Duration = Duration::from_secs(30); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(15 * 60); +const START_SETTLE_ATTEMPTS: usize = 20; +const START_SETTLE_DELAY: Duration = Duration::from_millis(500); +const MANAGED_FILE_TARGETS: [&str; 5] = [ + "/etc/machine-id", + "/etc/hosts", + "/etc/passwd", + "/etc/group", + "/calagopus-entrypoint", +]; + +const APPLY_RUNTIME_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::LXC::Config; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; + +my $overlay = $payload->{environment} // []; +die "environment must be an array\n" if ref($overlay) ne 'ARRAY'; + +my $managed_mounts = $payload->{managed_file_mounts}; +my $managed_targets = $payload->{managed_file_targets}; +my %managed_target; +if (defined($managed_mounts)) { + die "managed_file_mounts must be an array\n" if ref($managed_mounts) ne 'ARRAY'; + die "managed_file_targets must be an array\n" if ref($managed_targets) ne 'ARRAY'; + for my $target (@$managed_targets) { + die "invalid managed file target\n" if ref($target) || !defined($target) || !length($target); + $managed_target{$target} = 1; + } + for my $mount (@$managed_mounts) { + die "invalid managed file mount\n" if ref($mount) ne 'HASH'; + my $target = $mount->{target}; + my $entry = $mount->{entry}; + die "invalid managed file mount target\n" + if ref($target) || !defined($target) || !exists($managed_target{$target}); + die "invalid managed file mount entry\n" + if ref($entry) || !defined($entry) || !length($entry); + } +} + +sub mount_target { + my ($entry) = @_; + return undef if !defined($entry); + my @fields = grep { length($_) } split(/\s+/, $entry); + return scalar(@fields) >= 2 ? $fields[1] : undef; +} + +PVE::LXC::Config->lock_config($vmid, sub { + my $conf = PVE::LXC::Config->load_config($vmid); + my @lxc = @{$conf->{lxc} // []}; + my @environment; + + if (defined($conf->{env}) && length($conf->{env})) { + push @environment, split(/\0/, $conf->{env}, -1); + } + for my $entry (@lxc) { + next if ref($entry) ne 'ARRAY' || @$entry < 2; + push @environment, $entry->[1] + if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; + } + + # PVE exposes the same values through both `env` and low-level + # lxc.environment records. Collapse those views before applying the panel + # overlay so each synchronization is idempotent. + my @normalized; + my %index; + for my $variable (@environment) { + my ($name) = split(/=/, $variable, 2); + next if !defined($name) || !length($name); + if (exists($index{$name})) { + $normalized[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@normalized); + push @normalized, $variable; + } + } + @environment = @normalized; + for my $variable (@$overlay) { + my ($name) = split(/=/, $variable, 2); + die "invalid environment entry\n" if !defined($name) || !length($name); + if (exists($index{$name})) { + $environment[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@environment); + push @environment, $variable; + } + } + + @lxc = grep { + my $keep = ref($_) ne 'ARRAY' + || @$_ < 2 + || ($_->[0] ne 'lxc.environment' + && $_->[0] ne 'lxc.environment.runtime' + && $_->[0] ne 'lxc.signal.halt' + && $_->[0] ne 'lxc.init.cmd' + && $_->[0] ne 'lxc.console.logfile'); + if ($keep && defined($managed_mounts) && ref($_) eq 'ARRAY' && @$_ >= 2 + && $_->[0] eq 'lxc.mount.entry') { + my $target = mount_target($_->[1]); + $keep = 0 if defined($target) && exists($managed_target{$target}); + } + $keep; + } @lxc; + # PVE 9.2 serializes its `env` property as lxc.environment.runtime, but + # LXC 7 applies that subkey only to hooks. Mirror the merged environment + # through lxc.environment so PID 1 receives both OCI and panel variables. + $conf->{env} = join("\0", @environment); + push @lxc, map { ['lxc.environment', $_] } @environment; + if (defined($payload->{halt_signal}) && length($payload->{halt_signal})) { + push @lxc, ['lxc.signal.halt', $payload->{halt_signal}]; + } + if (defined($payload->{init_command}) && length($payload->{init_command})) { + push @lxc, ['lxc.init.cmd', $payload->{init_command}]; + } + if (defined($payload->{console_logfile}) && length($payload->{console_logfile})) { + push @lxc, ['lxc.console.logfile', $payload->{console_logfile}]; + } + if (defined($managed_mounts)) { + push @lxc, map { ['lxc.mount.entry', $_->{entry}] } @$managed_mounts; + } + + $conf->{lxc} = \@lxc; + PVE::LXC::Config->write_config($vmid, $conf); +}); +"#; + +const APPLY_FIREWALL_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::Firewall; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; +my $prefix = $payload->{managed_prefix} // ''; +die "invalid managed prefix\n" if ref($prefix) || !length($prefix); +my $rules = $payload->{rules} // []; +my $ipsets = $payload->{ipsets} // {}; +die "rules must be an array\n" if ref($rules) ne 'ARRAY'; +die "ipsets must be an object\n" if ref($ipsets) ne 'HASH'; + +for my $rule (@$rules) { + die "invalid firewall rule\n" if ref($rule) ne 'HASH'; + die "invalid firewall rule type\n" + if ref($rule->{type}) || ($rule->{type} // '') ne 'in'; + die "invalid firewall rule action\n" + if ref($rule->{action}) || ($rule->{action} // '') !~ /\A(?:ACCEPT|DROP)\z/; + die "invalid firewall rule interface\n" + if ref($rule->{iface}) || ($rule->{iface} // '') !~ /\A[A-Za-z0-9_.-]+\z/; + if (defined($rule->{proto})) { + die "invalid firewall rule protocol\n" + if ref($rule->{proto}) || $rule->{proto} !~ /\A(?:tcp|udp)\z/; + } + if (defined($rule->{dport})) { + die "invalid firewall rule destination port\n" + if ref($rule->{dport}) || $rule->{dport} !~ /\A[0-9]+(?:,[0-9]+)*\z/; + for my $port (split(/,/, $rule->{dport})) { + die "invalid firewall rule destination port\n" if $port < 1 || $port > 65535; + } + } + if (defined($rule->{source})) { + die "invalid firewall rule source\n" + if ref($rule->{source}) || $rule->{source} =~ /[\x00\r\n]/; + } +} + +# Match the pve-firewall daemon's initialization before loading and rewriting +# existing cluster or guest rules. +PVE::Firewall::init(); + +PVE::Firewall::lock_vmfw_conf($vmid, 10, sub { + my $cluster_conf = PVE::Firewall::load_clusterfw_conf(); + my $conf = PVE::Firewall::load_vmfw_conf($cluster_conf, 'ct', $vmid); + + $conf->{rules} = [grep { + my $comment = $_->{comment} // ''; + index($comment, $prefix) != 0; + } @{$conf->{rules} // []}]; + + for my $name (keys %{$conf->{ipset} // {}}) { + my $comment = $conf->{ipset_comments}->{$name} // ''; + next if index($comment, $prefix) != 0; + delete $conf->{ipset}->{$name}; + delete $conf->{ipset_comments}->{$name}; + } + + for my $name (keys %$ipsets) { + die "invalid ipset name\n" if $name !~ /^[A-Za-z][A-Za-z0-9_-]{0,30}$/; + my $set = $ipsets->{$name}; + die "ipset entries must be an array\n" if ref($set) ne 'ARRAY'; + $conf->{ipset}->{$name} = [map { { cidr => $_ } } @$set]; + $conf->{ipset_comments}->{$name} = "$prefix source file"; + } + + unshift @{$conf->{rules}}, @$rules; + if (@$rules) { + # Wings owns firewall mode for its tagged containers so the guest + # behavior matches the panel's default-allow rule model. + $conf->{options}->{enable} = 1; + $conf->{options}->{policy_in} = 'ACCEPT'; + } elsif (!@{$conf->{rules}}) { + $conf->{options}->{enable} = 0; + } + + PVE::Firewall::save_vmfw_conf($vmid, $conf); +}); +"#; + +#[derive(Debug, Clone)] +pub struct PveCli { + pct_path: PathBuf, + pvesh_path: PathBuf, + pveversion_path: PathBuf, + pvesm_path: PathBuf, + #[allow(dead_code)] + lxc_attach_path: PathBuf, + lxc_stop_path: PathBuf, + perl_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateContainerSpec { + pub vmid: u32, + pub template: String, + pub hostname: String, + pub rootfs_storage: String, + pub rootfs_size_gib: u64, + pub memory_mib: u64, + pub swap_mib: u64, + /// Panel CPU limit expressed as a percentage where 100 means one CPU of time. + pub cpu_limit_percent: Option, + pub cores: Option, + /// Complete Proxmox `net0` value, including static allocation details when + /// a server has a primary panel allocation. + pub network: String, + pub tags: Vec, + pub unprivileged: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataMountSpec { + pub vmid: u32, + pub data_path: String, + /// UID used by the game process inside the container. + pub container_uid: u32, + /// GID used by the game process inside the container. + pub container_gid: u32, + /// UID that owns the server data on the Proxmox host. + pub host_data_uid: u32, + /// GID that owns the server data on the Proxmox host. + pub host_data_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostBindMountSpec { + pub slot: u8, + pub source_path: String, + pub target_path: String, + pub read_only: bool, + /// UID that owns the bind source on the Proxmox host. + pub host_uid: u32, + /// GID that owns the bind source on the Proxmox host. + pub host_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BindMountSpec { + pub slot: u8, + pub source_path: String, + pub target_path: String, + pub read_only: bool, + pub container_uid: u32, + pub container_gid: u32, + pub host_uid: u32, + pub host_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostDevicePassthroughSpec { + pub slot: u8, + pub path: String, + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub deny_write: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DevicePassthroughSpec { + pub slot: u8, + pub path: String, + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub deny_write: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerOciUser { + pub uid: u32, + pub gid: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerResources { + pub memory_mib: u64, + pub swap_mib: u64, + pub cpu_limit_percent: Option, + pub cores: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AttachProcessSpec { + pub vmid: u32, + pub uid: u32, + pub gid: u32, + pub environment: Vec, + pub command: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedFileMountSpec { + pub source_path: String, + pub target_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConfigSpec { + pub vmid: u32, + pub environment: Vec, + pub halt_signal: Option, + pub init_command: Option, + pub console_logfile: Option, + /// `None` leaves low-level LXC file mounts untouched. `Some`, including an + /// empty vector, reconciles all Wings-reserved managed-file targets. + pub managed_file_mounts: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct FirewallRuleSpec { + pub r#type: String, + pub action: String, + pub iface: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub proto: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dport: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + pub comment: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct FirewallPolicySpec { + pub vmid: u32, + pub managed_prefix: String, + pub rules: Vec, + pub ipsets: std::collections::BTreeMap>, +} + +pub struct ConsoleSession { + pub child: tokio::process::Child, + pub reader: tokio::fs::File, + pub writer: tokio::fs::File, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClusterContainer { + pub vmid: u32, + pub node: String, + pub status: Option, + pub tags: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerRuntimeStatus { + pub status: ContainerStatus, + pub pid: Option, + pub uptime_seconds: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContainerInterface { + pub name: String, + pub addresses: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageContent { + pub volid: String, + pub content: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LxcStorage { + pub name: String, + pub available_bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OciTemplatePullSpec { + pub node: String, + pub storage: String, + pub reference: String, + pub filename: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OciTemplate { + pub volid: String, + pub revision: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskState { + Running, + Stopped, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TaskStatus { + pub state: TaskState, + pub exit_status: Option, +} + +impl PveCli { + pub fn from_config(config: &crate::config::PveLxcRuntime) -> Self { + Self { + pct_path: config.pct_path.clone().into(), + pvesh_path: config.pvesh_path.clone().into(), + pveversion_path: config.pveversion_path.clone().into(), + pvesm_path: config.pvesm_path.clone().into(), + lxc_attach_path: config.lxc_attach_path.clone().into(), + lxc_stop_path: config.lxc_stop_path.clone().into(), + perl_path: config.perl_path.clone().into(), + } + } + + pub fn panel_resources( + memory_limit: i64, + overhead_memory: i64, + swap: i64, + cpu_limit: i64, + threads: Option<&str>, + ) -> Result { + if memory_limit <= 0 { + return Err(anyhow::anyhow!( + "Proxmox LXC currently requires a positive server memory limit; unlimited memory is not mapped yet" + )); + } + if overhead_memory < 0 { + return Err(anyhow::anyhow!( + "server overhead memory cannot be negative for the Proxmox LXC runtime" + )); + } + let memory_mib = memory_limit + .checked_add(overhead_memory) + .context("server memory plus overhead overflowed")?; + if memory_mib < 16 { + return Err(anyhow::anyhow!( + "Proxmox LXC requires at least 16 MiB of memory, resolved server limit was {memory_mib} MiB" + )); + } + let memory_mib = u64::try_from(memory_mib).context("server memory limit was negative")?; + + let swap_mib = match swap { + -1 => { + return Err(anyhow::anyhow!( + "unlimited swap (-1) has no direct Proxmox LXC equivalent" + )); + } + value if value < 0 => { + return Err(anyhow::anyhow!( + "invalid negative swap limit for Proxmox LXC: {value}" + )); + } + value => u64::try_from(value).context("server swap limit exceeded u64")?, + }; + + let cpu_limit_percent = match cpu_limit { + value if value < 0 => { + return Err(anyhow::anyhow!( + "invalid negative CPU limit for Proxmox LXC: {value}" + )); + } + 0 => None, + value => Some(u64::try_from(value).context("server CPU limit exceeded u64")?), + }; + + if threads.is_some_and(|threads| !threads.trim().is_empty()) { + return Err(anyhow::anyhow!( + "Proxmox LXC CPU pinning for server build.threads is not implemented yet" + )); + } + + Ok(ContainerResources { + memory_mib, + swap_mib, + cpu_limit_percent, + cores: None, + }) + } + + async fn run(program: &std::path::Path, args: &[String]) -> Result { + let mut command = tokio::process::Command::new(program); + command.args(args).kill_on_drop(true); + let output = tokio::time::timeout(COMMAND_TIMEOUT, command.output()) + .await + .with_context(|| format!("timed out executing {}", program.display()))? + .with_context(|| format!("failed to execute {}", program.display()))?; + + if output.status.success() { + return Ok(output); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "{} exited with {}: {}", + program.display(), + output.status, + stderr.trim() + )) + } + + async fn run_with_stdin( + program: &std::path::Path, + args: &[String], + input: &[u8], + ) -> Result { + let mut child = tokio::process::Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to execute {}", program.display()))?; + + let mut stdin = child + .stdin + .take() + .context("spawned Proxmox helper did not expose stdin")?; + let operation = async move { + stdin.write_all(input).await?; + drop(stdin); + child.wait_with_output().await + }; + let output = tokio::time::timeout(CONFIG_HELPER_TIMEOUT, operation) + .await + .with_context(|| format!("timed out executing {}", program.display()))??; + if output.status.success() { + return Ok(output); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "{} exited with {}: {}", + program.display(), + output.status, + stderr.trim() + )) + } + + fn runtime_config_payload(spec: &RuntimeConfigSpec) -> Result, anyhow::Error> { + for variable in &spec.environment { + let Some((name, _)) = variable.split_once('=') else { + return Err(anyhow::anyhow!( + "runtime environment entry must be KEY=VALUE: {variable}" + )); + }; + if name.is_empty() + || variable.contains('\0') + || variable.contains('\n') + || variable.contains('\r') + { + return Err(anyhow::anyhow!( + "runtime environment entry cannot be represented safely in an LXC config: {variable}" + )); + } + } + if spec + .halt_signal + .as_deref() + .is_some_and(|signal| signal.is_empty() || signal.chars().any(char::is_control)) + { + return Err(anyhow::anyhow!("invalid LXC halt signal")); + } + if spec.init_command.as_deref().is_some_and(|command| { + command.is_empty() + || command.contains('\0') + || command.contains('\n') + || command.contains('\r') + }) { + return Err(anyhow::anyhow!("invalid LXC init command")); + } + if spec.console_logfile.as_deref().is_some_and(|path| { + !path.starts_with('/') + || path.contains('\0') + || path.contains('\n') + || path.contains('\r') + }) { + return Err(anyhow::anyhow!("invalid LXC console logfile path")); + } + + let (managed_file_mounts, managed_file_targets) = + if let Some(mounts) = &spec.managed_file_mounts { + let mut seen_targets = std::collections::HashSet::with_capacity(mounts.len()); + let mut encoded = Vec::with_capacity(mounts.len()); + + for mount in mounts { + if !MANAGED_FILE_TARGETS.contains(&mount.target_path.as_str()) { + return Err(anyhow::anyhow!( + "unsupported managed Proxmox file mount target: {}", + mount.target_path + )); + } + if !seen_targets.insert(mount.target_path.as_str()) { + return Err(anyhow::anyhow!( + "duplicate managed Proxmox file mount target: {}", + mount.target_path + )); + } + + let source = Self::encode_lxc_mount_field(&mount.source_path, false)?; + let target = Self::encode_lxc_mount_field(&mount.target_path, true)?; + encoded.push(serde_json::json!({ + "target": target, + "entry": format!("{source} {target} none bind,ro,create=file 0 0"), + })); + } + + let targets = MANAGED_FILE_TARGETS + .iter() + .map(|target| Self::encode_lxc_mount_field(target, true)) + .collect::, _>>()?; + (Some(encoded), Some(targets)) + } else { + (None, None) + }; + + serde_json::to_vec(&serde_json::json!({ + "vmid": spec.vmid, + "environment": spec.environment, + "halt_signal": spec.halt_signal, + "init_command": spec.init_command, + "console_logfile": spec.console_logfile, + "managed_file_mounts": managed_file_mounts, + "managed_file_targets": managed_file_targets, + })) + .context("failed to serialize Proxmox runtime configuration") + } + + fn encode_lxc_mount_field(path: &str, relative_target: bool) -> Result { + if !path.starts_with('/') + || path.contains('\0') + || path.contains('\n') + || path.contains('\r') + { + return Err(anyhow::anyhow!( + "invalid LXC managed file mount path: {path}" + )); + } + + let value = if relative_target { + path.strip_prefix('/').unwrap_or(path) + } else { + path + }; + if value.is_empty() { + return Err(anyhow::anyhow!( + "invalid empty LXC managed file mount target" + )); + } + + let mut encoded = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\\' => encoded.push_str("\\134"), + ' ' => encoded.push_str("\\040"), + '\t' => encoded.push_str("\\011"), + _ => encoded.push(character), + } + } + Ok(encoded) + } + + /// LXC's `lxc.init.cmd` parser supports whole-word single or double quotes, + /// but does not support escape sequences inside a quoted word. + pub fn encode_init_command(arguments: &[String]) -> Result { + if arguments.is_empty() { + return Err(anyhow::anyhow!("LXC init command cannot be empty")); + } + + let mut encoded = Vec::with_capacity(arguments.len()); + for argument in arguments { + if argument.contains('\0') || argument.contains('\n') || argument.contains('\r') { + return Err(anyhow::anyhow!( + "LXC init command argument contains an unsupported control character" + )); + } + + if !argument.is_empty() + && !argument.chars().any(char::is_whitespace) + && !argument.starts_with('\'') + && !argument.starts_with('"') + { + encoded.push(argument.clone()); + } else if !argument.contains('"') { + encoded.push(format!("\"{argument}\"")); + } else if !argument.contains('\'') { + encoded.push(format!("'{argument}'")); + } else { + return Err(anyhow::anyhow!( + "LXC init command argument cannot be represented safely: {argument}" + )); + } + } + + Ok(encoded.join(" ")) + } + + /// Applies dynamic Wings runtime settings without putting the NUL-separated + /// PVE `env` value on a Unix argv. The helper edits the regular PVE LXC + /// config under Proxmox's own config lock, preserving all unrelated options. + pub async fn apply_runtime_config( + &self, + spec: &RuntimeConfigSpec, + ) -> Result<(), anyhow::Error> { + let payload = Self::runtime_config_payload(spec)?; + Self::run_with_stdin( + &self.perl_path, + &["-e".to_string(), APPLY_RUNTIME_CONFIG_PERL.to_string()], + &payload, + ) + .await?; + Ok(()) + } + + pub async fn apply_firewall_config( + &self, + spec: &FirewallPolicySpec, + ) -> Result<(), anyhow::Error> { + let payload = serde_json::to_vec(spec).context("failed to serialize Proxmox firewall")?; + Self::run_with_stdin( + &self.perl_path, + &["-e".to_string(), APPLY_FIREWALL_CONFIG_PERL.to_string()], + &payload, + ) + .await?; + Ok(()) + } + + pub fn request_shutdown_args(vmid: u32) -> Vec { + vec![ + "-n".to_string(), + vmid.to_string(), + "--nowait".to_string(), + "--nokill".to_string(), + ] + } + + /// Requests the container init process to stop and returns immediately. + /// `lxc.signal.halt` determines which signal init receives. + pub async fn request_shutdown(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run(&self.lxc_stop_path, &Self::request_shutdown_args(vmid)).await?; + Ok(()) + } + + pub fn console_args(vmid: u32) -> Vec { + vec![ + "console".to_string(), + vmid.to_string(), + "--escape".to_string(), + "^z".to_string(), + ] + } + + /// Starts `pct console` on a real pseudo-terminal. `pct console` is an + /// interactive terminal client and does not behave correctly when wired to + /// ordinary pipes. + #[cfg(target_os = "linux")] + pub fn spawn_console(&self, vmid: u32) -> Result { + use rustix::pty::{OpenptFlags, grantpt, ioctl_tiocgptpeer, openpt, unlockpt}; + use std::os::unix::process::CommandExt; + + let flags = OpenptFlags::RDWR | OpenptFlags::NOCTTY | OpenptFlags::CLOEXEC; + let controller = openpt(flags).context("failed to allocate Proxmox console PTY")?; + grantpt(&controller).context("failed to grant Proxmox console PTY")?; + unlockpt(&controller).context("failed to unlock Proxmox console PTY")?; + let user = ioctl_tiocgptpeer(&controller, flags) + .context("failed to open Proxmox console PTY peer")?; + + let stdin = std::fs::File::from(user); + let stdout = stdin.try_clone()?; + let stderr = stdin.try_clone()?; + let controller = std::fs::File::from(controller); + let controller_writer = controller + .try_clone() + .context("failed to clone Proxmox console PTY controller")?; + + let mut command = tokio::process::Command::new(&self.pct_path); + command + .args(Self::console_args(vmid)) + .stdin(Stdio::from(stdin)) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .kill_on_drop(true); + + // `pct console` is an interactive terminal client. Merely connecting + // its stdio to a PTY is enough to receive output, but keyboard input is + // ignored unless the slave is also the child's controlling terminal. + unsafe { + command.as_std_mut().pre_exec(|| { + rustix::process::setsid()?; + rustix::process::ioctl_tiocsctty(std::io::stdin())?; + Ok(()) + }); + } + + let child = command + .spawn() + .with_context(|| format!("failed to execute {} console", self.pct_path.display()))?; + + Ok(ConsoleSession { + child, + reader: tokio::fs::File::from_std(controller), + writer: tokio::fs::File::from_std(controller_writer), + }) + } + + #[cfg(not(target_os = "linux"))] + pub fn spawn_console(&self, _vmid: u32) -> Result { + Err(anyhow::anyhow!( + "Proxmox LXC console attachment is only supported on Linux" + )) + } + + pub async fn version(&self) -> Result { + let output = Self::run(&self.pveversion_path, &[]).await?; + Ok(String::from_utf8(output.stdout) + .context("pveversion returned non-UTF8 output")? + .trim() + .to_string()) + } + + pub async fn next_vmid(&self) -> Result { + let args = vec![ + "get".to_string(), + "/cluster/nextid".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + let value: serde_json::Value = serde_json::from_slice(&output.stdout) + .context("failed to parse Proxmox nextid response")?; + let raw = value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + .context("Proxmox nextid response was not a VMID")?; + u32::try_from(raw).context("Proxmox nextid response exceeded the VMID range") + } + + pub async fn local_node(&self) -> Result { + let args = vec![ + "get".to_string(), + "/cluster/status".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_local_node(&output.stdout) + } + + pub async fn cluster_firewall_enabled(&self) -> Result { + let args = vec![ + "get".to_string(), + "/cluster/firewall/options".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + let value: serde_json::Value = serde_json::from_slice(&output.stdout) + .context("failed to parse Proxmox cluster firewall options")?; + Ok(value + .get("enable") + .and_then(|value| value.as_u64()) + .is_some_and(|value| value != 0) + || value + .get("enable") + .and_then(|value| value.as_bool()) + .unwrap_or(false)) + } + + /// Lists active LXC rootfs storages ordered by free capacity, so callers can + /// place a container on the largest pool that can satisfy its requested size. + pub async fn available_lxc_storages( + &self, + node: &str, + ) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/storage"), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_available_lxc_storages(&output.stdout) + } + + fn parse_local_node(output: &[u8]) -> Result { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster status response")?; + + rows.iter() + .find(|row| { + row.get("type").and_then(serde_json::Value::as_str) == Some("node") + && row.get("local").is_some_and(|local| { + local.as_bool() == Some(true) || local.as_u64() == Some(1) + }) + }) + .and_then(|row| row.get("name")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .context("Proxmox cluster status did not identify the local node") + } + + fn parse_available_lxc_storages(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox node storage response")?; + + let mut storages = rows + .iter() + .filter(|row| { + row.get("active").and_then(serde_json::Value::as_u64) == Some(1) + && row.get("enabled").and_then(serde_json::Value::as_u64) == Some(1) + && row + .get("content") + .and_then(serde_json::Value::as_str) + .is_some_and(|content| content.split(',').any(|kind| kind == "rootdir")) + }) + .map(|row| { + Ok(LxcStorage { + name: row + .get("storage") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC storage response did not include a name")? + .to_string(), + available_bytes: row + .get("avail") + .and_then(serde_json::Value::as_u64) + .context( + "Proxmox LXC storage response did not include available capacity", + )?, + }) + }) + .collect::, anyhow::Error>>()?; + storages.sort_unstable_by_key(|storage| std::cmp::Reverse(storage.available_bytes)); + Ok(storages) + } + + pub async fn list_containers(&self) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + "/cluster/resources".to_string(), + "--type".to_string(), + "vm".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_containers(&output.stdout) + } + + pub fn interfaces_args(node: &str, vmid: u32) -> Vec { + vec![ + "get".to_string(), + format!("/nodes/{node}/lxc/{vmid}/interfaces"), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn interfaces( + &self, + node: &str, + vmid: u32, + ) -> Result, anyhow::Error> { + let output = Self::run(&self.pvesh_path, &Self::interfaces_args(node, vmid)).await?; + Self::parse_interfaces(&output.stdout) + } + + pub fn runtime_status_args(node: &str, vmid: u32) -> Vec { + vec![ + "get".to_string(), + format!("/nodes/{node}/lxc/{vmid}/status/current"), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn runtime_status( + &self, + node: &str, + vmid: u32, + ) -> Result { + let output = Self::run(&self.pvesh_path, &Self::runtime_status_args(node, vmid)).await?; + Self::parse_runtime_status(&output.stdout) + } + + fn parse_runtime_status(output: &[u8]) -> Result { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox LXC runtime status response")?; + let status = value + .get("status") + .and_then(serde_json::Value::as_str) + .and_then(ContainerStatus::from_word) + .context("Proxmox LXC runtime status did not include a known status")?; + let pid = value.get("pid").and_then(|pid| { + pid.as_i64() + .or_else(|| pid.as_str().and_then(|pid| pid.parse().ok())) + }); + let uptime_seconds = value + .get("uptime") + .and_then(|uptime| { + uptime + .as_u64() + .or_else(|| uptime.as_str().and_then(|uptime| uptime.parse().ok())) + }) + .unwrap_or(0); + + Ok(ContainerRuntimeStatus { + status, + pid, + uptime_seconds, + }) + } + + fn parse_interfaces(output: &[u8]) -> Result, anyhow::Error> { + let rows: Option> = serde_json::from_slice(output) + .context("failed to parse Proxmox LXC interface response")?; + + rows.unwrap_or_default() + .into_iter() + .map(|row| { + let name = row + .get("name") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC interface row did not include a name")? + .to_string(); + let addresses = row + .get("ip-addresses") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|address| address.get("ip-address")) + .filter_map(serde_json::Value::as_str) + .filter_map(|address| address.parse::().ok()) + .collect(); + + Ok(ContainerInterface { name, addresses }) + }) + .collect() + } + + pub fn primary_container_address(interfaces: &[ContainerInterface]) -> Option { + let mut addresses = Self::container_addresses(interfaces).into_iter(); + + addresses + .clone() + .find(IpAddr::is_ipv4) + .or_else(|| addresses.find(IpAddr::is_ipv6)) + } + + pub fn container_addresses(interfaces: &[ContainerInterface]) -> Vec { + fn usable(address: IpAddr) -> bool { + match address { + IpAddr::V4(address) => { + !address.is_loopback() + && !address.is_unspecified() + && !address.is_multicast() + && !address.is_link_local() + } + IpAddr::V6(address) => { + !address.is_loopback() + && !address.is_unspecified() + && !address.is_multicast() + && !address.is_unicast_link_local() + } + } + } + + let mut addresses = interfaces + .iter() + .filter(|interface| interface.name != "lo") + .flat_map(|interface| interface.addresses.iter().copied()) + .filter(|address| usable(*address)) + .collect::>(); + addresses.sort_unstable(); + addresses.dedup(); + addresses + } + + pub async fn list_storage_content( + &self, + node: &str, + storage: &str, + ) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/storage/{storage}/content"), + "--content".to_string(), + "vztmpl".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_storage_content(&output.stdout) + } + + fn parse_storage_content(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox storage content response")?; + + rows.into_iter() + .map(|row| { + let volid = row + .get("volid") + .and_then(serde_json::Value::as_str) + .context("Proxmox storage content row did not include a volid")? + .to_string(); + let content = row + .get("content") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + + Ok(StorageContent { volid, content }) + }) + .collect() + } + + pub fn normalize_oci_reference(reference: &str) -> Result { + let reference = reference.trim(); + if reference.is_empty() { + return Err(anyhow::anyhow!("OCI image reference cannot be empty")); + } + if reference.contains('@') { + return Err(anyhow::anyhow!( + "Proxmox OCI registry pull currently requires a tagged image reference, not a digest reference: {reference}" + )); + } + + let last_slash = reference.rfind('/'); + let last_colon = reference.rfind(':'); + let has_tag = match (last_slash, last_colon) { + (_, None) => false, + (None, Some(_)) => true, + (Some(slash), Some(colon)) => colon > slash, + }; + + if has_tag { + Ok(reference.to_string()) + } else { + Ok(format!("{reference}:latest")) + } + } + + pub fn oci_cache_filename(reference: &str) -> String { + let digest = Sha256::digest(reference.as_bytes()); + format!("calagopus-oci-{}", hex::encode(digest)) + } + + pub fn oci_template_volid(storage: &str, reference: &str) -> String { + format!( + "{storage}:vztmpl/{}.tar", + Self::oci_cache_filename(reference) + ) + } + + pub fn oci_revision_tag(revision: &str) -> Result { + let revision = revision.strip_prefix("sha256:").unwrap_or(revision); + let short = revision + .get(..32) + .context("OCI image revision digest was unexpectedly short")?; + if !short.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow::anyhow!( + "OCI image revision was not a SHA-256 digest" + )); + } + Ok(format!("calagopus-image-{short}")) + } + + async fn template_path(&self, volid: &str) -> Result { + let output = Self::run(&self.pvesm_path, &["path".to_string(), volid.to_string()]).await?; + let path = + String::from_utf8(output.stdout).context("pvesm path returned non-UTF8 output")?; + let path = path.trim(); + if path.is_empty() { + return Err(anyhow::anyhow!( + "pvesm path returned an empty path for {volid}" + )); + } + Ok(path.into()) + } + + async fn template_revision(&self, volid: &str) -> Result { + let path = self.template_path(volid).await?; + tokio::task::spawn_blocking(move || { + use std::io::Read; + + let file = std::fs::File::open(&path) + .with_context(|| format!("failed to open OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(file); + for entry in archive + .entries() + .context("failed to read OCI template archive")? + { + let mut entry = entry.context("failed to read OCI template entry")?; + if entry.path()?.as_ref() != std::path::Path::new("index.json") { + continue; + } + let mut index = String::new(); + entry.read_to_string(&mut index)?; + let value: serde_json::Value = serde_json::from_str(&index) + .context("failed to parse OCI template index.json")?; + return value + .get("manifests") + .and_then(serde_json::Value::as_array) + .and_then(|manifests| manifests.first()) + .and_then(|manifest| manifest.get("digest")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .context("OCI template index did not include a manifest digest"); + } + Err(anyhow::anyhow!("OCI template did not contain index.json")) + }) + .await + .context("OCI template inspection task failed")? + } + + async fn template_is_fresh( + &self, + volid: &str, + max_age: Duration, + ) -> Result { + if max_age.is_zero() { + return Ok(false); + } + let path = self.template_path(volid).await?; + let modified = tokio::fs::metadata(&path).await?.modified()?; + Ok(modified.elapsed().is_ok_and(|age| age < max_age)) + } + + async fn remove_template(&self, volid: &str) -> Result<(), anyhow::Error> { + Self::run(&self.pvesm_path, &["free".to_string(), volid.to_string()]).await?; + Ok(()) + } + + pub fn pull_oci_args(spec: &OciTemplatePullSpec) -> Vec { + vec![ + "create".to_string(), + format!( + "/nodes/{}/storage/{}/oci-registry-pull", + spec.node, spec.storage + ), + "--reference".to_string(), + spec.reference.clone(), + "--filename".to_string(), + spec.filename.clone(), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn pull_oci_template( + &self, + spec: &OciTemplatePullSpec, + ) -> Result { + let output = Self::run(&self.pvesh_path, &Self::pull_oci_args(spec)).await?; + Self::parse_oci_pull_task(&output.stdout) + } + + fn parse_oci_pull_task(output: &[u8]) -> Result { + if let Ok(value) = serde_json::from_slice::(output) + && let Some(upid) = value.as_str().filter(|upid| upid.starts_with("UPID:")) + { + return Ok(upid.to_string()); + } + + let output = std::str::from_utf8(output) + .context("Proxmox OCI pull response contained non-UTF8 output")?; + for line in output + .lines() + .rev() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if let Ok(value) = serde_json::from_str::(line) + && let Some(upid) = value.as_str().filter(|upid| upid.starts_with("UPID:")) + { + return Ok(upid.to_string()); + } + if line.starts_with("UPID:") { + return Ok(line.to_string()); + } + } + + Err(anyhow::anyhow!( + "Proxmox OCI pull response did not contain a task UPID" + )) + } + + pub async fn task_status(&self, node: &str, upid: &str) -> Result { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/tasks/{upid}/status"), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_task_status(&output.stdout) + } + + fn parse_task_status(output: &[u8]) -> Result { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox task status response")?; + let state = match value.get("status").and_then(serde_json::Value::as_str) { + Some("running") => TaskState::Running, + Some("stopped") => TaskState::Stopped, + Some(status) => { + return Err(anyhow::anyhow!( + "unknown Proxmox task status response: {status}" + )); + } + None => { + return Err(anyhow::anyhow!( + "Proxmox task status response did not include status" + )); + } + }; + let exit_status = value + .get("exitstatus") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + + Ok(TaskStatus { state, exit_status }) + } + + pub async fn wait_for_task(&self, node: &str, upid: &str) -> Result<(), anyhow::Error> { + tokio::time::timeout(TASK_TIMEOUT, async { + loop { + let status = self.task_status(node, upid).await?; + match status.state { + TaskState::Running => tokio::time::sleep(TASK_POLL_INTERVAL).await, + TaskState::Stopped => { + return match status.exit_status.as_deref() { + Some("OK") => Ok(()), + Some(exit_status) => Err(anyhow::anyhow!( + "Proxmox task {upid} failed with exit status {exit_status}" + )), + None => Err(anyhow::anyhow!( + "Proxmox task {upid} stopped without an exit status" + )), + }; + } + } + } + }) + .await + .with_context(|| format!("timed out waiting for Proxmox task {upid}"))? + } + + fn validate_absolute_path(path: &str, label: &str) -> Result<(), anyhow::Error> { + let path = Path::new(path); + if !path.is_absolute() + || path + .components() + .any(|component| matches!(component, Component::ParentDir | Component::CurDir)) + || path + .as_os_str() + .as_encoded_bytes() + .iter() + .any(|byte| matches!(byte, 0 | b',' | b'\n' | b'\r')) + { + return Err(anyhow::anyhow!( + "invalid Proxmox {label} path: {}", + path.display() + )); + } + Ok(()) + } + + pub async fn ensure_oci_template( + &self, + node: &str, + storage: &str, + reference: &str, + max_age: Duration, + ) -> Result { + let reference = Self::normalize_oci_reference(reference)?; + let filename = Self::oci_cache_filename(&reference); + let volid = Self::oci_template_volid(storage, &reference); + let exists = self + .list_storage_content(node, storage) + .await? + .into_iter() + .any(|entry| entry.volid == volid); + if exists && self.template_is_fresh(&volid, max_age).await? { + return Ok(OciTemplate { + revision: self.template_revision(&volid).await?, + volid, + }); + } + if exists { + self.remove_template(&volid) + .await + .with_context(|| format!("failed to remove stale Proxmox OCI template {volid}"))?; + } + + let upid = self + .pull_oci_template(&OciTemplatePullSpec { + node: node.to_string(), + storage: storage.to_string(), + reference, + filename, + }) + .await?; + self.wait_for_task(node, &upid).await?; + + let exists = self + .list_storage_content(node, storage) + .await? + .into_iter() + .any(|entry| entry.volid == volid); + if exists { + Ok(OciTemplate { + revision: self.template_revision(&volid).await?, + volid, + }) + } else { + Err(anyhow::anyhow!( + "Proxmox OCI pull task completed but template {volid} was not present in storage" + )) + } + } + + fn parse_containers(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster resources response")?; + let mut containers = Vec::new(); + + for row in rows { + if row.get("type").and_then(serde_json::Value::as_str) != Some("lxc") { + continue; + } + + let raw_vmid = row + .get("vmid") + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + }) + .context("Proxmox LXC resource did not include a VMID")?; + let vmid = u32::try_from(raw_vmid).context("Proxmox LXC VMID exceeded u32")?; + let node = row + .get("node") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC resource did not include a node")? + .to_string(); + let status = row + .get("status") + .and_then(serde_json::Value::as_str) + .and_then(ContainerStatus::from_word); + let tags = row + .get("tags") + .and_then(serde_json::Value::as_str) + .map(|tags| { + tags.split(';') + .map(str::trim) + .filter(|tag| !tag.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + + containers.push(ClusterContainer { + vmid, + node, + status, + tags, + }); + } + + Ok(containers) + } + + pub async fn status(&self, vmid: u32) -> Result { + let args = vec!["status".to_string(), vmid.to_string()]; + let output = Self::run(&self.pct_path, &args).await?; + ContainerStatus::parse(&String::from_utf8_lossy(&output.stdout)) + } + + pub async fn start(&self, vmid: u32) -> Result<(), anyhow::Error> { + let args = ["start".to_string(), vmid.to_string()]; + let mut last_error = None; + + for attempt in 0..START_SETTLE_ATTEMPTS { + match Self::run(&self.pct_path, &args).await { + Ok(_) => return Ok(()), + Err(error) => { + let message = error.to_string(); + if !message.contains("monitor socket") + && !message.contains("got timeout") + && !message.contains("unable to get PID for CT") + { + return Err(error); + } + tokio::time::sleep(START_SETTLE_DELAY).await; + // A failed pct client can still have started the container. + // Reissuing pct start in that case would produce "already running". + if matches!(self.status(vmid).await, Ok(ContainerStatus::Running)) { + return Ok(()); + } + last_error = Some(error); + if attempt + 1 < START_SETTLE_ATTEMPTS { + tracing::warn!( + vmid, + attempt = attempt + 1, + "retrying Proxmox LXC start after monitor teardown" + ); + } + } + } + } + + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("failed to start LXC container"))) + } + + #[allow(dead_code)] + pub async fn shutdown(&self, vmid: u32, timeout_seconds: u64) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "shutdown".to_string(), + vmid.to_string(), + "--timeout".to_string(), + timeout_seconds.to_string(), + ], + ) + .await?; + Ok(()) + } + + pub async fn stop(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &["stop".to_string(), vmid.to_string()]).await?; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if self.status(vmid).await? == ContainerStatus::Stopped { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .with_context(|| format!("timed out waiting for Proxmox LXC {vmid} to stop"))? + } + + pub async fn destroy(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "destroy".to_string(), + vmid.to_string(), + "--purge".to_string(), + "1".to_string(), + ], + ) + .await?; + Ok(()) + } + + pub fn create_args(spec: &CreateContainerSpec) -> Vec { + let mut args = vec![ + "create".to_string(), + spec.vmid.to_string(), + spec.template.clone(), + "--hostname".to_string(), + spec.hostname.clone(), + "--rootfs".to_string(), + format!("{}:{}", spec.rootfs_storage, spec.rootfs_size_gib), + "--memory".to_string(), + spec.memory_mib.to_string(), + "--swap".to_string(), + spec.swap_mib.to_string(), + "--net0".to_string(), + spec.network.clone(), + "--unprivileged".to_string(), + u8::from(spec.unprivileged).to_string(), + "--cmode".to_string(), + "console".to_string(), + "--onboot".to_string(), + "0".to_string(), + ]; + + if let Some(cores) = spec.cores { + args.extend(["--cores".to_string(), cores.to_string()]); + } + if let Some(cpu_limit_percent) = spec.cpu_limit_percent { + args.extend([ + "--cpulimit".to_string(), + Self::format_cpu_limit_percent(cpu_limit_percent), + ]); + } + if !spec.tags.is_empty() { + args.extend(["--tags".to_string(), spec.tags.join(";")]); + } + + args + } + + fn format_cpu_limit_percent(percent: u64) -> String { + let whole = percent / 100; + let fractional = percent % 100; + if fractional == 0 { + return whole.to_string(); + } + if fractional.is_multiple_of(10) { + format!("{whole}.{}", fractional / 10) + } else { + format!("{whole}.{fractional:02}") + } + } + + pub async fn create(&self, spec: &CreateContainerSpec) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::create_args(spec)).await?; + Ok(()) + } + + pub fn set_resources_args(vmid: u32, resources: &ContainerResources) -> Vec { + let mut args = vec![ + "set".to_string(), + vmid.to_string(), + "--memory".to_string(), + resources.memory_mib.to_string(), + "--swap".to_string(), + resources.swap_mib.to_string(), + "--cpulimit".to_string(), + resources + .cpu_limit_percent + .map(Self::format_cpu_limit_percent) + .unwrap_or_else(|| "0".to_string()), + ]; + + if let Some(cores) = resources.cores { + args.extend(["--cores".to_string(), cores.to_string()]); + } + + args + } + + pub async fn set_resources( + &self, + vmid: u32, + resources: &ContainerResources, + ) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_resources_args(vmid, resources)).await?; + Ok(()) + } + + pub async fn set_network(&self, vmid: u32, network: &str) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "set".to_string(), + vmid.to_string(), + "--net0".to_string(), + network.to_string(), + ], + ) + .await?; + Ok(()) + } + + pub fn network_with_firewall(config: &str) -> Result, anyhow::Error> { + let network = config + .lines() + .find_map(|line| line.strip_prefix("net0:")) + .map(str::trim) + .context("Proxmox LXC config has no net0 interface")?; + let mut options: Vec<&str> = network + .split(',') + .filter(|option| !option.starts_with("firewall=")) + .collect(); + options.push("firewall=1"); + let wanted = options.join(","); + + Ok((wanted != network).then_some(wanted)) + } + + pub async fn enable_network_firewall(&self, vmid: u32) -> Result<(), anyhow::Error> { + let config = self.config(vmid).await?; + if let Some(network) = Self::network_with_firewall(&config)? { + self.set_network(vmid, &network).await?; + } + Ok(()) + } + + pub fn set_tags_args(vmid: u32, tags: &[String]) -> Vec { + vec![ + "set".to_string(), + vmid.to_string(), + "--tags".to_string(), + tags.join(";"), + ] + } + + pub async fn set_tags(&self, vmid: u32, tags: &[String]) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_tags_args(vmid, tags)).await?; + Ok(()) + } + + pub async fn config(&self, vmid: u32) -> Result { + let output = Self::run(&self.pct_path, &["config".to_string(), vmid.to_string()]).await?; + Self::decode_config_output(output.stdout) + } + + fn decode_config_output(stdout: Vec) -> Result { + let mut config = + String::from_utf8(stdout).context("pct config returned non-UTF8 output")?; + config.retain(|character| character != '\0'); + Ok(config) + } + + fn parse_oci_user(config: &str) -> Result { + let mut uid = None; + let mut gid = None; + + for line in config.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim(); + match key.trim() { + "lxc.init.uid" => { + uid = Some( + value + .parse::() + .with_context(|| format!("invalid lxc.init.uid value: {value}"))?, + ); + } + "lxc.init.gid" => { + gid = Some( + value + .parse::() + .with_context(|| format!("invalid lxc.init.gid value: {value}"))?, + ); + } + _ => {} + } + } + + Ok(ContainerOciUser { + uid: uid.unwrap_or(0), + gid: gid.unwrap_or(0), + }) + } + + pub async fn oci_user(&self, vmid: u32) -> Result { + let config = self.config(vmid).await?; + Self::parse_oci_user(&config) + } + + pub async fn oci_entrypoint(&self, vmid: u32) -> Result { + let config = self.config(vmid).await?; + let entrypoint = config + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + (key.trim() == "entrypoint").then(|| value.trim().to_string()) + }) + .filter(|value| !value.is_empty()) + .context("OCI-backed Proxmox container does not define an entrypoint")?; + if entrypoint.contains('\0') || entrypoint.contains('\n') || entrypoint.contains('\r') { + return Err(anyhow::anyhow!( + "OCI-backed Proxmox container entrypoint contains an unsupported control character" + )); + } + Ok(entrypoint) + } + + pub fn set_data_mount_args(spec: &DataMountSpec) -> Result, anyhow::Error> { + Self::set_bind_mount_args( + spec.vmid, + &[BindMountSpec { + slot: 0, + source_path: spec.data_path.clone(), + target_path: "/home/container".to_string(), + read_only: false, + container_uid: spec.container_uid, + container_gid: spec.container_gid, + host_uid: spec.host_data_uid, + host_gid: spec.host_data_gid, + }], + ) + } + + #[allow(dead_code)] + pub async fn set_data_mount(&self, spec: &DataMountSpec) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_data_mount_args(spec)?).await?; + Ok(()) + } + + pub fn set_bind_mount_args( + vmid: u32, + mounts: &[BindMountSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + let mut slots = std::collections::HashSet::new(); + + for mount in mounts { + if !slots.insert(mount.slot) { + return Err(anyhow::anyhow!( + "duplicate Proxmox bind mount slot mp{}", + mount.slot + )); + } + for (label, path) in [ + ("source", mount.source_path.as_str()), + ("target", mount.target_path.as_str()), + ] { + Self::validate_absolute_path(path, &format!("bind mount {label}"))?; + } + + let idmap = format!( + "u:{}:{}:1;g:{}:{}:1", + mount.container_uid, mount.host_uid, mount.container_gid, mount.host_gid + ); + let read_only = if mount.read_only { ",ro=1" } else { "" }; + args.extend([ + format!("--mp{}", mount.slot), + format!( + "{},mp={},backup=0,idmap={idmap}{read_only}", + mount.source_path, mount.target_path, + ), + ]); + } + + Ok(args) + } + + #[allow(dead_code)] + pub async fn set_bind_mounts( + &self, + vmid: u32, + mounts: &[BindMountSpec], + ) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_bind_mount_args(vmid, mounts)?).await?; + Ok(()) + } + + pub fn set_devices_args( + vmid: u32, + devices: &[DevicePassthroughSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + let mut slots = std::collections::HashSet::new(); + + for device in devices { + if !slots.insert(device.slot) { + return Err(anyhow::anyhow!( + "duplicate Proxmox device passthrough slot dev{}", + device.slot + )); + } + Self::validate_absolute_path(&device.path, "device passthrough")?; + if device.mode > 0o777 { + return Err(anyhow::anyhow!( + "invalid Proxmox device passthrough mode {:o}; only permission bits are supported", + device.mode + )); + } + + let mut value = format!( + "path={},uid={},gid={},mode={:04o}", + device.path, device.uid, device.gid, device.mode + ); + if device.deny_write { + value.push_str(",deny-write=1"); + } + args.extend([format!("--dev{}", device.slot), value]); + } + + Ok(args) + } + + #[allow(dead_code)] + pub async fn set_devices( + &self, + vmid: u32, + devices: &[DevicePassthroughSpec], + ) -> Result<(), anyhow::Error> { + if devices.is_empty() { + return Ok(()); + } + Self::run(&self.pct_path, &Self::set_devices_args(vmid, devices)?).await?; + Ok(()) + } + + fn configured_slots(config: &str, prefix: &str) -> std::collections::BTreeSet { + config + .lines() + .filter_map(|line| line.split_once(':').map(|(key, _)| key.trim())) + .filter_map(|key| key.strip_prefix(prefix)) + .filter_map(|slot| slot.parse::().ok()) + .collect() + } + + pub fn reconcile_mounts_and_devices_args( + vmid: u32, + current_config: &str, + mounts: &[BindMountSpec], + devices: &[DevicePassthroughSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + + let mount_args = Self::set_bind_mount_args(vmid, mounts)?; + args.extend(mount_args.into_iter().skip(2)); + let device_args = Self::set_devices_args(vmid, devices)?; + args.extend(device_args.into_iter().skip(2)); + + let desired_mounts = mounts + .iter() + .map(|mount| mount.slot) + .collect::>(); + let desired_devices = devices + .iter() + .map(|device| device.slot) + .collect::>(); + + let mut stale = Self::configured_slots(current_config, "mp") + .difference(&desired_mounts) + .map(|slot| format!("mp{slot}")) + .collect::>(); + stale.extend( + Self::configured_slots(current_config, "dev") + .difference(&desired_devices) + .map(|slot| format!("dev{slot}")), + ); + if !stale.is_empty() { + args.extend(["--delete".to_string(), stale.join(",")]); + } + + Ok(args) + } + + pub async fn create_with_bind_mounts( + &self, + spec: &CreateContainerSpec, + mounts: &[HostBindMountSpec], + ) -> Result { + self.create_with_bind_mounts_and_devices(spec, mounts, &[]) + .await + } + + pub async fn create_with_bind_mounts_and_devices( + &self, + spec: &CreateContainerSpec, + mounts: &[HostBindMountSpec], + devices: &[HostDevicePassthroughSpec], + ) -> Result { + self.create(spec).await?; + + let result = async { + let user = self.oci_user(spec.vmid).await?; + self.configure_mounts_and_devices(spec.vmid, user, mounts, devices) + .await?; + Ok(user) + } + .await; + + match result { + Ok(user) => Ok(user), + Err(error) => match self.destroy(spec.vmid).await { + Ok(()) => Err(error), + Err(cleanup_error) => Err(anyhow::anyhow!( + "{error:#}; additionally failed to destroy newly created container {}: {cleanup_error:#}", + spec.vmid + )), + }, + } + } + + pub async fn configure_mounts_and_devices( + &self, + vmid: u32, + user: ContainerOciUser, + mounts: &[HostBindMountSpec], + devices: &[HostDevicePassthroughSpec], + ) -> Result<(), anyhow::Error> { + let mounts = mounts + .iter() + .map(|mount| BindMountSpec { + slot: mount.slot, + source_path: mount.source_path.clone(), + target_path: mount.target_path.clone(), + read_only: mount.read_only, + container_uid: user.uid, + container_gid: user.gid, + host_uid: mount.host_uid, + host_gid: mount.host_gid, + }) + .collect::>(); + + let devices = devices + .iter() + .map(|device| DevicePassthroughSpec { + slot: device.slot, + path: device.path.clone(), + uid: device.uid, + gid: device.gid, + mode: device.mode, + deny_write: device.deny_write, + }) + .collect::>(); + + let current_config = self.config(vmid).await?; + let args = + Self::reconcile_mounts_and_devices_args(vmid, ¤t_config, &mounts, &devices)?; + if args.len() > 2 { + Self::run(&self.pct_path, &args).await?; + } + + Ok(()) + } + + #[allow(dead_code)] + pub async fn create_with_data_mount( + &self, + spec: &CreateContainerSpec, + data_path: &str, + host_data_uid: u32, + host_data_gid: u32, + ) -> Result { + self.create_with_bind_mounts( + spec, + &[HostBindMountSpec { + slot: 0, + source_path: data_path.to_string(), + target_path: "/home/container".to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }], + ) + .await + } + + pub fn attach_args(spec: &AttachProcessSpec) -> Result, anyhow::Error> { + if spec.command.is_empty() { + return Err(anyhow::anyhow!("attached process command cannot be empty")); + } + + let mut args = vec![ + "-n".to_string(), + spec.vmid.to_string(), + "--clear-env".to_string(), + "-u".to_string(), + spec.uid.to_string(), + "-g".to_string(), + spec.gid.to_string(), + ]; + + for variable in &spec.environment { + let Some((name, _)) = variable.split_once('=') else { + return Err(anyhow::anyhow!( + "attached process environment entry must be KEY=VALUE: {variable}" + )); + }; + if name.is_empty() || name.contains('\0') || variable.contains('\0') { + return Err(anyhow::anyhow!( + "attached process environment entry has an invalid name: {variable}" + )); + } + args.extend(["-v".to_string(), variable.clone()]); + } + + if spec.command.iter().any(|argument| argument.contains('\0')) { + return Err(anyhow::anyhow!( + "attached process command contains a NUL byte" + )); + } + + args.push("--".to_string()); + args.extend(spec.command.iter().cloned()); + + Ok(args) + } + + #[allow(dead_code)] + pub fn spawn_attached_process( + &self, + spec: &AttachProcessSpec, + ) -> Result { + let args = Self::attach_args(spec)?; + tokio::process::Command::new(&self.lxc_attach_path) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {}", self.lxc_attach_path.display())) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContainerStatus { + Running, + Stopped, +} + +impl ContainerStatus { + fn from_word(value: &str) -> Option { + match value { + "running" => Some(Self::Running), + "stopped" => Some(Self::Stopped), + _ => None, + } + } + + fn parse(value: &str) -> Result { + let value = value.trim(); + let status = value.strip_prefix("status: ").unwrap_or(value); + Self::from_word(status) + .ok_or_else(|| anyhow::anyhow!("unknown pct status response: {value}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_arguments_leave_data_mount_for_post_create_configuration() { + let args = PveCli::create_args(&CreateContainerSpec { + vmid: 1234, + template: "local:vztmpl/oci-minecraft.tar".to_string(), + hostname: "server-1234".to_string(), + rootfs_storage: "local-lvm".to_string(), + rootfs_size_gib: 8, + memory_mib: 4096, + swap_mib: 1024, + cpu_limit_percent: Some(250), + cores: Some(2), + network: "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1".to_string(), + tags: vec!["calagopus".to_string(), "calagopus-server".to_string()], + unprivileged: true, + }); + + assert_eq!(args.first().map(String::as_str), Some("create")); + assert!( + args.windows(2) + .any(|pair| pair == ["--rootfs", "local-lvm:8"]) + ); + assert!(args.windows(2).any(|pair| { + pair == [ + "--net0", + "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1", + ] + })); + assert!(!args.iter().any(|arg| arg == "--mp0")); + assert!(args.windows(2).any(|pair| pair == ["--cores", "2"])); + assert!(args.windows(2).any(|pair| pair == ["--cpulimit", "2.5"])); + assert!(args.windows(2).any(|pair| pair == ["--cmode", "console"])); + assert!( + args.windows(2) + .any(|pair| { pair == ["--tags", "calagopus;calagopus-server"] }) + ); + } + + #[test] + fn formats_panel_cpu_percent_for_proxmox_cpu_time() { + assert_eq!(PveCli::format_cpu_limit_percent(1), "0.01"); + assert_eq!(PveCli::format_cpu_limit_percent(50), "0.5"); + assert_eq!(PveCli::format_cpu_limit_percent(100), "1"); + assert_eq!(PveCli::format_cpu_limit_percent(250), "2.5"); + assert_eq!(PveCli::format_cpu_limit_percent(225), "2.25"); + } + + #[test] + fn enables_firewall_on_existing_network_configuration() { + let config = "arch: amd64\nnet0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(config).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let disabled = "net0: name=eth0,bridge=vmbr0,firewall=0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(disabled).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let enabled = "net0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1\n"; + assert_eq!(PveCli::network_with_firewall(enabled).unwrap(), None); + assert!(PveCli::network_with_firewall("arch: amd64\n").is_err()); + } + + #[test] + fn set_tags_replaces_container_tags_atomically() { + assert_eq!( + PveCli::set_tags_args( + 1234, + &[ + "calagopus".to_string(), + "calagopus-server-1234".to_string(), + "calagopus-image-deadbeef".to_string(), + ], + ), + vec![ + "set", + "1234", + "--tags", + "calagopus;calagopus-server-1234;calagopus-image-deadbeef", + ] + ); + } + + #[test] + fn maps_panel_resources_to_native_pve_limits() { + assert_eq!( + PveCli::panel_resources(2048, 256, 1024, 250, None).ok(), + Some(ContainerResources { + memory_mib: 2304, + swap_mib: 1024, + cpu_limit_percent: Some(250), + cores: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + cpu_limit_percent: None, + cores: None, + }) + ); + } + + #[test] + fn rejects_resource_limits_without_exact_pve_semantics() { + assert!(PveCli::panel_resources(0, 0, 0, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, -1, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3")).is_err()); + } + + #[test] + fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 4096, + swap_mib: 512, + cpu_limit_percent: Some(250), + cores: Some(4), + }, + ), + vec![ + "set", + "1234", + "--memory", + "4096", + "--swap", + "512", + "--cpulimit", + "2.5", + "--cores", + "4", + ] + ); + + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 2048, + swap_mib: 0, + cpu_limit_percent: None, + cores: None, + }, + ), + vec![ + "set", + "1234", + "--memory", + "2048", + "--swap", + "0", + "--cpulimit", + "0", + ] + ); + } + + #[test] + fn parses_non_root_oci_user_from_pct_config() { + assert_eq!( + PveCli::parse_oci_user( + "arch: amd64\nlxc.init.uid: 1000\nlxc.init.gid: 1001\nostype: unmanaged\n" + ) + .ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); + assert_eq!( + PveCli::parse_oci_user("lxc.init.uid: 0\nlxc.init.gid: 1000\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 1000 }) + ); + assert_eq!( + PveCli::parse_oci_user("arch: amd64\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 0 }) + ); + } + + #[test] + fn pct_config_text_tolerates_nul_bytes() { + let config = PveCli::decode_config_output( + b"arch: amd64\0\nlxc.init.uid: 1000\nlxc.init.gid: 1001\0\n".to_vec(), + ) + .unwrap_or_default(); + + assert_eq!( + PveCli::parse_oci_user(&config).ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); + } + + #[test] + fn data_mount_arguments_map_resolved_image_user_to_host_owner() { + let args = PveCli::set_data_mount_args(&DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 1000, + container_gid: 1001, + host_data_uid: 988, + host_data_gid: 989, + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "set", + "1234", + "--mp0", + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + ] + ); + } + + #[test] + fn data_mount_arguments_support_root_in_an_unprivileged_container() { + let spec = DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 0, + container_gid: 1000, + host_data_uid: 988, + host_data_gid: 988, + }; + + assert_eq!( + PveCli::set_data_mount_args(&spec).ok(), + Some(vec![ + "set".to_string(), + "1234".to_string(), + "--mp0".to_string(), + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:0:988:1;g:1000:988:1".to_string(), + ]) + ); + } + + #[test] + fn bind_mount_arguments_support_multiple_targets_and_reject_duplicates() { + let mounts = vec![ + BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/mnt/server".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + BindMountSpec { + slot: 1, + source_path: "/srv/install".to_string(), + target_path: "/mnt/install".to_string(), + read_only: true, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + ]; + + assert_eq!( + PveCli::set_bind_mount_args(1234, &mounts).unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/mnt/server,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--mp1", + "/srv/install,mp=/mnt/install,backup=0,idmap=u:1000:988:1;g:1001:989:1,ro=1", + ] + ); + + let mut duplicate = mounts.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_bind_mount_args(1234, &duplicate).is_err()); + + let mut unsafe_path = mounts; + unsafe_path[1].target_path = "/mnt/install,bad".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); + unsafe_path[1].target_path = "/mnt/../etc".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); + } + + #[test] + fn device_arguments_set_ownership_mode_and_write_policy() { + let devices = vec![ + DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }, + DevicePassthroughSpec { + slot: 1, + path: "/dev/dri/renderD128".to_string(), + uid: 1000, + gid: 1001, + mode: 0o640, + deny_write: true, + }, + ]; + + assert_eq!( + PveCli::set_devices_args(1234, &devices).unwrap_or_default(), + vec![ + "set", + "1234", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--dev1", + "path=/dev/dri/renderD128,uid=1000,gid=1001,mode=0640,deny-write=1", + ] + ); + + let mut duplicate = devices.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_devices_args(1234, &duplicate).is_err()); + + let mut unsafe_path = devices.clone(); + unsafe_path[1].path = "/dev/dri/renderD128,bad".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + unsafe_path[1].path = "/dev/../etc/shadow".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + + let mut root_owned = devices; + root_owned[0].uid = 0; + root_owned[0].gid = 0; + assert!(PveCli::set_devices_args(1234, &root_owned).is_ok()); + } + + #[test] + fn mount_and_device_reconciliation_removes_stale_slots() { + let mounts = vec![BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/home/container".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }]; + let devices = vec![DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }]; + let current_config = r#" +rootfs: local-lvm:vm-1234-disk-0,size=8G +mp0: /srv/old-server,mp=/home/container +mp2: /srv/removed,mp=/mnt/removed +dev0: path=/dev/kvm +dev3: path=/dev/dri/renderD128 +lxc.init.uid: 1000 +"#; + + assert_eq!( + PveCli::reconcile_mounts_and_devices_args(1234, current_config, &mounts, &devices,) + .unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--delete", + "mp2,dev3", + ] + ); + } + + #[test] + fn parses_pct_status() { + assert_eq!( + ContainerStatus::parse("status: running\n").ok(), + Some(ContainerStatus::Running) + ); + assert_eq!( + ContainerStatus::parse("status: stopped\n").ok(), + Some(ContainerStatus::Stopped) + ); + assert!(ContainerStatus::parse("status: mounted").is_err()); + } + + #[test] + fn parses_local_cluster_node() { + let output = br#"[ + {"type":"cluster","name":"og-cluster"}, + {"type":"node","name":"pve-a","local":0}, + {"type":"node","name":"pve-b","local":1} + ]"#; + + assert_eq!( + PveCli::parse_local_node(output).ok().as_deref(), + Some("pve-b") + ); + } + + #[test] + fn orders_available_active_lxc_storages() { + let output = br#"[ + {"storage":"local-lvm","active":1,"enabled":1,"content":"images,rootdir","avail":100}, + {"storage":"zfs","active":1,"enabled":1,"content":"rootdir,images","avail":200}, + {"storage":"local","active":1,"enabled":1,"content":"iso,vztmpl","avail":300}, + {"storage":"disabled","active":1,"enabled":0,"content":"rootdir","avail":400}, + {"storage":"offline","active":0,"enabled":1,"content":"rootdir","avail":500} + ]"#; + + assert_eq!( + PveCli::parse_available_lxc_storages(output).ok(), + Some(vec![ + LxcStorage { + name: "zfs".to_string(), + available_bytes: 200, + }, + LxcStorage { + name: "local-lvm".to_string(), + available_bytes: 100, + }, + ]) + ); + } + + #[test] + fn parses_lxc_resources_and_ignores_vms() { + let output = br#"[ + {"type":"qemu","vmid":101,"node":"pve-a","status":"running"}, + {"type":"lxc","vmid":102,"node":"pve-b","status":"running","tags":"calagopus;calagopus-server-abc"}, + {"type":"lxc","vmid":"103","node":"pve-b","status":"stopped"} + ]"#; + + let containers = PveCli::parse_containers(output).unwrap_or_default(); + assert_eq!(containers.len(), 2); + assert_eq!( + containers.first().map(|container| container.vmid), + Some(102) + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.first()) + .map(String::as_str), + Some("calagopus") + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.get(1)) + .map(String::as_str), + Some("calagopus-server-abc") + ); + assert_eq!(containers.get(1).map(|container| container.vmid), Some(103)); + } + + #[test] + fn parses_container_interfaces_and_prefers_usable_ipv4() { + let interfaces = PveCli::parse_interfaces( + br#"[ + {"name":"lo","ip-addresses":[ + {"ip-address":"127.0.0.1","ip-address-type":"inet","prefix":8}, + {"ip-address":"::1","ip-address-type":"inet6","prefix":128} + ]}, + {"name":"eth0","ip-addresses":[ + {"ip-address":"fe80::1234","ip-address-type":"inet6","prefix":64}, + {"ip-address":"fd12:3456::20","ip-address-type":"inet6","prefix":64}, + {"ip-address":"10.42.0.20","ip-address-type":"inet","prefix":24} + ]} + ]"#, + ) + .unwrap_or_default(); + + assert_eq!(interfaces.len(), 2); + assert_eq!( + PveCli::primary_container_address(&interfaces), + "10.42.0.20".parse().ok() + ); + assert_eq!( + PveCli::interfaces_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/interfaces", + "--output-format", + "json", + ] + ); + } + + #[test] + fn stopped_container_null_interface_response_is_empty() { + assert_eq!(PveCli::parse_interfaces(b"null").unwrap(), Vec::new()); + } + + #[test] + fn container_addresses_returns_all_usable_addresses_sorted_and_deduplicated() { + let interfaces = vec![ + ContainerInterface { + name: "lo".to_string(), + addresses: vec!["127.0.0.1".parse().unwrap(), "::1".parse().unwrap()], + }, + ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "fe80::1234".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "0.0.0.0".parse().unwrap(), + "ff02::1".parse().unwrap(), + ], + }, + ]; + + assert_eq!( + PveCli::container_addresses(&interfaces), + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); + } + + #[test] + fn parses_runtime_status_for_cgroup_discovery() { + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"running","pid":"4321","uptime":87}"#,).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Running, + pid: Some(4321), + uptime_seconds: 87, + }) + ); + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"stopped","uptime":"0"}"#).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Stopped, + pid: None, + uptime_seconds: 0, + }) + ); + assert!(PveCli::parse_runtime_status(br#"{"status":"mounted"}"#).is_err()); + assert_eq!( + PveCli::runtime_status_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/status/current", + "--output-format", + "json", + ] + ); + } + + #[test] + fn container_address_falls_back_to_non_link_local_ipv6() { + let interfaces = vec![ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fe80::1234" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + "fd12:3456::20" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + ], + }]; + + assert_eq!( + PveCli::primary_container_address(&interfaces), + "fd12:3456::20".parse().ok() + ); + } + + #[test] + fn normalizes_oci_references_for_proxmox() { + assert_eq!( + PveCli::normalize_oci_reference("ghcr.io/example/game").ok(), + Some("ghcr.io/example/game:latest".to_string()) + ); + assert_eq!( + PveCli::normalize_oci_reference("registry.example:5000/example/game:v2").ok(), + Some("registry.example:5000/example/game:v2".to_string()) + ); + assert!(PveCli::normalize_oci_reference("game@sha256:abc").is_err()); + } + + #[test] + fn oci_cache_name_is_stable_and_reference_specific() { + let first = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let again = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let other = PveCli::oci_cache_filename("ghcr.io/example/game:v2"); + + assert_eq!(first, again); + assert_ne!(first, other); + assert!(first.starts_with("calagopus-oci-")); + assert_eq!(first.len(), "calagopus-oci-".len() + 64); + } + + #[test] + fn oci_image_tag_uses_resolved_revision_digest() { + let first = PveCli::oci_revision_tag( + "sha256:052a0ffb2620d9d59dfd81caa280af01ca8e6fe744649b11da481c00a8562d40", + ) + .unwrap_or_default(); + let other = PveCli::oci_revision_tag( + "5c093e071615ce61063007dfb363fd5bb878d4673954445e2a4ee5e7de3fae3b", + ) + .unwrap_or_default(); + + assert_ne!(first, other); + assert_eq!(first, "calagopus-image-052a0ffb2620d9d59dfd81caa280af01"); + assert!(PveCli::oci_revision_tag("sha256:not-a-digest").is_err()); + } + + #[test] + fn oci_pull_arguments_use_proxmox_storage_endpoint() { + let args = PveCli::pull_oci_args(&OciTemplatePullSpec { + node: "pve-a".to_string(), + storage: "local".to_string(), + reference: "ghcr.io/example/game:latest".to_string(), + filename: "calagopus-oci-abc".to_string(), + }); + + assert_eq!( + args, + vec![ + "create", + "/nodes/pve-a/storage/local/oci-registry-pull", + "--reference", + "ghcr.io/example/game:latest", + "--filename", + "calagopus-oci-abc", + "--output-format", + "json", + ] + ); + } + + #[test] + fn parses_oci_pull_task_after_proxmox_progress_output() { + let upid = "UPID:pve4:000348B0:005B00EB:6ABACCF1:ociregistrypull:template.tar:root@pam:"; + let noisy = format!( + "Getting image source signatures\nCopying blob sha256:abc\nWriting manifest to image destination\n\"{upid}\"\n" + ); + + assert_eq!( + PveCli::parse_oci_pull_task(noisy.as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert_eq!( + PveCli::parse_oci_pull_task(format!("\"{upid}\"").as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert!(PveCli::parse_oci_pull_task(b"copy completed without task id\n").is_err()); + } + + #[test] + fn parses_storage_content_and_task_status() { + let content = PveCli::parse_storage_content( + br#"[{"volid":"local:vztmpl/calagopus-oci-abc.tar","content":"vztmpl"}]"#, + ) + .unwrap_or_default(); + assert_eq!( + content.first(), + Some(&StorageContent { + volid: "local:vztmpl/calagopus-oci-abc.tar".to_string(), + content: Some("vztmpl".to_string()), + }) + ); + + assert_eq!( + PveCli::parse_task_status(br#"{"status":"running"}"#).ok(), + Some(TaskStatus { + state: TaskState::Running, + exit_status: None, + }) + ); + assert_eq!( + PveCli::parse_task_status(br#"{"status":"stopped","exitstatus":"OK"}"#).ok(), + Some(TaskStatus { + state: TaskState::Stopped, + exit_status: Some("OK".to_string()), + }) + ); + } + + #[test] + fn attached_process_runs_as_explicit_non_root_user_with_clean_environment() { + let args = PveCli::attach_args(&AttachProcessSpec { + vmid: 1234, + uid: 1000, + gid: 1000, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + command: vec!["/bin/bash".to_string(), "/entrypoint.sh".to_string()], + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "-n", + "1234", + "--clear-env", + "-u", + "1000", + "-g", + "1000", + "-v", + "STARTUP=java -jar server.jar", + "-v", + "SERVER_MEMORY=4096", + "--", + "/bin/bash", + "/entrypoint.sh", + ] + ); + } + + #[test] + fn attached_process_supports_root_and_rejects_invalid_input() { + let spec = AttachProcessSpec { + vmid: 1234, + uid: 0, + gid: 1000, + environment: Vec::new(), + command: vec!["/bin/true".to_string()], + }; + assert!(PveCli::attach_args(&spec).is_ok()); + + let mut spec = spec; + spec.command.clear(); + assert!(PveCli::attach_args(&spec).is_err()); + + spec.command.push("/bin/true".to_string()); + spec.environment.push("BROKEN".to_string()); + assert!(PveCli::attach_args(&spec).is_err()); + } + + #[test] + fn runtime_configuration_uses_stdin_payload_and_validates_line_safety() { + let payload = PveCli::runtime_config_payload(&RuntimeConfigSpec { + vmid: 1234, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + halt_signal: Some("SIGINT".to_string()), + init_command: Some("/bin/bash /mnt/install/install.sh".to_string()), + console_logfile: Some("/var/log/calagopus/install.log".to_string()), + managed_file_mounts: Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus wings/vmount/example/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]), + }) + .unwrap_or_default(); + let value: serde_json::Value = serde_json::from_slice(&payload).unwrap_or_default(); + + assert_eq!( + value.get("vmid").and_then(serde_json::Value::as_u64), + Some(1234) + ); + assert_eq!( + value + .get("environment") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(2) + ); + assert_eq!( + value.get("halt_signal").and_then(serde_json::Value::as_str), + Some("SIGINT") + ); + assert_eq!( + value + .get("init_command") + .and_then(serde_json::Value::as_str), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + value + .get("console_logfile") + .and_then(serde_json::Value::as_str), + Some("/var/log/calagopus/install.log") + ); + assert_eq!( + value + .get("managed_file_mounts") + .and_then(serde_json::Value::as_array) + .and_then(|mounts| mounts.first()) + .and_then(|mount| mount.get("entry")) + .and_then(serde_json::Value::as_str), + Some( + "/var/lib/calagopus\\040wings/vmount/example/hosts etc/hosts none bind,ro,create=file 0 0" + ) + ); + assert_eq!( + value + .get("managed_file_targets") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(5) + ); + + let mut invalid = RuntimeConfigSpec { + vmid: 1234, + environment: vec!["BROKEN".to_string()], + halt_signal: None, + init_command: None, + console_logfile: None, + managed_file_mounts: None, + }; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment = vec!["GOOD=bad\nvalue".to_string()]; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment.clear(); + invalid.console_logfile = Some("relative.log".to_string()); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.console_logfile = None; + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts".to_string(), + target_path: "/etc/shadow".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "relative/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-a".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-b".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + } + + #[test] + fn init_command_encoding_matches_lxc_whole_word_quoting() { + assert_eq!( + PveCli::encode_init_command(&[ + "/bin/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + PveCli::encode_init_command(&[ + "/opt/my shell/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("\"/opt/my shell/bash\" /mnt/install/install.sh") + ); + assert!(PveCli::encode_init_command(&["both'\"quotes here".to_string()]).is_err()); + } + + #[test] + fn lifecycle_arguments_request_nonblocking_graceful_shutdown_and_console() { + assert_eq!( + PveCli::request_shutdown_args(1234), + vec!["-n", "1234", "--nowait", "--nokill"] + ); + assert_eq!( + PveCli::console_args(1234), + vec!["console", "1234", "--escape", "^z"] + ); + } +} diff --git a/application/src/server/executor/pve_lxc/firewall.rs b/application/src/server/executor/pve_lxc/firewall.rs new file mode 100644 index 00000000..75718469 --- /dev/null +++ b/application/src/server/executor/pve_lxc/firewall.rs @@ -0,0 +1,651 @@ +use super::cli::{FirewallPolicySpec, FirewallRuleSpec, PveCli}; +use crate::server::firewall::{ + FirewallBackend, FirewallRuleAction, FirewallRuleProtocol, FirewallServerSpec, FirewallTarget, + sets, +}; +use anyhow::Context; +use std::{ + collections::{BTreeMap, BTreeSet, HashSet}, + sync::Arc, +}; + +const MANAGED_COMMENT_PREFIX: &str = "calagopus-wings:"; + +#[derive(Clone)] +struct StoredPolicy { + vmid: u32, + server: uuid::Uuid, + ports: Vec, + rules: Vec, + files: Option, + ipsets: BTreeMap>, +} + +struct Watch { + changed: Arc, + task: tokio::task::AbortHandle, + notifier: crate::server::filesystem::inotify::InotifyServerNotifier, +} + +impl Drop for Watch { + fn drop(&mut self) { + self.task.abort(); + self.notifier + .watch_firewall_files(Vec::new(), Arc::clone(&self.changed)); + } +} + +#[derive(Default)] +struct State { + policies: BTreeMap, + watches: BTreeMap, +} + +struct Inner { + cli: PveCli, + node: String, + tag_prefix: String, + limits: sets::SourceFileLimits, + state: tokio::sync::Mutex, + operation: tokio::sync::Mutex<()>, +} + +pub struct ProxmoxFirewall { + inner: Arc, +} + +impl ProxmoxFirewall { + pub fn new( + cli: PveCli, + node: String, + tag_prefix: String, + limits: sets::SourceFileLimits, + ) -> Self { + Self { + inner: Arc::new(Inner { + cli, + node, + tag_prefix, + limits, + state: tokio::sync::Mutex::new(State::default()), + operation: tokio::sync::Mutex::new(()), + }), + } + } + + fn stored(spec: &FirewallServerSpec) -> Result { + let FirewallTarget::ProxmoxLxc { vmid, interface } = spec + .target + .as_ref() + .context("Proxmox firewall requires an LXC target")?; + if interface != "net0" { + anyhow::bail!("unsupported Proxmox firewall interface {interface}"); + } + + let mut ports = spec.container_ports.clone(); + ports.sort_unstable(); + ports.dedup(); + Ok(StoredPolicy { + vmid: *vmid, + server: spec.server, + ports, + rules: spec.rules.clone(), + files: spec.files.clone(), + ipsets: BTreeMap::new(), + }) + } + + fn managed_prefix(server: uuid::Uuid) -> String { + format!("{MANAGED_COMMENT_PREFIX}{server}:") + } + + fn rule_sources( + server: uuid::Uuid, + rule: &crate::server::firewall::FirewallRule, + ) -> Vec> { + let mut sources: Vec> = rule + .sources + .iter() + .map(|source| Some(format!("{source:#}"))) + .collect(); + sources.sort(); + if let Some(file) = &rule.source_file { + let path = sets::source_file_path(file); + sources.push(Some(format!("+{}", sets::set_base_name(server, &path)))); + } + if sources.is_empty() { + sources.push(None); + } + sources + } + + fn translate(policy: &StoredPolicy) -> Vec { + let mut translated = Vec::new(); + let prefix = Self::managed_prefix(policy.server); + + for (rule_index, rule) in policy.rules.iter().enumerate() { + if rule.action == FirewallRuleAction::Deny + && rule.protocols.is_empty() + && rule.sources.is_empty() + && rule.ports.is_none() + && rule.source_file.is_none() + { + let sequence = translated.len(); + translated.push(FirewallRuleSpec { + r#type: "in".to_string(), + action: "DROP".to_string(), + iface: "net0".to_string(), + proto: None, + dport: None, + source: None, + comment: format!("{prefix}{rule_index}:{sequence} managed by Wings"), + }); + continue; + } + + let allowed_ports: BTreeSet = match &rule.ports { + Some(restricted) => policy + .ports + .iter() + .copied() + .filter(|port| restricted.contains(port)) + .collect(), + None => policy.ports.iter().copied().collect(), + }; + if allowed_ports.is_empty() { + continue; + } + let dport = allowed_ports + .into_iter() + .map(|port| port.to_string()) + .collect::>() + .join(","); + let protocols: Vec = + [FirewallRuleProtocol::Tcp, FirewallRuleProtocol::Udp] + .into_iter() + .filter(|protocol| { + rule.protocols.is_empty() || rule.protocols.contains(protocol) + }) + .collect(); + + for protocol in protocols { + for source in Self::rule_sources(policy.server, rule) { + let sequence = translated.len(); + translated.push(FirewallRuleSpec { + r#type: "in".to_string(), + action: match rule.action { + FirewallRuleAction::Allow => "ACCEPT", + FirewallRuleAction::Deny => "DROP", + } + .to_string(), + iface: "net0".to_string(), + proto: Some(protocol.as_str().to_string()), + dport: Some(dport.clone()), + source, + comment: format!("{prefix}{rule_index}:{sequence} managed by Wings"), + }); + } + } + } + + translated + } + + async fn load_ipsets(&self, policy: &mut StoredPolicy) { + let Some(access) = &policy.files else { + policy.ipsets.clear(); + return; + }; + + let referenced = policy.rules.iter().filter_map(|rule| { + rule.source_file.as_ref().map(|file| { + let path = sets::source_file_path(file); + (sets::set_base_name(policy.server, &path), path) + }) + }); + let mut wanted = BTreeSet::new(); + for (name, path) in referenced { + wanted.insert(name.clone()); + policy.ipsets.entry(name.clone()).or_default(); + match sets::read_source_file(&access.filesystem, &path, self.inner.limits).await { + Ok((entries, stats)) => { + policy + .ipsets + .insert(name, entries.into_iter().map(|entry| format!("{entry:#}")).collect()); + let message = format!( + "Firewall source file {} loaded with {} entries{}.", + path.display(), + stats.entries, + if stats.invalid == 0 { + String::new() + } else { + format!(", {} invalid lines skipped", stats.invalid) + } + ); + if stats.entries == 0 || stats.invalid > 0 { + access.log_error(&message); + } else { + access.log(&message); + } + } + Err(error) => access.log_error(&format!( + "Firewall source file {} could not be loaded, the previous entries are retained: {error}", + path.display() + )), + } + } + policy.ipsets.retain(|name, _| wanted.contains(name)); + } + + async fn apply_policy(&self, policy: &mut StoredPolicy) -> Result<(), anyhow::Error> { + self.load_ipsets(policy).await; + // Enable filtering first. Applying the managed rules is then the + // committing operation, so a crash cannot leave new rules ineffective. + self.inner.cli.enable_network_firewall(policy.vmid).await?; + self.inner + .cli + .apply_firewall_config(&FirewallPolicySpec { + vmid: policy.vmid, + managed_prefix: Self::managed_prefix(policy.server), + rules: Self::translate(policy), + ipsets: policy.ipsets.clone(), + }) + .await + } + + async fn apply_empty(&self, vmid: u32, server: uuid::Uuid) -> Result<(), anyhow::Error> { + self.inner + .cli + .apply_firewall_config(&FirewallPolicySpec { + vmid, + managed_prefix: Self::managed_prefix(server), + rules: Vec::new(), + ipsets: BTreeMap::new(), + }) + .await + } + + async fn reload(inner: Arc, server: uuid::Uuid) { + let firewall = Self { inner }; + let _operation = firewall.inner.operation.lock().await; + let policy = firewall + .inner + .state + .lock() + .await + .policies + .get(&server) + .cloned(); + let Some(mut policy) = policy else { + return; + }; + if let Err(error) = firewall.apply_policy(&mut policy).await { + tracing::error!(server = %server, "failed to reload Proxmox firewall source files: {error:#}"); + return; + } + firewall + .inner + .state + .lock() + .await + .policies + .insert(server, policy); + } + + async fn update_watch(&self, policy: &StoredPolicy) { + let mut state = self.inner.state.lock().await; + state.watches.remove(&policy.server); + let Some(access) = &policy.files else { + return; + }; + let paths: Vec<_> = policy + .rules + .iter() + .filter_map(|rule| rule.source_file.as_ref()) + .map(|file| { + access + .filesystem + .base_path + .join(sets::source_file_path(file)) + }) + .collect(); + if paths.is_empty() { + return; + } + + let changed = Arc::new(tokio::sync::Notify::new()); + let Some(notifier) = access.notifier.clone() else { + return; + }; + notifier.watch_firewall_files(paths, Arc::clone(&changed)); + let inner = Arc::clone(&self.inner); + let server = policy.server; + let task = tokio::spawn({ + let changed = Arc::clone(&changed); + async move { + loop { + changed.notified().await; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + Self::reload(Arc::clone(&inner), server).await; + } + } + }) + .abort_handle(); + state.watches.insert( + policy.server, + Watch { + changed, + task, + notifier, + }, + ); + } + + async fn owned_vmids(&self, server: uuid::Uuid) -> Result, anyhow::Error> { + let server_tag = format!("{}-server-{server}", self.inner.tag_prefix); + let replacement_tag = format!("{}-repl-{server}", self.inner.tag_prefix); + Ok(self + .inner + .cli + .list_containers() + .await? + .into_iter() + .filter(|container| { + container.node == self.inner.node + && container + .tags + .iter() + .any(|tag| tag == &server_tag || tag == &replacement_tag) + }) + .map(|container| container.vmid) + .collect()) + } + + fn managed_vmids_for( + containers: Vec, + node: &str, + tag_prefix: &str, + ) -> BTreeMap> { + let server_prefix = format!("{tag_prefix}-server-"); + let replacement_prefix = format!("{tag_prefix}-repl-"); + let mut managed = BTreeMap::>::new(); + for container in containers { + if container.node != node { + continue; + } + for tag in &container.tags { + let value = tag + .strip_prefix(&server_prefix) + .or_else(|| tag.strip_prefix(&replacement_prefix)); + let Some(server) = value.and_then(|value| uuid::Uuid::parse_str(value).ok()) else { + continue; + }; + managed.entry(server).or_default().push(container.vmid); + break; + } + } + managed + } + + async fn managed_vmids(&self) -> Result>, anyhow::Error> { + Ok(Self::managed_vmids_for( + self.inner.cli.list_containers().await?, + &self.inner.node, + &self.inner.tag_prefix, + )) + } + + /// Removes Wings-owned guest rules before switching to a host-local + /// backend. Administrator rules and containers on other cluster nodes are + /// left untouched. + pub async fn clear_managed(&self) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + for (server, vmids) in self.managed_vmids().await? { + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + } + self.inner.state.lock().await.policies.clear(); + self.inner.state.lock().await.watches.clear(); + Ok(()) + } +} + +#[async_trait::async_trait] +impl FirewallBackend for ProxmoxFirewall { + async fn boot(&self) -> Result<(), anyhow::Error> { + if !self.inner.cli.cluster_firewall_enabled().await? { + anyhow::bail!( + "the Proxmox datacenter firewall is disabled; enable it before using the Proxmox LXC firewall backend" + ); + } + tracing::info!("using Proxmox guest firewall backend"); + Ok(()) + } + + async fn sync(&self, spec: &FirewallServerSpec) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + let mut policy = Self::stored(spec)?; + if let Some(previous) = self.inner.state.lock().await.policies.get(&spec.server) { + policy.ipsets = previous.ipsets.clone(); + } + self.apply_policy(&mut policy).await?; + self.inner + .state + .lock() + .await + .policies + .insert(spec.server, policy.clone()); + drop(_operation); + self.update_watch(&policy).await; + Ok(()) + } + + async fn clear(&self, server: uuid::Uuid) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + let remembered = self + .inner + .state + .lock() + .await + .policies + .remove(&server) + .map(|policy| policy.vmid); + self.inner.state.lock().await.watches.remove(&server); + let mut vmids: HashSet = self.owned_vmids(server).await?.into_iter().collect(); + if let Some(vmid) = remembered { + vmids.insert(vmid); + } + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + Ok(()) + } + + async fn reconcile(&self, specs: &[FirewallServerSpec]) -> Result<(), anyhow::Error> { + let wanted: HashSet = specs.iter().map(|spec| spec.server).collect(); + let mut stale: HashSet = self + .inner + .state + .lock() + .await + .policies + .keys() + .filter(|server| !wanted.contains(server)) + .copied() + .collect(); + let discovered = self.managed_vmids().await?; + stale.extend( + discovered + .keys() + .filter(|server| !wanted.contains(server)) + .copied(), + ); + for server in stale { + let _operation = self.inner.operation.lock().await; + let remembered = self + .inner + .state + .lock() + .await + .policies + .remove(&server) + .map(|policy| policy.vmid); + self.inner.state.lock().await.watches.remove(&server); + let mut vmids: HashSet = discovered + .get(&server) + .into_iter() + .flatten() + .copied() + .collect(); + if let Some(vmid) = remembered { + vmids.insert(vmid); + } + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + } + for spec in specs { + self.sync(spec).await?; + } + crate::server::firewall::clear_host_local(self.inner.limits).await; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn managed_inventory_excludes_other_cluster_nodes() { + let local = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let remote = uuid::Uuid::parse_str("12345678-3456-7890-abcd-ef1234567890").unwrap(); + let containers = vec![ + super::super::cli::ClusterContainer { + vmid: 100, + node: "pve-a".to_string(), + status: None, + tags: vec![format!("calagopus-server-{local}")], + }, + super::super::cli::ClusterContainer { + vmid: 200, + node: "pve-b".to_string(), + status: None, + tags: vec![format!("calagopus-server-{remote}")], + }, + ]; + + let managed = ProxmoxFirewall::managed_vmids_for(containers, "pve-a", "calagopus"); + assert_eq!(managed.get(&local), Some(&vec![100])); + assert!(!managed.contains_key(&remote)); + } + + #[test] + fn translates_panel_rules_to_vm_scoped_proxmox_rules() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565, 25566], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::from([FirewallRuleProtocol::Tcp]), + sources: vec!["10.0.0.0/8".parse().unwrap()], + ports: Some(vec![25565]), + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].iface, "net0"); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[0].dport.as_deref(), Some("25565")); + assert_eq!(rules[0].source.as_deref(), Some("10.0.0.0/8")); + } + + #[test] + fn translates_empty_protocols_and_source_files() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let source_file = "blocked.txt".to_string(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Allow, + protocols: HashSet::new(), + sources: Vec::new(), + ports: None, + source_file: Some(source_file.clone().into()), + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 2); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[1].proto.as_deref(), Some("udp")); + let path = sets::source_file_path(&source_file); + let expected = format!("+{}", sets::set_base_name(server, &path)); + assert_eq!(rules[0].source.as_deref(), Some(expected.as_str())); + assert_eq!(rules[1].source.as_deref(), Some(expected.as_str())); + } + + #[test] + fn does_not_emit_synthetic_fallback_rules() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565, 25566], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::from([FirewallRuleProtocol::Tcp]), + sources: Vec::new(), + ports: Some(vec![25565]), + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[0].dport.as_deref(), Some("25565")); + } + + #[test] + fn translates_terminal_deny_everything_as_an_unrestricted_drop() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::new(), + sources: Vec::new(), + ports: None, + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].iface, "net0"); + assert_eq!(rules[0].proto, None); + assert_eq!(rules[0].dport, None); + assert_eq!(rules[0].source, None); + } +} diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs new file mode 100644 index 00000000..409f59e5 --- /dev/null +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -0,0 +1,4595 @@ +//! Proxmox VE LXC runtime support. +//! +//! This module intentionally sits behind the existing `ServerExecutor` boundary so +//! Docker remains the default runtime and upstream changes to the Docker executor +//! do not have to be carried into the PVE implementation. + +pub mod cli; +pub mod firewall; + +use super::{ProcessHandle, ServerExecutor, StatusReceiver, UsedPort}; +use crate::io::line_buffer::LineBuffer; +use anyhow::Context; +use compact_str::ToCompactString; +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + io::Cursor, + net::{IpAddr, Ipv4Addr}, + path::PathBuf, + sync::{ + Arc, OnceLock, Weak, + atomic::{AtomicBool, Ordering}, + }, +}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +pub use cli::PveCli; + +pub async fn create_firewall( + app_config: &crate::config::Config, + cli: &PveCli, +) -> Result, anyhow::Error> { + let (backend, limits, tag_prefix, configured_node) = { + let config = app_config.load(); + let firewall = &config.runtime.pve_lxc.firewall; + ( + firewall.backend, + crate::server::firewall::sets::SourceFileLimits { + max_entries: usize::try_from(firewall.source_file_max_entries) + .context("Proxmox firewall source entry limit exceeds usize")?, + max_bytes: firewall.source_file_max_bytes, + }, + config.runtime.pve_lxc.tag_prefix.clone(), + config.runtime.pve_lxc.node.clone(), + ) + }; + let local_node = cli.local_node().await?; + let node = PveLxcExecutor::runtime_node(&configured_node, &local_node)?; + + let proxmox_enabled = match cli.cluster_firewall_enabled().await { + Ok(enabled) => enabled, + Err(error) if backend == crate::config::PveLxcFirewallBackend::Auto => { + tracing::warn!( + "failed to probe the Proxmox datacenter firewall, falling back to host firewalling: {error:#}" + ); + false + } + Err(error) => return Err(error.context("failed to probe the Proxmox datacenter firewall")), + }; + + let firewall: Arc = match backend { + crate::config::PveLxcFirewallBackend::Auto if proxmox_enabled => Arc::new( + firewall::ProxmoxFirewall::new(cli.clone(), node.clone(), tag_prefix, limits), + ), + crate::config::PveLxcFirewallBackend::Proxmox => Arc::new(firewall::ProxmoxFirewall::new( + cli.clone(), + node, + tag_prefix, + limits, + )), + crate::config::PveLxcFirewallBackend::Auto => { + tracing::info!( + "Proxmox datacenter firewall is disabled; using the host-local firewall backend" + ); + firewall::ProxmoxFirewall::new(cli.clone(), node, tag_prefix, limits) + .clear_managed() + .await + .context("failed to clear stale Proxmox guest firewall rules before selecting the host-local backend")?; + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Auto, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Nftables => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Nftables, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Iptables => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Iptables, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Disabled => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Disabled, + limits, + ) + .await + } + }; + Ok(firewall) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProcNetFamily { + Ipv4, + Ipv6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BoundPort { + family: ProcNetFamily, + address: Option, + port: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EdgePeer { + public_ip: Ipv4Addr, + tunnel_ip: Ipv4Addr, +} + +struct PveProcessHandle { + vmid: u32, + node: String, + cli: PveCli, + server: Weak, + app_config: Arc, + firewall: Arc, + log_path: PathBuf, + stdin_tx: tokio::sync::mpsc::Sender>, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, + started: Arc, + status_task: tokio::task::JoinHandle<()>, + stats_task: tokio::task::JoinHandle<()>, + console_task: tokio::task::JoinHandle<()>, +} + +impl PveProcessHandle { + fn observed_status( + status: cli::ContainerStatus, + start_succeeded: bool, + seen_running: &mut bool, + ) -> Option { + match status { + cli::ContainerStatus::Running => { + *seen_running = true; + Some(super::ProcessStatus::Running) + } + cli::ContainerStatus::Stopped if !*seen_running && !start_succeeded => None, + cli::ContainerStatus::Stopped => Some(super::ProcessStatus::Stopped { + exit_code: -1, + oom_killed: false, + }), + } + } + + fn managed_file_stage_name(target: &str) -> Result<&'static str, anyhow::Error> { + match target { + "/etc/machine-id" => Ok("machine-id"), + "/etc/hosts" => Ok("hosts"), + "/etc/passwd" => Ok("passwd"), + "/etc/group" => Ok("group"), + "/calagopus-entrypoint" => Ok("entrypoint"), + _ => Err(anyhow::anyhow!( + "unsupported managed Proxmox file mount target: {target}" + )), + } + } + + fn managed_file_stage_directory( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result { + let config = app_config.load(); + let root = config + .runtime + .pve_lxc + .managed_file_directory + .as_path(&config); + if !root.is_absolute() { + return Err(anyhow::anyhow!( + "runtime.pve_lxc.managed_file_directory must be an absolute path" + )); + } + Ok(root.join(server_uuid.to_compact_string())) + } + + async fn cleanup_managed_file_staging( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result<(), anyhow::Error> { + let path = Self::managed_file_stage_directory(app_config, server_uuid)?; + match tokio::fs::remove_dir_all(&path).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| { + format!( + "failed to remove managed Proxmox file staging directory {}", + path.display() + ) + }), + } + } + + #[allow(clippy::too_many_arguments)] + async fn new( + vmid: u32, + node: String, + cli: PveCli, + stats_sampler: Arc, + server: &crate::server::Server, + app_config: Arc, + firewall: Arc, + status_tx: tokio::sync::mpsc::Sender, + fresh_log: bool, + suppress_initial_stopped: bool, + ) -> Result { + let log_path = Self::runtime_log_path(&app_config, server.uuid); + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + if fresh_log { + tokio::fs::File::create(&log_path).await?; + } else { + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + } + + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + + let started = Arc::new(AtomicBool::new(!suppress_initial_stopped)); + let state_started = Arc::clone(&started); + let state_cli = cli.clone(); + let state_server_uuid = server.uuid; + let status_task = tokio::spawn(async move { + let mut seen_running = state_started.load(Ordering::Acquire); + let mut tick = tokio::time::interval(std::time::Duration::from_secs(2)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + let process_status = match state_cli.status(vmid).await { + Ok(status) => match Self::observed_status( + status, + state_started.load(Ordering::Acquire), + &mut seen_running, + ) { + Some(status) => status, + None => continue, + }, + Err(error) => { + tracing::warn!( + server = %state_server_uuid, + vmid, + "failed to read Proxmox LXC process state: {error:#}" + ); + continue; + } + }; + + if status_tx.send(process_status).await.is_err() { + break; + } + } + }); + + let stats_cli = cli.clone(); + let stats_node = node.clone(); + let stats_usage = server.resource_usage.clone(); + let stats_server = Arc::downgrade(&**server); + let stats_server_uuid = server.uuid; + let disk_bytes = server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(server.state.get_state()); + usage.disk_bytes = disk_bytes; + }); + let stats_task = tokio::spawn(async move { + let mut samples: Option = None; + let mut started_at: Option = None; + let mut previous_cpu_total = 0u64; + let mut previous_sample_at: Option = None; + let mut warned_unresolved_cgroup = false; + let mut resolve_tick = tokio::time::interval(std::time::Duration::from_secs(2)); + resolve_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + let Some(stats_server) = stats_server.upgrade() else { + break; + }; + let state = stats_server.state.get_state(); + if state == crate::server::state::ServerState::Offline { + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + resolve_tick.tick().await; + continue; + } + + if samples.is_none() { + resolve_tick.tick().await; + + let runtime_status = match stats_cli.runtime_status(&stats_node, vmid).await { + Ok(status) => status, + Err(error) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to resolve Proxmox LXC runtime stats process: {error:#}" + ); + continue; + } + }; + + if runtime_status.status != cli::ContainerStatus::Running { + let state = stats_server.state.get_state(); + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + continue; + } + + let Some(pid) = runtime_status.pid else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "running Proxmox LXC status did not include an init PID; resource stats are unavailable until it does" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + let Some(files) = super::cgroup::StatFiles::resolve(pid) else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + pid, + "could not resolve the Proxmox LXC cgroup-v2 files; resource stats are unavailable until they can be resolved" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + + samples = Some(stats_sampler.register(files)); + started_at = std::time::Instant::now().checked_sub( + std::time::Duration::from_secs(runtime_status.uptime_seconds), + ); + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + } + + let Some(receiver) = samples.as_mut() else { + continue; + }; + let sample = match tokio::time::timeout( + std::time::Duration::from_secs(3), + receiver.recv(), + ) + .await + { + Ok(Some(Ok(sample))) => sample, + Ok(Some(Err(error))) if error.kind() == std::io::ErrorKind::NotFound => { + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + continue; + } + Ok(Some(Err(error))) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to read Proxmox LXC cgroup stats: {error}" + ); + samples = None; + continue; + } + Ok(None) | Err(_) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "Proxmox LXC cgroup stats sampler stopped delivering; resolving the container again" + ); + samples = None; + continue; + } + }; + + let cpu_absolute = if let Some(previous_at) = previous_sample_at { + let cpu_delta_ns = + sample.cpu_total_ns.saturating_sub(previous_cpu_total) as f64; + let wall_delta_ns = sample.at.duration_since(previous_at).as_nanos() as f64; + + if wall_delta_ns > 0.0 && cpu_delta_ns > 0.0 { + ((cpu_delta_ns / wall_delta_ns) * 100.0 * 1000.0).round() / 1000.0 + } else { + 0.0 + } + } else { + 0.0 + }; + previous_cpu_total = sample.cpu_total_ns; + previous_sample_at = Some(sample.at); + + let disk_bytes = stats_server.filesystem.limiter_usage().await; + let cpu_limit = stats_server.configuration.read().await.build.cpu_limit; + let cpu_limit_absolute = if cpu_limit > 0 { + u32::try_from(cpu_limit).unwrap_or(u32::MAX) + } else { + std::thread::available_parallelism().map_or(1, |threads| threads.get()) as u32 + * 100 + }; + let uptime = started_at + .map(|started_at| { + u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX) + }) + .unwrap_or(0); + let state = stats_server.state.get_state(); + + stats_usage.send_modify(|usage| { + usage.memory_bytes = sample.memory_bytes; + usage.memory_limit_bytes = sample.memory_limit_bytes; + usage.disk_bytes = disk_bytes; + usage.state = state; + usage.cpu_absolute = cpu_absolute; + usage.cpu_limit_absolute = cpu_limit_absolute; + usage.uptime = uptime; + + if let Some((rx_bytes, rx_packets, tx_bytes, tx_packets)) = sample.network { + usage.network.rx_bytes = rx_bytes; + usage.network.rx_packets = rx_packets; + usage.network.tx_bytes = tx_bytes; + usage.network.tx_packets = tx_packets; + } + }); + } + }); + + let console_cli = cli.clone(); + let console_app_config = Arc::clone(&app_config); + let console_log_max_bytes = app_config.load().runtime.pve_lxc.console_log_max_bytes; + let console_server = Arc::downgrade(&**server); + let console_log_path = log_path.clone(); + let console_task = tokio::spawn(async move { + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + match console_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => {} + Ok(cli::ContainerStatus::Stopped) => { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + Err(error) => { + tracing::warn!( + vmid, + "failed to read Proxmox LXC state before console attach: {error:#}" + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + } + + let session = match console_cli.spawn_console(vmid) { + Ok(session) => session, + Err(error) => { + tracing::warn!(vmid, "failed to attach Proxmox LXC console: {error:#}"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + // `tokio::fs::File` serializes operations on a single handle. + // A blocking PTY read would therefore prevent stdin writes + // indefinitely if both directions shared one Tokio file. + let cli::ConsoleSession { + mut child, + mut reader, + mut writer, + } = session; + let mut log = match tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&console_log_path) + .await + { + Ok(log) => log, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox LXC runtime log: {error:#}"); + child.start_kill().ok(); + break; + } + }; + let mut log_size = log.metadata().await.map_or(0, |metadata| metadata.len()); + let mut buffer = vec![0u8; 8192]; + + loop { + tokio::select! { + read = reader.read(&mut buffer) => { + match read { + Ok(0) => break, + Ok(read) => { + let Some(data) = buffer.get(..read) else { + tracing::error!(vmid, read, "Proxmox console returned an invalid read length"); + break; + }; + line_buffer.extend(data); + while let Some(line) = line_buffer.next_line() { + if Self::is_pct_console_banner(line) { + continue; + } + if let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + true, + ).await { + tracing::error!(vmid, "failed to append Proxmox LXC runtime log: {error:#}"); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = console_app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = console_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox LXC console: {error:#}"); + break; + } + } + } + data = stdin_rx.recv() => { + let Some(data) = data else { + child.start_kill().ok(); + return; + }; + if let Err(error) = writer.write_all(&data).await { + tracing::warn!(vmid, "failed to write Proxmox LXC console stdin: {error:#}"); + break; + } + } + result = child.wait() => { + if let Err(error) = result { + tracing::warn!(vmid, "failed waiting for Proxmox LXC console: {error:#}"); + } + break; + } + } + } + + child.start_kill().ok(); + if !matches!( + console_cli.status(vmid).await, + Ok(cli::ContainerStatus::Running) + ) { + if let Some(line) = line_buffer.flush() { + if !Self::is_pct_console_banner(line) + && let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + false, + ) + .await + { + tracing::error!( + vmid, + "failed to append final Proxmox LXC runtime log line: {error:#}" + ); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + if !Self::is_pct_console_banner(line.as_bytes()) { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + line_buffer = LineBuffer::new(); + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + }); + + Ok(Self { + vmid, + node, + cli, + server: Arc::downgrade(&**server), + app_config, + firewall, + log_path, + stdin_tx, + stdout_ratelimited_rx, + stdout_rx, + started, + status_task, + stats_task, + console_task, + }) + } + + fn runtime_log_path(app_config: &crate::config::Config, server_uuid: uuid::Uuid) -> PathBuf { + app_config + .resolve_as_path(|config| &config.system.log_directory) + .join("pve-lxc") + .join(format!("{server_uuid}.log")) + } + + async fn write_bounded_log( + log: &mut tokio::fs::File, + size: &mut u64, + max_bytes: u64, + line: &[u8], + newline: bool, + ) -> Result<(), std::io::Error> { + let mut line = line; + let mut write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + if max_bytes > 0 && write_size > max_bytes { + let newline_size = usize::from(newline); + let keep = usize::try_from(max_bytes) + .unwrap_or(usize::MAX) + .saturating_sub(newline_size); + let start = line.len().saturating_sub(keep); + line = line.get(start..).unwrap_or_default(); + write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + } + if max_bytes > 0 && size.saturating_add(write_size) > max_bytes { + log.set_len(0).await?; + log.seek(std::io::SeekFrom::Start(0)).await?; + *size = 0; + } + log.write_all(line).await?; + if newline { + log.write_all(b"\n").await?; + } + *size = size.saturating_add(write_size); + Ok(()) + } + + /// `pct console` forwards terminal key input and expects Enter as a carriage + /// return. Wings command producers use line feeds because Docker stdin is a + /// byte stream, so normalize line endings only at the PVE PTY boundary. + fn normalize_console_input(data: Vec) -> Vec { + let mut normalized = Vec::with_capacity(data.len()); + let mut previous_was_carriage_return = false; + + for byte in data { + if byte == b'\n' { + if !previous_was_carriage_return { + normalized.push(b'\r'); + } + previous_was_carriage_return = false; + } else { + normalized.push(byte); + previous_was_carriage_return = byte == b'\r'; + } + } + + normalized + } + + fn is_pct_console_banner(line: &[u8]) -> bool { + let line = String::from_utf8_lossy(line); + let line = line.trim_matches(|character: char| character.is_whitespace()); + line.starts_with("Connected to tty ") + || line.starts_with("Type to exit the console") + } + + fn get_server(&self) -> Result, anyhow::Error> { + self.server + .upgrade() + .ok_or_else(|| anyhow::anyhow!("server has been dropped")) + } + + fn halt_signal(stop_type: &str, stop_value: Option<&str>) -> Option { + if stop_type != "signal" { + return None; + } + + Some( + match stop_value.map(str::to_uppercase).as_deref() { + Some("SIGABRT") => "SIGABRT", + Some("SIGINT") | Some("C") => "SIGINT", + Some("SIGTERM") => "SIGTERM", + Some("SIGQUIT") => "SIGQUIT", + _ => "SIGKILL", + } + .to_string(), + ) + } + + async fn managed_file_mounts( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = (server, app_config); + return Err(anyhow::anyhow!( + "managed Proxmox container files require a Unix host" + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let server_uuid = { + let configuration = server.configuration.read().await; + configuration.ensure_vmounts(app_config).await?; + configuration.uuid + }; + let (machine_id_enabled, tundra_enabled, passwd_enabled, passwd_directory) = { + let config = app_config.load(); + ( + config.system.machine_id.enabled, + config.tundra.enabled, + config.system.passwd.enabled, + config.system.passwd.directory.as_path(&config), + ) + }; + + let vmount_path = app_config.vmount_path(server_uuid); + let mut files = Vec::with_capacity(4); + if machine_id_enabled { + files.push((vmount_path.join("machine-id"), "/etc/machine-id")); + } + if tundra_enabled { + files.push((vmount_path.join("hosts"), "/etc/hosts")); + } + if passwd_enabled { + files.push((passwd_directory.join("passwd"), "/etc/passwd")); + files.push((passwd_directory.join("group"), "/etc/group")); + } + + let stage_directory = Self::managed_file_stage_directory(app_config, server_uuid)?; + let stage_root = stage_directory + .parent() + .context("managed Proxmox file staging directory did not have a parent")?; + tokio::fs::create_dir_all(stage_root) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging root {}", + stage_root.display() + ) + })?; + tokio::fs::set_permissions(stage_root, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging root permissions on {}", + stage_root.display() + ) + })?; + tokio::fs::create_dir_all(&stage_directory) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging directory {}", + stage_directory.display() + ) + })?; + tokio::fs::set_permissions(&stage_directory, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging permissions on {}", + stage_directory.display() + ) + })?; + + let mut mounts = Vec::with_capacity(files.len() + 1); + for (source, target) in files { + let source = tokio::fs::canonicalize(&source).await.with_context(|| { + format!( + "failed to resolve managed Proxmox container file {}", + source.display() + ) + })?; + let metadata = tokio::fs::metadata(&source).await.with_context(|| { + format!( + "failed to inspect managed Proxmox container file {}", + source.display() + ) + })?; + if !metadata.is_file() { + return Err(anyhow::anyhow!( + "managed Proxmox container file {} is not a regular file", + source.display() + )); + } + + let staged_source = stage_directory.join(Self::managed_file_stage_name(target)?); + tokio::fs::copy(&source, &staged_source) + .await + .with_context(|| { + format!( + "failed to stage managed Proxmox container file {} at {}", + source.display(), + staged_source.display() + ) + })?; + tokio::fs::set_permissions(&staged_source, std::fs::Permissions::from_mode(0o644)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file permissions on {}", + staged_source.display() + ) + })?; + let staged_source = + tokio::fs::canonicalize(&staged_source) + .await + .with_context(|| { + format!( + "failed to resolve staged managed Proxmox container file {}", + staged_source.display() + ) + })?; + let source_path = staged_source.into_os_string().into_string().map_err(|_| { + anyhow::anyhow!("managed Proxmox container file path is not valid UTF-8") + })?; + + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: target.to_string(), + }); + } + + let entrypoint_target = "/calagopus-entrypoint"; + let entrypoint_source = + stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); + tokio::fs::write( + &entrypoint_source, + r#"#!/bin/sh +attempt=0 +while [ "$attempt" -lt 120 ]; do + if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then + exec "$@" + fi + attempt=$((attempt + 1)) + sleep 0.25 +done +echo 'timed out waiting for Proxmox DHCP' >&2 +exit 1 +"#, + ) + .await + .context("failed to stage Proxmox network-ready entrypoint")?; + tokio::fs::set_permissions(&entrypoint_source, std::fs::Permissions::from_mode(0o755)) + .await + .context("failed to make Proxmox network-ready entrypoint executable")?; + let entrypoint_source = tokio::fs::canonicalize(&entrypoint_source) + .await + .context("failed to resolve staged Proxmox network-ready entrypoint")?; + let source_path = entrypoint_source + .into_os_string() + .into_string() + .map_err(|_| { + anyhow::anyhow!("managed Proxmox entrypoint path is not valid UTF-8") + })?; + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: entrypoint_target.to_string(), + }); + + Ok(mounts) + } + } + + async fn runtime_config( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + cli: &PveCli, + vmid: u32, + ) -> Result { + let environment = server.configuration.read().await.environment(app_config); + let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; + let process = server.process_configuration.read().await; + let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); + let image_entrypoint = cli.oci_entrypoint(vmid).await?; + let mut init_arguments = vec!["/calagopus-entrypoint".to_string()]; + init_arguments.extend( + shell_words::split(&image_entrypoint) + .context("failed to parse OCI entrypoint for Proxmox LXC")?, + ); + let init_command = PveCli::encode_init_command(&init_arguments)?; + + Ok(cli::RuntimeConfigSpec { + vmid, + environment, + halt_signal, + init_command: Some(init_command), + console_logfile: None, + managed_file_mounts: Some(managed_file_mounts), + }) + } + + async fn sync_firewall(&self, wait_for_address: bool) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let has_rules = !server.configuration.read().await.firewall.is_empty(); + let edge_forwarding = PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some(); + let running = self.cli.status(self.vmid).await? == cli::ContainerStatus::Running; + + let container_ips = if running && (has_rules || edge_forwarding) { + let mut last_error = None; + let mut addresses = Vec::new(); + let attempts = if wait_for_address { 40 } else { 1 }; + + for attempt in 0..attempts { + match self.cli.interfaces(&self.node, self.vmid).await { + Ok(interfaces) => { + addresses = PveCli::container_addresses(&interfaces); + if !addresses.is_empty() { + break; + } + last_error = None; + } + Err(error) => last_error = Some(error), + } + + if attempt + 1 < attempts { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + + if addresses.is_empty() { + if let Some(error) = last_error { + return Err(error.context( + "failed to resolve Proxmox LXC address for firewall and edge forwarding", + )); + } + return Err(anyhow::anyhow!( + "running Proxmox LXC container {} has no usable bridged address; refusing to leave firewall or edge forwarding unapplied", + self.vmid + )); + } + + addresses + } else { + Vec::new() + }; + + let (mut spec, edge_forward) = { + let configuration = server.configuration.read().await; + let destination = container_ips.iter().find_map(|address| match address { + IpAddr::V4(address) => Some(*address), + IpAddr::V6(_) => None, + }); + let edge_forward = if edge_forwarding && running { + Some(PveLxcExecutor::edge_forward_payload( + &configuration, + destination + .context("running Proxmox LXC has no IPv4 address for edge forwarding")?, + )?) + } else { + None + }; + ( + PveLxcExecutor::firewall_spec(&configuration, container_ips, Some(self.vmid)), + edge_forward, + ) + }; + spec.files = Some(PveLxcExecutor::firewall_file_access(&server)); + self.firewall.sync(&spec).await?; + if edge_forwarding { + let (public_ip, payload) = edge_forward.flatten().unzip(); + PveLxcExecutor::sync_edge_forwarding(&self.app_config, server.uuid, public_ip, payload) + .await?; + } + Ok(()) + } + + async fn stop_after_firewall_failure( + &self, + error: anyhow::Error, + context: &'static str, + ) -> anyhow::Error { + match self.cli.stop(self.vmid).await { + Ok(()) => error.context(format!("{context}; container was stopped")), + Err(stop_error) => error.context(format!( + "{context} and failed to stop the exposed container: {stop_error:#}" + )), + } + } +} + +impl Drop for PveProcessHandle { + fn drop(&mut self) { + self.status_task.abort(); + self.stats_task.abort(); + self.console_task.abort(); + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = match tokio::fs::File::open(&self.log_path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Box::new(Cursor::new(Vec::new()))); + } + Err(error) => return Err(error.into()), + }; + return Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)); + } + + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + + async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { + self.stdin_tx + .send(Self::normalize_console_input(data)) + .await + .map_err(Into::into) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let resources = { + let configuration = server.configuration.read().await; + PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )? + }; + self.cli.set_resources(self.vmid, &resources).await?; + self.cli + .apply_runtime_config( + &Self::runtime_config(&server, &self.app_config, &self.cli, self.vmid).await?, + ) + .await?; + if let Err(error) = self.sync_firewall(true).await { + let configuration = server.configuration.read().await; + let requires_network_policy = !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()); + drop(configuration); + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to synchronize Proxmox LXC network policy after configuration update", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state after configuration update: {error:#}" + ); + } + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + + if let Err(error) = self.sync_firewall(true).await { + let requires_network_policy = match self.get_server() { + Ok(server) => { + let configuration = server.configuration.read().await; + !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()) + } + Err(_) => true, + }; + + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to apply Proxmox LXC network policy", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let process = server.process_configuration.read().await; + let stop_type = process.stop.r#type.clone(); + let stop_value = process.stop.value.clone(); + drop(process); + + if stop_type == "command" { + let mut command = stop_value + .map(|value| value.as_bytes().to_vec()) + .unwrap_or_default(); + command.push(b'\n'); + self.stdin_tx.send(command).await.map_err(Into::into) + } else { + self.cli.request_shutdown(self.vmid).await + } + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } +} + +struct PveHelperProcessHandle { + vmid: u32, + cli: PveCli, + log_path: PathBuf, + started: Arc, + cancelled: Arc, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, +} + +impl PveHelperProcessHandle { + #[allow(clippy::too_many_arguments)] + async fn new( + vmid: u32, + cli: PveCli, + server: &crate::server::Server, + app_config: Arc, + log_path: PathBuf, + status_tx: tokio::sync::mpsc::Sender, + already_started: bool, + start_at_end: bool, + auto_cleanup: bool, + cleanup_path: Option, + ) -> Result { + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + + let started = Arc::new(AtomicBool::new(already_started)); + let cancelled = Arc::new(AtomicBool::new(false)); + + let status_cli = cli.clone(); + let status_started = Arc::clone(&started); + let status_cancelled = Arc::clone(&cancelled); + let status_server = server.uuid; + tokio::spawn(async move { + let mut sent_running = false; + let mut tick = tokio::time::interval(std::time::Duration::from_millis(500)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + if status_cancelled.load(Ordering::Acquire) { + break; + } + + match status_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => { + if !sent_running { + sent_running = true; + if status_tx.send(super::ProcessStatus::Running).await.is_err() { + break; + } + } + } + Ok(cli::ContainerStatus::Stopped) if status_started.load(Ordering::Acquire) => { + if !sent_running + && status_tx.send(super::ProcessStatus::Running).await.is_err() + { + break; + } + let _ = status_tx + .send(super::ProcessStatus::Stopped { + exit_code: -1, + oom_killed: false, + }) + .await; + break; + } + Ok(cli::ContainerStatus::Stopped) => {} + Err(error) => { + tracing::warn!( + server = %status_server, + vmid, + "failed to read Proxmox helper LXC process state: {error:#}" + ); + } + } + } + }); + + let output_cli = cli.clone(); + let output_started = Arc::clone(&started); + let output_cancelled = Arc::clone(&cancelled); + let output_server = Arc::downgrade(&**server); + let output_log_path = log_path.clone(); + tokio::spawn(async move { + let mut file = match tokio::fs::OpenOptions::new() + .read(true) + .open(&output_log_path) + .await + { + Ok(file) => file, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox helper log: {error:#}"); + return; + } + }; + if start_at_end + && let Ok(metadata) = file.metadata().await + && let Err(error) = file.seek(std::io::SeekFrom::Start(metadata.len())).await + { + tracing::warn!(vmid, "failed to seek Proxmox helper log: {error:#}"); + } + + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + if output_cancelled.load(Ordering::Acquire) { + break; + } + + if let (Ok(position), Ok(metadata)) = + (file.stream_position().await, file.metadata().await) + && metadata.len() < position + { + if let Err(error) = file.seek(std::io::SeekFrom::Start(0)).await { + tracing::warn!( + vmid, + "failed to rewind truncated Proxmox helper log: {error:#}" + ); + } + line_buffer = LineBuffer::new(); + } + + let mut data = Vec::new(); + match file.read_to_end(&mut data).await { + Ok(_) => {} + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox helper log: {error:#}"); + } + } + + if !data.is_empty() { + line_buffer.extend(&data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = output_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + + let stopped = output_started.load(Ordering::Acquire) + && matches!( + output_cli.status(vmid).await, + Ok(cli::ContainerStatus::Stopped) + ); + if stopped { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let mut final_data = Vec::new(); + if file.read_to_end(&mut final_data).await.is_ok() && !final_data.is_empty() { + line_buffer.extend(&final_data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + if let Some(line) = line_buffer.flush() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + break; + } + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + if auto_cleanup { + output_cancelled.store(true, Ordering::Release); + if let Err(error) = output_cli.destroy(vmid).await { + tracing::warn!( + vmid, + "failed to destroy completed Proxmox helper LXC: {error:#}" + ); + } + if let Some(path) = cleanup_path + && let Err(error) = tokio::fs::remove_dir_all(&path).await + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + path = %path.display(), + "failed to remove Proxmox helper staging directory: {error:#}" + ); + } + } + }); + + Ok(Self { + vmid, + cli, + log_path, + started, + cancelled, + stdout_ratelimited_rx, + stdout_rx, + }) + } +} + +impl Drop for PveHelperProcessHandle { + fn drop(&mut self) { + if !self.started.load(Ordering::Acquire) { + self.cancelled.store(true, Ordering::Release); + } + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveHelperProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = tokio::fs::File::open(&self.log_path).await?; + Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)) + } else { + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + } + + async fn send_stdin(&self, _data: Vec) -> Result<(), anyhow::Error> { + Err(anyhow::anyhow!( + "stdin is not supported for Proxmox helper containers" + )) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PveVersion { + major: u64, + minor: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ServerContainerPlan { + Reuse { + vmid: u32, + stale_replacement: Option, + }, + ReplaceWithStaged { + old_vmid: u32, + staged_vmid: u32, + }, + ReplaceFresh { + old_vmid: u32, + stale_replacement: Option, + }, + RecoverStaged { + staged_vmid: u32, + }, + CreateFresh { + stale_replacement: Option, + }, +} + +impl PveVersion { + fn parse(value: &str) -> Result { + let (_, version_and_rest) = value + .trim() + .split_once('/') + .context("pveversion output did not include a version")?; + let version = version_and_rest + .split('/') + .next() + .context("pveversion output did not include a version number")?; + let mut parts = version.split('.'); + let major = parts + .next() + .context("pveversion output did not include a major version")? + .parse() + .context("failed to parse Proxmox major version")?; + let minor = parts + .next() + .context("pveversion output did not include a minor version")? + .parse() + .context("failed to parse Proxmox minor version")?; + + Ok(Self { major, minor }) + } + + fn is_supported(self) -> bool { + self.major > 9 || (self.major == 9 && self.minor >= 2) + } +} + +pub struct PveLxcExecutor { + cli: PveCli, + app_config: Arc, + firewall: Arc, + stats_sampler: Arc, + node: OnceLock, + template_provisioning: tokio::sync::Mutex<()>, +} + +impl PveLxcExecutor { + fn image_cache_max_age(config: &crate::config::InnerConfig) -> std::time::Duration { + let cache = config.docker.registry_image_fetch_cache; + if cache.enabled { + std::time::Duration::from_secs(cache.duration) + } else { + std::time::Duration::ZERO + } + } + + fn network_option( + bridge: &str, + vlan_tag: Option, + address: Option<(Ipv4Addr, u8, Ipv4Addr)>, + ) -> Result { + if bridge.is_empty() + || !bridge + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) + { + return Err(anyhow::anyhow!("invalid Proxmox LXC bridge: {bridge}")); + } + if let Some(tag) = vlan_tag + && !(1..=4094).contains(&tag) + { + return Err(anyhow::anyhow!( + "Proxmox LXC VLAN tag must be between 1 and 4094, got {tag}" + )); + } + + let mut options = vec![ + format!("name=eth0,bridge={bridge}"), + "firewall=1".to_string(), + ]; + if let Some(tag) = vlan_tag { + options.push(format!("tag={tag}")); + } + match address { + Some((ip, prefix, gateway)) => { + if prefix > 32 { + return Err(anyhow::anyhow!( + "Proxmox LXC network prefix must be between 0 and 32, got {prefix}" + )); + } + options.push(format!("ip={ip}/{prefix}")); + options.push(format!("gw={gateway}")); + } + None => options.push("ip=dhcp".to_string()), + } + options.push("type=veth".to_string()); + if address.is_none() { + options.push("host-managed=1".to_string()); + } + Ok(options.join(",")) + } + + fn dhcp_network(bridge: &str, vlan_tag: Option) -> Result { + Self::network_option(bridge, vlan_tag, None) + } + + fn network_for_allocation( + bridge: &str, + vlan_tag: Option, + network_prefix: Option, + gateway: Option<&str>, + allocation: Option<&str>, + ) -> Result { + let Some(allocation) = allocation else { + return Self::dhcp_network(bridge, vlan_tag); + }; + let allocation = allocation.parse::().with_context(|| { + format!("LXC primary allocation {allocation} is not an IPv4 address") + })?; + let prefix = network_prefix.context( + "Proxmox LXC static allocations require runtime.pve_lxc.network_prefix; configure the isolated LXC subnet before assigning panel allocations", + )?; + let gateway = gateway + .filter(|gateway| !gateway.trim().is_empty()) + .context( + "Proxmox LXC static allocations require runtime.pve_lxc.gateway; configure the isolated LXC subnet before assigning panel allocations", + )? + .parse::() + .context("Proxmox LXC gateway is not an IPv4 address")?; + Self::network_option(bridge, vlan_tag, Some((allocation, prefix, gateway))) + } + + async fn rootfs_storage( + &self, + node: &str, + configured_storage: String, + rootfs_size_gib: u64, + ) -> Result { + if !configured_storage.is_empty() && configured_storage != "auto" { + return Ok(configured_storage); + } + + let required_bytes = rootfs_size_gib + .checked_mul(1024 * 1024 * 1024) + .context("Proxmox LXC rootfs size overflowed u64")?; + self.cli + .available_lxc_storages(node) + .await? + .into_iter() + .find(|storage| storage.available_bytes >= required_bytes) + .map(|storage| storage.name) + .context(format!( + "no active Proxmox LXC storage has {} GiB free for the requested rootfs", + rootfs_size_gib + )) + } + + const INSTALLER_HELPER_ROLE: &'static str = "installer"; + const SCRIPT_HELPER_ROLE: &'static str = "script"; + + pub fn new( + app_config: Arc, + cli: PveCli, + firewall: Arc, + ) -> Self { + Self { + cli, + app_config, + firewall, + stats_sampler: Arc::new(super::cgroup::StatsSampler::default()), + node: OnceLock::new(), + template_provisioning: tokio::sync::Mutex::new(()), + } + } + + #[cfg(not(target_os = "linux"))] + fn unsupported(operation: &str) -> Result { + Err(anyhow::anyhow!( + "Proxmox VE LXC runtime does not implement {operation} yet" + )) + } + + fn runtime_node(configured_node: &str, local_node: &str) -> Result { + if configured_node.trim().is_empty() { + return Ok(local_node.to_string()); + } + if configured_node != local_node { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime must target the local node ({local_node}); configured node was {configured_node}" + )); + } + + Ok(configured_node.to_string()) + } + + fn vmid_conflict(error: &anyhow::Error) -> bool { + let message = error.to_string().to_ascii_lowercase(); + message.contains("already exists") || message.contains("already in use") + } + + fn parse_proc_net_address(raw: &str, family: ProcNetFamily) -> Result { + match family { + ProcNetFamily::Ipv4 => { + if raw.len() != 8 { + return Err(anyhow::anyhow!( + "invalid IPv4 address in Linux socket table: {raw}" + )); + } + let word = u32::from_str_radix(raw, 16) + .context("failed to parse IPv4 address from Linux socket table")?; + Ok(std::net::Ipv4Addr::from(word.to_le_bytes()).into()) + } + ProcNetFamily::Ipv6 => { + if raw.len() != 32 { + return Err(anyhow::anyhow!( + "invalid IPv6 address in Linux socket table: {raw}" + )); + } + + let mut bytes = [0u8; 16]; + for (index, chunk) in raw.as_bytes().chunks_exact(8).enumerate() { + let chunk = std::str::from_utf8(chunk) + .context("Linux IPv6 socket table contained non-UTF8 data")?; + let word = u32::from_str_radix(chunk, 16) + .context("failed to parse IPv6 address from Linux socket table")?; + bytes + .get_mut(index * 4..index * 4 + 4) + .context("IPv6 socket-table word exceeded its destination buffer")? + .copy_from_slice(&word.to_le_bytes()); + } + Ok(std::net::Ipv6Addr::from(bytes).into()) + } + } + } + + fn parse_proc_net_ports( + contents: &str, + family: ProcNetFamily, + tcp: bool, + ) -> Result, anyhow::Error> { + let mut ports = Vec::new(); + + for line in contents.lines().skip(1) { + let fields = line.split_whitespace().collect::>(); + if fields.len() < 4 { + continue; + } + let state = fields + .get(3) + .context("Linux socket table row did not include a state")?; + if tcp && *state != "0A" { + continue; + } + + let endpoint = fields + .get(1) + .context("Linux socket table row did not include a local endpoint")?; + let (raw_address, raw_port) = endpoint + .rsplit_once(':') + .context("Linux socket table local address did not include a port")?; + let port = u16::from_str_radix(raw_port, 16) + .context("failed to parse port from Linux socket table")?; + if port == 0 { + continue; + } + + let address = Self::parse_proc_net_address(raw_address, family)?; + ports.push(BoundPort { + family, + address: (!address.is_unspecified()).then_some(address), + port, + }); + } + + Ok(ports) + } + + async fn container_bound_ports(pid: i64) -> Result, anyhow::Error> { + if pid <= 0 { + return Err(anyhow::anyhow!( + "Proxmox LXC runtime returned invalid PID {pid}" + )); + } + + let tables = [ + ("tcp", ProcNetFamily::Ipv4, true), + ("tcp6", ProcNetFamily::Ipv6, true), + ("udp", ProcNetFamily::Ipv4, false), + ("udp6", ProcNetFamily::Ipv6, false), + ]; + let mut ports = Vec::new(); + let mut tables_read = 0usize; + + for (table, family, tcp) in tables { + let path = format!("/proc/{pid}/net/{table}"); + match tokio::fs::read_to_string(&path).await { + Ok(contents) => { + tables_read += 1; + ports.extend( + Self::parse_proc_net_ports(&contents, family, tcp).with_context(|| { + format!("failed to parse Proxmox LXC socket table {path}") + })?, + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("failed to read Proxmox LXC socket table {path}") + }); + } + } + } + + if tables_read == 0 { + return Err(anyhow::anyhow!( + "Proxmox LXC process {pid} disappeared before its socket tables could be read" + )); + } + + Ok(ports) + } + + fn bound_port_matches_address(bound: BoundPort, address: IpAddr) -> bool { + match bound.address { + None => matches!( + (bound.family, address), + (ProcNetFamily::Ipv4, IpAddr::V4(_)) | (ProcNetFamily::Ipv6, IpAddr::V6(_)) + ), + Some(IpAddr::V6(bound_v6)) => { + bound_v6 + .to_ipv4_mapped() + .map(IpAddr::V4) + .is_some_and(|mapped| mapped == address) + || IpAddr::V6(bound_v6) == address + } + Some(bound_address) => bound_address == address, + } + } + + fn server_tag(prefix: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-server-{uuid}") + } + + fn replacement_tag(prefix: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-repl-{uuid}") + } + + fn helper_tag(prefix: &str, role: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-helper-{role}-{uuid}") + } + + fn server_from_tags(prefix: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let server_tag_prefix = format!("{prefix}-server-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&server_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn replacement_from_tags(prefix: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let replacement_tag_prefix = format!("{prefix}-repl-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&replacement_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn has_image_tag(container: &cli::ClusterContainer, image_tag: &str) -> bool { + container.tags.iter().any(|tag| tag == image_tag) + } + + fn plan_server_container( + existing: Option<&cli::ClusterContainer>, + replacement: Option<&cli::ClusterContainer>, + image_tag: &str, + ) -> ServerContainerPlan { + match (existing, replacement) { + (Some(existing), replacement) if Self::has_image_tag(existing, image_tag) => { + ServerContainerPlan::Reuse { + vmid: existing.vmid, + stale_replacement: replacement.map(|container| container.vmid), + } + } + (Some(existing), Some(replacement)) if Self::has_image_tag(replacement, image_tag) => { + ServerContainerPlan::ReplaceWithStaged { + old_vmid: existing.vmid, + staged_vmid: replacement.vmid, + } + } + (Some(existing), replacement) => ServerContainerPlan::ReplaceFresh { + old_vmid: existing.vmid, + stale_replacement: replacement.map(|container| container.vmid), + }, + (None, Some(replacement)) if Self::has_image_tag(replacement, image_tag) => { + ServerContainerPlan::RecoverStaged { + staged_vmid: replacement.vmid, + } + } + (None, replacement) => ServerContainerPlan::CreateFresh { + stale_replacement: replacement.map(|container| container.vmid), + }, + } + } + + fn helper_from_tags(prefix: &str, role: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let helper_tag_prefix = format!("{prefix}-helper-{role}-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&helper_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn owned_server_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::server_from_tags(prefix, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple Proxmox LXC containers claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + fn owned_replacement_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::replacement_from_tags(prefix, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple staged Proxmox LXC replacements claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + async fn ensure_container_stopped( + &self, + container: &cli::ClusterContainer, + purpose: &str, + ) -> Result<(), anyhow::Error> { + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + return Err(anyhow::anyhow!( + "Proxmox LXC container {} must be stopped before {purpose}", + container.vmid + )); + } + Ok(()) + } + + async fn remove_staged_replacement( + &self, + replacement: &cli::ClusterContainer, + server_uuid: uuid::Uuid, + ) -> Result<(), anyhow::Error> { + self.ensure_container_stopped(replacement, "discarding a staged image replacement") + .await?; + self.cli.destroy(replacement.vmid).await.with_context(|| { + format!( + "failed to remove stale staged Proxmox LXC replacement {} for server {server_uuid}", + replacement.vmid + ) + }) + } + + fn owned_helper_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + role: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::helper_from_tags(prefix, role, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple Proxmox LXC {role} helpers claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + async fn server_container( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + Self::owned_server_container( + &self.cli.list_containers().await?, + node, + &tag_prefix, + server_uuid, + ) + } + + async fn helper_container( + &self, + server_uuid: uuid::Uuid, + role: &str, + ) -> Result, anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + Self::owned_helper_container( + &self.cli.list_containers().await?, + node, + &tag_prefix, + role, + server_uuid, + ) + } + + async fn server_address( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let Some(container) = self.server_container(server_uuid).await? else { + return Ok(None); + }; + if container.status != Some(cli::ContainerStatus::Running) { + return Ok(None); + } + + let interfaces = self.cli.interfaces(&container.node, container.vmid).await?; + Ok(PveCli::primary_container_address(&interfaces)) + } + + async fn server_addresses( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let Some(container) = self.server_container(server_uuid).await? else { + return Ok(Vec::new()); + }; + if container.status != Some(cli::ContainerStatus::Running) { + return Ok(Vec::new()); + } + + let interfaces = self.cli.interfaces(&container.node, container.vmid).await?; + Ok(PveCli::container_addresses(&interfaces)) + } + + fn direct_bridge_firewall_spec( + server: uuid::Uuid, + mappings: &HashMap>, + rules: &[crate::server::firewall::FirewallRule], + mut container_ips: Vec, + vmid: Option, + ) -> crate::server::firewall::FirewallServerSpec { + let mut ports = BTreeSet::new(); + for allocation_ports in mappings.values() { + ports.extend(allocation_ports.iter().copied()); + } + container_ips.sort_unstable(); + container_ips.dedup(); + + crate::server::firewall::FirewallServerSpec { + server, + target: vmid.map(|vmid| crate::server::firewall::FirewallTarget::ProxmoxLxc { + vmid, + interface: "net0".to_string(), + }), + // Bridged PVE containers do not publish Docker-style host bindings. + bindings: Vec::new(), + container_ports: ports.into_iter().collect(), + container_ips, + rules: rules.to_vec(), + files: None, + } + } + + fn firewall_spec( + configuration: &crate::server::configuration::ServerConfiguration, + container_ips: Vec, + vmid: Option, + ) -> crate::server::firewall::FirewallServerSpec { + Self::direct_bridge_firewall_spec( + configuration.uuid, + &configuration.allocations.mappings, + &configuration.firewall, + container_ips, + vmid, + ) + } + + fn edge_forward_payload( + configuration: &crate::server::configuration::ServerConfiguration, + destination: Ipv4Addr, + ) -> Result, anyhow::Error> { + let Some(allocation) = configuration.allocations.default.as_ref() else { + return Ok(None); + }; + let public_ip = allocation + .ip + .to_string() + .parse::() + .with_context(|| { + format!( + "Proxmox edge forwarding requires an IPv4 primary allocation, got {}", + allocation.ip + ) + })?; + let ports = configuration + .allocations + .mappings + .get(&allocation.ip.to_compact_string()) + .cloned() + .unwrap_or_default(); + Ok(Some(( + public_ip, + serde_json::json!({ + "ip": destination.to_string(), + // Panel allocations are protocol-neutral. Publish both transports so + // TCP games and UDP games preserve the same allocation contract. + "tcp": ports, + "udp": ports, + }), + ))) + } + + fn edge_forwarding_config( + app_config: &crate::config::Config, + ) -> Result, anyhow::Error> { + let config = app_config.load(); + let pve = &config.runtime.pve_lxc; + let interface = pve.edge_wireguard_interface.trim().to_string(); + let identity_path = pve.edge_ssh_identity_path.trim().to_string(); + let known_hosts_path = pve.edge_known_hosts_path.trim().to_string(); + if identity_path.is_empty() && known_hosts_path.is_empty() { + return Ok(None); + } + if interface.is_empty() || identity_path.is_empty() || known_hosts_path.is_empty() { + return Err(anyhow::anyhow!( + "Proxmox edge forwarding requires edge_wireguard_interface, edge_ssh_identity_path, and edge_known_hosts_path" + )); + } + Ok(Some((interface, identity_path, known_hosts_path))) + } + + async fn validate_edge_forwarding_config( + app_config: &crate::config::Config, + ) -> Result<(), anyhow::Error> { + let Some((_, identity_path, _)) = Self::edge_forwarding_config(app_config)? else { + return Ok(()); + }; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let metadata = tokio::fs::metadata(&identity_path).await.with_context(|| { + format!("failed to inspect Proxmox edge SSH identity {identity_path}") + })?; + if !metadata.is_file() { + anyhow::bail!("Proxmox edge SSH identity is not a regular file: {identity_path}"); + } + if metadata.uid() != 0 || metadata.permissions().mode() & 0o077 != 0 { + anyhow::bail!( + "Proxmox edge SSH identity must be owned by root and inaccessible to group and other users: {identity_path}" + ); + } + } + Ok(()) + } + + async fn edge_peers(interface: &str) -> Result, anyhow::Error> { + let mut command = tokio::process::Command::new("/usr/bin/wg"); + command.args(["show", interface, "dump"]).kill_on_drop(true); + let output = tokio::time::timeout(std::time::Duration::from_secs(5), command.output()) + .await + .context("timed out inspecting WireGuard peers for Proxmox edge forwarding")? + .context("failed to inspect WireGuard peers for Proxmox edge forwarding")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to inspect WireGuard interface {interface}: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + + let dump = String::from_utf8(output.stdout) + .context("WireGuard peer output was not valid UTF-8")?; + let mut peers = Vec::new(); + for line in dump.lines().skip(1) { + let fields = line.split('\t').collect::>(); + let Some([_, _, endpoint, allowed_ips, ..]) = fields.get(..) else { + continue; + }; + if *endpoint == "(none)" { + continue; + } + let endpoint = endpoint + .parse::() + .with_context(|| format!("WireGuard peer endpoint was invalid: {endpoint}"))?; + let IpAddr::V4(public_ip) = endpoint.ip() else { + tracing::warn!(endpoint = %endpoint, "ignoring IPv6 WireGuard edge endpoint because panel allocations require IPv4"); + continue; + }; + let Some(tunnel_ip) = allowed_ips + .split(',') + .filter_map(|cidr| cidr.split_once('/')) + .find_map(|(ip, prefix)| { + (prefix == "32") + .then(|| ip.parse::().ok()) + .flatten() + }) + else { + continue; + }; + peers.push(EdgePeer { + public_ip, + tunnel_ip, + }); + } + peers.sort_unstable_by_key(|peer| (peer.public_ip, peer.tunnel_ip)); + peers.dedup(); + Ok(peers) + } + + async fn edge_ssh( + host: Ipv4Addr, + identity_path: &str, + known_hosts_path: &str, + server: uuid::Uuid, + payload: Option, + ) -> Result<(), anyhow::Error> { + let action = if payload.is_some() { "sync" } else { "clear" }; + let mut command = tokio::process::Command::new("/usr/bin/ssh"); + command + .args([ + "-i", + identity_path, + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=5", + "-o", + "StrictHostKeyChecking=yes", + "-o", + &format!("UserKnownHostsFile={known_hosts_path}"), + &format!("root@{host}"), + action, + &server.to_string(), + ]) + .stdin(if payload.is_some() { + std::process::Stdio::piped() + } else { + std::process::Stdio::null() + }) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let mut child = command + .spawn() + .context("failed to start Proxmox edge forwarding SSH command")?; + if let Some(payload) = payload { + let bytes = serde_json::to_vec(&payload) + .context("failed to serialize Proxmox edge forwarding payload")?; + child + .stdin + .as_mut() + .context("Proxmox edge forwarding SSH stdin was unavailable")? + .write_all(&bytes) + .await?; + } + let output = + tokio::time::timeout(std::time::Duration::from_secs(15), child.wait_with_output()) + .await + .context("timed out synchronizing Proxmox edge forwarding")? + .context("failed while synchronizing Proxmox edge forwarding")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "Proxmox edge forwarding {action} for {server} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(()) + } + + async fn sync_edge_forwarding( + app_config: &crate::config::Config, + server: uuid::Uuid, + public_ip: Option, + payload: Option, + ) -> Result<(), anyhow::Error> { + let Some((interface, identity_path, known_hosts_path)) = + Self::edge_forwarding_config(app_config)? + else { + return Ok(()); + }; + let peers = Self::edge_peers(&interface).await?; + for peer in &peers { + Self::edge_ssh( + peer.tunnel_ip, + &identity_path, + &known_hosts_path, + server, + None, + ) + .await?; + } + if let Some(payload) = payload { + let public_ip = public_ip.context( + "Proxmox edge forwarding payload did not include a public allocation IP", + )?; + let peer = peers + .iter() + .find(|peer| peer.public_ip == public_ip) + .with_context(|| { + format!("no WireGuard peer on {interface} has endpoint address {public_ip}") + })?; + Self::edge_ssh( + peer.tunnel_ip, + &identity_path, + &known_hosts_path, + server, + Some(payload), + ) + .await?; + } + Ok(()) + } + + fn firewall_file_access( + server: &Arc, + ) -> crate::server::firewall::sets::FirewallFileAccess { + crate::server::firewall::sets::FirewallFileAccess { + filesystem: (*server.filesystem).clone(), + notifier: Some(server.filesystem.server_notifier().clone()), + server: Some(Arc::downgrade(server)), + } + } + + async fn sync_server_firewall( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + let container = self.server_container(server.uuid).await?; + let addresses = self.server_addresses(server.uuid).await?; + let (has_rules, mut spec) = { + let configuration = server.configuration.read().await; + ( + !configuration.firewall.is_empty(), + Self::firewall_spec( + &configuration, + addresses, + container.as_ref().map(|container| container.vmid), + ), + ) + }; + if has_rules && spec.container_ips.is_empty() { + let running = self + .server_container(server.uuid) + .await? + .is_some_and(|container| container.status == Some(cli::ContainerStatus::Running)); + if running { + return Err(anyhow::anyhow!( + "running Proxmox LXC server {} has no usable bridged address; refusing to leave configured firewall rules unapplied", + server.uuid + )); + } + } + spec.files = Some(Self::firewall_file_access(server)); + self.firewall.sync(&spec).await + } + + fn helper_resources( + configuration: &crate::server::configuration::ServerConfiguration, + app_config: &crate::config::Config, + ) -> Result { + let mut resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + let config = app_config.load(); + let limits = &config.docker.installer_limits; + + resources.memory_mib = resources.memory_mib.max(limits.memory.as_mib()); + if let Some(cpu_limit_percent) = resources.cpu_limit_percent + && cpu_limit_percent < limits.cpu + { + resources.cpu_limit_percent = Some(limits.cpu); + } + + Ok(resources) + } + + async fn server_mount_specs( + &self, + server: &crate::server::Server, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = server; + return Self::unsupported("bind mounts on this host platform"); + } + + #[cfg(unix)] + { + let (server_uuid, hugepages_passthrough_enabled, configured_mounts) = { + let configuration = server.configuration.read().await; + ( + configuration.uuid, + configuration.container.hugepages_passthrough_enabled, + configuration.mounts.clone(), + ) + }; + let (host_data_uid, host_data_gid) = { + let config = self.app_config.load(); + (config.system.user.uid, config.system.user.gid) + }; + + let mut mounts = Vec::with_capacity(configured_mounts.len() + 2); + mounts.push(( + server.filesystem.get_base_fs_mount_path().await, + "/home/container".to_string(), + false, + )); + + if hugepages_passthrough_enabled { + mounts.push(( + PathBuf::from("/dev/hugepages"), + "/dev/hugepages".to_string(), + false, + )); + } + + if !configured_mounts.is_empty() { + let allowed = + crate::server::configuration::AllowedMounts::load(&self.app_config).await; + for mount in configured_mounts { + let source = match mount.resolve_allowed_source(&allowed).await { + Ok(source) => source, + Err(error) => { + tracing::warn!( + server = %server_uuid, + "not mounting {} -> {} in Proxmox LXC: {error:#}", + mount.source, + mount.target, + ); + continue; + } + }; + mounts.push((source, mount.target.to_string(), mount.read_only)); + } + } + + let mut specs = Vec::with_capacity(mounts.len()); + for (slot, (source, target, read_only)) in mounts.into_iter().enumerate() { + let slot = u8::try_from(slot).map_err(|_| { + anyhow::anyhow!("Proxmox LXC supports at most 256 bind mount slots") + })?; + let source = tokio::fs::canonicalize(&source).await.with_context(|| { + format!( + "failed to resolve Proxmox bind mount source {}", + source.display() + ) + })?; + let metadata = tokio::fs::metadata(&source).await.with_context(|| { + format!( + "failed to inspect Proxmox bind mount source {}", + source.display() + ) + })?; + if !metadata.is_dir() { + return Err(anyhow::anyhow!( + "Proxmox LXC native mpN bind mounts require a directory source; {} -> {target} is not a directory", + source.display() + )); + } + let source_path = source + .into_os_string() + .into_string() + .map_err(|_| anyhow::anyhow!("Proxmox bind mount source is not valid UTF-8"))?; + + specs.push(cli::HostBindMountSpec { + slot, + source_path, + target_path: target, + read_only, + host_uid: host_data_uid, + host_gid: host_data_gid, + }); + } + + Ok(specs) + } + } + + fn device_deny_write(permissions: &str, path: &std::path::Path) -> Result { + let readable = permissions.contains('r'); + let writable = permissions.contains('w'); + let invalid = permissions + .chars() + .any(|permission| !matches!(permission, 'r' | 'w' | 'm')); + + if !readable || invalid { + return Err(anyhow::anyhow!( + "Proxmox LXC devN passthrough for {} requires read access and accepts r, w, and m permission flags; requested {permissions}", + path.display() + )); + } + + Ok(!writable) + } + + async fn server_device_specs( + &self, + server: &crate::server::Server, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = server; + return Self::unsupported("device passthrough on this host platform"); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + + let (server_uuid, kvm_passthrough_enabled, configured_devices) = { + let configuration = server.configuration.read().await; + ( + configuration.uuid, + configuration.container.kvm_passthrough_enabled, + configuration.devices.clone(), + ) + }; + + let mut devices = Vec::<(PathBuf, String)>::with_capacity( + configured_devices.len() + usize::from(kvm_passthrough_enabled), + ); + if kvm_passthrough_enabled { + devices.push((PathBuf::from("/dev/kvm"), "rw".to_string())); + } + + if !configured_devices.is_empty() { + let allowed = + crate::server::configuration::AllowedDevices::load(&self.app_config).await; + for device in configured_devices { + let source = match device.resolve_allowed_source(&allowed).await { + Ok(source) => source, + Err(error) => { + tracing::warn!( + server = %server_uuid, + "not passing device {} -> {} into Proxmox LXC: {error:#}", + device.source, + device.target, + ); + continue; + } + }; + let source_string = source.to_string_lossy(); + if device.target.as_str() != source_string.as_ref() { + return Err(anyhow::anyhow!( + "Proxmox LXC devN passthrough cannot remap device {} to {}; source and target must be identical", + source.display(), + device.target + )); + } + devices.push((source, device.permissions.to_string())); + } + } + + let mut specs = Vec::with_capacity(devices.len()); + for (slot, (path, permissions)) in devices.into_iter().enumerate() { + let slot = u8::try_from(slot).map_err(|_| { + anyhow::anyhow!("Proxmox LXC supports at most 256 device passthrough slots") + })?; + let deny_write = Self::device_deny_write(&permissions, &path)?; + + let metadata = tokio::fs::metadata(&path).await.with_context(|| { + format!( + "failed to inspect Proxmox device passthrough source {}", + path.display() + ) + })?; + let file_type = metadata.file_type(); + if !file_type.is_char_device() && !file_type.is_block_device() { + return Err(anyhow::anyhow!( + "Proxmox device passthrough source {} is not a character or block device", + path.display() + )); + } + let path = path + .into_os_string() + .into_string() + .map_err(|_| anyhow::anyhow!("Proxmox device path is not valid UTF-8"))?; + + specs.push(cli::HostDevicePassthroughSpec { + slot, + path, + uid: metadata.uid(), + gid: metadata.gid(), + mode: metadata.mode() & 0o777, + deny_write, + }); + } + + Ok(specs) + } + } + + #[allow(clippy::too_many_arguments)] + async fn create_helper_container( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + role: &str, + vmid: u32, + resources: cli::ContainerResources, + environment: Vec, + staging_path: &std::path::Path, + staging_target: &str, + script_target: &str, + log_path: &std::path::Path, + ) -> Result<(), anyhow::Error> { + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + let image = PveCli::normalize_oci_reference(script.container_image.trim_end_matches('~'))?; + let ( + template_storage, + rootfs_storage, + bridge, + vlan_tag, + rootfs_size_gib, + tag_prefix, + unprivileged, + host_data_uid, + host_data_gid, + image_cache_max_age, + ) = { + let config = self.app_config.load(); + let pve = &config.runtime.pve_lxc; + ( + pve.template_storage.clone(), + pve.rootfs_storage.clone(), + pve.bridge.clone(), + pve.vlan_tag, + pve.rootfs_size_gib, + pve.tag_prefix.clone(), + pve.unprivileged, + config.system.user.uid, + config.system.user.gid, + Self::image_cache_max_age(&config), + ) + }; + + let template = { + let _template = self.template_provisioning.lock().await; + self.cli + .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) + .await? + }; + let image_tag = PveCli::oci_revision_tag(&template.revision)?; + let rootfs_storage = self + .rootfs_storage(&node, rootfs_storage, rootfs_size_gib) + .await?; + let network = Self::dhcp_network(&bridge, vlan_tag)?; + let staging_path_utf8 = staging_path + .to_str() + .context("Proxmox helper staging path is not valid UTF-8")?; + let log_path = log_path + .to_str() + .context("Proxmox helper log path is not valid UTF-8")?; + let create = cli::CreateContainerSpec { + vmid, + template: template.volid, + hostname: format!("{role}-{vmid}"), + rootfs_storage, + rootfs_size_gib, + memory_mib: resources.memory_mib, + swap_mib: resources.swap_mib, + cpu_limit_percent: resources.cpu_limit_percent, + cores: resources.cores, + network, + tags: vec![ + tag_prefix.clone(), + Self::helper_tag(&tag_prefix, role, server.uuid), + image_tag, + ], + unprivileged, + }; + let mounts = [ + cli::HostBindMountSpec { + slot: 0, + source_path: server.filesystem.base().to_string(), + target_path: "/mnt/server".to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }, + cli::HostBindMountSpec { + slot: 1, + source_path: staging_path_utf8.to_string(), + target_path: staging_target.to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }, + ]; + + let helper_wrapper_name = "calagopus-helper-entrypoint"; + let helper_wrapper_path = staging_path.join(helper_wrapper_name); + tokio::fs::write( + &helper_wrapper_path, + r#"#!/bin/sh +attempt=0 +while [ "$attempt" -lt 120 ]; do + if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then + "$@" + status=$? + if [ -n "${INSTALL_STATUS_FILE:-}" ]; then + if [ "$status" -eq 0 ]; then + : > "$INSTALL_STATUS_FILE" + else + printf 'installer exited with status %s\n' "$status" > "$INSTALL_STATUS_FILE" + fi + fi + exit "$status" + fi + attempt=$((attempt + 1)) + sleep 0.25 +done +echo 'timed out waiting for Proxmox DHCP' >&2 +if [ -n "${INSTALL_STATUS_FILE:-}" ]; then + printf '%s\n' 'timed out waiting for Proxmox DHCP' > "$INSTALL_STATUS_FILE" +fi +exit 1 +"#, + ) + .await + .context("failed to write Proxmox helper entrypoint wrapper")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions( + &helper_wrapper_path, + std::fs::Permissions::from_mode(0o755), + ) + .await + .context("failed to make Proxmox helper entrypoint wrapper executable")?; + } + + self.cli.create_with_bind_mounts(&create, &mounts).await?; + // PVE applies a host-managed DHCP lease immediately after CT init starts. + // Keep PID 1 alive until that lease installs a default route; otherwise a + // fast installer can attempt its first download and exit before `pct` + // has a chance to run its DHCP hook. + let init_command = PveCli::encode_init_command(&[ + format!("{staging_target}/{helper_wrapper_name}"), + script.entrypoint.to_string(), + script_target.to_string(), + ])?; + let runtime_config = cli::RuntimeConfigSpec { + vmid, + environment, + halt_signal: None, + init_command: Some(init_command), + console_logfile: Some(log_path.to_string()), + managed_file_mounts: None, + }; + + if let Err(error) = self.cli.apply_runtime_config(&runtime_config).await { + if let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox helper after runtime configuration failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to configure Proxmox helper LXC")); + } + + Ok(()) + } + + async fn prepare_server_container( + &self, + server: &crate::server::Server, + ) -> Result<(u32, cli::ContainerOciUser, bool), anyhow::Error> { + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + + let (server_uuid, image, resources, primary_allocation) = { + let configuration = server.configuration.read().await; + let resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + + ( + configuration.uuid, + configuration + .container + .image + .trim_end_matches('~') + .to_string(), + resources, + configuration + .allocations + .default + .as_ref() + .map(|allocation| allocation.ip.to_string()), + ) + }; + let image = PveCli::normalize_oci_reference(&image)?; + let mounts = self.server_mount_specs(server).await?; + let devices = self.server_device_specs(server).await?; + + let ( + template_storage, + rootfs_storage, + bridge, + vlan_tag, + network_prefix, + gateway, + rootfs_size_gib, + tag_prefix, + unprivileged, + edge_forwarding, + image_cache_max_age, + ) = { + let config = self.app_config.load(); + let pve = &config.runtime.pve_lxc; + ( + pve.template_storage.clone(), + pve.rootfs_storage.clone(), + pve.bridge.clone(), + pve.vlan_tag, + pve.network_prefix, + pve.gateway.clone(), + pve.rootfs_size_gib, + pve.tag_prefix.clone(), + pve.unprivileged, + Self::edge_forwarding_config(&self.app_config)?.is_some(), + Self::image_cache_max_age(&config), + ) + }; + let network = if edge_forwarding { + Self::dhcp_network(&bridge, vlan_tag)? + } else { + Self::network_for_allocation( + &bridge, + vlan_tag, + network_prefix, + gateway.as_deref(), + primary_allocation.as_deref(), + )? + }; + + let template = { + let _template = self.template_provisioning.lock().await; + self.cli + .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) + .await? + }; + let image_tag = PveCli::oci_revision_tag(&template.revision)?; + let containers = self.cli.list_containers().await?; + let existing = Self::owned_server_container(&containers, &node, &tag_prefix, server_uuid)?; + let replacement = + Self::owned_replacement_container(&containers, &node, &tag_prefix, server_uuid)?; + let plan = Self::plan_server_container(existing.as_ref(), replacement.as_ref(), &image_tag); + + let final_tags = vec![ + tag_prefix.clone(), + Self::server_tag(&tag_prefix, server_uuid), + image_tag.clone(), + ]; + + match plan { + ServerContainerPlan::Reuse { + vmid, + stale_replacement, + } => { + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + + self.cli.set_resources(vmid, &resources).await?; + self.cli.set_network(vmid, &network).await?; + let user = self.cli.oci_user(vmid).await?; + self.cli + .configure_mounts_and_devices(vmid, user, &mounts, &devices) + .await?; + return Ok((vmid, user, false)); + } + ServerContainerPlan::RecoverStaged { staged_vmid } => { + let staged = replacement + .as_ref() + .filter(|container| container.vmid == staged_vmid) + .context("staged Proxmox replacement disappeared from the inventory plan")?; + self.ensure_container_stopped(staged, "recovering a staged image replacement") + .await?; + self.cli.set_resources(staged_vmid, &resources).await?; + self.cli.set_network(staged_vmid, &network).await?; + let user = self.cli.oci_user(staged_vmid).await?; + self.cli + .configure_mounts_and_devices(staged_vmid, user, &mounts, &devices) + .await?; + self.cli + .set_tags(staged_vmid, &final_tags) + .await + .with_context(|| { + format!( + "failed to promote staged Proxmox LXC replacement {staged_vmid} for server {server_uuid}" + ) + })?; + return Ok((staged_vmid, user, true)); + } + ServerContainerPlan::CreateFresh { stale_replacement } => { + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + } + ServerContainerPlan::ReplaceFresh { + old_vmid, + stale_replacement, + } => { + let old = existing + .as_ref() + .filter(|container| container.vmid == old_vmid) + .context("owned Proxmox container disappeared from the inventory plan")?; + self.ensure_container_stopped(old, "replacing its OCI image") + .await?; + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + } + ServerContainerPlan::ReplaceWithStaged { + old_vmid, + staged_vmid, + } => { + let old = existing + .as_ref() + .filter(|container| container.vmid == old_vmid) + .context("owned Proxmox container disappeared from the inventory plan")?; + let staged = replacement + .as_ref() + .filter(|container| container.vmid == staged_vmid) + .context("staged Proxmox replacement disappeared from the inventory plan")?; + self.ensure_container_stopped(old, "replacing its OCI image") + .await?; + self.ensure_container_stopped(staged, "promoting a staged image replacement") + .await?; + + self.cli.set_resources(staged_vmid, &resources).await?; + self.cli.set_network(staged_vmid, &network).await?; + let user = self.cli.oci_user(staged_vmid).await?; + self.cli + .configure_mounts_and_devices(staged_vmid, user, &mounts, &devices) + .await?; + + if let Err(error) = self.cli.destroy(old_vmid).await { + return Err(error.context(format!( + "failed to remove old Proxmox LXC container {old_vmid} while staged replacement {staged_vmid} remains recoverable" + ))); + } + self.cli + .set_tags(staged_vmid, &final_tags) + .await + .with_context(|| { + format!( + "old Proxmox LXC container {old_vmid} was removed, but failed to promote staged replacement {staged_vmid}; the staged replacement remains recoverable on the next setup" + ) + })?; + return Ok((staged_vmid, user, true)); + } + } + + let rootfs_storage = self + .rootfs_storage(&node, rootfs_storage, rootfs_size_gib) + .await?; + let replacing = matches!(plan, ServerContainerPlan::ReplaceFresh { .. }); + let create_tags = if replacing { + vec![ + tag_prefix.clone(), + Self::replacement_tag(&tag_prefix, server_uuid), + image_tag, + ] + } else { + final_tags.clone() + }; + let mut created = None; + for _ in 0..3 { + let vmid = self.cli.next_vmid().await?; + let create = cli::CreateContainerSpec { + vmid, + template: template.volid.clone(), + hostname: server_uuid.to_string(), + rootfs_storage: rootfs_storage.clone(), + rootfs_size_gib, + memory_mib: resources.memory_mib, + swap_mib: resources.swap_mib, + cpu_limit_percent: resources.cpu_limit_percent, + cores: resources.cores, + network: network.clone(), + tags: create_tags.clone(), + unprivileged, + }; + match self + .cli + .create_with_bind_mounts_and_devices(&create, &mounts, &devices) + .await + { + Ok(user) => { + created = Some((vmid, user)); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!(vmid, "retrying Proxmox LXC create after VMID collision"); + } + Err(error) => return Err(error), + } + } + let (vmid, user) = created + .context("failed to create Proxmox LXC after three VMID allocation collisions")?; + + if let ServerContainerPlan::ReplaceFresh { old_vmid, .. } = plan { + if let Err(error) = self.cli.destroy(old_vmid).await { + return match self.cli.destroy(vmid).await { + Ok(()) => Err(error.context(format!( + "failed to remove old Proxmox LXC container {old_vmid}; discarded prepared replacement {vmid}" + ))), + Err(cleanup_error) => Err(anyhow::anyhow!( + "failed to remove old Proxmox LXC container {old_vmid}: {error:#}; additionally failed to discard staged replacement {vmid}: {cleanup_error:#}" + )), + }; + } + + self.cli.set_tags(vmid, &final_tags).await.with_context(|| { + format!( + "old Proxmox LXC container {old_vmid} was removed, but failed to promote staged replacement {vmid}; the staged replacement remains recoverable on the next setup" + ) + })?; + } + + Ok((vmid, user, true)) + } +} + +#[async_trait::async_trait] +impl ServerExecutor for PveLxcExecutor { + async fn boot(&self) -> Result<(), anyhow::Error> { + if !cfg!(target_os = "linux") { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime requires a Linux host running Proxmox VE" + )); + } + if !self.app_config.load().runtime.pve_lxc.unprivileged { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime currently requires unprivileged containers because per-mount idmap is required for Wings-owned server data" + )); + } + Self::validate_edge_forwarding_config(&self.app_config).await?; + + let raw_version = self.cli.version().await?; + let version = PveVersion::parse(&raw_version)?; + if !version.is_supported() { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime requires Proxmox VE 9.2 or newer, found {raw_version}" + )); + } + + let configured_node = self.app_config.load().runtime.pve_lxc.node.clone(); + let local_node = self.cli.local_node().await?; + let node = Self::runtime_node(&configured_node, &local_node)?; + self.node + .set(node.clone()) + .map_err(|_| anyhow::anyhow!("Proxmox VE LXC runtime was booted more than once"))?; + + self.firewall + .boot() + .await + .context("failed to initialize Proxmox LXC host firewall")?; + + let next_vmid = self.cli.next_vmid().await?; + tracing::info!( + pve_version = %raw_version, + node = %node, + next_vmid, + "Proxmox VE LXC runtime is available" + ); + + Ok(()) + } + + async fn reconcile_firewall( + &self, + servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + let mut specs = Vec::with_capacity(servers.len()); + for server in servers { + let addresses = self + .server_addresses(server.settings.uuid) + .await + .with_context(|| { + format!( + "failed to resolve Proxmox LXC addresses while reconciling firewall for {}", + server.settings.uuid + ) + })?; + let container = self.server_container(server.settings.uuid).await?; + let mut spec = Self::firewall_spec( + &server.settings, + addresses, + container.as_ref().map(|container| container.vmid), + ); + + if !server.settings.firewall.is_empty() && spec.container_ips.is_empty() { + let running = self + .server_container(server.settings.uuid) + .await? + .is_some_and(|container| { + container.status == Some(cli::ContainerStatus::Running) + }); + if running { + return Err(anyhow::anyhow!( + "running Proxmox LXC server {} has no usable bridged address; refusing to reconcile configured firewall rules without a destination", + server.settings.uuid + )); + } + } + + if spec.references_files() { + match crate::server::filesystem::cap::CapFilesystem::new( + &self.app_config.data_path(spec.server), + ) + .await + { + Ok(filesystem) => { + spec.files = Some(crate::server::firewall::sets::FirewallFileAccess { + filesystem, + notifier: None, + server: None, + }); + } + Err(error) => { + tracing::warn!( + server = %spec.server, + "failed to open the server directory for its firewall source files: {error}" + ); + } + } + } + + specs.push(spec); + } + + self.firewall.reconcile(&specs).await + } + + async fn setup_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + if server.configuration.read().await.entrypoint.is_some() { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime does not support panel entrypoint overrides yet; use the OCI image entrypoint" + )); + } + + let (vmid, _oci_user, created) = self.prepare_server_container(server).await?; + let runtime_config = match PveProcessHandle::runtime_config( + server, + &self.app_config, + &self.cli, + vmid, + ) + .await + { + Ok(runtime_config) => runtime_config, + Err(error) => { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after managed file preparation failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to prepare managed Proxmox container files")); + } + }; + + if let Err(error) = self.cli.apply_runtime_config(&runtime_config).await { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after runtime configuration failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to apply Proxmox LXC runtime configuration")); + } + + if let Err(error) = self.sync_server_firewall(server).await { + let has_rules = !server.configuration.read().await.firewall.is_empty(); + if has_rules { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after firewall setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to prepare Proxmox LXC server firewall")); + } + + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + let handle = match PveProcessHandle::new( + vmid, + node, + self.cli.clone(), + Arc::clone(&self.stats_sampler), + server, + Arc::clone(&self.app_config), + Arc::clone(&self.firewall), + status_tx, + true, + true, + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after process setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to create Proxmox LXC process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn attach_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let container = self + .server_container(server.uuid) + .await? + .context("no Proxmox LXC container found for server")?; + if container.status != Some(cli::ContainerStatus::Running) { + return Err(anyhow::anyhow!( + "Proxmox LXC container {} for server {} is not running", + container.vmid, + server.uuid + )); + } + + // Reconcile runtime-owned files and mount entries when Wings adopts an + // already-running container. This covers features enabled after the + // container was created, including Tundra's managed /etc/hosts file. + // LXC applies a newly-added mount entry on the container's next start, + // while creating the backing file immediately lets Tundra reconcile + // its state without racing a missing host path. + self.cli + .apply_runtime_config( + &PveProcessHandle::runtime_config( + server, + &self.app_config, + &self.cli, + container.vmid, + ) + .await?, + ) + .await + .context("failed to reconcile an adopted Proxmox LXC runtime configuration")?; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = Arc::new( + PveProcessHandle::new( + container.vmid, + container.node.clone(), + self.cli.clone(), + Arc::clone(&self.stats_sampler), + server, + Arc::clone(&self.app_config), + Arc::clone(&self.firewall), + status_tx, + false, + false, + ) + .await?, + ); + + if let Err(error) = handle.sync_firewall(true).await { + let configuration = server.configuration.read().await; + let requires_network_policy = !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && Self::edge_forwarding_config(&self.app_config)?.is_some()); + drop(configuration); + if requires_network_policy { + return Err(handle + .stop_after_firewall_failure( + error, + "failed to attach Proxmox LXC network policy", + ) + .await); + } + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + Ok((handle, status_rx)) + } + + async fn cleanup_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + if let Err(error) = self.firewall.clear(server.uuid).await { + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall rules during cleanup: {error:#}" + ); + } + Self::sync_edge_forwarding(&self.app_config, server.uuid, None, None).await?; + + let Some(container) = self.server_container(server.uuid).await? else { + PveProcessHandle::cleanup_managed_file_staging(&self.app_config, server.uuid).await?; + return Ok(()); + }; + + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + self.cli.stop(container.vmid).await?; + } + self.cli.destroy(container.vmid).await?; + PveProcessHandle::cleanup_managed_file_staging(&self.app_config, server.uuid).await?; + + Ok(()) + } + + async fn setup_installation_process( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + if let Some(container) = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + { + return Err(anyhow::anyhow!( + "Proxmox installer helper already exists for server {} as VMID {}", + server.uuid, + container.vmid + )); + } + + let (resources, mut environment, host_data_uid, host_data_gid) = { + let configuration = server.configuration.read().await; + let config = self.app_config.load(); + ( + Self::helper_resources(&configuration, &self.app_config)?, + configuration.environment(&self.app_config), + config.system.user.uid, + config.system.user.gid, + ) + }; + for (key, value) in &script.environment { + environment.push(format!( + "{key}={}", + match value { + serde_json::Value::String(value) => value.clone(), + other => other.to_string(), + } + )); + } + environment.push(format!( + "INSTALL_STATUS_FILE=/mnt/install/{}", + crate::server::installation::INSTALL_STATUS_FILE_NAME + )); + environment.push(format!( + "INSTALL_PROGRESS_FILE=/mnt/install/{}", + crate::server::installation::INSTALL_PROGRESS_FILE_NAME + )); + + let staging_path = self.app_config.tmp_data_path(server.uuid); + let status_path = staging_path.join(crate::server::installation::INSTALL_STATUS_FILE_NAME); + let progress_path = + staging_path.join(crate::server::installation::INSTALL_PROGRESS_FILE_NAME); + let log_path = staging_path.join("pve-installer-console.log"); + async { + tokio::fs::create_dir_all(&staging_path).await?; + tokio::fs::write( + staging_path.join("install.sh"), + script.script.replace("\r\n", "\n"), + ) + .await?; + // The helper clears this marker only after the installer exits + // successfully. A killed helper therefore cannot be mistaken for a + // completed installation. + tokio::fs::write(&status_path, "installation process did not complete\n").await?; + tokio::fs::write(&progress_path, "").await?; + tokio::fs::File::create(&log_path).await?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o755)) + .await?; + tokio::fs::set_permissions(&status_path, std::fs::Permissions::from_mode(0o644)) + .await?; + tokio::fs::set_permissions(&progress_path, std::fs::Permissions::from_mode(0o644)) + .await?; + let status_path = status_path.clone(); + let progress_path = progress_path.clone(); + tokio::task::spawn_blocking(move || { + std::os::unix::fs::chown( + &status_path, + Some(host_data_uid), + Some(host_data_gid), + )?; + std::os::unix::fs::chown( + &progress_path, + Some(host_data_uid), + Some(host_data_gid), + ) + }) + .await + .map_err(std::io::Error::other)??; + } + + Ok::<(), std::io::Error>(()) + } + .await + .with_context(|| { + format!( + "failed to prepare Proxmox installer staging directory {}", + staging_path.display() + ) + })?; + + let mut created_vmid = None; + for _ in 0..3 { + let vmid = self.cli.next_vmid().await?; + match self + .create_helper_container( + server, + script, + Self::INSTALLER_HELPER_ROLE, + vmid, + resources, + environment.clone(), + &staging_path, + "/mnt/install", + "/mnt/install/install.sh", + &log_path, + ) + .await + { + Ok(()) => { + created_vmid = Some(vmid); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!( + vmid, + "retrying Proxmox installer create after VMID collision" + ); + } + Err(error) => return Err(error), + } + } + let vmid = created_vmid + .context("failed to create Proxmox installer after three VMID allocation collisions")?; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = match PveHelperProcessHandle::new( + vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + false, + false, + false, + None, + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + if let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove Proxmox installer helper after handle setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to create Proxmox installer process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn attach_installation_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let container = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + .context("no Proxmox installer helper found")?; + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status != cli::ContainerStatus::Running { + return Err(anyhow::anyhow!( + "Proxmox installer helper {} for server {} is not running", + container.vmid, + server.uuid + )); + } + + let log_path = self + .app_config + .tmp_data_path(server.uuid) + .join("pve-installer-console.log"); + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = Arc::new( + PveHelperProcessHandle::new( + container.vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + true, + true, + false, + None, + ) + .await?, + ); + + Ok((handle, status_rx)) + } + + async fn cleanup_installation_process( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + let Some(container) = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + else { + return Ok(()); + }; + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + self.cli.stop(container.vmid).await?; + } + self.cli.destroy(container.vmid).await?; + Ok(()) + } + + async fn setup_script_process( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let (resources, mut environment) = { + let configuration = server.configuration.read().await; + ( + Self::helper_resources(&configuration, &self.app_config)?, + configuration.environment(&self.app_config), + ) + }; + for (key, value) in &script.environment { + environment.push(format!( + "{key}={}", + match value { + serde_json::Value::String(value) => value.clone(), + other => other.to_string(), + } + )); + } + + let staging_path = self + .app_config + .tmp_data_path(server.uuid) + .join(format!("pve-script-{}", uuid::Uuid::new_v4())); + let log_path = staging_path.join("console.log"); + tokio::fs::create_dir_all(&staging_path).await?; + tokio::fs::write( + staging_path.join("script.sh"), + script.script.replace("\r\n", "\n"), + ) + .await?; + tokio::fs::File::create(&log_path).await?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o755)) + .await?; + } + + let mut created_vmid = None; + for _ in 0..3 { + let vmid = self.cli.next_vmid().await?; + match self + .create_helper_container( + server, + script, + Self::SCRIPT_HELPER_ROLE, + vmid, + resources, + environment.clone(), + &staging_path, + "/mnt/script", + "/mnt/script/script.sh", + &log_path, + ) + .await + { + Ok(()) => { + created_vmid = Some(vmid); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!(vmid, "retrying Proxmox script create after VMID collision"); + } + Err(error) => { + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(error); + } + } + } + let Some(vmid) = created_vmid else { + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(anyhow::anyhow!( + "failed to create Proxmox script helper after three VMID allocation collisions" + )); + }; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = match PveHelperProcessHandle::new( + vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + false, + false, + true, + Some(staging_path.clone()), + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + self.cli.destroy(vmid).await.ok(); + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(error.context("failed to create Proxmox script process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn resolve_internal_target( + &self, + server: &crate::server::Server, + port: u16, + ) -> Result, anyhow::Error> { + Ok(self + .server_address(server.uuid) + .await? + .map(|address| std::net::SocketAddr::new(address, port))) + } + + async fn resolve_published_address(&self, server: &crate::server::Server) -> Option { + match self.server_address(server.uuid).await { + Ok(address) => address, + Err(error) => { + tracing::error!( + server = %server.uuid, + "failed to resolve Proxmox LXC address: {error:#}" + ); + None + } + } + } + + async fn container_refs( + &self, + servers: &[crate::server::Server], + ) -> HashMap { + let Some(node) = self.node.get() else { + tracing::error!("cannot inventory Proxmox containers before runtime boot"); + return HashMap::new(); + }; + let containers = match self.cli.list_containers().await { + Ok(containers) => containers, + Err(err) => { + tracing::error!("failed to list Proxmox LXC containers: {err:#}"); + return HashMap::new(); + } + }; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let requested_servers: HashSet<_> = servers.iter().map(|server| server.uuid).collect(); + + let owned = containers + .into_iter() + .filter(|container| container.node == *node) + .filter_map(|container| { + let server_uuid = Self::server_from_tags(&tag_prefix, &container.tags)?; + requested_servers + .contains(&server_uuid) + .then_some((server_uuid, container)) + }) + .collect::>(); + + let mut references = HashMap::new(); + for (server_uuid, container) in owned { + if container.status != Some(cli::ContainerStatus::Running) { + continue; + } + match self + .cli + .runtime_status(&container.node, container.vmid) + .await + { + Ok(status) if status.status == cli::ContainerStatus::Running => { + if let Some(pid) = status.pid.filter(|pid| *pid > 0) { + references.insert(server_uuid, format!("pid:{pid}")); + } else { + tracing::warn!( + server = %server_uuid, + vmid = container.vmid, + "running Proxmox LXC did not expose an init PID for private networking" + ); + } + } + Ok(_) => {} + Err(error) => tracing::warn!( + server = %server_uuid, + vmid = container.vmid, + "failed to resolve Proxmox LXC init PID for private networking: {error:#}" + ), + } + } + + references + } + + async fn used_ports( + &self, + ips: &[IpAddr], + ) -> Result>, anyhow::Error> { + if ips.is_empty() { + return Ok(HashMap::new()); + } + + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let requested = ips.iter().copied().collect::>(); + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let mut used = ips + .iter() + .copied() + .map(|ip| (ip, HashMap::>::new())) + .collect::>(); + + let mut containers = self.cli.list_containers().await?; + containers.sort_unstable_by_key(|container| container.vmid); + + for container in containers.into_iter().filter(|container| { + container.node == *node && container.status == Some(cli::ContainerStatus::Running) + }) { + let interfaces = match self.cli.interfaces(&container.node, container.vmid).await { + Ok(interfaces) => interfaces, + Err(error) => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC during used-port discovery after interface lookup failed: {error:#}" + ); + continue; + } + }; + let matching_addresses = PveCli::container_addresses(&interfaces) + .into_iter() + .filter(|address| requested.contains(address)) + .collect::>(); + if matching_addresses.is_empty() { + continue; + } + + let runtime = match self + .cli + .runtime_status(&container.node, container.vmid) + .await + { + Ok(runtime) => runtime, + Err(error) => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC during used-port discovery after runtime lookup failed: {error:#}" + ); + continue; + } + }; + if runtime.status != cli::ContainerStatus::Running { + continue; + } + let Some(pid) = runtime.pid.filter(|pid| *pid > 0) else { + tracing::warn!( + vmid = container.vmid, + "skipping running Proxmox LXC without an init PID during used-port discovery" + ); + continue; + }; + let bound_ports = match Self::container_bound_ports(pid).await { + Ok(bound_ports) => bound_ports, + Err(error) => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC during used-port discovery after namespace inspection failed: {error:#}" + ); + continue; + } + }; + let server = Self::server_from_tags(&tag_prefix, &container.tags); + + for address in matching_addresses { + let Some(address_ports) = used.get_mut(&address) else { + continue; + }; + for bound in bound_ports + .iter() + .copied() + .filter(|bound| Self::bound_port_matches_address(*bound, address)) + { + address_ports + .entry(bound.port) + .and_modify(|owner| { + if *owner != server { + *owner = None; + } + }) + .or_insert(server); + } + } + } + + Ok(used + .into_iter() + .map(|(ip, ports)| { + let mut ports = ports + .into_iter() + .map(|(port, server)| UsedPort { port, server }) + .collect::>(); + ports.sort_unstable_by_key(|port| port.port); + (ip, ports) + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn successful_start_followed_by_first_stopped_probe_emits_stopped() { + let mut seen_running = false; + let status = PveProcessHandle::observed_status( + cli::ContainerStatus::Stopped, + true, + &mut seen_running, + ); + assert!(matches!( + status, + Some(crate::server::executor::ProcessStatus::Stopped { .. }) + )); + } + + #[tokio::test] + async fn runtime_log_never_exceeds_its_retention_limit() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("console.log"); + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await + .unwrap(); + let mut size = 0; + + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"first", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"second", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"oversized-line", true) + .await + .unwrap(); + file.flush().await.unwrap(); + + let data = tokio::fs::read(&path).await.unwrap(); + assert_eq!(data, b"ed-line\n"); + assert_eq!(data.len(), 8); + assert_eq!(size, 8); + } + + #[test] + fn static_allocation_uses_explicit_isolated_vlan_network() { + assert_eq!( + PveLxcExecutor::network_for_allocation( + "vmbr0", + Some(30), + Some(24), + Some("10.0.30.1"), + Some("10.0.30.42"), + ) + .unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=10.0.30.42/24,gw=10.0.30.1,type=veth" + ); + } + + #[test] + fn static_allocation_never_derives_the_management_network() { + let error = PveLxcExecutor::network_for_allocation( + "vmbr0", + Some(30), + None, + None, + Some("10.0.30.42"), + ) + .unwrap_err(); + assert!(error.to_string().contains("network_prefix")); + + assert_eq!( + PveLxcExecutor::dhcp_network("vmbr0", Some(30)).unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=dhcp,type=veth,host-managed=1" + ); + } + + #[test] + fn network_bridge_rejects_option_injection() { + assert!(PveLxcExecutor::dhcp_network("vmbr0 firewall=0", None).is_err()); + assert!(PveLxcExecutor::dhcp_network("vmbr0,firewall=0", None).is_err()); + } + + #[test] + fn accepts_pve_9_2_and_newer() { + assert_eq!( + PveVersion::parse("pve-manager/9.2.1/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); + assert_eq!( + PveVersion::parse("pve-manager/10.0.0/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); + } + + #[test] + fn rejects_pve_older_than_9_2() { + assert_eq!( + PveVersion::parse("pve-manager/9.1.7/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(false) + ); + } + + #[test] + fn pve_runtime_only_targets_the_local_node() { + assert_eq!( + PveLxcExecutor::runtime_node("", "pve-a").ok().as_deref(), + Some("pve-a") + ); + assert_eq!( + PveLxcExecutor::runtime_node("pve-a", "pve-a") + .ok() + .as_deref(), + Some("pve-a") + ); + assert!(PveLxcExecutor::runtime_node("pve-b", "pve-a").is_err()); + } + + #[test] + fn proc_net_parser_reports_listeners_and_udp_reservations() { + fn proc_ipv4(address: std::net::Ipv4Addr) -> String { + format!("{:08X}", u32::from_le_bytes(address.octets())) + } + + let any = proc_ipv4(std::net::Ipv4Addr::UNSPECIFIED); + let loopback = proc_ipv4(std::net::Ipv4Addr::LOCALHOST); + let tcp = format!( + " sl local_address rem_address st\n 0: {any}:63DD 00000000:0000 0A\n 1: {loopback}:C350 0100007F:9C40 01\n" + ); + let udp = format!( + " sl local_address rem_address st\n 0: {any}:4ABC 00000000:0000 07\n 1: {loopback}:D431 0100007F:0035 01\n 2: {any}:0000 00000000:0000 07\n" + ); + + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&tcp, ProcNetFamily::Ipv4, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 25565, + }] + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&udp, ProcNetFamily::Ipv4, false).unwrap(), + vec![ + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + BoundPort { + family: ProcNetFamily::Ipv4, + address: Some(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + port: 54321, + }, + ] + ); + } + + #[test] + fn proc_net_parser_decodes_ipv6_and_matches_ipv4_mapped_bindings() { + fn proc_ipv6(address: std::net::Ipv6Addr) -> String { + address + .octets() + .chunks_exact(4) + .map(|chunk| { + format!( + "{:08X}", + u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) + ) + }) + .collect::() + } + + let address: std::net::Ipv6Addr = "fd12:3456::20".parse().unwrap(); + let raw = proc_ipv6(address); + let table = format!( + " sl local_address rem_address st\n 0: {raw}:63DD 00000000000000000000000000000000:0000 0A\n" + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&table, ProcNetFamily::Ipv6, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6(address)), + port: 25565, + }] + ); + + let mapped = BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6("::ffff:10.42.0.20".parse().unwrap())), + port: 25565, + }; + assert!(PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.21".parse().unwrap() + )); + assert!(PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv6, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); + } + + #[test] + fn ownership_tags_require_marker_and_server_uuid() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let server_tag = PveLxcExecutor::server_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), server_tag]; + + assert_eq!( + PveLxcExecutor::server_from_tags("calagopus", &tags), + Some(server_uuid) + ); + assert_eq!(PveLxcExecutor::server_from_tags("other", &tags), None); + } + + #[test] + fn helper_tags_are_distinct_from_servers_and_roles() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let installer_tag = PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ); + let tags = vec!["calagopus".to_string(), installer_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::helper_from_tags( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + &tags, + ), + Some(server_uuid) + ); + assert_eq!( + PveLxcExecutor::helper_from_tags( + "calagopus", + PveLxcExecutor::SCRIPT_HELPER_ROLE, + &tags, + ), + None + ); + assert_eq!( + PveLxcExecutor::helper_from_tags("other", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), + None + ); + } + + #[test] + fn replacement_tags_are_distinct_and_recoverable() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let replacement_tag = PveLxcExecutor::replacement_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), replacement_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::replacement_from_tags("calagopus", &tags), + Some(server_uuid) + ); + } + + #[test] + fn image_drift_plan_recovers_staged_replacements() { + let desired = "calagopus-image-new"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old".to_string()], + }; + let staged = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&staged), desired), + ServerContainerPlan::ReplaceWithStaged { + old_vmid: 101, + staged_vmid: 102, + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&staged), desired), + ServerContainerPlan::RecoverStaged { staged_vmid: 102 } + ); + } + + #[test] + fn image_drift_plan_discards_stale_staging_without_replacing_good_container() { + let desired = "calagopus-image-current"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + let stale = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old-attempt".to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&stale), desired), + ServerContainerPlan::Reuse { + vmid: 101, + stale_replacement: Some(102), + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&stale), desired), + ServerContainerPlan::CreateFresh { + stale_replacement: Some(102), + } + ); + } + + #[test] + fn owned_container_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::server_tag("calagopus", server_uuid), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_server_container(&containers, "pve-a", "calagopus", server_uuid,) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_server_container(&containers, "pve-b", "calagopus", server_uuid,) + .ok() + .flatten(), + None + ); + } + + #[test] + fn owned_helper_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 201, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Running), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 202, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::SCRIPT_HELPER_ROLE, + server_uuid, + ) + .ok() + .flatten(), + None + ); + } + + #[test] + fn direct_bridge_firewall_spec_targets_container_addresses_without_published_bindings() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let mappings = HashMap::from([ + ( + compact_str::CompactString::from("10.42.0.20"), + vec![25566, 25565, 25565], + ), + (compact_str::CompactString::from("10.42.0.21"), vec![19132]), + ]); + let rules = vec![crate::server::firewall::FirewallRule { + action: crate::server::firewall::FirewallRuleAction::Deny, + protocols: HashSet::from([crate::server::firewall::FirewallRuleProtocol::Tcp]), + sources: Vec::new(), + ports: None, + source_file: None, + }]; + let spec = PveLxcExecutor::direct_bridge_firewall_spec( + server_uuid, + &mappings, + &rules, + vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + ], + None, + ); + + assert!(spec.bindings.is_empty()); + assert_eq!(spec.container_ports, vec![19132, 25565, 25566]); + assert_eq!( + spec.container_ips, + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); + assert_eq!(spec.rules, rules); + + let concrete = crate::server::firewall::expand_rules(&spec); + assert_eq!(concrete.len(), 6); + assert!( + concrete + .iter() + .all(|rule| matches!(rule.dst, crate::server::firewall::RuleDst::Container { .. })) + ); + } + + #[test] + fn console_input_uses_terminal_enter_without_doubling_crlf() { + assert_eq!( + PveProcessHandle::normalize_console_input(b"list\n".to_vec()), + b"list\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(b"first\r\nsecond\n".to_vec()), + b"first\rsecond\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(vec![0x03]), + vec![0x03] + ); + assert!(PveProcessHandle::is_pct_console_banner( + b"Connected to tty 0\r" + )); + assert!(PveProcessHandle::is_pct_console_banner( + b"Type to exit the console, to enter Ctrl+z itself\r" + )); + assert!(!PveProcessHandle::is_pct_console_banner( + b"[Server thread/INFO]: Connected to tty 0" + )); + } + + #[test] + fn halt_signal_matches_wings_stop_semantics() { + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGABRT")), + Some("SIGABRT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("c")), + Some("SIGINT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGTERM")), + Some("SIGTERM".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGQUIT")), + Some("SIGQUIT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("unexpected")), + Some("SIGKILL".to_string()) + ); + assert_eq!(PveProcessHandle::halt_signal("command", Some("stop")), None); + } + + #[test] + fn device_permissions_require_exact_pve_devn_semantics() { + let path = std::path::Path::new("/dev/kvm"); + + assert_eq!( + PveLxcExecutor::device_deny_write("r", path).ok(), + Some(true) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("rw", path).ok(), + Some(false) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("wr", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("w", path).is_err()); + assert!(PveLxcExecutor::device_deny_write("m", path).is_err()); + assert_eq!( + PveLxcExecutor::device_deny_write("rwm", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("rwx", path).is_err()); + } +} diff --git a/application/src/server/firewall/iptables.rs b/application/src/server/firewall/iptables.rs index 42cc8863..1bf78f49 100644 --- a/application/src/server/firewall/iptables.rs +++ b/application/src/server/firewall/iptables.rs @@ -775,6 +775,7 @@ mod tests { fn rules() -> BTreeMap> { let spec = FirewallServerSpec { server: uuid::Uuid::from_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(), + target: None, bindings: vec![ FirewallBinding { ip: Some("192.168.1.5".parse().unwrap()), diff --git a/application/src/server/firewall/mod.rs b/application/src/server/firewall/mod.rs index 5a631175..d0f4f6a7 100644 --- a/application/src/server/firewall/mod.rs +++ b/application/src/server/firewall/mod.rs @@ -133,6 +133,9 @@ pub struct FirewallBinding { pub struct FirewallServerSpec { pub server: uuid::Uuid, + /// Runtime-specific destination for backends which attach policy to a + /// workload instead of the host forwarding path. + pub target: Option, pub bindings: Vec, pub container_ports: Vec, pub container_ips: Vec, @@ -140,6 +143,11 @@ pub struct FirewallServerSpec { pub files: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FirewallTarget { + ProxmoxLxc { vmid: u32, interface: String }, +} + impl FirewallServerSpec { #[inline] pub fn references_files(&self) -> bool { @@ -301,6 +309,99 @@ pub trait FirewallBackend: Send + Sync { async fn reconcile(&self, specs: &[FirewallServerSpec]) -> Result<(), anyhow::Error>; } +pub async fn create_host_local_with( + backend: FirewallBackendKind, + limits: sets::SourceFileLimits, +) -> Arc { + if backend == FirewallBackendKind::Disabled { + return Arc::new(noop::NoopFirewall::new(false)); + } + + if !cfg!(target_os = "linux") { + if backend != FirewallBackendKind::Auto { + tracing::warn!( + "docker.firewall.backend is set to {:?}, but host-local server firewalls are only supported on linux", + backend + ); + } + + return Arc::new(noop::NoopFirewall::new(true)); + } + + if std::path::Path::new("/.dockerenv").exists() || std::env::var("OCI_CONTAINER").is_ok() { + tracing::warn!( + "host-local server firewalls require Wings to run in the host network namespace; containerized Wings is not supported for this runtime" + ); + + return Arc::new(noop::NoopFirewall::new(true)); + } + + let runner = CommandRunner::Local; + match backend { + FirewallBackendKind::Nftables => { + warn_bridge_nf(&runner).await; + Arc::new(nftables::NftablesFirewall::new(Vec::new(), runner, limits)) + } + FirewallBackendKind::Iptables => { + warn_bridge_nf(&runner).await; + Arc::new(iptables::IptablesFirewall::new(Vec::new(), runner, limits).await) + } + FirewallBackendKind::Container => { + tracing::warn!( + "the container firewall backend is only available with the Docker runtime; host-local server firewall rules will not be applied" + ); + Arc::new(noop::NoopFirewall::new(true)) + } + FirewallBackendKind::Auto => { + if runner + .run( + "nft", + &["--check", "-f", "-"], + Some(b"add table inet wings\n"), + ) + .await + .is_ok() + { + tracing::info!("using nftables host-local server firewall backend"); + warn_bridge_nf(&runner).await; + + Arc::new(nftables::NftablesFirewall::new(Vec::new(), runner, limits)) + } else if runner + .run("iptables", &["-w", "-S", "FORWARD"], None) + .await + .is_ok() + { + tracing::info!("using iptables host-local server firewall backend"); + warn_bridge_nf(&runner).await; + + Arc::new(iptables::IptablesFirewall::new(Vec::new(), runner, limits).await) + } else { + tracing::warn!( + "neither nftables nor iptables are usable on this host, host-local server firewall rules will not be applied" + ); + + Arc::new(noop::NoopFirewall::new(true)) + } + } + FirewallBackendKind::Disabled => Arc::new(noop::NoopFirewall::new(false)), + } +} + +/// Removes Wings-owned host firewall state after another backend has committed +/// its replacement policy. Both host implementations are checked because the +/// previously selected `auto` backend may differ from the one usable now. +pub async fn clear_host_local(limits: sets::SourceFileLimits) { + let nftables = nftables::NftablesFirewall::new(Vec::new(), CommandRunner::Local, limits); + if let Err(error) = nftables.reconcile(&[]).await { + tracing::debug!("failed to clear stale Wings nftables policy: {error:#}"); + } + + let iptables = iptables::IptablesFirewall::new(Vec::new(), CommandRunner::Local, limits).await; + if let Err(error) = iptables.reconcile(&[]).await { + tracing::debug!("failed to clear stale Wings iptables policy: {error:#}"); + } +} + pub async fn create( config: &crate::config::Config, docker: &Arc, @@ -681,6 +782,7 @@ mod tests { fn spec(bindings: Vec, rules: Vec) -> FirewallServerSpec { FirewallServerSpec { server: uuid::Uuid::new_v4(), + target: None, bindings, container_ports: Vec::new(), container_ips: Vec::new(), diff --git a/application/src/server/firewall/nftables.rs b/application/src/server/firewall/nftables.rs index 082c02a0..44e9a86c 100644 --- a/application/src/server/firewall/nftables.rs +++ b/application/src/server/firewall/nftables.rs @@ -650,6 +650,7 @@ mod tests { fn rules() -> BTreeMap> { let spec = FirewallServerSpec { server: server(), + target: None, bindings: vec![ FirewallBinding { ip: Some("192.168.1.5".parse().unwrap()), @@ -730,6 +731,7 @@ mod tests { fn render_ruleset_declares_sets_and_matches_them_per_family() { let spec = FirewallServerSpec { server: server(), + target: None, bindings: vec![FirewallBinding { ip: None, port: 25565, diff --git a/application/src/server/firewall/sets.rs b/application/src/server/firewall/sets.rs index 512be652..b3ce5ef3 100644 --- a/application/src/server/firewall/sets.rs +++ b/application/src/server/firewall/sets.rs @@ -122,6 +122,45 @@ async fn fingerprint( })) } +pub async fn read_source_file( + filesystem: &crate::server::filesystem::cap::CapFilesystem, + path: &Path, + limits: SourceFileLimits, +) -> Result<(Vec, LoadStats), LoadError> { + let file = filesystem.async_open(path).await?; + let mut lines = tokio::io::BufReader::new(file).lines(); + let mut entries = Vec::new(); + let mut stats = LoadStats::default(); + let mut bytes: u64 = 0; + + while let Some(line) = lines.next_line().await? { + bytes += line.len() as u64 + 1; + if bytes > limits.max_bytes { + return Err(LoadError::TooLarge { + max_bytes: limits.max_bytes, + }); + } + + match parse_line(&line) { + None => {} + Some(Err(())) => stats.invalid += 1, + Some(Ok(entry)) => { + stats.entries += 1; + if stats.entries > limits.max_entries { + return Err(LoadError::TooManyEntries { + max_entries: limits.max_entries, + }); + } + entries.push(entry); + } + } + } + + entries.sort_unstable(); + entries.dedup(); + Ok((entries, stats)) +} + #[derive(Debug, Default, Clone, Copy)] pub struct LoadStats { pub entries: usize, diff --git a/application/src/server/mod.rs b/application/src/server/mod.rs index 2d0dd947..7cf6bcbc 100644 --- a/application/src/server/mod.rs +++ b/application/src/server/mod.rs @@ -511,6 +511,13 @@ impl Server { None => break, }; + #[cfg(unix)] + if let Some(tundra) = server.app_state.tundra.as_ref() + && tundra.hub.connected() + { + tundra.rebroadcast(); + } + if !server.filesystem.write_tracking_active() { server.filesystem.disk_checker_state_dirty.store(true, Ordering::Relaxed); } diff --git a/application/src/setup/mod.rs b/application/src/setup/mod.rs index b4958e41..e1bc5024 100644 --- a/application/src/setup/mod.rs +++ b/application/src/setup/mod.rs @@ -171,7 +171,7 @@ async fn probe( let container = std::env::var("OCI_CONTAINER").is_ok(); let data_directory = state.config.system.data_directory.as_path(&state.config); - let (cpu_count, memory_bytes, disk_bytes) = tokio::task::spawn_blocking(move || { + let (cpu_count, memory_bytes, local_disk_bytes) = tokio::task::spawn_blocking(move || { let mut system = sysinfo::System::new(); system.refresh_memory(); system.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing()); @@ -188,6 +188,10 @@ async fn probe( "failed to read system information", )?; + // Server data is bind-mounted from this filesystem for every backend, so + // its capacity is the only capacity the panel can safely allocate. + let disk_bytes = local_disk_bytes; + let ips = if container { BTreeSet::new() } else { diff --git a/application/src/stats.rs b/application/src/stats.rs index 6e1d6b5a..dc3933b6 100644 --- a/application/src/stats.rs +++ b/application/src/stats.rs @@ -1,5 +1,5 @@ use serde::Serialize; -use std::{path::Path, sync::Arc}; +use std::sync::Arc; use sysinfo::{Disks, Networks, System}; use utoipa::ToSchema; @@ -62,8 +62,12 @@ impl StatsManager { } } -impl Default for StatsManager { - fn default() -> Self { +impl StatsManager { + pub fn new(config: Arc) -> Self { + Self::new_with_config(Some(config)) + } + + fn new_with_config(config: Option>) -> Self { let stats = Arc::new(arc_swap::ArcSwap::new(Arc::new(SystemStats::default()))); std::thread::spawn({ @@ -77,6 +81,10 @@ impl Default for StatsManager { let mut sys = System::new_with_specifics(refresh_kind); let mut disks = Disks::new_with_refreshed_list(); let mut networks = Networks::new_with_refreshed_list(); + let data_directory = config.as_ref().map_or_else( + || std::path::PathBuf::from("/"), + |config| config.resolve_as_path(|inner| &inner.system.data_directory), + ); let cpu_model = sys .cpus() @@ -89,7 +97,7 @@ impl Default for StatsManager { sys.refresh_specifics(refresh_kind); networks.refresh(true); for disk in disks.list_mut() { - if disk.mount_point() == Path::new("/") { + if data_directory.starts_with(disk.mount_point()) { disk.refresh(); } } @@ -117,7 +125,11 @@ impl Default for StatsManager { let total_memory = sys.total_memory(); let used_memory = sys.used_memory(); - let disk = match disks.iter().find(|d| d.mount_point() == Path::new("/")) { + let disk = match disks + .iter() + .filter(|disk| data_directory.starts_with(disk.mount_point())) + .max_by_key(|disk| disk.mount_point().as_os_str().len()) + { Some(d) => d, None => match disks.first() { Some(d) => d, @@ -183,3 +195,9 @@ impl Default for StatsManager { Self { stats } } } + +impl Default for StatsManager { + fn default() -> Self { + Self::new_with_config(None) + } +} diff --git a/application/src/tundra/daemon.rs b/application/src/tundra/daemon.rs index 68417b33..9349c547 100644 --- a/application/src/tundra/daemon.rs +++ b/application/src/tundra/daemon.rs @@ -1,5 +1,6 @@ use super::TundraManager; use crate::{io::hash_reader::HashReader, routes::State}; +use anyhow::Context; use futures::StreamExt; use sha2::Digest; use std::{ @@ -15,6 +16,7 @@ const BINARY_NAME: &str = "tundra-node"; const SERVER_PLACEHOLDER: &str = "{server}"; const EXTRACT_CONTAINER_NAME: &str = "calagopus-wings-tundra-extract"; const SOURCE_ENTRYPOINT: &str = "/usr/bin/calagopus-tundra"; +const PIDFILE_NAME: &str = "tundra-node.pid"; const CREATE_ATTEMPTS: u32 = 5; const CREATE_BACKOFF: Duration = Duration::from_millis(200); @@ -441,7 +443,9 @@ impl Applied { } pub async fn stop(manager: &TundraManager) -> Result<(), anyhow::Error> { - let docker = manager.docker(); + let Some(docker) = manager.docker() else { + return stop_native(manager).await; + }; let inspect = match docker.inspect_container(CONTAINER_NAME, None).await { Ok(inspect) => inspect, Err(bollard::errors::Error::DockerResponseServerError { @@ -472,6 +476,124 @@ pub async fn stop(manager: &TundraManager) -> Result<(), anyhow::Error> { Ok(()) } +fn native_pid(manager: &TundraManager) -> Option { + let pid = std::fs::read_to_string(manager.data_dir.join("node").join(PIDFILE_NAME)) + .ok()? + .trim() + .parse::() + .ok()?; + let command = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?; + let expected_config = config_path(manager); + let command_matches = command + .split(|byte| *byte == 0) + .any(|argument| argument == expected_config.as_os_str().as_encoded_bytes()); + let executable = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?; + let expected_binary = binary_path(manager); + let executable_bytes = executable.as_os_str().as_encoded_bytes(); + let expected_bytes = expected_binary.as_os_str().as_encoded_bytes(); + let executable_matches = executable_bytes == expected_bytes + || executable_bytes.strip_suffix(b" (deleted)") == Some(expected_bytes); + (command_matches && executable_matches).then_some(pid) +} + +async fn stop_native(manager: &TundraManager) -> Result<(), anyhow::Error> { + if !manager.disabled() { + return Ok(()); + } + let Some(pid) = native_pid(manager) else { + return Ok(()); + }; + + let pid = rustix::process::Pid::from_raw(pid as i32) + .context("native tundra daemon PID was invalid")?; + rustix::process::kill_process(pid, rustix::process::Signal::TERM) + .context("failed to stop the native tundra daemon")?; + manager.hub.disconnect(); + Ok(()) +} + +async fn restart_native(manager: &TundraManager) -> Result<(), anyhow::Error> { + let output = tokio::process::Command::new(binary_path(manager)) + .arg("restart") + .arg("--config") + .arg(config_path(manager)) + .output() + .await + .context("failed to request a native tundra restart")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "native tundra restart failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(()) +} + +async fn ensure_native( + manager: &TundraManager, + wanted: &Applied, + tunnel_port: u16, +) -> Result<(), anyhow::Error> { + if let Some(pid) = native_pid(manager) { + let applied = manager.hub.request_metrics().await; + if applied + .as_ref() + .is_ok_and(|metrics| wanted.matches(metrics)) + { + return Ok(()); + } + + if manager.serving() && manager.restart_due() { + tracing::info!( + pid, + "requesting the native tundra daemon to apply its current binary and config" + ); + tokio::time::timeout(Duration::from_secs(30), restart_native(manager)).await??; + } + return Ok(()); + } + + let mut child = tokio::process::Command::new(binary_path(manager)) + .arg("--config") + .arg(config_path(manager)) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .context("failed to start the native tundra daemon")?; + let pid = child + .id() + .context("native tundra daemon did not expose a process ID")?; + let pid_path = manager.data_dir.join("node").join(PIDFILE_NAME); + if let Some(parent) = pid_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&pid_path, format!("{pid}\n")) + .await + .context("failed to write native tundra daemon PID file")?; + tokio::spawn(async move { + match child.wait().await { + Ok(status) if status.success() => { + tracing::info!(?pid, "native tundra daemon stopped"); + } + Ok(status) => { + tracing::error!(?pid, %status, "native tundra daemon exited"); + } + Err(error) => { + tracing::error!(?pid, "failed to wait for native tundra daemon: {error:#}"); + } + } + if tokio::fs::read_to_string(&pid_path) + .await + .is_ok_and(|contents| contents.trim() == pid.to_string()) + { + tokio::fs::remove_file(pid_path).await.ok(); + } + }); + tracing::info!(pid, tunnel_port, "started the native tundra daemon"); + Ok(()) +} + pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow::Error> { if !manager.serving() { return Ok(()); @@ -493,23 +615,34 @@ pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow let source = config.tundra.binary.as_path(&config); let source_image = config.tundra.source_image.clone(); let metrics_port = config.tundra.metrics_port; - let hosts_path = config - .system - .vmount_directory - .as_path(&config) - .join(SERVER_PLACEHOLDER) - .join("hosts"); + let hosts_path = match config.runtime.backend { + crate::config::RuntimeBackend::PveLxc => config + .runtime + .pve_lxc + .managed_file_directory + .as_path(&config) + .join(SERVER_PLACEHOLDER) + .join("hosts"), + crate::config::RuntimeBackend::Auto | crate::config::RuntimeBackend::Docker => config + .system + .vmount_directory + .as_path(&config) + .join(SERVER_PLACEHOLDER) + .join("hosts"), + }; let image = config.tundra.image.clone(); let docker = manager.docker(); - let desired = create_body(state, manager, &image); drop(config); let refresh = !manager.images_refreshed.load(Ordering::Relaxed); if source.as_os_str().is_empty() { - ensure_image(&docker, state, &source_image, "source", refresh).await?; - extract_binary(&docker, manager, &source_image).await?; + let docker = docker + .as_ref() + .context("native tundra requires tundra.binary to point to a tundra-node executable")?; + ensure_image(docker, state, &source_image, "source", refresh).await?; + extract_binary(docker, manager, &source_image).await?; } else { let dest = binary_path(manager); let digest_path = image_digest_path(manager); @@ -545,6 +678,12 @@ pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow return Ok(()); } + let Some(docker) = docker else { + return ensure_native(manager, &wanted, own.tunnel_port).await; + }; + + let desired = create_body(state, manager, &image); + ensure_image(&docker, state, &image, "daemon", refresh).await?; manager.images_refreshed.store(true, Ordering::Relaxed); diff --git a/application/src/tundra/mod.rs b/application/src/tundra/mod.rs index 240e2bab..77824e59 100644 --- a/application/src/tundra/mod.rs +++ b/application/src/tundra/mod.rs @@ -71,7 +71,7 @@ pub struct TundraManager { pub data_dir: PathBuf, pub hub: hub::Hub, ca: ca::LocalCa, - docker: Arc, + docker: Option>, token: parking_lot::RwLock, control: parking_lot::Mutex, @@ -88,7 +88,7 @@ pub struct TundraManager { impl TundraManager { pub fn create( config: &crate::config::Config, - docker: Arc, + docker: Option>, ) -> Result, anyhow::Error> { let cfg = config.load(); let data_dir = cfg.tundra.data_directory.as_path(&cfg); @@ -134,8 +134,8 @@ impl TundraManager { } #[inline] - pub fn docker(&self) -> Arc { - Arc::clone(&self.docker) + pub fn docker(&self) -> Option> { + self.docker.as_ref().map(Arc::clone) } #[inline] @@ -203,10 +203,37 @@ impl TundraManager { self.control.lock().enriched.as_deref().cloned() } + fn supports_process_refs(metrics: &serde_json::Value) -> bool { + metrics + .get("capabilities") + .and_then(serde_json::Value::as_array) + .is_some_and(|capabilities| { + capabilities + .iter() + .any(|capability| capability.as_str() == Some("process_container_refs")) + }) + } + async fn rebuild(&self, state: &State) { let Some(cached) = self.cached() else { return; }; + + if state.config.load().runtime.backend == crate::config::RuntimeBackend::PveLxc { + let supported = self + .hub + .request_metrics() + .await + .as_ref() + .is_ok_and(Self::supports_process_refs); + if !supported { + tracing::error!( + "refusing to publish Proxmox LXC references because the connected tundra-node does not advertise process_container_refs support" + ); + return; + } + } + let mut snapshot = Snapshot::clone(&cached); let cfg = state.config.load(); @@ -388,6 +415,19 @@ pub async fn run(state: State) { #[cfg(test)] mod tests { use super::*; + + #[test] + fn process_reference_capability_is_explicit() { + assert!(TundraManager::supports_process_refs(&serde_json::json!({ + "capabilities": ["process_container_refs"] + }))); + assert!(!TundraManager::supports_process_refs( + &serde_json::json!({}) + )); + assert!(!TundraManager::supports_process_refs(&serde_json::json!({ + "capabilities": ["some_future_capability"] + }))); + } use crate::remote::tundra::{Disabled, TunnelState}; fn manager(dir: &Path) -> TundraManager { @@ -395,7 +435,9 @@ mod tests { data_dir: dir.to_path_buf(), hub: hub::Hub::default(), ca: ca::LocalCa::load_or_create(dir).unwrap(), - docker: Arc::new(bollard::Docker::connect_with_local_defaults().unwrap()), + docker: Some(Arc::new( + bollard::Docker::connect_with_local_defaults().unwrap(), + )), token: parking_lot::RwLock::new(String::new()), control: parking_lot::Mutex::new(ControlState { cached: None, diff --git a/docs/pve-lxc-runtime.md b/docs/pve-lxc-runtime.md new file mode 100644 index 00000000..c528d7ef --- /dev/null +++ b/docs/pve-lxc-runtime.md @@ -0,0 +1,512 @@ +# Proxmox VE LXC runtime + +This branch is introducing Proxmox VE 9.2 LXC as an alternate Wings runtime while +keeping Docker as the preferred runtime. The implementation is deliberately placed behind +the existing `ServerExecutor` boundary so the fork can continue to merge upstream +Docker changes with minimal conflict. + +Runtime selection now happens behind `ServerExecutor`. New installations start with +`runtime.backend: auto`: Wings selects a reachable Docker daemon when one is +present, otherwise selects a local Proxmox VE LXC runtime. It persists that +selection to `config.yml`, so installing Docker later does not change an already +configured LXC node. `runtime.backend: pve_lxc` remains available to force the +PVE executor without connecting to Docker. PVE boot validates that Wings is running on Linux, requires Proxmox VE 9.2 +or newer, discovers the local node, rejects a configured remote cluster node, and +checks `/cluster/nextid` as a read-only capability probe. The local-node requirement +matches the runtime's use of local `pct`/LXC processes, host bind paths, host +firewalling, and cgroup/procfs telemetry. + +The typed `pct`, `pvesh`, `pvesm`, and `pveversion` wrapper covers VMID allocation, cluster +LXC inventory, container status, start/shutdown/stop/destroy, and deterministic +LXC creation arguments. It also covers Proxmox's OCI registry pull endpoint, +storage-content discovery, asynchronous UPID polling, and deterministic OCI +template cache names. Inventory uses a stable marker tag plus server UUID to +recover Wings ownership after restart without another mapping database. Server +data is intended to stay on the Wings host and be bind-mounted at +`/home/container`, matching the path expected by existing eggs while allowing an +LXC root filesystem to be replaced during image updates. + +Example configuration: + +```yaml +runtime: + backend: pve_lxc + pve_lxc: + template_storage: local + # "auto" chooses the active rootdir-capable storage with the most free + # space that can fit the requested rootfs. Set a storage name to pin it. + rootfs_storage: auto + # VLAN-aware bridge carrying the isolated game-server subnet. + bridge: vmbr0 + # Applied to every LXC veth. Leave unset only when the bridge itself is + # dedicated to the game-server network. + vlan_tag: 30 + # Used for direct private allocations when edge forwarding is disabled. + network_prefix: 24 + gateway: 10.0.30.1 + # Optional edge publishing. The panel allocation is the VPS public IP. + # Wings matches it to a WireGuard peer endpoint and derives the control IP + # from that peer's IPv4 /32 AllowedIPs entry. + edge_wireguard_interface: wg-calagopus + edge_ssh_identity_path: /etc/calagopus-wings/edge-forward + edge_known_hosts_path: /etc/calagopus-wings/edge-known-hosts + rootfs_size_gib: 8 + console_log_max_bytes: 5242880 + tag_prefix: calagopus + managed_file_directory: /var/lib/lxc/calagopus-wings-managed + unprivileged: true + firewall: + # Prefer PVE guest rules when the datacenter firewall is enabled, then + # fall back to host-local nftables or iptables. + backend: auto + source_file_max_entries: 10000 + source_file_max_bytes: 1048576 +``` + +With `rootfs_storage: auto`, Wings orders active, enabled Proxmox storages that +support `rootdir` by free capacity. It creates the LXC on the largest pool that +has at least the requested rootfs size available, then falls through to the +next pool if the preferred pool cannot fit it. + +Panel setup and node statistics report the filesystem that contains +`system.data_directory`. Persistent game data is bind-mounted from that path, so +rootfs pools do not increase allocatable server storage. Rootfs selection and +panel game-data capacity are intentionally reported separately. + +PVE's low-level unprivileged bind-mount setup must be able to traverse every host +directory component used as a raw file source. Wings therefore stages managed +`/etc/machine-id`, `/etc/hosts`, `/etc/passwd`, and `/etc/group` files under the +PVE-specific `managed_file_directory` with execute-only traversal for non-root +users, instead of weakening the existing 0700 `system.vmount_directory`. The +configured staging path must be absolute and its parent components must already +be traversable on the PVE host. + +## Compatibility plan + +The PVE runtime should preserve the contracts already exposed by `ProcessHandle` +and `ServerExecutor`: + +1. Pull the configured OCI image into Proxmox template storage, then create one LXC + per Calagopus server using a cluster-assigned VMID. +2. Tag each CT with a stable Calagopus marker and server UUID so Wings can recover + ownership after restart without maintaining a second mapping database. +3. Map memory, swap and CPU limits to native LXC settings and mount the existing + Wings server data directory at `/home/container`. +4. Preserve the OCI image process semantics and expose them through a Wings-owned + process session. The server process uses the OCI image entrypoint as the LXC + init process and Wings attaches to the PVE console for stdin/stdout. +5. Run install and script jobs through the same execution layer, with explicit + resource/time limits. +6. Resolve PVE networking separately from Docker port publishing. The first target + is bridged per-container networking; host DNAT can be added as a separate policy + when allocations need host-port translation. + +## Implemented boundary + +- Runtime selection is isolated in `server::executor::create_runtime`; existing + Docker behavior remains the default path. +- PVE boot validates the host/PVE version, resolves the target node, and verifies + that a cluster VMID can be allocated. +- PVE cluster inventory filters to the selected node and recovers requested server + UUIDs from Calagopus ownership tags. +- Automatic runtime probing is bounded to three seconds per candidate. Proxmox + asynchronous tasks have a 15-minute deadline, and locked configuration helpers + have a 30-second deadline, so a stuck Docker socket, OCI pull, or PVE config + lock cannot hold Wings startup or an install forever. +- The LXC network is configured entirely in Wings' runtime config. `bridge` + selects the PVE bridge and `vlan_tag` optionally tags each veth. Without edge + publishing, `network_prefix` plus `gateway` define the isolated game subnet and + a primary panel allocation becomes the static `net0` address. With edge + publishing enabled, panel allocations are public VPS addresses and every main + or helper CT uses DHCP on the hidden bridge instead. +- Edge publishing needs no per-VPS address in Wings configuration. Wings reads + `wg show dump`, matches the panel allocation IP to a peer's current + public endpoint, and uses that peer's IPv4 `/32` AllowedIPs address for its + restricted SSH control connection. This continues to work when a hypervisor's + home WAN address changes and lets the panel allocation alone choose among + multiple edge VPS peers. Before applying a mapping, Wings clears the server UUID + from every discovered edge peer so allocation changes cannot leave stale DNAT. + Once the CT starts, Wings resolves its DHCP IPv4 address through the Proxmox + interface API and sends that hidden address plus the panel ports to the selected + edge controller. Cleanup clears the mapping on every edge peer. The SSH + identity must be a root-owned regular file with mode `0600` or more restrictive. + Each edge should restrict that key to the forwarding controller with an OpenSSH + `command="...",restrict` authorized-key entry. + For a running owned CT, Wings can query the Proxmox LXC interface API, ignore loopback + and link-local-only addresses, collect all usable IPv4/IPv6 addresses, prefer + IPv4 when resolving a single tunnel target, and resolve that target directly to + the CT address and requested game port. The same primary address is exposed as + the runtime's published address for this initial bridged model. +- Used-port discovery now follows that direct-bridge model. Wings inventories all + running LXC containers on the selected local node, matches their actual bridged + addresses against the queried allocation IPs, resolves each CT's init PID, and + reads the CT network namespace through `/proc//net/{tcp,tcp6,udp,udp6}`. + TCP reports listening sockets; UDP reports bound/connected reservations. Wildcard + sockets apply to every queried usable address on that CT, IPv4-mapped IPv6 + bindings are normalized, Calagopus-owned server CTs are attributed to their + server UUID, and other running LXCs are returned as used with no server owner. + This detects direct-address collisions without adding Docker-style host-port + publishing or DNAT. A container that disappears or becomes unreadable during + inventory is logged and skipped instead of aborting allocation discovery for + every other container. +- Server firewall policy is now wired into the PVE runtime without introducing a + Docker-style host NAT layer. Wings creates a host-local nftables or iptables + backend and expands configured allocation ports directly against every usable + bridged CT address. PVE firewall specs intentionally contain no published host + bindings, so rules target the CT destination IP/port rather than an emulated + host port. +- Firewall state participates in the PVE lifecycle: the backend boots with the + runtime, startup reconciliation rebuilds rules for known servers, setup and + process attachment synchronize rules, configuration sync refreshes them, and + server cleanup clears them. A running CT with configured firewall rules fails + closed if Wings cannot resolve a usable bridged address or cannot use the + selected host-local firewall backend. Starting such a CT stops it again when + its rules cannot be applied. +- `runtime.pve_lxc.firewall.backend: auto` now prefers VM-scoped Proxmox + firewall rules when the datacenter firewall is enabled and otherwise retains + the host-local fallback. `proxmox` requires the datacenter firewall and fails + clearly when it is disabled; Wings never changes the cluster-wide setting. + Managed rules and source-file IP sets are written atomically under Proxmox's + VM firewall lock, preserve unrelated administrator rules, appear in the PVE + UI, and carry stable comments so reconciliation removes only Wings-owned + entries. Managed LXC veths enable PVE firewall processing with `firewall=1`. + Reconciliation is restricted to tagged containers on this Wings instance's + configured local PVE node, so another cluster node's Wings instance cannot + clear its rules. If `auto` falls back from Proxmox to a host-local backend, + Wings first removes its own stale guest rules while preserving administrator + rules. Wings owns guest firewall mode for its tagged containers and + sets the guest input policy to `ACCEPT` because panel firewall policy + allows unmatched traffic, while Proxmox otherwise defaults unmatched guest + input to `DROP`. Only the ordered panel rules appear in the guest rule list. + The panel's terminal unrestricted deny is emitted as a protocol-free, + port-free PVE `DROP`, so "Deny Everything Else" covers all remaining guest + input rather than only the server's allocated ports. + When the configured bridge uses a host-local DHCP/DNS service, the PVE host + firewall must allow those service ports from the game subnet. A masqueraded + bridge used with the legacy PVE firewall also needs the standard `fwbr+` + conntrack-zone rule so reply traffic returns through the same zone. +- OCI image references without a tag are normalized to `:latest`. Tagged + references are pulled through `/nodes/{node}/storage/{storage}/oci-registry-pull`, + and Wings waits for the returned Proxmox task to stop with `exitstatus: OK`. +- Live Proxmox VE 9.2.2 validation confirmed that `oci-registry-pull` can emit + image-copy progress on stdout before the JSON task UPID even when + `--output-format json` is requested. The CLI parser therefore accepts both a + clean JSON response and the real PVE progress-plus-UPID response instead of + assuming stdout contains only JSON. +- OCI templates use a deterministic `calagopus-oci-.tar` cache key derived + from the normalized image reference. The Docker image-fetch cache settings also + control LXC template freshness. Once stale, a mutable tag is pulled again and + Wings reads the resolved manifest digest from the OCI archive. Container image + ownership tags use that content digest, so a changed upstream tag triggers the + normal staged rootfs replacement flow. +- Digest (`@sha256:...`) references are rejected clearly for now because the PVE + OCI pull API currently accepts tagged references. +- `pct create` arguments model memory, swap, cores, bridge selection, rootfs + storage, and ownership tags. Bind mounts are deliberately configured in a + second step: after the OCI CT rootfs is created, Wings reads PVE's resolved + `lxc.init.uid`/`lxc.init.gid`, rejects a root/default image user, then configures + `/home/container` with `pct set --mp0` and PVE 9.2's per-mount `idmap` option. + That maps the image UID/GID directly to the UID/GID Wings already uses to own + server files on the host without changing container-wide `lxc.idmap` or host + `/etc/subuid` and `/etc/subgid` ranges. Because PVE's per-mount idmap only + applies to unprivileged CTs, this runtime currently rejects + `pve_lxc.unprivileged: false` at boot. +- Server mounts now use the same allowed-mount validation as the Docker runtime. + PVE `mpN` entries support the main `/home/container` data mount, optional + `/dev/hugepages`, configured directory mounts, and read-only mounts. Each source + is canonicalized and maps the OCI process UID/GID to Wings' configured host + UID/GID, preserving the host-side identity used by the Docker runtime instead of + granting the process ownership based on the mounted directory's owner. + Native `mpN` is directory-oriented, so Wings' generated per-file mounts use + low-level `lxc.mount.entry` bind mounts with `ro,create=file`. Managed + `machine-id`, `hosts`, `passwd`, and `group` files are copied from Wings' vmount + directory into a PVE-specific staging directory with traversable parent paths, + then mounted read-only into the CT. Changes to those source files are reflected + when Wings next reconciles the runtime configuration. + Runtime config reconciliation owns only those four target paths and preserves + unrelated low-level LXC entries. Disabling one of the corresponding Wings + features removes a stale managed entry on the next config reconciliation. +- KVM and configured character/block devices can be passed through with PVE 9.2 + `devN`. The host device's numeric UID/GID and mode are preserved on the CT device + node. Configured devices must use the same source and target path. + `devN` can express read-only or read/write access here; Docker's independent + `m` (mknod) permission and write-only access are rejected instead of being + silently approximated. +- Mount and device configuration is reconciled as one `pct set` operation. Desired + `mpN`/`devN` entries are replaced in place and stale slots owned by the managed + CT are deleted, so removing an egg mount/device cannot leave old passthrough + configuration behind after Wings restarts and reuses the CT. +- The two-stage create helper is transactional for this preparation phase: if OCI + user inspection or mount/device configuration fails, it destroys the newly + created CT. A cleanup failure is returned alongside the original preparation + failure so an orphaned CT cannot be hidden. +- Server configuration can now be translated into that preparation flow: Wings + resolves the configured OCI image/cache, allocates a VMID, applies the runtime + storage/bridge/tag settings, uses the existing server data path and host + UID/GID, then performs the transactional create + mount sequence from + `setup_server_process` before returning the usable PVE `ProcessHandle`. +- Preparation now reconciles ownership before allocating a VMID. Each new CT gets + a deterministic OCI-image identity tag derived from the resolved manifest + digest. A single existing CT with the same server and image tags is reused, + its mutable resource limits are synchronized, and its OCI user is re-read from + PVE. Duplicate permanent or staged CT claims fail explicitly. +- A stopped owned CT whose OCI image identity changed is replaced transactionally. + Wings first resolves the target template and creates/configures a second CT with + a server-specific staging tag, while the old CT and host-owned server data stay + intact. Only after OCI-user discovery plus mount/device reconciliation succeeds + does Wings destroy the old rootfs and promote the staged CT to the normal server + ownership tag. If removing the old CT fails, the new staging CT is discarded + when possible. If Wings exits or tag promotion fails after the old CT is gone, + the staging tag is discoverable and the next setup resumes promotion instead of + creating an untracked duplicate. Stale staging CTs from abandoned image changes + are cleaned up only while stopped. A running old or staging CT is never killed + implicitly by image reconciliation; it must be stopped before replacement. + `/home/container` remains the same Wings-host directory throughout this flow, so + replacement affects the OCI rootfs rather than persistent server data. +- PVE resource mapping uses `memory_limit + overhead_memory` directly as the CT RAM + limit, maps panel swap as PVE's additional swap amount, and maps CPU percent to + `cpulimit` CPU-time units (`100% = 1`, `250% = 2.5`). Unlimited memory, unlimited + swap (`-1`), and CPU pinning via `build.threads` currently fail explicitly + because they do not yet have exact PVE mappings in this runtime. +- The CLI resource update path uses `pct set` for memory, swap, and CPU time. It + writes `cpulimit=0` when the panel CPU limit is disabled so an old limit cannot + survive a configuration change. `cores` is updated when an explicit value is + present. +- The CLI layer can spawn a Wings-owned `lxc-attach` session with piped standard + I/O, a clean explicitly supplied environment, and an explicit UID/GID, + including OCI images whose configured process user is root. +- Server setup now creates or reuses the owned CT, applies runtime environment + variables and `lxc.signal.halt` under PVE's config lock, then exposes it through + a PVE `ProcessHandle`. Environment keys supplied by Wings replace matching + existing values while unrelated low-level entries are preserved. PVE serializes + its high-level `env` property as `lxc.environment.runtime`, which reaches hooks + but not CT init on the tested LXC 7 host. Wings therefore retains the PVE `env` + metadata and also writes reconciled `lxc.environment` entries for the init + process. Both PVE representations are normalized by variable name before the + panel overlay is applied, making repeated synchronization idempotent. +- New CTs are created with `cmode=console`. The process handle attaches with + `pct console` through a PTY, creates a session with the PTY slave as the + controlling terminal, and forwards Wings stdin. The PTY master uses separate + Tokio file handles for reading and writing so an idle blocking read cannot + prevent console commands or graceful-stop input. Wings converts producer line + feeds to terminal carriage returns at this boundary, line-buffers stdout into + both existing broadcast paths, applies the same console throttling policy used + by the Docker path, and reconnects the console while the CT remains running. + `pct console` connection banners are excluded from broadcasts and runtime logs. +- Filtered console lines are retained on the Wings host at + `/pve-lxc/.log`. Fresh server setup truncates the + file; process reattachment preserves it. `console_log_max_bytes` bounds each + file and starts a fresh retained log at the limit. `logs(Some(n))` seeks + backward to the requested trailing lines without loading the entire file; + `logs(None)` streams the full retained log. +- Process state is polled from `pct status`. A newly prepared CT suppresses the + initial stopped state until `pct start` succeeds. A CT that starts and exits + between status polls still emits stopped and enters normal crash handling. PVE does not expose + Docker-equivalent process exit-code/OOM metadata through this path, so a stopped + CT currently reports `exit_code: -1` and `oom_killed: false`. +- Signal-based stops set `lxc.signal.halt` from the configured Wings stop signal + and request a graceful `lxc-stop --nowait --nokill`. Command-based stops are + sent over the attached console. Hard kill uses `pct stop`. Immediate restart + after a graceful stop can hit PVE's transient monitor-socket teardown window; + `pct start` is retried only for that known failure, after checking whether the + CT actually started despite the client error. +- `sync_configuration` updates mutable PVE resource limits plus runtime environment + and halt-signal configuration. Panel entrypoint overrides are rejected for now; + the OCI image entrypoint is used as the server process. +- Normal server cleanup hard-stops a running CT and destroys the owned container, + so the next normal setup starts from a fresh OCI root filesystem while keeping + the host-owned server data directory. +- Installation and ad-hoc script execution now use short-lived helper CTs rather + than Docker. Helpers have role-specific ownership tags so they cannot be + mistaken for the main server CT. They reuse the configured OCI image pull/cache + path, mount the server data at `/mnt/server`, and mount a host staging directory + at `/mnt/install` or `/mnt/script` with the same host-managed UID/GID mapping used + by the main data mount. +- Helper CTs execute the requested entrypoint plus staged script through + `lxc.init.cmd`, making the job itself PID 1. `lxc.console.logfile` writes console + output to a durable Wings-host file, which lets Wings stream output without a + long-lived `lxc-attach` child and lets an in-progress installer be reattached + after a Wings restart. Installer status/progress files remain host-backed and + use the existing Wings installation result flow. The status file begins with a + failure marker and is cleared only after a successful installer exit, so a + killed or OOM-terminated helper cannot be reported as successful completion. +- Both helper and game CTs use a small host-managed init wrapper that waits for + PVE's host-side DHCP hook to install a default route before invoking the OCI + entrypoint. The game wrapper then `exec`s the original OCI command. Installer + wrappers also write a nonzero script exit to the existing installation status + file so an early helper failure cannot be reported as a successful install. +- Installer helpers persist until normal installation cleanup so their retained + console/status data remains available for result evaluation. Script helpers are + automatically destroyed after the CT stops and their final console bytes have + been drained; their per-run staging directory is removed at the same time. +- Helper process state uses the same PVE status polling limitation as the main + process (`exit_code: -1`, `oom_killed: false`). A helper that starts and exits + between polls still emits a running transition before stopped so very short + installers cannot strand the existing installation state machine waiting for a + transition it never observed. +- OCI template preparation is serialized across main and helper provisioning so + concurrent requests cannot corrupt the shared cache. Container creation itself + remains concurrent. A VMID collision from another cluster client is retried + with a newly allocated ID up to three times. +- Main-server resource telemetry now uses Proxmox + `/nodes/{node}/lxc/{vmid}/status/current` to discover the running CT init PID, + then reuses Wings' existing cgroup-v2 sampler against that process. It publishes + memory usage/limit, CPU percent, configured CPU limit, per-container network + byte/packet counters, uptime, disk usage, and current server state through the + existing resource-usage watch channel. CPU percent is calculated from cgroup + CPU-time deltas using the same semantics as the Docker path. +- The PVE stats worker re-resolves the runtime PID and cgroup if the CT restarts, + its cgroup files disappear, or the sampler stops delivering. Offline state + clears live-container metrics while preserving disk usage. A running CT with no + usable PID or cgroup-v2 path is retried and reported as unavailable rather than + falling back to guessed host-wide values. +- Tundra can run as a native child process when Docker is unavailable. Wings + renders the same control-plane config and Unix socket used by the Docker path, + installs the configured `tundra-node` binary into its managed data directory, + tracks the child with its PID file, and reconnects to an existing owned process + after a Wings restart. PVE inventory publishes each running CT as + `pid:`; the Tundra node resolves that reference directly through + procfs, so Docker is not required to discover the CT network namespace, address, + or managed hosts file. Docker remains the preferred runtime when both backends + are available. The matching Tundra source change is preserved in + `patches/tundra-pve-lxc-process-ref.patch` until it can be merged upstream. + The patched daemon advertises `process_container_refs` in its metrics response; + Wings refuses to publish LXC references unless that capability is present. + CI verifies the patch against pinned Tundra commit + `ccb05e1406232f8c39f05a984133277026cd79d6`. +- Adopting an already-running LXC reconciles Wings-owned runtime files and + `lxc.mount.entry` records. This creates Tundra's managed hosts file immediately; + when the feature was enabled after the CT started, the new read-only + `/etc/hosts` bind mount takes effect on that CT's next restart. + +## Initial support contract + +- Native Tundra requires a `tundra-node` built with + `patches/tundra-pve-lxc-process-ref.patch`. Wings does not build or install that + patched binary; operators must supply the compatible executable configured for + the node. Capability negotiation fails closed when an unpatched daemon connects. +- Panel entrypoint overrides, CPU pinning, unlimited memory, unlimited swap, and + digest-form image references are rejected explicitly. PVE OCI pulls currently + do not receive the Docker private-registry credentials from Wings. +- One primary IPv4 allocation selects the CT address or edge peer. Additional + allocation ports are supported on that address; multiple allocation IPs are + not yet mapped to extra LXC interfaces. +- PID limits, block-I/O weight, and a writable-rootfs toggle do not yet have LXC + mappings. The OCI rootfs remains writable and Wings enforces the implemented + memory, swap, CPU-time, mount, device, network, and firewall settings. +- Edge publishing is an optional deployment component. It discovers peers from + the configured WireGuard interface and invokes the restricted edge controller + over SSH with pinned host keys. Controller commands have bounded SSH process + execution and failures fail the required network-policy synchronization; Wings + stops a newly started server when its required edge mapping cannot be applied. + The forced-command controller receives `clear ` with no stdin, or + `sync ` with JSON stdin containing `ip`, `tcp`, and `udp`. WireGuard + discovery times out after five seconds and each SSH operation after 15 seconds. + Sites that do not configure the edge identity use direct bridged allocations + and have no dependency on that controller. + +## Disposable PVE smoke test + +`scripts/pve-lxc-smoke-test.sh` exercises the host contracts the runtime depends +on without requiring a running Wings instance. Run it as root on a disposable PVE +9.2+ node. By default it uses `local` for OCI templates, `local-lvm` for rootfs, +`vmbr0` for networking, and the non-root +`docker.io/nginxinc/nginx-unprivileged:alpine` image. Those values can be +overridden with `TEMPLATE_STORAGE`, `ROOTFS_STORAGE`, `BRIDGE`, and `IMAGE`. + +The smoke test pulls/reuses the OCI template, creates an unprivileged CT with the +same `pct create` shape used by Wings, reads PVE's resolved `lxc.init.uid/gid`, +applies the per-mount idmap for `/home/container`, starts the CT, verifies the +interfaces and `status/current` APIs, checks the init PID's cgroup and network +namespace proc files, writes through the mapped data mount as the OCI user, +checks init command, environment, console logging and a read-only managed file, +then tests graceful stop, immediate restart with the same transient-error retry +policy, hard stop, and rootfs recreation with persistent host-backed data. +The CT and temporary host data are cleaned up automatically unless `KEEP_CT=1` is +set. The OCI template cache is intentionally retained for repeat runs. + +The smoke test passed end-to-end on a disposable node running PVE 9.2.2 +(`pve-manager/9.2.2`, kernel `7.0.2-6-pve`) on September 28, 2026. The immediate +restart encountered one monitor teardown error and succeeded on retry. The run +also verified OCI template reuse, DHCP/interface discovery, `status/current` +init-PID discovery, the `/home/container` per-mount idmap, host ownership +translation, cgroup-v2 and `/proc/{pid}/net/{tcp,tcp6,udp,udp6}` visibility, +runtime environment reaching init, durable console output, read-only managed +`/etc/hosts`, graceful and hard stops, rootfs recreation, and persistent data. +The tagged CT and its two temporary host directories were removed. PVE's +`pct config` output also contained a NUL byte during this OCI flow; the runtime +and smoke test strip NULs before parsing the otherwise textual configuration. + +### Panel-to-edge Minecraft validation + +The deployed Wings binary passed an end-to-end panel-created Paper server test on +a disposable PVE node: + +- The installer helper acquired `10.70.0.20` from the host-only + `vmbr-calagopus` DHCP service and downloaded the Paper server jar. +- The game CT ran unprivileged on the same bridge and acquired `10.70.0.21`. + Its OCI entrypoint waited for DHCP, downloaded the Mojang runtime jar, generated + a world, and reported `Done (27.464s)` while listening on `0.0.0.0:25565`. +- Wings selected the WireGuard peer whose endpoint matched the panel allocation + `192.0.2.10`, then installed TCP and UDP DNAT rules on that edge for + `192.0.2.10:25565 -> 10.70.0.21:25565`. +- A TCP probe from the development host to `192.0.2.10:25565` and a second + probe on the edge to `10.70.0.21:25565` both succeeded. The panel reported the + server running with live CPU, memory, disk, uptime, and network telemetry. +- A panel console `list` command reached Paper and returned its online-player + response. The panel Stop action then sent Paper's configured stop command, + Paper saved all three dimensions, VMID 100 reached `stopped`, and a subsequent + panel Start returned the server to `Done (22.133s)`. No `pct console` connection + banners appeared in the panel or persisted runtime log. A fresh probe to the + public allocation succeeded after restart. +- PVE can temporarily return JSON `null` from the interfaces endpoint while a CT + finishes shutting down. Wings treats that response as an empty interface list, + allowing an immediate panel Restart to clear stale firewall state and continue. + A live restart after this change returned Paper to `Done (23.479s)`. +- Paper 26.3 with Java 25 exhausted the original 1 GiB test limit during initial + world creation. PVE's kernel log confirmed a memory-cgroup OOM kill. Raising + the panel limit to 4 GiB produced a stable server at about 1.4 GiB resident + memory. This was a workload sizing issue rather than an LXC networking failure. + +### Native Tundra validation + +During live validation, the PVE node was enrolled in the panel private network at its +Tailscale address on UDP 7100 with Docker absent. Wings launched the native +`tundra-node`, issued and published its node certificate through the existing +control-plane shim, and the panel reported the daemon connected with its snapshot +applied. The running Minecraft LXC joined with hostname `test-mc.tunnel` and +offered TCP 25565. Tundra accepted the Wings `pid:` reference, adopted +the CT without Docker, and applied a snapshot containing the LXC server. Metrics +reported the control link up and the daemon listening on UDP 7100. A second +private-network member is still required to exercise an inter-node QUIC relay and +end-to-end hostname connection. + +## Remaining work before production use + +- Define a verified PVE-safe equivalent for the optional managed + `/sys/class/dmi/id/product_uuid` file. The regular generated files use live + read-only LXC file bind mounts, but sysfs behavior still needs real-PVE + validation before claiming parity for this path. +- Confirm private-registry authentication behavior for PVE's OCI pull path. +- Continue live PVE 9.2 validation for less common Wings behavior: helper + reattachment after a daemon restart, one-shot script cleanup, and + host-local nftables/iptables filtering on bridged CT traffic. Main process + startup, helper installation, DHCP, telemetry, WireGuard peer selection, edge + DNAT synchronization, and public connectivity have now been exercised through + the Rust runtime against the live panel. + +A cached Rust 1.97 toolchain is available on the current macOS development host; +formatting passes there. The macOS test build cannot produce a Wings binary because +existing filesystem code uses Linux-only `rustix` APIs and the PVE console-spawn +method is intentionally compiled only on Linux. On September 28, 2026, the branch +was archived into a disposable Rust 1.97.1 LXC on a disposable PVE build container with locked Cargo caches. +`cargo check --tests --offline --target x86_64-unknown-linux-gnu` passed, followed +by `cargo test --offline --target x86_64-unknown-linux-gnu`: 743 passed, 9 ignored, +0 failed. `CARGO_GIT_BRANCH` was supplied because the source archive omitted +`.git`. The tagged build CT, uploaded archives, and temporary Rust OCI template +were removed after the run. This validates Linux compilation and unit tests, but +it does not run a configured Wings daemon or the full PVE executor against a live +panel. The later live validation above fills that gap for the primary install, +start, telemetry, console-command, graceful-stop, and edge-forwarding paths. The +current PVE-specific unit suite also passes all 60 targeted tests on the PVE test +node before the deployed release build. diff --git a/patches/tundra-pve-lxc-process-ref.patch b/patches/tundra-pve-lxc-process-ref.patch new file mode 100644 index 00000000..479a28be --- /dev/null +++ b/patches/tundra-pve-lxc-process-ref.patch @@ -0,0 +1,169 @@ +From 14667f635b0fcf263d2de44f03d92fae6e7bf9e6 Mon Sep 17 00:00:00 2001 +From: Philip Ulrich +Date: Tue, 29 Sep 2026 10:53:42 -0400 +Subject: [PATCH] support process-backed containers in tundra node + +--- + tundra-node/build.rs | 2 ++ + tundra-node/src/main.rs | 2 +- + tundra-node/src/node/docker.rs | 60 ++++++++++++++++++++++++++++++++++ + tundra-node/src/node/mod.rs | 18 +++++++--- + tundra-node/src/metrics.rs | 1 + + 5 files changed, 77 insertions(+), 6 deletions(-) + +diff --git a/tundra-node/build.rs b/tundra-node/build.rs +index 729a5c7..514bc95 100644 +--- a/tundra-node/build.rs ++++ b/tundra-node/build.rs +@@ -46,6 +46,8 @@ fn handle_git_info() { + { + git_hash = String::from_utf8_lossy(&output.stdout).trim().to_string(); + } ++ } else { ++ println!("cargo:rustc-env=CARGO_GIT_BRANCH=unknown"); + } + println!("cargo:rustc-env=CARGO_GIT_COMMIT={git_hash}"); + } +diff --git a/tundra-node/src/main.rs b/tundra-node/src/main.rs +index 4d64e8e..24bfd67 100644 +--- a/tundra-node/src/main.rs ++++ b/tundra-node/src/main.rs +@@ -108,7 +108,7 @@ async fn main() -> Result<(), anyhow::Error> { + Ok(d) => Some(d), + Err(err) => { + tracing::warn!( +- "docker is unavailable, no containers will be adopted: {:?}", ++ "docker is unavailable; Docker container references cannot be adopted, but process-backed references remain available: {:?}", + err + ); + None +diff --git a/tundra-node/src/node/docker.rs b/tundra-node/src/node/docker.rs +index 3054d12..7917fca 100644 +--- a/tundra-node/src/node/docker.rs ++++ b/tundra-node/src/node/docker.rs +@@ -6,6 +6,8 @@ use bollard::{ + use futures::{Stream, StreamExt}; + use std::{collections::HashMap, net::Ipv4Addr, path::PathBuf, pin::Pin}; + ++pub const PROCESS_REF_PREFIX: &str = "pid:"; ++ + fn classify(msg: bollard::models::EventMessage) -> Option { + let actor = msg.actor?; + let id = actor.id?; +@@ -52,6 +54,48 @@ pub struct ContainerInfo { + pub hosts_path: PathBuf, + } + ++impl ContainerInfo { ++ /// Adopt a container described by its init PID instead of a Docker object. ++ /// Wings uses this for runtimes such as Proxmox LXC. ++ pub fn from_process_ref( ++ container_ref: &str, ++ dial_addr: Option<&str>, ++ ) -> Result, anyhow::Error> { ++ let Some(raw_pid) = container_ref.strip_prefix(PROCESS_REF_PREFIX) else { ++ return Ok(None); ++ }; ++ let pid: i32 = raw_pid.parse().context(format!( ++ "invalid process container reference {container_ref}" ++ ))?; ++ if pid <= 0 { ++ return Err(anyhow::anyhow!( ++ "process container reference {container_ref} has an invalid PID" ++ )); ++ } ++ ++ let running = PathBuf::from(format!("/proc/{pid}/ns/net")).exists(); ++ let ip = dial_addr.and_then(|dial| { ++ dial.parse::() ++ .map(|address| address.ip()) ++ .or_else(|_| dial.parse::()) ++ .ok() ++ .and_then(|address| match address { ++ std::net::IpAddr::V4(address) => Some(address), ++ std::net::IpAddr::V6(_) => None, ++ }) ++ }); ++ ++ Ok(Some(Self { ++ id: container_ref.to_owned(), ++ pid, ++ running, ++ ip, ++ // Process-backed runtimes configure a control-plane hosts template. ++ hosts_path: PathBuf::new(), ++ })) ++ } ++} ++ + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum ContainerEvent { + Started { id: String, name: Option }, +@@ -136,6 +180,22 @@ mod tests { + use super::*; + use bollard::models::{EndpointSettings, EventActor, EventMessage}; + ++ #[test] ++ fn process_references_are_runtime_neutral() { ++ let reference = format!("pid:{}", std::process::id()); ++ let info = ContainerInfo::from_process_ref(&reference, Some("10.70.0.58:25565")) ++ .unwrap() ++ .unwrap(); ++ assert!(info.running); ++ assert_eq!(info.pid, std::process::id() as i32); ++ assert_eq!(info.ip, Some(Ipv4Addr::new(10, 70, 0, 58))); ++ assert!( ++ ContainerInfo::from_process_ref("docker-id", None) ++ .unwrap() ++ .is_none() ++ ); ++ } ++ + fn endpoint(ip: &str) -> EndpointSettings { + EndpointSettings { + ip_address: Some(ip.to_owned()), +diff --git a/tundra-node/src/node/mod.rs b/tundra-node/src/node/mod.rs +index 56f4f35..d737afb 100644 +--- a/tundra-node/src/node/mod.rs ++++ b/tundra-node/src/node/mod.rs +@@ -387,17 +387,25 @@ impl Node { + } + + pub async fn refresh_containers(&self, index: &SnapshotIndex) { +- let Some(docker) = &self.docker else { +- return; +- }; +- + let mut found = HashMap::new(); + for server in index.servers_on(&self.uuid) { + let Some(entry) = index.server(server) else { + continue; + }; + +- match docker.inspect(&entry.container_ref).await { ++ let inspected = match docker::ContainerInfo::from_process_ref( ++ &entry.container_ref, ++ entry.dial_addr.as_deref(), ++ ) { ++ Ok(Some(info)) => Ok(info), ++ Ok(None) => match &self.docker { ++ Some(docker) => docker.inspect(&entry.container_ref).await, ++ None => continue, ++ }, ++ Err(err) => Err(err), ++ }; ++ ++ match inspected { + Ok(info) if info.running => { + found.insert(*server, info); + } +diff --git a/tundra-node/src/metrics.rs b/tundra-node/src/metrics.rs +--- a/tundra-node/src/metrics.rs ++++ b/tundra-node/src/metrics.rs +@@ -284,3 +284,4 @@ impl Metrics { + json!({ + "applied": self.applied, ++ "capabilities": ["process_container_refs"], + "node": { +-- +2.50.1 (Apple Git-155) diff --git a/scripts/pve-lxc-smoke-test.sh b/scripts/pve-lxc-smoke-test.sh new file mode 100755 index 00000000..00ec73f2 --- /dev/null +++ b/scripts/pve-lxc-smoke-test.sh @@ -0,0 +1,513 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="${IMAGE:-docker.io/nginxinc/nginx-unprivileged:alpine}" +TEMPLATE_STORAGE="${TEMPLATE_STORAGE:-local}" +ROOTFS_STORAGE="${ROOTFS_STORAGE:-local-lvm}" +BRIDGE="${BRIDGE:-vmbr0}" +ROOTFS_SIZE_GIB="${ROOTFS_SIZE_GIB:-2}" +MEMORY_MIB="${MEMORY_MIB:-256}" +SWAP_MIB="${SWAP_MIB:-256}" +HOST_DATA_UID="${HOST_DATA_UID:-1000}" +HOST_DATA_GID="${HOST_DATA_GID:-1000}" +KEEP_CT="${KEEP_CT:-0}" + +if [[ ${EUID} -ne 0 ]]; then + echo "error: run this smoke test as root on the Proxmox node" >&2 + exit 1 +fi + +for command in pveversion pvesh pct lxc-attach lxc-stop sha256sum perl tr awk stat; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "error: required command not found: $command" >&2 + exit 1 + fi +done + +node="$(pvesh get /cluster/status --output-format json | perl -MJSON::PP -0777 -e ' + my $rows = decode_json(); + for my $row (@$rows) { + next if ($row->{type} // q{}) ne q{node}; + next if !($row->{local} // 0); + print $row->{name} // q{}; + last; + } +')" +if [[ -z "$node" ]]; then + echo "error: Proxmox cluster status did not identify the local node" >&2 + exit 1 +fi +version="$(pveversion | sed -n 's#^pve-manager/\([0-9][0-9.]*\)/.*#\1#p')" +if [[ ! "$version" =~ ^([0-9]+)\.([0-9]+)(\.|$) ]]; then + echo "error: Proxmox VE 9.2 or newer is required; found ${version:-unknown}" >&2 + exit 1 +fi +major="${BASH_REMATCH[1]}" +minor="${BASH_REMATCH[2]}" +if [[ "$major" -lt 9 || ( "$major" -eq 9 && "$minor" -lt 2 ) ]]; then + echo "error: Proxmox VE 9.2 or newer is required; found ${version:-unknown}" >&2 + exit 1 +fi + +vmid="$(pvesh get /cluster/nextid)" +if [[ ! "$vmid" =~ ^[0-9]+$ || "$vmid" -le 0 ]]; then + echo "error: Proxmox returned an invalid next VMID: ${vmid:-empty}" >&2 + exit 1 +fi +server_uuid="$(cat /proc/sys/kernel/random/uuid)" +run_tag="calagopus-smoke" +server_tag="calagopus-server-${server_uuid}" +normalized_image="$IMAGE" +last_component="${normalized_image##*/}" +if [[ "$last_component" != *:* ]]; then + normalized_image="${normalized_image}:latest" +fi +if [[ "$normalized_image" == *@* ]]; then + echo "error: digest image references are not supported by the current runtime" >&2 + exit 1 +fi + +image_digest="$(printf '%s' "$normalized_image" | sha256sum | awk '{print $1}')" +image_tag="calagopus-image-${image_digest:0:32}" +template_name="calagopus-oci-${image_digest}" +template_volid="${TEMPLATE_STORAGE}:vztmpl/${template_name}.tar" +data_dir="/var/lib/calagopus-pve-smoke/${server_uuid}" +managed_dir="/var/lib/calagopus-pve-smoke-managed/${server_uuid}" +managed_parent="${managed_dir%/*}" +managed_parent_created=0 +managed_parent_mode="" +created_ct=0 +config_file="$(mktemp /tmp/calagopus-smoke-config.XXXXXX)" + +cleanup() { + local status=0 + if [[ "$KEEP_CT" == "1" ]]; then + echo "KEEP_CT=1: leaving CT $vmid and $data_dir in place" + return + fi + + if pct config "$vmid" >"$config_file" 2>/dev/null; then + if grep -Eq "^tags: .*${run_tag}" "$config_file"; then + if pct status "$vmid" 2>/dev/null | grep -q running; then + pct stop "$vmid" >/dev/null 2>&1 || status=1 + fi + pct destroy "$vmid" --purge 1 >/dev/null 2>&1 || status=1 + else + echo "warning: refusing to clean CT $vmid because the smoke-test tag is missing" >&2 + status=1 + fi + fi + rm -f "$config_file" + rm -rf "$data_dir" + rm -rf "$managed_dir" + if [[ "$managed_parent_created" == "1" ]]; then + rmdir "$managed_parent" 2>/dev/null || true + elif [[ -n "$managed_parent_mode" && -d "$managed_parent" ]]; then + chmod "$managed_parent_mode" "$managed_parent" || status=1 + fi + if [[ "$status" -ne 0 ]]; then + echo "warning: smoke-test cleanup was incomplete" >&2 + fi +} +trap cleanup EXIT INT TERM + +json_field() { + local field="$1" + perl -MJSON::PP -0777 -e ' + my $field = shift @ARGV; + local $/; + my $json = decode_json(); + my $value = $json->{$field}; + print $value if defined $value; + ' "$field" +} + +ensure_template() { + if pvesh get "/nodes/${node}/storage/${TEMPLATE_STORAGE}/content" --content vztmpl --output-format json \ + | grep -Fq "\"volid\":\"${template_volid}\""; then + echo "template: reusing $template_volid" + return + fi + + echo "template: pulling $normalized_image" + local output upid task_json task_status exit_status + output="$(pvesh create "/nodes/${node}/storage/${TEMPLATE_STORAGE}/oci-registry-pull" \ + --reference "$normalized_image" \ + --filename "$template_name" \ + --output-format json)" + printf '%s\n' "$output" + + upid="$(printf '%s\n' "$output" \ + | sed -n -e 's/^"\(UPID:.*\)"$/\1/p' -e '/^UPID:/p' \ + | tail -n 1)" + if [[ -z "$upid" ]]; then + echo "error: OCI pull completed without a parseable UPID" >&2 + exit 1 + fi + + while true; do + task_json="$(pvesh get "/nodes/${node}/tasks/${upid}/status" --output-format json)" + task_status="$(printf '%s' "$task_json" | json_field status)" + if [[ "$task_status" == "stopped" ]]; then + exit_status="$(printf '%s' "$task_json" | json_field exitstatus)" + if [[ "$exit_status" != "OK" ]]; then + echo "error: OCI pull task failed: ${exit_status:-unknown}" >&2 + exit 1 + fi + break + fi + sleep 1 + done + + if ! pvesh get "/nodes/${node}/storage/${TEMPLATE_STORAGE}/content" --content vztmpl --output-format json \ + | grep -Fq "\"volid\":\"${template_volid}\""; then + echo "error: OCI pull task succeeded but $template_volid is absent" >&2 + exit 1 + fi +} + +create_ct() { + echo "ct: creating VMID $vmid" + pct create "$vmid" "$template_volid" \ + --hostname "calagopus-smoke-${vmid}" \ + --rootfs "${ROOTFS_STORAGE}:${ROOTFS_SIZE_GIB}" \ + --memory "$MEMORY_MIB" \ + --swap "$SWAP_MIB" \ + --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp,type=veth,host-managed=1" \ + --unprivileged 1 \ + --cmode console \ + --onboot 0 \ + --cores 1 \ + --cpulimit 1 \ + --tags "calagopus;${run_tag};${server_tag};${image_tag}" + created_ct=1 +} + +configure_data_mount() { + local config uid gid + config="$(pct config "$vmid" | tr -d '\000')" + uid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.uid" {print $2; exit}')" + gid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.gid" {print $2; exit}')" + if [[ -z "$uid" || -z "$gid" || "$uid" == "0" || "$gid" == "0" ]]; then + echo "error: OCI image did not resolve to a non-root lxc.init.uid/gid" >&2 + printf '%s\n' "$config" >&2 + exit 1 + fi + + echo "oci user: uid=$uid gid=$gid" + mkdir -p "$data_dir" + chown "$HOST_DATA_UID:$HOST_DATA_GID" "$data_dir" + chmod 0750 "$data_dir" + + pct set "$vmid" --mp0 \ + "${data_dir},mp=/home/container,backup=0,idmap=u:${uid}:${HOST_DATA_UID}:1;g:${gid}:${HOST_DATA_GID}:1" + + OCI_UID="$uid" + OCI_GID="$gid" +} + +wait_for_address() { + local attempt interfaces usable + for attempt in $(seq 1 30); do + interfaces="$(pvesh get "/nodes/${node}/lxc/${vmid}/interfaces" --output-format json 2>/dev/null || true)" + usable="$(printf '%s' "$interfaces" | perl -MJSON::PP -0777 -e ' + my $rows = eval { decode_json() } // []; + for my $row (@$rows) { + next if ($row->{name} // q{}) eq q{lo}; + for my $address (@{$row->{q{ip-addresses}} // []}) { + my $ip = $address->{q{ip-address}}; + next if !defined($ip); + next if $ip eq q{0.0.0.0} || $ip eq q{::} || $ip eq q{::1}; + next if $ip =~ /^127\./ || $ip =~ /^169\.254\./ || $ip =~ /^fe[89ab][0-9a-f]:/i; + print 1; + exit; + } + } + ')" + if [[ "$usable" == "1" ]]; then + printf '%s\n' "$interfaces" + return + fi + sleep 1 + done + echo "error: container did not report an interface address" >&2 + exit 1 +} + +verify_running_contracts() { + local runtime_json pid cgroup_file host_owner + runtime_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json)" + pid="$(printf '%s' "$runtime_json" | json_field pid)" + if [[ -z "$pid" || "$pid" == "0" || ! -d "/proc/$pid" ]]; then + echo "error: status/current did not expose a usable CT init PID" >&2 + printf '%s\n' "$runtime_json" >&2 + exit 1 + fi + echo "runtime status: $runtime_json" + echo "interfaces:" + wait_for_address + + lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'printf "calagopus-pve-smoke\n" > /home/container/smoke-marker && cat /home/container/smoke-marker' + + if [[ "$(cat "$data_dir/smoke-marker")" != "calagopus-pve-smoke" ]]; then + echo "error: host-backed /home/container marker mismatch" >&2 + exit 1 + fi + host_owner="$(stat -c '%u:%g' "$data_dir/smoke-marker")" + if [[ "$host_owner" != "${HOST_DATA_UID}:${HOST_DATA_GID}" ]]; then + echo "error: mount idmap produced host owner $host_owner, expected ${HOST_DATA_UID}:${HOST_DATA_GID}" >&2 + exit 1 + fi + echo "mount idmap: container ${OCI_UID}:${OCI_GID} -> host $host_owner" + + cgroup_file="/proc/${pid}/cgroup" + if [[ ! -r "$cgroup_file" ]]; then + echo "error: cannot read $cgroup_file" >&2 + exit 1 + fi + echo "cgroup: $(cat "$cgroup_file")" + + if [[ ! -r "/proc/${pid}/net/tcp" || ! -r "/proc/${pid}/net/tcp6" \ + || ! -r "/proc/${pid}/net/udp" || ! -r "/proc/${pid}/net/udp6" ]]; then + echo "error: CT network namespace proc files are unavailable through /proc/$pid/net" >&2 + exit 1 + fi + if ! awk 'NR > 1 && $4 == "0A" { found = 1 } END { exit !found }' \ + "/proc/${pid}/net/tcp" "/proc/${pid}/net/tcp6"; then + echo "error: no TCP listener was visible through the CT init PID network namespace" >&2 + exit 1 + fi + echo "proc net: tcp/tcp6/udp/udp6 are readable for init PID $pid" +} + +configure_runtime_contracts() { + local wrapper_path console_log managed_hosts + wrapper_path="$data_dir/runtime-wrapper.sh" + console_log="$data_dir/runtime-console.log" + managed_hosts="$managed_dir/hosts" + + if pct status "$vmid" | grep -q running; then + pct stop "$vmid" + fi + + cat >"$wrapper_path" <<'EOF' +#!/bin/sh +printf 'calagopus-init-wrapper %s\n' "${CALAGOPUS_SMOKE:-missing}" +exec nginx -g 'daemon off;' +EOF + chown "$HOST_DATA_UID:$HOST_DATA_GID" "$wrapper_path" + chmod 0755 "$wrapper_path" + + if [[ -d "$managed_parent" ]]; then + managed_parent_mode="$(stat -c '%a' "$managed_parent")" + else + mkdir -p "$managed_parent" + managed_parent_created=1 + fi + mkdir -p "$managed_dir" + chmod 0711 "$managed_parent" "$managed_dir" + cat >"$managed_hosts" <<'EOF' +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +# calagopus-managed-hosts +EOF + chmod 0644 "$managed_hosts" + rm -f "$console_log" + + perl -MPVE::LXC::Config - "$vmid" "$console_log" "$managed_hosts" <<'PERL' +use strict; +use warnings; + +my ($vmid, $console_log, $managed_hosts) = @ARGV; + +sub mount_target { + my ($entry) = @_; + return undef if !defined($entry); + my @fields = grep { length($_) } split(/\s+/, $entry); + return scalar(@fields) >= 2 ? $fields[1] : undef; +} + +PVE::LXC::Config->lock_config($vmid, sub { + my $conf = PVE::LXC::Config->load_config($vmid); + my @environment; + push @environment, split(/\0/, $conf->{env}, -1) + if defined($conf->{env}) && length($conf->{env}); + for my $entry (@{$conf->{lxc} // []}) { + next if ref($entry) ne 'ARRAY' || @$entry < 2; + push @environment, $entry->[1] + if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; + } + my %index; + for my $i (0 .. $#environment) { + my ($name) = split(/=/, $environment[$i], 2); + $index{$name} = $i if defined($name) && length($name); + } + my $smoke_environment = 'CALAGOPUS_SMOKE=runtime-config'; + if (exists($index{CALAGOPUS_SMOKE})) { + $environment[$index{CALAGOPUS_SMOKE}] = $smoke_environment; + } else { + push @environment, $smoke_environment; + } + + my @lxc = grep { + my $keep = ref($_) ne 'ARRAY' + || @$_ < 2 + || ($_->[0] ne 'lxc.environment' + && $_->[0] ne 'lxc.environment.runtime' + && $_->[0] ne 'lxc.signal.halt' + && $_->[0] ne 'lxc.init.cmd' + && $_->[0] ne 'lxc.console.logfile'); + if ($keep && ref($_) eq 'ARRAY' && @$_ >= 2 && $_->[0] eq 'lxc.mount.entry') { + my $target = mount_target($_->[1]); + $keep = 0 if defined($target) && $target eq 'etc/hosts'; + } + $keep; + } @{$conf->{lxc} // []}; + + $conf->{env} = join("\0", @environment); + push @lxc, map { ['lxc.environment', $_] } @environment; + push @lxc, ['lxc.signal.halt', 'SIGTERM']; + push @lxc, ['lxc.init.cmd', '/bin/sh /home/container/runtime-wrapper.sh']; + push @lxc, ['lxc.console.logfile', $console_log]; + push @lxc, [ + 'lxc.mount.entry', + "$managed_hosts etc/hosts none bind,ro,create=file 0 0", + ]; + $conf->{lxc} = \@lxc; + PVE::LXC::Config->write_config($vmid, $conf); +}); +PERL + + CONSOLE_LOG="$console_log" +} + +wait_for_stopped() { + local attempt + for attempt in $(seq 1 40); do + if pct status "$vmid" | grep -q stopped; then + return + fi + sleep 0.25 + done + echo "error: CT $vmid did not stop after lxc-stop request" >&2 + exit 1 +} + +wait_for_running_pid() { + local attempt status_json pid + for attempt in $(seq 1 40); do + status_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json 2>/dev/null || true)" + pid="$(printf '%s' "$status_json" | json_field pid 2>/dev/null || true)" + if [[ -n "$pid" ]] && pct status "$vmid" 2>/dev/null | grep -q running; then + return + fi + sleep 0.25 + done + echo "error: CT $vmid did not expose a stable running PID" >&2 + exit 1 +} + +start_ct() { + local attempt output + for attempt in $(seq 1 20); do + if output="$(pct start "$vmid" 2>&1)"; then + return + fi + if [[ "$output" != *"monitor socket"* && "$output" != *"got timeout"* \ + && "$output" != *"unable to get PID for CT"* ]]; then + printf '%s\n' "$output" >&2 + return 1 + fi + if pct status "$vmid" 2>/dev/null | grep -q running; then + return + fi + echo "start: retrying after transient monitor teardown (attempt $attempt)" >&2 + sleep 0.5 + done + printf '%s\n' "$output" >&2 + return 1 +} + +verify_runtime_contracts() { + local attempt + configure_runtime_contracts + start_ct + + for attempt in $(seq 1 40); do + if [[ -f "$CONSOLE_LOG" ]] \ + && grep -Fq 'calagopus-init-wrapper runtime-config' "$CONSOLE_LOG"; then + break + fi + sleep 0.25 + done + if [[ ! -f "$CONSOLE_LOG" ]] \ + || ! grep -Fq 'calagopus-init-wrapper runtime-config' "$CONSOLE_LOG"; then + echo "error: lxc.init.cmd/environment output was not captured by lxc.console.logfile" >&2 + exit 1 + fi + echo "runtime config: init command, environment, and console logfile are active" + + if ! lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'grep -Fq "calagopus-managed-hosts" /etc/hosts'; then + echo "error: managed read-only /etc/hosts bind was not visible in the CT" >&2 + exit 1 + fi + if lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'printf "unexpected-write\n" >> /etc/hosts' >/dev/null 2>&1; then + echo "error: managed /etc/hosts bind unexpectedly allowed writes" >&2 + exit 1 + fi + echo "managed file bind: /etc/hosts is visible and read-only" + + verify_running_contracts + + lxc-stop -n "$vmid" --nowait --nokill + wait_for_stopped + echo "shutdown: lxc-stop --nowait --nokill honored lxc.signal.halt" + + start_ct + wait_for_running_pid + pct stop "$vmid" + if ! pct status "$vmid" | grep -q stopped; then + echo "error: pct stop did not hard-stop CT $vmid" >&2 + exit 1 + fi + echo "shutdown: pct stop hard-stop path works" +} + +stop_and_destroy_ct() { + if pct status "$vmid" | grep -q running; then + if ! pct shutdown "$vmid" --timeout 10; then + pct stop "$vmid" + fi + fi + pct destroy "$vmid" --purge 1 + created_ct=0 +} + +echo "PVE $version node=$node vmid=$vmid image=$normalized_image" +ensure_template +create_ct +configure_data_mount +start_ct +verify_running_contracts +verify_runtime_contracts + +# Recreate the rootfs while keeping the host-backed data path. This exercises the +# persistence assumption used by staged image replacement. +echo "replacement: recreating CT rootfs while preserving $data_dir" +stop_and_destroy_ct +create_ct +configure_data_mount +start_ct +persisted_marker="$(lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'cat /home/container/smoke-marker')" +if [[ "$persisted_marker" != "calagopus-pve-smoke" ]]; then + echo "error: persistent server data did not survive CT rootfs replacement" >&2 + exit 1 +fi +echo "replacement: persistent marker survived rootfs recreation" +verify_running_contracts + +echo "PASS: PVE LXC OCI lifecycle/runtime-config/idmap/procfs/replacement contracts" From af015a8643d6278fee04544d8c3b905dabe3a25a Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 02/13] move PVE LXC runtime guide to repository root --- docs/pve-lxc-runtime.md => PVE_LXC_RUNTIME.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/pve-lxc-runtime.md => PVE_LXC_RUNTIME.md (100%) diff --git a/docs/pve-lxc-runtime.md b/PVE_LXC_RUNTIME.md similarity index 100% rename from docs/pve-lxc-runtime.md rename to PVE_LXC_RUNTIME.md From 86f5436644603c6ef1c3a9a7d5a62fa17e46fa7e Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 03/13] move runtime docs and Tundra changes to companion repositories --- .github/workflows/test.yml | 10 - PVE_LXC_RUNTIME.md | 512 ----------------------- patches/tundra-pve-lxc-process-ref.patch | 169 -------- 3 files changed, 691 deletions(-) delete mode 100644 PVE_LXC_RUNTIME.md delete mode 100644 patches/tundra-pve-lxc-process-ref.patch diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3555a7f8..e2e25045 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,16 +30,6 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check - - name: Check Tundra process-reference patch - env: - TUNDRA_BASE: ccb05e1406232f8c39f05a984133277026cd79d6 - run: | - git init "$RUNNER_TEMP/tundra" - git -C "$RUNNER_TEMP/tundra" remote add origin https://github.com/calagopus/tundra.git - git -C "$RUNNER_TEMP/tundra" fetch --depth=1 origin "$TUNDRA_BASE" - git -C "$RUNNER_TEMP/tundra" checkout --detach FETCH_HEAD - git -C "$RUNNER_TEMP/tundra" apply --check "$GITHUB_WORKSPACE/patches/tundra-pve-lxc-process-ref.patch" - - name: Install cross run: cargo install cross --git https://github.com/cross-rs/cross --rev 8c1a8aa4b661711f4b7b6ac07c2e8929ce2f7d27 --locked diff --git a/PVE_LXC_RUNTIME.md b/PVE_LXC_RUNTIME.md deleted file mode 100644 index c528d7ef..00000000 --- a/PVE_LXC_RUNTIME.md +++ /dev/null @@ -1,512 +0,0 @@ -# Proxmox VE LXC runtime - -This branch is introducing Proxmox VE 9.2 LXC as an alternate Wings runtime while -keeping Docker as the preferred runtime. The implementation is deliberately placed behind -the existing `ServerExecutor` boundary so the fork can continue to merge upstream -Docker changes with minimal conflict. - -Runtime selection now happens behind `ServerExecutor`. New installations start with -`runtime.backend: auto`: Wings selects a reachable Docker daemon when one is -present, otherwise selects a local Proxmox VE LXC runtime. It persists that -selection to `config.yml`, so installing Docker later does not change an already -configured LXC node. `runtime.backend: pve_lxc` remains available to force the -PVE executor without connecting to Docker. PVE boot validates that Wings is running on Linux, requires Proxmox VE 9.2 -or newer, discovers the local node, rejects a configured remote cluster node, and -checks `/cluster/nextid` as a read-only capability probe. The local-node requirement -matches the runtime's use of local `pct`/LXC processes, host bind paths, host -firewalling, and cgroup/procfs telemetry. - -The typed `pct`, `pvesh`, `pvesm`, and `pveversion` wrapper covers VMID allocation, cluster -LXC inventory, container status, start/shutdown/stop/destroy, and deterministic -LXC creation arguments. It also covers Proxmox's OCI registry pull endpoint, -storage-content discovery, asynchronous UPID polling, and deterministic OCI -template cache names. Inventory uses a stable marker tag plus server UUID to -recover Wings ownership after restart without another mapping database. Server -data is intended to stay on the Wings host and be bind-mounted at -`/home/container`, matching the path expected by existing eggs while allowing an -LXC root filesystem to be replaced during image updates. - -Example configuration: - -```yaml -runtime: - backend: pve_lxc - pve_lxc: - template_storage: local - # "auto" chooses the active rootdir-capable storage with the most free - # space that can fit the requested rootfs. Set a storage name to pin it. - rootfs_storage: auto - # VLAN-aware bridge carrying the isolated game-server subnet. - bridge: vmbr0 - # Applied to every LXC veth. Leave unset only when the bridge itself is - # dedicated to the game-server network. - vlan_tag: 30 - # Used for direct private allocations when edge forwarding is disabled. - network_prefix: 24 - gateway: 10.0.30.1 - # Optional edge publishing. The panel allocation is the VPS public IP. - # Wings matches it to a WireGuard peer endpoint and derives the control IP - # from that peer's IPv4 /32 AllowedIPs entry. - edge_wireguard_interface: wg-calagopus - edge_ssh_identity_path: /etc/calagopus-wings/edge-forward - edge_known_hosts_path: /etc/calagopus-wings/edge-known-hosts - rootfs_size_gib: 8 - console_log_max_bytes: 5242880 - tag_prefix: calagopus - managed_file_directory: /var/lib/lxc/calagopus-wings-managed - unprivileged: true - firewall: - # Prefer PVE guest rules when the datacenter firewall is enabled, then - # fall back to host-local nftables or iptables. - backend: auto - source_file_max_entries: 10000 - source_file_max_bytes: 1048576 -``` - -With `rootfs_storage: auto`, Wings orders active, enabled Proxmox storages that -support `rootdir` by free capacity. It creates the LXC on the largest pool that -has at least the requested rootfs size available, then falls through to the -next pool if the preferred pool cannot fit it. - -Panel setup and node statistics report the filesystem that contains -`system.data_directory`. Persistent game data is bind-mounted from that path, so -rootfs pools do not increase allocatable server storage. Rootfs selection and -panel game-data capacity are intentionally reported separately. - -PVE's low-level unprivileged bind-mount setup must be able to traverse every host -directory component used as a raw file source. Wings therefore stages managed -`/etc/machine-id`, `/etc/hosts`, `/etc/passwd`, and `/etc/group` files under the -PVE-specific `managed_file_directory` with execute-only traversal for non-root -users, instead of weakening the existing 0700 `system.vmount_directory`. The -configured staging path must be absolute and its parent components must already -be traversable on the PVE host. - -## Compatibility plan - -The PVE runtime should preserve the contracts already exposed by `ProcessHandle` -and `ServerExecutor`: - -1. Pull the configured OCI image into Proxmox template storage, then create one LXC - per Calagopus server using a cluster-assigned VMID. -2. Tag each CT with a stable Calagopus marker and server UUID so Wings can recover - ownership after restart without maintaining a second mapping database. -3. Map memory, swap and CPU limits to native LXC settings and mount the existing - Wings server data directory at `/home/container`. -4. Preserve the OCI image process semantics and expose them through a Wings-owned - process session. The server process uses the OCI image entrypoint as the LXC - init process and Wings attaches to the PVE console for stdin/stdout. -5. Run install and script jobs through the same execution layer, with explicit - resource/time limits. -6. Resolve PVE networking separately from Docker port publishing. The first target - is bridged per-container networking; host DNAT can be added as a separate policy - when allocations need host-port translation. - -## Implemented boundary - -- Runtime selection is isolated in `server::executor::create_runtime`; existing - Docker behavior remains the default path. -- PVE boot validates the host/PVE version, resolves the target node, and verifies - that a cluster VMID can be allocated. -- PVE cluster inventory filters to the selected node and recovers requested server - UUIDs from Calagopus ownership tags. -- Automatic runtime probing is bounded to three seconds per candidate. Proxmox - asynchronous tasks have a 15-minute deadline, and locked configuration helpers - have a 30-second deadline, so a stuck Docker socket, OCI pull, or PVE config - lock cannot hold Wings startup or an install forever. -- The LXC network is configured entirely in Wings' runtime config. `bridge` - selects the PVE bridge and `vlan_tag` optionally tags each veth. Without edge - publishing, `network_prefix` plus `gateway` define the isolated game subnet and - a primary panel allocation becomes the static `net0` address. With edge - publishing enabled, panel allocations are public VPS addresses and every main - or helper CT uses DHCP on the hidden bridge instead. -- Edge publishing needs no per-VPS address in Wings configuration. Wings reads - `wg show dump`, matches the panel allocation IP to a peer's current - public endpoint, and uses that peer's IPv4 `/32` AllowedIPs address for its - restricted SSH control connection. This continues to work when a hypervisor's - home WAN address changes and lets the panel allocation alone choose among - multiple edge VPS peers. Before applying a mapping, Wings clears the server UUID - from every discovered edge peer so allocation changes cannot leave stale DNAT. - Once the CT starts, Wings resolves its DHCP IPv4 address through the Proxmox - interface API and sends that hidden address plus the panel ports to the selected - edge controller. Cleanup clears the mapping on every edge peer. The SSH - identity must be a root-owned regular file with mode `0600` or more restrictive. - Each edge should restrict that key to the forwarding controller with an OpenSSH - `command="...",restrict` authorized-key entry. - For a running owned CT, Wings can query the Proxmox LXC interface API, ignore loopback - and link-local-only addresses, collect all usable IPv4/IPv6 addresses, prefer - IPv4 when resolving a single tunnel target, and resolve that target directly to - the CT address and requested game port. The same primary address is exposed as - the runtime's published address for this initial bridged model. -- Used-port discovery now follows that direct-bridge model. Wings inventories all - running LXC containers on the selected local node, matches their actual bridged - addresses against the queried allocation IPs, resolves each CT's init PID, and - reads the CT network namespace through `/proc//net/{tcp,tcp6,udp,udp6}`. - TCP reports listening sockets; UDP reports bound/connected reservations. Wildcard - sockets apply to every queried usable address on that CT, IPv4-mapped IPv6 - bindings are normalized, Calagopus-owned server CTs are attributed to their - server UUID, and other running LXCs are returned as used with no server owner. - This detects direct-address collisions without adding Docker-style host-port - publishing or DNAT. A container that disappears or becomes unreadable during - inventory is logged and skipped instead of aborting allocation discovery for - every other container. -- Server firewall policy is now wired into the PVE runtime without introducing a - Docker-style host NAT layer. Wings creates a host-local nftables or iptables - backend and expands configured allocation ports directly against every usable - bridged CT address. PVE firewall specs intentionally contain no published host - bindings, so rules target the CT destination IP/port rather than an emulated - host port. -- Firewall state participates in the PVE lifecycle: the backend boots with the - runtime, startup reconciliation rebuilds rules for known servers, setup and - process attachment synchronize rules, configuration sync refreshes them, and - server cleanup clears them. A running CT with configured firewall rules fails - closed if Wings cannot resolve a usable bridged address or cannot use the - selected host-local firewall backend. Starting such a CT stops it again when - its rules cannot be applied. -- `runtime.pve_lxc.firewall.backend: auto` now prefers VM-scoped Proxmox - firewall rules when the datacenter firewall is enabled and otherwise retains - the host-local fallback. `proxmox` requires the datacenter firewall and fails - clearly when it is disabled; Wings never changes the cluster-wide setting. - Managed rules and source-file IP sets are written atomically under Proxmox's - VM firewall lock, preserve unrelated administrator rules, appear in the PVE - UI, and carry stable comments so reconciliation removes only Wings-owned - entries. Managed LXC veths enable PVE firewall processing with `firewall=1`. - Reconciliation is restricted to tagged containers on this Wings instance's - configured local PVE node, so another cluster node's Wings instance cannot - clear its rules. If `auto` falls back from Proxmox to a host-local backend, - Wings first removes its own stale guest rules while preserving administrator - rules. Wings owns guest firewall mode for its tagged containers and - sets the guest input policy to `ACCEPT` because panel firewall policy - allows unmatched traffic, while Proxmox otherwise defaults unmatched guest - input to `DROP`. Only the ordered panel rules appear in the guest rule list. - The panel's terminal unrestricted deny is emitted as a protocol-free, - port-free PVE `DROP`, so "Deny Everything Else" covers all remaining guest - input rather than only the server's allocated ports. - When the configured bridge uses a host-local DHCP/DNS service, the PVE host - firewall must allow those service ports from the game subnet. A masqueraded - bridge used with the legacy PVE firewall also needs the standard `fwbr+` - conntrack-zone rule so reply traffic returns through the same zone. -- OCI image references without a tag are normalized to `:latest`. Tagged - references are pulled through `/nodes/{node}/storage/{storage}/oci-registry-pull`, - and Wings waits for the returned Proxmox task to stop with `exitstatus: OK`. -- Live Proxmox VE 9.2.2 validation confirmed that `oci-registry-pull` can emit - image-copy progress on stdout before the JSON task UPID even when - `--output-format json` is requested. The CLI parser therefore accepts both a - clean JSON response and the real PVE progress-plus-UPID response instead of - assuming stdout contains only JSON. -- OCI templates use a deterministic `calagopus-oci-.tar` cache key derived - from the normalized image reference. The Docker image-fetch cache settings also - control LXC template freshness. Once stale, a mutable tag is pulled again and - Wings reads the resolved manifest digest from the OCI archive. Container image - ownership tags use that content digest, so a changed upstream tag triggers the - normal staged rootfs replacement flow. -- Digest (`@sha256:...`) references are rejected clearly for now because the PVE - OCI pull API currently accepts tagged references. -- `pct create` arguments model memory, swap, cores, bridge selection, rootfs - storage, and ownership tags. Bind mounts are deliberately configured in a - second step: after the OCI CT rootfs is created, Wings reads PVE's resolved - `lxc.init.uid`/`lxc.init.gid`, rejects a root/default image user, then configures - `/home/container` with `pct set --mp0` and PVE 9.2's per-mount `idmap` option. - That maps the image UID/GID directly to the UID/GID Wings already uses to own - server files on the host without changing container-wide `lxc.idmap` or host - `/etc/subuid` and `/etc/subgid` ranges. Because PVE's per-mount idmap only - applies to unprivileged CTs, this runtime currently rejects - `pve_lxc.unprivileged: false` at boot. -- Server mounts now use the same allowed-mount validation as the Docker runtime. - PVE `mpN` entries support the main `/home/container` data mount, optional - `/dev/hugepages`, configured directory mounts, and read-only mounts. Each source - is canonicalized and maps the OCI process UID/GID to Wings' configured host - UID/GID, preserving the host-side identity used by the Docker runtime instead of - granting the process ownership based on the mounted directory's owner. - Native `mpN` is directory-oriented, so Wings' generated per-file mounts use - low-level `lxc.mount.entry` bind mounts with `ro,create=file`. Managed - `machine-id`, `hosts`, `passwd`, and `group` files are copied from Wings' vmount - directory into a PVE-specific staging directory with traversable parent paths, - then mounted read-only into the CT. Changes to those source files are reflected - when Wings next reconciles the runtime configuration. - Runtime config reconciliation owns only those four target paths and preserves - unrelated low-level LXC entries. Disabling one of the corresponding Wings - features removes a stale managed entry on the next config reconciliation. -- KVM and configured character/block devices can be passed through with PVE 9.2 - `devN`. The host device's numeric UID/GID and mode are preserved on the CT device - node. Configured devices must use the same source and target path. - `devN` can express read-only or read/write access here; Docker's independent - `m` (mknod) permission and write-only access are rejected instead of being - silently approximated. -- Mount and device configuration is reconciled as one `pct set` operation. Desired - `mpN`/`devN` entries are replaced in place and stale slots owned by the managed - CT are deleted, so removing an egg mount/device cannot leave old passthrough - configuration behind after Wings restarts and reuses the CT. -- The two-stage create helper is transactional for this preparation phase: if OCI - user inspection or mount/device configuration fails, it destroys the newly - created CT. A cleanup failure is returned alongside the original preparation - failure so an orphaned CT cannot be hidden. -- Server configuration can now be translated into that preparation flow: Wings - resolves the configured OCI image/cache, allocates a VMID, applies the runtime - storage/bridge/tag settings, uses the existing server data path and host - UID/GID, then performs the transactional create + mount sequence from - `setup_server_process` before returning the usable PVE `ProcessHandle`. -- Preparation now reconciles ownership before allocating a VMID. Each new CT gets - a deterministic OCI-image identity tag derived from the resolved manifest - digest. A single existing CT with the same server and image tags is reused, - its mutable resource limits are synchronized, and its OCI user is re-read from - PVE. Duplicate permanent or staged CT claims fail explicitly. -- A stopped owned CT whose OCI image identity changed is replaced transactionally. - Wings first resolves the target template and creates/configures a second CT with - a server-specific staging tag, while the old CT and host-owned server data stay - intact. Only after OCI-user discovery plus mount/device reconciliation succeeds - does Wings destroy the old rootfs and promote the staged CT to the normal server - ownership tag. If removing the old CT fails, the new staging CT is discarded - when possible. If Wings exits or tag promotion fails after the old CT is gone, - the staging tag is discoverable and the next setup resumes promotion instead of - creating an untracked duplicate. Stale staging CTs from abandoned image changes - are cleaned up only while stopped. A running old or staging CT is never killed - implicitly by image reconciliation; it must be stopped before replacement. - `/home/container` remains the same Wings-host directory throughout this flow, so - replacement affects the OCI rootfs rather than persistent server data. -- PVE resource mapping uses `memory_limit + overhead_memory` directly as the CT RAM - limit, maps panel swap as PVE's additional swap amount, and maps CPU percent to - `cpulimit` CPU-time units (`100% = 1`, `250% = 2.5`). Unlimited memory, unlimited - swap (`-1`), and CPU pinning via `build.threads` currently fail explicitly - because they do not yet have exact PVE mappings in this runtime. -- The CLI resource update path uses `pct set` for memory, swap, and CPU time. It - writes `cpulimit=0` when the panel CPU limit is disabled so an old limit cannot - survive a configuration change. `cores` is updated when an explicit value is - present. -- The CLI layer can spawn a Wings-owned `lxc-attach` session with piped standard - I/O, a clean explicitly supplied environment, and an explicit UID/GID, - including OCI images whose configured process user is root. -- Server setup now creates or reuses the owned CT, applies runtime environment - variables and `lxc.signal.halt` under PVE's config lock, then exposes it through - a PVE `ProcessHandle`. Environment keys supplied by Wings replace matching - existing values while unrelated low-level entries are preserved. PVE serializes - its high-level `env` property as `lxc.environment.runtime`, which reaches hooks - but not CT init on the tested LXC 7 host. Wings therefore retains the PVE `env` - metadata and also writes reconciled `lxc.environment` entries for the init - process. Both PVE representations are normalized by variable name before the - panel overlay is applied, making repeated synchronization idempotent. -- New CTs are created with `cmode=console`. The process handle attaches with - `pct console` through a PTY, creates a session with the PTY slave as the - controlling terminal, and forwards Wings stdin. The PTY master uses separate - Tokio file handles for reading and writing so an idle blocking read cannot - prevent console commands or graceful-stop input. Wings converts producer line - feeds to terminal carriage returns at this boundary, line-buffers stdout into - both existing broadcast paths, applies the same console throttling policy used - by the Docker path, and reconnects the console while the CT remains running. - `pct console` connection banners are excluded from broadcasts and runtime logs. -- Filtered console lines are retained on the Wings host at - `/pve-lxc/.log`. Fresh server setup truncates the - file; process reattachment preserves it. `console_log_max_bytes` bounds each - file and starts a fresh retained log at the limit. `logs(Some(n))` seeks - backward to the requested trailing lines without loading the entire file; - `logs(None)` streams the full retained log. -- Process state is polled from `pct status`. A newly prepared CT suppresses the - initial stopped state until `pct start` succeeds. A CT that starts and exits - between status polls still emits stopped and enters normal crash handling. PVE does not expose - Docker-equivalent process exit-code/OOM metadata through this path, so a stopped - CT currently reports `exit_code: -1` and `oom_killed: false`. -- Signal-based stops set `lxc.signal.halt` from the configured Wings stop signal - and request a graceful `lxc-stop --nowait --nokill`. Command-based stops are - sent over the attached console. Hard kill uses `pct stop`. Immediate restart - after a graceful stop can hit PVE's transient monitor-socket teardown window; - `pct start` is retried only for that known failure, after checking whether the - CT actually started despite the client error. -- `sync_configuration` updates mutable PVE resource limits plus runtime environment - and halt-signal configuration. Panel entrypoint overrides are rejected for now; - the OCI image entrypoint is used as the server process. -- Normal server cleanup hard-stops a running CT and destroys the owned container, - so the next normal setup starts from a fresh OCI root filesystem while keeping - the host-owned server data directory. -- Installation and ad-hoc script execution now use short-lived helper CTs rather - than Docker. Helpers have role-specific ownership tags so they cannot be - mistaken for the main server CT. They reuse the configured OCI image pull/cache - path, mount the server data at `/mnt/server`, and mount a host staging directory - at `/mnt/install` or `/mnt/script` with the same host-managed UID/GID mapping used - by the main data mount. -- Helper CTs execute the requested entrypoint plus staged script through - `lxc.init.cmd`, making the job itself PID 1. `lxc.console.logfile` writes console - output to a durable Wings-host file, which lets Wings stream output without a - long-lived `lxc-attach` child and lets an in-progress installer be reattached - after a Wings restart. Installer status/progress files remain host-backed and - use the existing Wings installation result flow. The status file begins with a - failure marker and is cleared only after a successful installer exit, so a - killed or OOM-terminated helper cannot be reported as successful completion. -- Both helper and game CTs use a small host-managed init wrapper that waits for - PVE's host-side DHCP hook to install a default route before invoking the OCI - entrypoint. The game wrapper then `exec`s the original OCI command. Installer - wrappers also write a nonzero script exit to the existing installation status - file so an early helper failure cannot be reported as a successful install. -- Installer helpers persist until normal installation cleanup so their retained - console/status data remains available for result evaluation. Script helpers are - automatically destroyed after the CT stops and their final console bytes have - been drained; their per-run staging directory is removed at the same time. -- Helper process state uses the same PVE status polling limitation as the main - process (`exit_code: -1`, `oom_killed: false`). A helper that starts and exits - between polls still emits a running transition before stopped so very short - installers cannot strand the existing installation state machine waiting for a - transition it never observed. -- OCI template preparation is serialized across main and helper provisioning so - concurrent requests cannot corrupt the shared cache. Container creation itself - remains concurrent. A VMID collision from another cluster client is retried - with a newly allocated ID up to three times. -- Main-server resource telemetry now uses Proxmox - `/nodes/{node}/lxc/{vmid}/status/current` to discover the running CT init PID, - then reuses Wings' existing cgroup-v2 sampler against that process. It publishes - memory usage/limit, CPU percent, configured CPU limit, per-container network - byte/packet counters, uptime, disk usage, and current server state through the - existing resource-usage watch channel. CPU percent is calculated from cgroup - CPU-time deltas using the same semantics as the Docker path. -- The PVE stats worker re-resolves the runtime PID and cgroup if the CT restarts, - its cgroup files disappear, or the sampler stops delivering. Offline state - clears live-container metrics while preserving disk usage. A running CT with no - usable PID or cgroup-v2 path is retried and reported as unavailable rather than - falling back to guessed host-wide values. -- Tundra can run as a native child process when Docker is unavailable. Wings - renders the same control-plane config and Unix socket used by the Docker path, - installs the configured `tundra-node` binary into its managed data directory, - tracks the child with its PID file, and reconnects to an existing owned process - after a Wings restart. PVE inventory publishes each running CT as - `pid:`; the Tundra node resolves that reference directly through - procfs, so Docker is not required to discover the CT network namespace, address, - or managed hosts file. Docker remains the preferred runtime when both backends - are available. The matching Tundra source change is preserved in - `patches/tundra-pve-lxc-process-ref.patch` until it can be merged upstream. - The patched daemon advertises `process_container_refs` in its metrics response; - Wings refuses to publish LXC references unless that capability is present. - CI verifies the patch against pinned Tundra commit - `ccb05e1406232f8c39f05a984133277026cd79d6`. -- Adopting an already-running LXC reconciles Wings-owned runtime files and - `lxc.mount.entry` records. This creates Tundra's managed hosts file immediately; - when the feature was enabled after the CT started, the new read-only - `/etc/hosts` bind mount takes effect on that CT's next restart. - -## Initial support contract - -- Native Tundra requires a `tundra-node` built with - `patches/tundra-pve-lxc-process-ref.patch`. Wings does not build or install that - patched binary; operators must supply the compatible executable configured for - the node. Capability negotiation fails closed when an unpatched daemon connects. -- Panel entrypoint overrides, CPU pinning, unlimited memory, unlimited swap, and - digest-form image references are rejected explicitly. PVE OCI pulls currently - do not receive the Docker private-registry credentials from Wings. -- One primary IPv4 allocation selects the CT address or edge peer. Additional - allocation ports are supported on that address; multiple allocation IPs are - not yet mapped to extra LXC interfaces. -- PID limits, block-I/O weight, and a writable-rootfs toggle do not yet have LXC - mappings. The OCI rootfs remains writable and Wings enforces the implemented - memory, swap, CPU-time, mount, device, network, and firewall settings. -- Edge publishing is an optional deployment component. It discovers peers from - the configured WireGuard interface and invokes the restricted edge controller - over SSH with pinned host keys. Controller commands have bounded SSH process - execution and failures fail the required network-policy synchronization; Wings - stops a newly started server when its required edge mapping cannot be applied. - The forced-command controller receives `clear ` with no stdin, or - `sync ` with JSON stdin containing `ip`, `tcp`, and `udp`. WireGuard - discovery times out after five seconds and each SSH operation after 15 seconds. - Sites that do not configure the edge identity use direct bridged allocations - and have no dependency on that controller. - -## Disposable PVE smoke test - -`scripts/pve-lxc-smoke-test.sh` exercises the host contracts the runtime depends -on without requiring a running Wings instance. Run it as root on a disposable PVE -9.2+ node. By default it uses `local` for OCI templates, `local-lvm` for rootfs, -`vmbr0` for networking, and the non-root -`docker.io/nginxinc/nginx-unprivileged:alpine` image. Those values can be -overridden with `TEMPLATE_STORAGE`, `ROOTFS_STORAGE`, `BRIDGE`, and `IMAGE`. - -The smoke test pulls/reuses the OCI template, creates an unprivileged CT with the -same `pct create` shape used by Wings, reads PVE's resolved `lxc.init.uid/gid`, -applies the per-mount idmap for `/home/container`, starts the CT, verifies the -interfaces and `status/current` APIs, checks the init PID's cgroup and network -namespace proc files, writes through the mapped data mount as the OCI user, -checks init command, environment, console logging and a read-only managed file, -then tests graceful stop, immediate restart with the same transient-error retry -policy, hard stop, and rootfs recreation with persistent host-backed data. -The CT and temporary host data are cleaned up automatically unless `KEEP_CT=1` is -set. The OCI template cache is intentionally retained for repeat runs. - -The smoke test passed end-to-end on a disposable node running PVE 9.2.2 -(`pve-manager/9.2.2`, kernel `7.0.2-6-pve`) on September 28, 2026. The immediate -restart encountered one monitor teardown error and succeeded on retry. The run -also verified OCI template reuse, DHCP/interface discovery, `status/current` -init-PID discovery, the `/home/container` per-mount idmap, host ownership -translation, cgroup-v2 and `/proc/{pid}/net/{tcp,tcp6,udp,udp6}` visibility, -runtime environment reaching init, durable console output, read-only managed -`/etc/hosts`, graceful and hard stops, rootfs recreation, and persistent data. -The tagged CT and its two temporary host directories were removed. PVE's -`pct config` output also contained a NUL byte during this OCI flow; the runtime -and smoke test strip NULs before parsing the otherwise textual configuration. - -### Panel-to-edge Minecraft validation - -The deployed Wings binary passed an end-to-end panel-created Paper server test on -a disposable PVE node: - -- The installer helper acquired `10.70.0.20` from the host-only - `vmbr-calagopus` DHCP service and downloaded the Paper server jar. -- The game CT ran unprivileged on the same bridge and acquired `10.70.0.21`. - Its OCI entrypoint waited for DHCP, downloaded the Mojang runtime jar, generated - a world, and reported `Done (27.464s)` while listening on `0.0.0.0:25565`. -- Wings selected the WireGuard peer whose endpoint matched the panel allocation - `192.0.2.10`, then installed TCP and UDP DNAT rules on that edge for - `192.0.2.10:25565 -> 10.70.0.21:25565`. -- A TCP probe from the development host to `192.0.2.10:25565` and a second - probe on the edge to `10.70.0.21:25565` both succeeded. The panel reported the - server running with live CPU, memory, disk, uptime, and network telemetry. -- A panel console `list` command reached Paper and returned its online-player - response. The panel Stop action then sent Paper's configured stop command, - Paper saved all three dimensions, VMID 100 reached `stopped`, and a subsequent - panel Start returned the server to `Done (22.133s)`. No `pct console` connection - banners appeared in the panel or persisted runtime log. A fresh probe to the - public allocation succeeded after restart. -- PVE can temporarily return JSON `null` from the interfaces endpoint while a CT - finishes shutting down. Wings treats that response as an empty interface list, - allowing an immediate panel Restart to clear stale firewall state and continue. - A live restart after this change returned Paper to `Done (23.479s)`. -- Paper 26.3 with Java 25 exhausted the original 1 GiB test limit during initial - world creation. PVE's kernel log confirmed a memory-cgroup OOM kill. Raising - the panel limit to 4 GiB produced a stable server at about 1.4 GiB resident - memory. This was a workload sizing issue rather than an LXC networking failure. - -### Native Tundra validation - -During live validation, the PVE node was enrolled in the panel private network at its -Tailscale address on UDP 7100 with Docker absent. Wings launched the native -`tundra-node`, issued and published its node certificate through the existing -control-plane shim, and the panel reported the daemon connected with its snapshot -applied. The running Minecraft LXC joined with hostname `test-mc.tunnel` and -offered TCP 25565. Tundra accepted the Wings `pid:` reference, adopted -the CT without Docker, and applied a snapshot containing the LXC server. Metrics -reported the control link up and the daemon listening on UDP 7100. A second -private-network member is still required to exercise an inter-node QUIC relay and -end-to-end hostname connection. - -## Remaining work before production use - -- Define a verified PVE-safe equivalent for the optional managed - `/sys/class/dmi/id/product_uuid` file. The regular generated files use live - read-only LXC file bind mounts, but sysfs behavior still needs real-PVE - validation before claiming parity for this path. -- Confirm private-registry authentication behavior for PVE's OCI pull path. -- Continue live PVE 9.2 validation for less common Wings behavior: helper - reattachment after a daemon restart, one-shot script cleanup, and - host-local nftables/iptables filtering on bridged CT traffic. Main process - startup, helper installation, DHCP, telemetry, WireGuard peer selection, edge - DNAT synchronization, and public connectivity have now been exercised through - the Rust runtime against the live panel. - -A cached Rust 1.97 toolchain is available on the current macOS development host; -formatting passes there. The macOS test build cannot produce a Wings binary because -existing filesystem code uses Linux-only `rustix` APIs and the PVE console-spawn -method is intentionally compiled only on Linux. On September 28, 2026, the branch -was archived into a disposable Rust 1.97.1 LXC on a disposable PVE build container with locked Cargo caches. -`cargo check --tests --offline --target x86_64-unknown-linux-gnu` passed, followed -by `cargo test --offline --target x86_64-unknown-linux-gnu`: 743 passed, 9 ignored, -0 failed. `CARGO_GIT_BRANCH` was supplied because the source archive omitted -`.git`. The tagged build CT, uploaded archives, and temporary Rust OCI template -were removed after the run. This validates Linux compilation and unit tests, but -it does not run a configured Wings daemon or the full PVE executor against a live -panel. The later live validation above fills that gap for the primary install, -start, telemetry, console-command, graceful-stop, and edge-forwarding paths. The -current PVE-specific unit suite also passes all 60 targeted tests on the PVE test -node before the deployed release build. diff --git a/patches/tundra-pve-lxc-process-ref.patch b/patches/tundra-pve-lxc-process-ref.patch deleted file mode 100644 index 479a28be..00000000 --- a/patches/tundra-pve-lxc-process-ref.patch +++ /dev/null @@ -1,169 +0,0 @@ -From 14667f635b0fcf263d2de44f03d92fae6e7bf9e6 Mon Sep 17 00:00:00 2001 -From: Philip Ulrich -Date: Tue, 29 Sep 2026 10:53:42 -0400 -Subject: [PATCH] support process-backed containers in tundra node - ---- - tundra-node/build.rs | 2 ++ - tundra-node/src/main.rs | 2 +- - tundra-node/src/node/docker.rs | 60 ++++++++++++++++++++++++++++++++++ - tundra-node/src/node/mod.rs | 18 +++++++--- - tundra-node/src/metrics.rs | 1 + - 5 files changed, 77 insertions(+), 6 deletions(-) - -diff --git a/tundra-node/build.rs b/tundra-node/build.rs -index 729a5c7..514bc95 100644 ---- a/tundra-node/build.rs -+++ b/tundra-node/build.rs -@@ -46,6 +46,8 @@ fn handle_git_info() { - { - git_hash = String::from_utf8_lossy(&output.stdout).trim().to_string(); - } -+ } else { -+ println!("cargo:rustc-env=CARGO_GIT_BRANCH=unknown"); - } - println!("cargo:rustc-env=CARGO_GIT_COMMIT={git_hash}"); - } -diff --git a/tundra-node/src/main.rs b/tundra-node/src/main.rs -index 4d64e8e..24bfd67 100644 ---- a/tundra-node/src/main.rs -+++ b/tundra-node/src/main.rs -@@ -108,7 +108,7 @@ async fn main() -> Result<(), anyhow::Error> { - Ok(d) => Some(d), - Err(err) => { - tracing::warn!( -- "docker is unavailable, no containers will be adopted: {:?}", -+ "docker is unavailable; Docker container references cannot be adopted, but process-backed references remain available: {:?}", - err - ); - None -diff --git a/tundra-node/src/node/docker.rs b/tundra-node/src/node/docker.rs -index 3054d12..7917fca 100644 ---- a/tundra-node/src/node/docker.rs -+++ b/tundra-node/src/node/docker.rs -@@ -6,6 +6,8 @@ use bollard::{ - use futures::{Stream, StreamExt}; - use std::{collections::HashMap, net::Ipv4Addr, path::PathBuf, pin::Pin}; - -+pub const PROCESS_REF_PREFIX: &str = "pid:"; -+ - fn classify(msg: bollard::models::EventMessage) -> Option { - let actor = msg.actor?; - let id = actor.id?; -@@ -52,6 +54,48 @@ pub struct ContainerInfo { - pub hosts_path: PathBuf, - } - -+impl ContainerInfo { -+ /// Adopt a container described by its init PID instead of a Docker object. -+ /// Wings uses this for runtimes such as Proxmox LXC. -+ pub fn from_process_ref( -+ container_ref: &str, -+ dial_addr: Option<&str>, -+ ) -> Result, anyhow::Error> { -+ let Some(raw_pid) = container_ref.strip_prefix(PROCESS_REF_PREFIX) else { -+ return Ok(None); -+ }; -+ let pid: i32 = raw_pid.parse().context(format!( -+ "invalid process container reference {container_ref}" -+ ))?; -+ if pid <= 0 { -+ return Err(anyhow::anyhow!( -+ "process container reference {container_ref} has an invalid PID" -+ )); -+ } -+ -+ let running = PathBuf::from(format!("/proc/{pid}/ns/net")).exists(); -+ let ip = dial_addr.and_then(|dial| { -+ dial.parse::() -+ .map(|address| address.ip()) -+ .or_else(|_| dial.parse::()) -+ .ok() -+ .and_then(|address| match address { -+ std::net::IpAddr::V4(address) => Some(address), -+ std::net::IpAddr::V6(_) => None, -+ }) -+ }); -+ -+ Ok(Some(Self { -+ id: container_ref.to_owned(), -+ pid, -+ running, -+ ip, -+ // Process-backed runtimes configure a control-plane hosts template. -+ hosts_path: PathBuf::new(), -+ })) -+ } -+} -+ - #[derive(Debug, Clone, PartialEq, Eq)] - pub enum ContainerEvent { - Started { id: String, name: Option }, -@@ -136,6 +180,22 @@ mod tests { - use super::*; - use bollard::models::{EndpointSettings, EventActor, EventMessage}; - -+ #[test] -+ fn process_references_are_runtime_neutral() { -+ let reference = format!("pid:{}", std::process::id()); -+ let info = ContainerInfo::from_process_ref(&reference, Some("10.70.0.58:25565")) -+ .unwrap() -+ .unwrap(); -+ assert!(info.running); -+ assert_eq!(info.pid, std::process::id() as i32); -+ assert_eq!(info.ip, Some(Ipv4Addr::new(10, 70, 0, 58))); -+ assert!( -+ ContainerInfo::from_process_ref("docker-id", None) -+ .unwrap() -+ .is_none() -+ ); -+ } -+ - fn endpoint(ip: &str) -> EndpointSettings { - EndpointSettings { - ip_address: Some(ip.to_owned()), -diff --git a/tundra-node/src/node/mod.rs b/tundra-node/src/node/mod.rs -index 56f4f35..d737afb 100644 ---- a/tundra-node/src/node/mod.rs -+++ b/tundra-node/src/node/mod.rs -@@ -387,17 +387,25 @@ impl Node { - } - - pub async fn refresh_containers(&self, index: &SnapshotIndex) { -- let Some(docker) = &self.docker else { -- return; -- }; -- - let mut found = HashMap::new(); - for server in index.servers_on(&self.uuid) { - let Some(entry) = index.server(server) else { - continue; - }; - -- match docker.inspect(&entry.container_ref).await { -+ let inspected = match docker::ContainerInfo::from_process_ref( -+ &entry.container_ref, -+ entry.dial_addr.as_deref(), -+ ) { -+ Ok(Some(info)) => Ok(info), -+ Ok(None) => match &self.docker { -+ Some(docker) => docker.inspect(&entry.container_ref).await, -+ None => continue, -+ }, -+ Err(err) => Err(err), -+ }; -+ -+ match inspected { - Ok(info) if info.running => { - found.insert(*server, info); - } -diff --git a/tundra-node/src/metrics.rs b/tundra-node/src/metrics.rs ---- a/tundra-node/src/metrics.rs -+++ b/tundra-node/src/metrics.rs -@@ -284,3 +284,4 @@ impl Metrics { - json!({ - "applied": self.applied, -+ "capabilities": ["process_container_refs"], - "node": { --- -2.50.1 (Apple Git-155) From e892a9e7fd851c78400597f6aa1868ba246c509f Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 04/13] map panel runtime limits to native LXC settings --- .../src/server/executor/pve_lxc/cli.rs | 243 ++++++++++++++++-- .../src/server/executor/pve_lxc/mod.rs | 79 +++++- 2 files changed, 285 insertions(+), 37 deletions(-) diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index 8ce922c4..180ec9cc 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -117,7 +117,17 @@ PVE::LXC::Config->lock_config($vmid, sub { && $_->[0] ne 'lxc.environment.runtime' && $_->[0] ne 'lxc.signal.halt' && $_->[0] ne 'lxc.init.cmd' - && $_->[0] ne 'lxc.console.logfile'); + && $_->[0] ne 'lxc.console.logfile' + && $_->[0] ne 'lxc.cgroup.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup2.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup.pids.max' + && $_->[0] ne 'lxc.cgroup2.pids.max' + && $_->[0] ne 'lxc.cgroup.blkio.weight' + && $_->[0] ne 'lxc.cgroup2.io.weight' + && $_->[0] ne 'lxc.cgroup.memory.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.max' + && $_->[0] ne 'lxc.cgroup.memory.memsw.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); if ($keep && defined($managed_mounts) && ref($_) eq 'ARRAY' && @$_ >= 2 && $_->[0] eq 'lxc.mount.entry') { my $target = mount_target($_->[1]); @@ -139,6 +149,21 @@ PVE::LXC::Config->lock_config($vmid, sub { if (defined($payload->{console_logfile}) && length($payload->{console_logfile})) { push @lxc, ['lxc.console.logfile', $payload->{console_logfile}]; } + if (defined($payload->{cpuset_cpus}) && length($payload->{cpuset_cpus})) { + push @lxc, ['lxc.cgroup2.cpuset.cpus', $payload->{cpuset_cpus}]; + } + if (defined($payload->{pids_limit}) && $payload->{pids_limit} > 0) { + push @lxc, ['lxc.cgroup2.pids.max', $payload->{pids_limit}]; + } + if (defined($payload->{io_weight}) && $payload->{io_weight} > 0) { + push @lxc, ['lxc.cgroup2.io.weight', $payload->{io_weight}]; + } + if ($payload->{memory_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.max', 'max']; + } + if ($payload->{swap_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; + } if (defined($managed_mounts)) { push @lxc, map { ['lxc.mount.entry', $_->{entry}] } @$managed_mounts; } @@ -335,6 +360,8 @@ pub struct ContainerOciUser { pub struct ContainerResources { pub memory_mib: u64, pub swap_mib: u64, + pub memory_unlimited: bool, + pub swap_unlimited: bool, pub cpu_limit_percent: Option, pub cores: Option, } @@ -361,6 +388,16 @@ pub struct RuntimeConfigSpec { pub halt_signal: Option, pub init_command: Option, pub console_logfile: Option, + /// Exact host CPUs assigned through the panel's `build.threads` setting. + /// `None` removes a previously managed CPU set. + pub cpuset_cpus: Option, + /// Maximum process count for the container. `None` removes the managed + /// cgroup limit. + pub pids_limit: Option, + /// cgroup v2 I/O weight in the kernel's 1..=10000 range. + pub io_weight: Option, + pub memory_unlimited: bool, + pub swap_unlimited: bool, /// `None` leaves low-level LXC file mounts untouched. `Some`, including an /// empty vector, reconciles all Wings-reserved managed-file targets. pub managed_file_mounts: Option>, @@ -473,9 +510,9 @@ impl PveCli { cpu_limit: i64, threads: Option<&str>, ) -> Result { - if memory_limit <= 0 { + if memory_limit < 0 { return Err(anyhow::anyhow!( - "Proxmox LXC currently requires a positive server memory limit; unlimited memory is not mapped yet" + "invalid negative memory limit for Proxmox LXC: {memory_limit}" )); } if overhead_memory < 0 { @@ -483,22 +520,26 @@ impl PveCli { "server overhead memory cannot be negative for the Proxmox LXC runtime" )); } - let memory_mib = memory_limit - .checked_add(overhead_memory) - .context("server memory plus overhead overflowed")?; - if memory_mib < 16 { - return Err(anyhow::anyhow!( - "Proxmox LXC requires at least 16 MiB of memory, resolved server limit was {memory_mib} MiB" - )); - } - let memory_mib = u64::try_from(memory_mib).context("server memory limit was negative")?; - - let swap_mib = match swap { - -1 => { + let memory_unlimited = memory_limit == 0; + let memory_mib = if memory_unlimited { + // PVE requires a regular memory property even when a low-level LXC + // cgroup override removes the limit before the first start. + 16 + } else { + let memory_mib = memory_limit + .checked_add(overhead_memory) + .context("server memory plus overhead overflowed")?; + if memory_mib < 16 { return Err(anyhow::anyhow!( - "unlimited swap (-1) has no direct Proxmox LXC equivalent" + "Proxmox LXC requires at least 16 MiB of memory, resolved server limit was {memory_mib} MiB" )); } + u64::try_from(memory_mib).context("server memory limit was negative")? + }; + + let swap_unlimited = memory_unlimited || swap == -1; + let swap_mib = match swap { + -1 => 0, value if value < 0 => { return Err(anyhow::anyhow!( "invalid negative swap limit for Proxmox LXC: {value}" @@ -517,20 +558,67 @@ impl PveCli { value => Some(u64::try_from(value).context("server CPU limit exceeded u64")?), }; - if threads.is_some_and(|threads| !threads.trim().is_empty()) { - return Err(anyhow::anyhow!( - "Proxmox LXC CPU pinning for server build.threads is not implemented yet" - )); - } + let _cpuset_cpus = threads + .filter(|threads| !threads.trim().is_empty()) + .map(Self::normalize_cpuset) + .transpose()?; Ok(ContainerResources { memory_mib, swap_mib, + memory_unlimited, + swap_unlimited, cpu_limit_percent, cores: None, }) } + pub(crate) fn normalize_cpuset(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err(anyhow::anyhow!("CPU set cannot be empty")); + } + + for part in value.split(',') { + let part = part.trim(); + if part.is_empty() { + return Err(anyhow::anyhow!("CPU set contains an empty item")); + } + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start, Some(end)), + None => (part, None), + }; + let start = start + .parse::() + .with_context(|| format!("invalid CPU set item: {part}"))?; + if let Some(end) = end { + let end = end + .parse::() + .with_context(|| format!("invalid CPU set item: {part}"))?; + if end < start { + return Err(anyhow::anyhow!("invalid descending CPU set range: {part}")); + } + } + } + + Ok(value.to_string()) + } + + pub(crate) fn cgroup2_io_weight(weight: Option) -> Result, anyhow::Error> { + let Some(weight) = weight else { + return Ok(None); + }; + if !(10..=1000).contains(&weight) { + return Err(anyhow::anyhow!( + "block I/O weight must be between 10 and 1000, got {weight}" + )); + } + + // Match opencontainers/cgroups' conversion from the cgroup v1 API + // range used by Docker to the cgroup v2 io.weight range. + Ok(Some(1 + (u64::from(weight) - 10) * 9999 / 990)) + } + async fn run(program: &std::path::Path, args: &[String]) -> Result { let mut command = tokio::process::Command::new(program); command.args(args).kill_on_drop(true); @@ -631,6 +719,20 @@ impl PveCli { }) { return Err(anyhow::anyhow!("invalid LXC console logfile path")); } + if let Some(cpuset) = spec.cpuset_cpus.as_deref() { + Self::normalize_cpuset(cpuset)?; + } + if spec.pids_limit == Some(0) { + return Err(anyhow::anyhow!("LXC PID limit must be positive")); + } + if spec + .io_weight + .is_some_and(|weight| !(1..=10_000).contains(&weight)) + { + return Err(anyhow::anyhow!( + "cgroup v2 I/O weight must be between 1 and 10000" + )); + } let (managed_file_mounts, managed_file_targets) = if let Some(mounts) = &spec.managed_file_mounts { @@ -674,6 +776,11 @@ impl PveCli { "halt_signal": spec.halt_signal, "init_command": spec.init_command, "console_logfile": spec.console_logfile, + "cpuset_cpus": spec.cpuset_cpus, + "pids_limit": spec.pids_limit, + "io_weight": spec.io_weight, + "memory_unlimited": spec.memory_unlimited, + "swap_unlimited": spec.swap_unlimited, "managed_file_mounts": managed_file_mounts, "managed_file_targets": managed_file_targets, })) @@ -2262,6 +2369,8 @@ mod tests { Some(ContainerResources { memory_mib: 2304, swap_mib: 1024, + memory_unlimited: false, + swap_unlimited: false, cpu_limit_percent: Some(250), cores: None, }) @@ -2271,6 +2380,41 @@ mod tests { Some(ContainerResources { memory_mib: 2048, swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: None, + cores: None, + }) + ); + } + + #[test] + fn rejects_invalid_resource_limits() { + assert!(PveCli::panel_resources(-1, 0, 0, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, -2, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("3-1")).is_err()); + } + + #[test] + fn maps_unlimited_memory_and_swap_to_lxc_overrides() { + assert_eq!( + PveCli::panel_resources(0, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 16, + swap_mib: 0, + memory_unlimited: true, + swap_unlimited: true, + cpu_limit_percent: None, + cores: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, -1, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: true, cpu_limit_percent: None, cores: None, }) @@ -2278,10 +2422,21 @@ mod tests { } #[test] - fn rejects_resource_limits_without_exact_pve_semantics() { - assert!(PveCli::panel_resources(0, 0, 0, 0, None).is_err()); - assert!(PveCli::panel_resources(2048, 0, -1, 0, None).is_err()); - assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3")).is_err()); + fn accepts_panel_cpu_pinning_syntax() { + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).is_ok()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0,,2")).is_err()); + } + + #[test] + fn converts_docker_block_io_weight_to_cgroup_v2() { + assert_eq!(PveCli::cgroup2_io_weight(None).ok(), Some(None)); + assert_eq!(PveCli::cgroup2_io_weight(Some(10)).ok(), Some(Some(1))); + assert_eq!(PveCli::cgroup2_io_weight(Some(500)).ok(), Some(Some(4950))); + assert_eq!( + PveCli::cgroup2_io_weight(Some(1000)).ok(), + Some(Some(10_000)) + ); + assert!(PveCli::cgroup2_io_weight(Some(9)).is_err()); } #[test] @@ -2292,6 +2447,8 @@ mod tests { &ContainerResources { memory_mib: 4096, swap_mib: 512, + memory_unlimited: false, + swap_unlimited: false, cpu_limit_percent: Some(250), cores: Some(4), }, @@ -2316,6 +2473,8 @@ mod tests { &ContainerResources { memory_mib: 2048, swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, cpu_limit_percent: None, cores: None, }, @@ -2941,6 +3100,11 @@ lxc.init.uid: 1000 halt_signal: Some("SIGINT".to_string()), init_command: Some("/bin/bash /mnt/install/install.sh".to_string()), console_logfile: Some("/var/log/calagopus/install.log".to_string()), + cpuset_cpus: Some("0-3,8".to_string()), + pids_limit: Some(5120), + io_weight: Some(4950), + memory_unlimited: true, + swap_unlimited: true, managed_file_mounts: Some(vec![ManagedFileMountSpec { source_path: "/var/lib/calagopus wings/vmount/example/hosts".to_string(), target_path: "/etc/hosts".to_string(), @@ -2976,6 +3140,30 @@ lxc.init.uid: 1000 .and_then(serde_json::Value::as_str), Some("/var/log/calagopus/install.log") ); + assert_eq!( + value.get("cpuset_cpus").and_then(serde_json::Value::as_str), + Some("0-3,8") + ); + assert_eq!( + value.get("pids_limit").and_then(serde_json::Value::as_u64), + Some(5120) + ); + assert_eq!( + value.get("io_weight").and_then(serde_json::Value::as_u64), + Some(4950) + ); + assert_eq!( + value + .get("memory_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + value + .get("swap_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); assert_eq!( value .get("managed_file_mounts") @@ -3001,6 +3189,11 @@ lxc.init.uid: 1000 halt_signal: None, init_command: None, console_logfile: None, + cpuset_cpus: None, + pids_limit: None, + io_weight: None, + memory_unlimited: false, + swap_unlimited: false, managed_file_mounts: None, }; assert!(PveCli::runtime_config_payload(&invalid).is_err()); diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index 409f59e5..9c44f958 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -941,16 +941,57 @@ exit 1 cli: &PveCli, vmid: u32, ) -> Result { - let environment = server.configuration.read().await.environment(app_config); + let ( + environment, + panel_entrypoint, + cpuset_cpus, + io_weight, + memory_unlimited, + swap_unlimited, + ) = { + let configuration = server.configuration.read().await; + let resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + ( + configuration.environment(app_config), + configuration.entrypoint.clone(), + configuration + .build + .threads + .as_deref() + .filter(|threads| !threads.trim().is_empty()) + .map(PveCli::normalize_cpuset) + .transpose()?, + PveCli::cgroup2_io_weight(configuration.build.io_weight)?, + resources.memory_unlimited, + resources.swap_unlimited, + ) + }; + let pids_limit = match app_config.load().docker.container_pid_limit { + 0 => None, + limit => Some(limit), + }; let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; let process = server.process_configuration.read().await; let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); - let image_entrypoint = cli.oci_entrypoint(vmid).await?; let mut init_arguments = vec!["/calagopus-entrypoint".to_string()]; - init_arguments.extend( - shell_words::split(&image_entrypoint) - .context("failed to parse OCI entrypoint for Proxmox LXC")?, - ); + if let Some(entrypoint) = panel_entrypoint { + if entrypoint.is_empty() { + return Err(anyhow::anyhow!("panel entrypoint override cannot be empty")); + } + init_arguments.extend(entrypoint); + } else { + let image_entrypoint = cli.oci_entrypoint(vmid).await?; + init_arguments.extend( + shell_words::split(&image_entrypoint) + .context("failed to parse OCI entrypoint for Proxmox LXC")?, + ); + } let init_command = PveCli::encode_init_command(&init_arguments)?; Ok(cli::RuntimeConfigSpec { @@ -959,6 +1000,11 @@ exit 1 halt_signal, init_command: Some(init_command), console_logfile: None, + cpuset_cpus, + pids_limit, + io_weight, + memory_unlimited, + swap_unlimited, managed_file_mounts: Some(managed_file_mounts), }) } @@ -2878,12 +2924,27 @@ exit 1 script.entrypoint.to_string(), script_target.to_string(), ])?; + let (pids_limit, io_weight) = { + let configuration = server.configuration.read().await; + ( + match self.app_config.load().docker.container_pid_limit { + 0 => None, + limit => Some(limit), + }, + PveCli::cgroup2_io_weight(configuration.build.io_weight)?, + ) + }; let runtime_config = cli::RuntimeConfigSpec { vmid, environment, halt_signal: None, init_command: Some(init_command), console_logfile: Some(log_path.to_string()), + cpuset_cpus: None, + pids_limit, + io_weight, + memory_unlimited: resources.memory_unlimited, + swap_unlimited: resources.swap_unlimited, managed_file_mounts: None, }; @@ -3304,12 +3365,6 @@ impl ServerExecutor for PveLxcExecutor { &self, server: &crate::server::Server, ) -> Result<(Arc, StatusReceiver), anyhow::Error> { - if server.configuration.read().await.entrypoint.is_some() { - return Err(anyhow::anyhow!( - "Proxmox VE LXC runtime does not support panel entrypoint overrides yet; use the OCI image entrypoint" - )); - } - let (vmid, _oci_user, created) = self.prepare_server_container(server).await?; let runtime_config = match PveProcessHandle::runtime_config( server, From f3f23d3bc333cfd5eaad3058e6611bee6894af39 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 05/13] report LXC process exit and OOM status --- .../src/server/executor/pve_lxc/mod.rs | 173 ++++++++++++++++-- 1 file changed, 157 insertions(+), 16 deletions(-) diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index 9c44f958..a27b9e31 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -143,10 +143,13 @@ struct PveProcessHandle { } impl PveProcessHandle { + const EXIT_STATUS_FILE: &'static str = ".wings-exit-status"; + fn observed_status( status: cli::ContainerStatus, start_succeeded: bool, seen_running: &mut bool, + exit_status: Option<(i32, bool)>, ) -> Option { match status { cli::ContainerStatus::Running => { @@ -154,11 +157,43 @@ impl PveProcessHandle { Some(super::ProcessStatus::Running) } cli::ContainerStatus::Stopped if !*seen_running && !start_succeeded => None, - cli::ContainerStatus::Stopped => Some(super::ProcessStatus::Stopped { - exit_code: -1, - oom_killed: false, - }), + cli::ContainerStatus::Stopped => { + let (exit_code, oom_killed) = exit_status.unwrap_or((-1, false)); + Some(super::ProcessStatus::Stopped { + exit_code, + oom_killed, + }) + } + } + } + + fn parse_exit_status(contents: &str) -> Option<(i32, bool)> { + let mut exit_code = None; + let mut oom_killed = None; + for line in contents.lines() { + let (key, value) = line.split_once('=')?; + match key { + "exit_code" => exit_code = value.parse::().ok(), + "oom_killed" => { + oom_killed = match value { + "0" => Some(false), + "1" => Some(true), + _ => None, + } + } + _ => {} + } } + Some((exit_code?, oom_killed?)) + } + + async fn read_exit_status(path: &std::path::Path) -> Option<(i32, bool)> { + let contents = tokio::fs::read_to_string(path).await.ok()?; + Self::parse_exit_status(&contents) + } + + fn exit_status_path(server: &crate::server::InnerServer) -> PathBuf { + std::path::Path::new(server.filesystem.base().as_str()).join(Self::EXIT_STATUS_FILE) } fn managed_file_stage_name(target: &str) -> Result<&'static str, anyhow::Error> { @@ -248,6 +283,7 @@ impl PveProcessHandle { let state_started = Arc::clone(&started); let state_cli = cli.clone(); let state_server_uuid = server.uuid; + let state_exit_status_path = Self::exit_status_path(server); let status_task = tokio::spawn(async move { let mut seen_running = state_started.load(Ordering::Acquire); let mut tick = tokio::time::interval(std::time::Duration::from_secs(2)); @@ -256,14 +292,22 @@ impl PveProcessHandle { loop { tick.tick().await; let process_status = match state_cli.status(vmid).await { - Ok(status) => match Self::observed_status( - status, - state_started.load(Ordering::Acquire), - &mut seen_running, - ) { - Some(status) => status, - None => continue, - }, + Ok(status) => { + let exit_status = if status == cli::ContainerStatus::Stopped { + Self::read_exit_status(&state_exit_status_path).await + } else { + None + }; + match Self::observed_status( + status, + state_started.load(Ordering::Acquire), + &mut seen_running, + exit_status, + ) { + Some(status) => status, + None => continue, + } + } Err(error) => { tracing::warn!( server = %state_server_uuid, @@ -903,13 +947,65 @@ impl PveProcessHandle { attempt=0 while [ "$attempt" -lt 120 ]; do if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then - exec "$@" + break fi attempt=$((attempt + 1)) sleep 0.25 done -echo 'timed out waiting for Proxmox DHCP' >&2 -exit 1 +if [ "$attempt" -ge 120 ]; then + echo 'timed out waiting for Proxmox DHCP' >&2 + exit 1 +fi + +read_oom_kills() { + value=0 + if [ -r /sys/fs/cgroup/memory.events ]; then + while read -r key count; do + if [ "$key" = oom_kill ]; then + value=$count + break + fi + done < /sys/fs/cgroup/memory.events + fi + printf '%s' "$value" +} + +child=0 +forward_signal() { + if [ "$child" -gt 0 ]; then + kill "-$1" "$child" 2>/dev/null || true + fi +} +trap 'forward_signal HUP' HUP +trap 'forward_signal INT' INT +trap 'forward_signal QUIT' QUIT +trap 'forward_signal ABRT' ABRT +trap 'forward_signal TERM' TERM + +oom_before=$(read_oom_kills) +"$@" <&0 & +child=$! +while :; do + wait "$child" + status=$? + if ! kill -0 "$child" 2>/dev/null; then + break + fi +done +oom_after=$(read_oom_kills) +oom_killed=0 +if [ "$oom_after" -gt "$oom_before" ]; then + oom_killed=1 +fi + +marker=/home/container/.wings-exit-status +temporary="${marker}.tmp.$$" +( + umask 077 + printf 'exit_code=%s\noom_killed=%s\n' "$status" "$oom_killed" > "$temporary" +) +mv -f "$temporary" "$marker" +exit "$status" "#, ) .await @@ -1189,6 +1285,20 @@ impl ProcessHandle for PveProcessHandle { } async fn start(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + match tokio::fs::remove_file(&exit_status_path).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to clear stale Proxmox LXC exit status {}", + exit_status_path.display() + ) + }); + } + } self.cli.start(self.vmid).await?; self.started.store(true, Ordering::Release); @@ -1240,6 +1350,16 @@ impl ProcessHandle for PveProcessHandle { } async fn kill(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + tokio::fs::write(&exit_status_path, "exit_code=137\noom_killed=0\n") + .await + .with_context(|| { + format!( + "failed to record forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + })?; self.cli.stop(self.vmid).await } } @@ -4105,13 +4225,34 @@ mod tests { cli::ContainerStatus::Stopped, true, &mut seen_running, + Some((42, true)), ); assert!(matches!( status, - Some(crate::server::executor::ProcessStatus::Stopped { .. }) + Some(crate::server::executor::ProcessStatus::Stopped { + exit_code: 42, + oom_killed: true, + }) )); } + #[test] + fn parses_managed_exit_status_strictly() { + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=137\noom_killed=1\n"), + Some((137, true)) + ); + assert_eq!( + PveProcessHandle::parse_exit_status("oom_killed=0\nexit_code=0\n"), + Some((0, false)) + ); + assert_eq!(PveProcessHandle::parse_exit_status("exit_code=0\n"), None); + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=nope\noom_killed=0\n"), + None + ); + } + #[tokio::test] async fn runtime_log_never_exceeds_its_retention_limit() { let directory = tempfile::tempdir().unwrap(); From e8bea8522c9f9422550d50c59b6711e3a2fc4610 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 06/13] split PVE LXC runtime into focused modules --- .../src/server/executor/pve_lxc/cli.rs | 1229 +--------- .../server/executor/pve_lxc/cli/scripts.rs | 237 ++ .../src/server/executor/pve_lxc/cli/tests.rs | 988 ++++++++ .../server/executor/pve_lxc/helper_process.rs | 310 +++ .../src/server/executor/pve_lxc/mod.rs | 2129 +---------------- .../server/executor/pve_lxc/server_process.rs | 1240 ++++++++++ .../src/server/executor/pve_lxc/tests.rs | 559 +++++ 7 files changed, 3343 insertions(+), 3349 deletions(-) create mode 100644 application/src/server/executor/pve_lxc/cli/scripts.rs create mode 100644 application/src/server/executor/pve_lxc/cli/tests.rs create mode 100644 application/src/server/executor/pve_lxc/helper_process.rs create mode 100644 application/src/server/executor/pve_lxc/server_process.rs create mode 100644 application/src/server/executor/pve_lxc/tests.rs diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index 180ec9cc..21d30c2f 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -23,243 +23,9 @@ const MANAGED_FILE_TARGETS: [&str; 5] = [ "/calagopus-entrypoint", ]; -const APPLY_RUNTIME_CONFIG_PERL: &str = r#" -use strict; -use warnings; -use JSON::PP qw(decode_json); -use PVE::LXC::Config; - -my $input; -{ - local $/; - $input = ; -} -my $payload = decode_json($input // '{}'); -my $vmid = int($payload->{vmid} // 0); -die "invalid VMID\n" if $vmid <= 0; - -my $overlay = $payload->{environment} // []; -die "environment must be an array\n" if ref($overlay) ne 'ARRAY'; - -my $managed_mounts = $payload->{managed_file_mounts}; -my $managed_targets = $payload->{managed_file_targets}; -my %managed_target; -if (defined($managed_mounts)) { - die "managed_file_mounts must be an array\n" if ref($managed_mounts) ne 'ARRAY'; - die "managed_file_targets must be an array\n" if ref($managed_targets) ne 'ARRAY'; - for my $target (@$managed_targets) { - die "invalid managed file target\n" if ref($target) || !defined($target) || !length($target); - $managed_target{$target} = 1; - } - for my $mount (@$managed_mounts) { - die "invalid managed file mount\n" if ref($mount) ne 'HASH'; - my $target = $mount->{target}; - my $entry = $mount->{entry}; - die "invalid managed file mount target\n" - if ref($target) || !defined($target) || !exists($managed_target{$target}); - die "invalid managed file mount entry\n" - if ref($entry) || !defined($entry) || !length($entry); - } -} - -sub mount_target { - my ($entry) = @_; - return undef if !defined($entry); - my @fields = grep { length($_) } split(/\s+/, $entry); - return scalar(@fields) >= 2 ? $fields[1] : undef; -} - -PVE::LXC::Config->lock_config($vmid, sub { - my $conf = PVE::LXC::Config->load_config($vmid); - my @lxc = @{$conf->{lxc} // []}; - my @environment; - - if (defined($conf->{env}) && length($conf->{env})) { - push @environment, split(/\0/, $conf->{env}, -1); - } - for my $entry (@lxc) { - next if ref($entry) ne 'ARRAY' || @$entry < 2; - push @environment, $entry->[1] - if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; - } - - # PVE exposes the same values through both `env` and low-level - # lxc.environment records. Collapse those views before applying the panel - # overlay so each synchronization is idempotent. - my @normalized; - my %index; - for my $variable (@environment) { - my ($name) = split(/=/, $variable, 2); - next if !defined($name) || !length($name); - if (exists($index{$name})) { - $normalized[$index{$name}] = $variable; - } else { - $index{$name} = scalar(@normalized); - push @normalized, $variable; - } - } - @environment = @normalized; - for my $variable (@$overlay) { - my ($name) = split(/=/, $variable, 2); - die "invalid environment entry\n" if !defined($name) || !length($name); - if (exists($index{$name})) { - $environment[$index{$name}] = $variable; - } else { - $index{$name} = scalar(@environment); - push @environment, $variable; - } - } - - @lxc = grep { - my $keep = ref($_) ne 'ARRAY' - || @$_ < 2 - || ($_->[0] ne 'lxc.environment' - && $_->[0] ne 'lxc.environment.runtime' - && $_->[0] ne 'lxc.signal.halt' - && $_->[0] ne 'lxc.init.cmd' - && $_->[0] ne 'lxc.console.logfile' - && $_->[0] ne 'lxc.cgroup.cpuset.cpus' - && $_->[0] ne 'lxc.cgroup2.cpuset.cpus' - && $_->[0] ne 'lxc.cgroup.pids.max' - && $_->[0] ne 'lxc.cgroup2.pids.max' - && $_->[0] ne 'lxc.cgroup.blkio.weight' - && $_->[0] ne 'lxc.cgroup2.io.weight' - && $_->[0] ne 'lxc.cgroup.memory.limit_in_bytes' - && $_->[0] ne 'lxc.cgroup2.memory.max' - && $_->[0] ne 'lxc.cgroup.memory.memsw.limit_in_bytes' - && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); - if ($keep && defined($managed_mounts) && ref($_) eq 'ARRAY' && @$_ >= 2 - && $_->[0] eq 'lxc.mount.entry') { - my $target = mount_target($_->[1]); - $keep = 0 if defined($target) && exists($managed_target{$target}); - } - $keep; - } @lxc; - # PVE 9.2 serializes its `env` property as lxc.environment.runtime, but - # LXC 7 applies that subkey only to hooks. Mirror the merged environment - # through lxc.environment so PID 1 receives both OCI and panel variables. - $conf->{env} = join("\0", @environment); - push @lxc, map { ['lxc.environment', $_] } @environment; - if (defined($payload->{halt_signal}) && length($payload->{halt_signal})) { - push @lxc, ['lxc.signal.halt', $payload->{halt_signal}]; - } - if (defined($payload->{init_command}) && length($payload->{init_command})) { - push @lxc, ['lxc.init.cmd', $payload->{init_command}]; - } - if (defined($payload->{console_logfile}) && length($payload->{console_logfile})) { - push @lxc, ['lxc.console.logfile', $payload->{console_logfile}]; - } - if (defined($payload->{cpuset_cpus}) && length($payload->{cpuset_cpus})) { - push @lxc, ['lxc.cgroup2.cpuset.cpus', $payload->{cpuset_cpus}]; - } - if (defined($payload->{pids_limit}) && $payload->{pids_limit} > 0) { - push @lxc, ['lxc.cgroup2.pids.max', $payload->{pids_limit}]; - } - if (defined($payload->{io_weight}) && $payload->{io_weight} > 0) { - push @lxc, ['lxc.cgroup2.io.weight', $payload->{io_weight}]; - } - if ($payload->{memory_unlimited}) { - push @lxc, ['lxc.cgroup2.memory.max', 'max']; - } - if ($payload->{swap_unlimited}) { - push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; - } - if (defined($managed_mounts)) { - push @lxc, map { ['lxc.mount.entry', $_->{entry}] } @$managed_mounts; - } +mod scripts; - $conf->{lxc} = \@lxc; - PVE::LXC::Config->write_config($vmid, $conf); -}); -"#; - -const APPLY_FIREWALL_CONFIG_PERL: &str = r#" -use strict; -use warnings; -use JSON::PP qw(decode_json); -use PVE::Firewall; - -my $input; -{ - local $/; - $input = ; -} -my $payload = decode_json($input // '{}'); -my $vmid = int($payload->{vmid} // 0); -die "invalid VMID\n" if $vmid <= 0; -my $prefix = $payload->{managed_prefix} // ''; -die "invalid managed prefix\n" if ref($prefix) || !length($prefix); -my $rules = $payload->{rules} // []; -my $ipsets = $payload->{ipsets} // {}; -die "rules must be an array\n" if ref($rules) ne 'ARRAY'; -die "ipsets must be an object\n" if ref($ipsets) ne 'HASH'; - -for my $rule (@$rules) { - die "invalid firewall rule\n" if ref($rule) ne 'HASH'; - die "invalid firewall rule type\n" - if ref($rule->{type}) || ($rule->{type} // '') ne 'in'; - die "invalid firewall rule action\n" - if ref($rule->{action}) || ($rule->{action} // '') !~ /\A(?:ACCEPT|DROP)\z/; - die "invalid firewall rule interface\n" - if ref($rule->{iface}) || ($rule->{iface} // '') !~ /\A[A-Za-z0-9_.-]+\z/; - if (defined($rule->{proto})) { - die "invalid firewall rule protocol\n" - if ref($rule->{proto}) || $rule->{proto} !~ /\A(?:tcp|udp)\z/; - } - if (defined($rule->{dport})) { - die "invalid firewall rule destination port\n" - if ref($rule->{dport}) || $rule->{dport} !~ /\A[0-9]+(?:,[0-9]+)*\z/; - for my $port (split(/,/, $rule->{dport})) { - die "invalid firewall rule destination port\n" if $port < 1 || $port > 65535; - } - } - if (defined($rule->{source})) { - die "invalid firewall rule source\n" - if ref($rule->{source}) || $rule->{source} =~ /[\x00\r\n]/; - } -} - -# Match the pve-firewall daemon's initialization before loading and rewriting -# existing cluster or guest rules. -PVE::Firewall::init(); - -PVE::Firewall::lock_vmfw_conf($vmid, 10, sub { - my $cluster_conf = PVE::Firewall::load_clusterfw_conf(); - my $conf = PVE::Firewall::load_vmfw_conf($cluster_conf, 'ct', $vmid); - - $conf->{rules} = [grep { - my $comment = $_->{comment} // ''; - index($comment, $prefix) != 0; - } @{$conf->{rules} // []}]; - - for my $name (keys %{$conf->{ipset} // {}}) { - my $comment = $conf->{ipset_comments}->{$name} // ''; - next if index($comment, $prefix) != 0; - delete $conf->{ipset}->{$name}; - delete $conf->{ipset_comments}->{$name}; - } - - for my $name (keys %$ipsets) { - die "invalid ipset name\n" if $name !~ /^[A-Za-z][A-Za-z0-9_-]{0,30}$/; - my $set = $ipsets->{$name}; - die "ipset entries must be an array\n" if ref($set) ne 'ARRAY'; - $conf->{ipset}->{$name} = [map { { cidr => $_ } } @$set]; - $conf->{ipset_comments}->{$name} = "$prefix source file"; - } - - unshift @{$conf->{rules}}, @$rules; - if (@$rules) { - # Wings owns firewall mode for its tagged containers so the guest - # behavior matches the panel's default-allow rule model. - $conf->{options}->{enable} = 1; - $conf->{options}->{policy_in} = 'ACCEPT'; - } elsif (!@{$conf->{rules}}) { - $conf->{options}->{enable} = 0; - } - - PVE::Firewall::save_vmfw_conf($vmid, $conf); -}); -"#; +use scripts::{APPLY_FIREWALL_CONFIG_PERL, APPLY_RUNTIME_CONFIG_PERL}; #[derive(Debug, Clone)] pub struct PveCli { @@ -2273,993 +2039,4 @@ impl ContainerStatus { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn create_arguments_leave_data_mount_for_post_create_configuration() { - let args = PveCli::create_args(&CreateContainerSpec { - vmid: 1234, - template: "local:vztmpl/oci-minecraft.tar".to_string(), - hostname: "server-1234".to_string(), - rootfs_storage: "local-lvm".to_string(), - rootfs_size_gib: 8, - memory_mib: 4096, - swap_mib: 1024, - cpu_limit_percent: Some(250), - cores: Some(2), - network: "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1".to_string(), - tags: vec!["calagopus".to_string(), "calagopus-server".to_string()], - unprivileged: true, - }); - - assert_eq!(args.first().map(String::as_str), Some("create")); - assert!( - args.windows(2) - .any(|pair| pair == ["--rootfs", "local-lvm:8"]) - ); - assert!(args.windows(2).any(|pair| { - pair == [ - "--net0", - "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1", - ] - })); - assert!(!args.iter().any(|arg| arg == "--mp0")); - assert!(args.windows(2).any(|pair| pair == ["--cores", "2"])); - assert!(args.windows(2).any(|pair| pair == ["--cpulimit", "2.5"])); - assert!(args.windows(2).any(|pair| pair == ["--cmode", "console"])); - assert!( - args.windows(2) - .any(|pair| { pair == ["--tags", "calagopus;calagopus-server"] }) - ); - } - - #[test] - fn formats_panel_cpu_percent_for_proxmox_cpu_time() { - assert_eq!(PveCli::format_cpu_limit_percent(1), "0.01"); - assert_eq!(PveCli::format_cpu_limit_percent(50), "0.5"); - assert_eq!(PveCli::format_cpu_limit_percent(100), "1"); - assert_eq!(PveCli::format_cpu_limit_percent(250), "2.5"); - assert_eq!(PveCli::format_cpu_limit_percent(225), "2.25"); - } - - #[test] - fn enables_firewall_on_existing_network_configuration() { - let config = "arch: amd64\nnet0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth\n"; - assert_eq!( - PveCli::network_with_firewall(config).unwrap().as_deref(), - Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") - ); - - let disabled = "net0: name=eth0,bridge=vmbr0,firewall=0,ip=dhcp,type=veth\n"; - assert_eq!( - PveCli::network_with_firewall(disabled).unwrap().as_deref(), - Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") - ); - - let enabled = "net0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1\n"; - assert_eq!(PveCli::network_with_firewall(enabled).unwrap(), None); - assert!(PveCli::network_with_firewall("arch: amd64\n").is_err()); - } - - #[test] - fn set_tags_replaces_container_tags_atomically() { - assert_eq!( - PveCli::set_tags_args( - 1234, - &[ - "calagopus".to_string(), - "calagopus-server-1234".to_string(), - "calagopus-image-deadbeef".to_string(), - ], - ), - vec![ - "set", - "1234", - "--tags", - "calagopus;calagopus-server-1234;calagopus-image-deadbeef", - ] - ); - } - - #[test] - fn maps_panel_resources_to_native_pve_limits() { - assert_eq!( - PveCli::panel_resources(2048, 256, 1024, 250, None).ok(), - Some(ContainerResources { - memory_mib: 2304, - swap_mib: 1024, - memory_unlimited: false, - swap_unlimited: false, - cpu_limit_percent: Some(250), - cores: None, - }) - ); - assert_eq!( - PveCli::panel_resources(2048, 0, 0, 0, None).ok(), - Some(ContainerResources { - memory_mib: 2048, - swap_mib: 0, - memory_unlimited: false, - swap_unlimited: false, - cpu_limit_percent: None, - cores: None, - }) - ); - } - - #[test] - fn rejects_invalid_resource_limits() { - assert!(PveCli::panel_resources(-1, 0, 0, 0, None).is_err()); - assert!(PveCli::panel_resources(2048, 0, -2, 0, None).is_err()); - assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("3-1")).is_err()); - } - - #[test] - fn maps_unlimited_memory_and_swap_to_lxc_overrides() { - assert_eq!( - PveCli::panel_resources(0, 0, 0, 0, None).ok(), - Some(ContainerResources { - memory_mib: 16, - swap_mib: 0, - memory_unlimited: true, - swap_unlimited: true, - cpu_limit_percent: None, - cores: None, - }) - ); - assert_eq!( - PveCli::panel_resources(2048, 0, -1, 0, None).ok(), - Some(ContainerResources { - memory_mib: 2048, - swap_mib: 0, - memory_unlimited: false, - swap_unlimited: true, - cpu_limit_percent: None, - cores: None, - }) - ); - } - - #[test] - fn accepts_panel_cpu_pinning_syntax() { - assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).is_ok()); - assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0,,2")).is_err()); - } - - #[test] - fn converts_docker_block_io_weight_to_cgroup_v2() { - assert_eq!(PveCli::cgroup2_io_weight(None).ok(), Some(None)); - assert_eq!(PveCli::cgroup2_io_weight(Some(10)).ok(), Some(Some(1))); - assert_eq!(PveCli::cgroup2_io_weight(Some(500)).ok(), Some(Some(4950))); - assert_eq!( - PveCli::cgroup2_io_weight(Some(1000)).ok(), - Some(Some(10_000)) - ); - assert!(PveCli::cgroup2_io_weight(Some(9)).is_err()); - } - - #[test] - fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { - assert_eq!( - PveCli::set_resources_args( - 1234, - &ContainerResources { - memory_mib: 4096, - swap_mib: 512, - memory_unlimited: false, - swap_unlimited: false, - cpu_limit_percent: Some(250), - cores: Some(4), - }, - ), - vec![ - "set", - "1234", - "--memory", - "4096", - "--swap", - "512", - "--cpulimit", - "2.5", - "--cores", - "4", - ] - ); - - assert_eq!( - PveCli::set_resources_args( - 1234, - &ContainerResources { - memory_mib: 2048, - swap_mib: 0, - memory_unlimited: false, - swap_unlimited: false, - cpu_limit_percent: None, - cores: None, - }, - ), - vec![ - "set", - "1234", - "--memory", - "2048", - "--swap", - "0", - "--cpulimit", - "0", - ] - ); - } - - #[test] - fn parses_non_root_oci_user_from_pct_config() { - assert_eq!( - PveCli::parse_oci_user( - "arch: amd64\nlxc.init.uid: 1000\nlxc.init.gid: 1001\nostype: unmanaged\n" - ) - .ok(), - Some(ContainerOciUser { - uid: 1000, - gid: 1001, - }) - ); - assert_eq!( - PveCli::parse_oci_user("lxc.init.uid: 0\nlxc.init.gid: 1000\n").ok(), - Some(ContainerOciUser { uid: 0, gid: 1000 }) - ); - assert_eq!( - PveCli::parse_oci_user("arch: amd64\n").ok(), - Some(ContainerOciUser { uid: 0, gid: 0 }) - ); - } - - #[test] - fn pct_config_text_tolerates_nul_bytes() { - let config = PveCli::decode_config_output( - b"arch: amd64\0\nlxc.init.uid: 1000\nlxc.init.gid: 1001\0\n".to_vec(), - ) - .unwrap_or_default(); - - assert_eq!( - PveCli::parse_oci_user(&config).ok(), - Some(ContainerOciUser { - uid: 1000, - gid: 1001, - }) - ); - } - - #[test] - fn data_mount_arguments_map_resolved_image_user_to_host_owner() { - let args = PveCli::set_data_mount_args(&DataMountSpec { - vmid: 1234, - data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), - container_uid: 1000, - container_gid: 1001, - host_data_uid: 988, - host_data_gid: 989, - }) - .unwrap_or_default(); - - assert_eq!( - args, - vec![ - "set", - "1234", - "--mp0", - "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", - ] - ); - } - - #[test] - fn data_mount_arguments_support_root_in_an_unprivileged_container() { - let spec = DataMountSpec { - vmid: 1234, - data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), - container_uid: 0, - container_gid: 1000, - host_data_uid: 988, - host_data_gid: 988, - }; - - assert_eq!( - PveCli::set_data_mount_args(&spec).ok(), - Some(vec![ - "set".to_string(), - "1234".to_string(), - "--mp0".to_string(), - "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:0:988:1;g:1000:988:1".to_string(), - ]) - ); - } - - #[test] - fn bind_mount_arguments_support_multiple_targets_and_reject_duplicates() { - let mounts = vec![ - BindMountSpec { - slot: 0, - source_path: "/srv/server".to_string(), - target_path: "/mnt/server".to_string(), - read_only: false, - container_uid: 1000, - container_gid: 1001, - host_uid: 988, - host_gid: 989, - }, - BindMountSpec { - slot: 1, - source_path: "/srv/install".to_string(), - target_path: "/mnt/install".to_string(), - read_only: true, - container_uid: 1000, - container_gid: 1001, - host_uid: 988, - host_gid: 989, - }, - ]; - - assert_eq!( - PveCli::set_bind_mount_args(1234, &mounts).unwrap_or_default(), - vec![ - "set", - "1234", - "--mp0", - "/srv/server,mp=/mnt/server,backup=0,idmap=u:1000:988:1;g:1001:989:1", - "--mp1", - "/srv/install,mp=/mnt/install,backup=0,idmap=u:1000:988:1;g:1001:989:1,ro=1", - ] - ); - - let mut duplicate = mounts.clone(); - duplicate[1].slot = 0; - assert!(PveCli::set_bind_mount_args(1234, &duplicate).is_err()); - - let mut unsafe_path = mounts; - unsafe_path[1].target_path = "/mnt/install,bad".to_string(); - assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); - unsafe_path[1].target_path = "/mnt/../etc".to_string(); - assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); - } - - #[test] - fn device_arguments_set_ownership_mode_and_write_policy() { - let devices = vec![ - DevicePassthroughSpec { - slot: 0, - path: "/dev/kvm".to_string(), - uid: 1000, - gid: 1001, - mode: 0o660, - deny_write: false, - }, - DevicePassthroughSpec { - slot: 1, - path: "/dev/dri/renderD128".to_string(), - uid: 1000, - gid: 1001, - mode: 0o640, - deny_write: true, - }, - ]; - - assert_eq!( - PveCli::set_devices_args(1234, &devices).unwrap_or_default(), - vec![ - "set", - "1234", - "--dev0", - "path=/dev/kvm,uid=1000,gid=1001,mode=0660", - "--dev1", - "path=/dev/dri/renderD128,uid=1000,gid=1001,mode=0640,deny-write=1", - ] - ); - - let mut duplicate = devices.clone(); - duplicate[1].slot = 0; - assert!(PveCli::set_devices_args(1234, &duplicate).is_err()); - - let mut unsafe_path = devices.clone(); - unsafe_path[1].path = "/dev/dri/renderD128,bad".to_string(); - assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); - unsafe_path[1].path = "/dev/../etc/shadow".to_string(); - assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); - - let mut root_owned = devices; - root_owned[0].uid = 0; - root_owned[0].gid = 0; - assert!(PveCli::set_devices_args(1234, &root_owned).is_ok()); - } - - #[test] - fn mount_and_device_reconciliation_removes_stale_slots() { - let mounts = vec![BindMountSpec { - slot: 0, - source_path: "/srv/server".to_string(), - target_path: "/home/container".to_string(), - read_only: false, - container_uid: 1000, - container_gid: 1001, - host_uid: 988, - host_gid: 989, - }]; - let devices = vec![DevicePassthroughSpec { - slot: 0, - path: "/dev/kvm".to_string(), - uid: 1000, - gid: 1001, - mode: 0o660, - deny_write: false, - }]; - let current_config = r#" -rootfs: local-lvm:vm-1234-disk-0,size=8G -mp0: /srv/old-server,mp=/home/container -mp2: /srv/removed,mp=/mnt/removed -dev0: path=/dev/kvm -dev3: path=/dev/dri/renderD128 -lxc.init.uid: 1000 -"#; - - assert_eq!( - PveCli::reconcile_mounts_and_devices_args(1234, current_config, &mounts, &devices,) - .unwrap_or_default(), - vec![ - "set", - "1234", - "--mp0", - "/srv/server,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", - "--dev0", - "path=/dev/kvm,uid=1000,gid=1001,mode=0660", - "--delete", - "mp2,dev3", - ] - ); - } - - #[test] - fn parses_pct_status() { - assert_eq!( - ContainerStatus::parse("status: running\n").ok(), - Some(ContainerStatus::Running) - ); - assert_eq!( - ContainerStatus::parse("status: stopped\n").ok(), - Some(ContainerStatus::Stopped) - ); - assert!(ContainerStatus::parse("status: mounted").is_err()); - } - - #[test] - fn parses_local_cluster_node() { - let output = br#"[ - {"type":"cluster","name":"og-cluster"}, - {"type":"node","name":"pve-a","local":0}, - {"type":"node","name":"pve-b","local":1} - ]"#; - - assert_eq!( - PveCli::parse_local_node(output).ok().as_deref(), - Some("pve-b") - ); - } - - #[test] - fn orders_available_active_lxc_storages() { - let output = br#"[ - {"storage":"local-lvm","active":1,"enabled":1,"content":"images,rootdir","avail":100}, - {"storage":"zfs","active":1,"enabled":1,"content":"rootdir,images","avail":200}, - {"storage":"local","active":1,"enabled":1,"content":"iso,vztmpl","avail":300}, - {"storage":"disabled","active":1,"enabled":0,"content":"rootdir","avail":400}, - {"storage":"offline","active":0,"enabled":1,"content":"rootdir","avail":500} - ]"#; - - assert_eq!( - PveCli::parse_available_lxc_storages(output).ok(), - Some(vec![ - LxcStorage { - name: "zfs".to_string(), - available_bytes: 200, - }, - LxcStorage { - name: "local-lvm".to_string(), - available_bytes: 100, - }, - ]) - ); - } - - #[test] - fn parses_lxc_resources_and_ignores_vms() { - let output = br#"[ - {"type":"qemu","vmid":101,"node":"pve-a","status":"running"}, - {"type":"lxc","vmid":102,"node":"pve-b","status":"running","tags":"calagopus;calagopus-server-abc"}, - {"type":"lxc","vmid":"103","node":"pve-b","status":"stopped"} - ]"#; - - let containers = PveCli::parse_containers(output).unwrap_or_default(); - assert_eq!(containers.len(), 2); - assert_eq!( - containers.first().map(|container| container.vmid), - Some(102) - ); - assert_eq!( - containers - .first() - .and_then(|container| container.tags.first()) - .map(String::as_str), - Some("calagopus") - ); - assert_eq!( - containers - .first() - .and_then(|container| container.tags.get(1)) - .map(String::as_str), - Some("calagopus-server-abc") - ); - assert_eq!(containers.get(1).map(|container| container.vmid), Some(103)); - } - - #[test] - fn parses_container_interfaces_and_prefers_usable_ipv4() { - let interfaces = PveCli::parse_interfaces( - br#"[ - {"name":"lo","ip-addresses":[ - {"ip-address":"127.0.0.1","ip-address-type":"inet","prefix":8}, - {"ip-address":"::1","ip-address-type":"inet6","prefix":128} - ]}, - {"name":"eth0","ip-addresses":[ - {"ip-address":"fe80::1234","ip-address-type":"inet6","prefix":64}, - {"ip-address":"fd12:3456::20","ip-address-type":"inet6","prefix":64}, - {"ip-address":"10.42.0.20","ip-address-type":"inet","prefix":24} - ]} - ]"#, - ) - .unwrap_or_default(); - - assert_eq!(interfaces.len(), 2); - assert_eq!( - PveCli::primary_container_address(&interfaces), - "10.42.0.20".parse().ok() - ); - assert_eq!( - PveCli::interfaces_args("pve-a", 1234), - vec![ - "get", - "/nodes/pve-a/lxc/1234/interfaces", - "--output-format", - "json", - ] - ); - } - - #[test] - fn stopped_container_null_interface_response_is_empty() { - assert_eq!(PveCli::parse_interfaces(b"null").unwrap(), Vec::new()); - } - - #[test] - fn container_addresses_returns_all_usable_addresses_sorted_and_deduplicated() { - let interfaces = vec![ - ContainerInterface { - name: "lo".to_string(), - addresses: vec!["127.0.0.1".parse().unwrap(), "::1".parse().unwrap()], - }, - ContainerInterface { - name: "eth0".to_string(), - addresses: vec![ - "fd12:3456::20".parse().unwrap(), - "10.42.0.20".parse().unwrap(), - "fe80::1234".parse().unwrap(), - "10.42.0.20".parse().unwrap(), - "0.0.0.0".parse().unwrap(), - "ff02::1".parse().unwrap(), - ], - }, - ]; - - assert_eq!( - PveCli::container_addresses(&interfaces), - vec![ - "10.42.0.20".parse::().unwrap(), - "fd12:3456::20".parse::().unwrap(), - ] - ); - } - - #[test] - fn parses_runtime_status_for_cgroup_discovery() { - assert_eq!( - PveCli::parse_runtime_status(br#"{"status":"running","pid":"4321","uptime":87}"#,).ok(), - Some(ContainerRuntimeStatus { - status: ContainerStatus::Running, - pid: Some(4321), - uptime_seconds: 87, - }) - ); - assert_eq!( - PveCli::parse_runtime_status(br#"{"status":"stopped","uptime":"0"}"#).ok(), - Some(ContainerRuntimeStatus { - status: ContainerStatus::Stopped, - pid: None, - uptime_seconds: 0, - }) - ); - assert!(PveCli::parse_runtime_status(br#"{"status":"mounted"}"#).is_err()); - assert_eq!( - PveCli::runtime_status_args("pve-a", 1234), - vec![ - "get", - "/nodes/pve-a/lxc/1234/status/current", - "--output-format", - "json", - ] - ); - } - - #[test] - fn container_address_falls_back_to_non_link_local_ipv6() { - let interfaces = vec![ContainerInterface { - name: "eth0".to_string(), - addresses: vec![ - "fe80::1234" - .parse() - .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), - "fd12:3456::20" - .parse() - .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), - ], - }]; - - assert_eq!( - PveCli::primary_container_address(&interfaces), - "fd12:3456::20".parse().ok() - ); - } - - #[test] - fn normalizes_oci_references_for_proxmox() { - assert_eq!( - PveCli::normalize_oci_reference("ghcr.io/example/game").ok(), - Some("ghcr.io/example/game:latest".to_string()) - ); - assert_eq!( - PveCli::normalize_oci_reference("registry.example:5000/example/game:v2").ok(), - Some("registry.example:5000/example/game:v2".to_string()) - ); - assert!(PveCli::normalize_oci_reference("game@sha256:abc").is_err()); - } - - #[test] - fn oci_cache_name_is_stable_and_reference_specific() { - let first = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); - let again = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); - let other = PveCli::oci_cache_filename("ghcr.io/example/game:v2"); - - assert_eq!(first, again); - assert_ne!(first, other); - assert!(first.starts_with("calagopus-oci-")); - assert_eq!(first.len(), "calagopus-oci-".len() + 64); - } - - #[test] - fn oci_image_tag_uses_resolved_revision_digest() { - let first = PveCli::oci_revision_tag( - "sha256:052a0ffb2620d9d59dfd81caa280af01ca8e6fe744649b11da481c00a8562d40", - ) - .unwrap_or_default(); - let other = PveCli::oci_revision_tag( - "5c093e071615ce61063007dfb363fd5bb878d4673954445e2a4ee5e7de3fae3b", - ) - .unwrap_or_default(); - - assert_ne!(first, other); - assert_eq!(first, "calagopus-image-052a0ffb2620d9d59dfd81caa280af01"); - assert!(PveCli::oci_revision_tag("sha256:not-a-digest").is_err()); - } - - #[test] - fn oci_pull_arguments_use_proxmox_storage_endpoint() { - let args = PveCli::pull_oci_args(&OciTemplatePullSpec { - node: "pve-a".to_string(), - storage: "local".to_string(), - reference: "ghcr.io/example/game:latest".to_string(), - filename: "calagopus-oci-abc".to_string(), - }); - - assert_eq!( - args, - vec![ - "create", - "/nodes/pve-a/storage/local/oci-registry-pull", - "--reference", - "ghcr.io/example/game:latest", - "--filename", - "calagopus-oci-abc", - "--output-format", - "json", - ] - ); - } - - #[test] - fn parses_oci_pull_task_after_proxmox_progress_output() { - let upid = "UPID:pve4:000348B0:005B00EB:6ABACCF1:ociregistrypull:template.tar:root@pam:"; - let noisy = format!( - "Getting image source signatures\nCopying blob sha256:abc\nWriting manifest to image destination\n\"{upid}\"\n" - ); - - assert_eq!( - PveCli::parse_oci_pull_task(noisy.as_bytes()) - .ok() - .as_deref(), - Some(upid) - ); - assert_eq!( - PveCli::parse_oci_pull_task(format!("\"{upid}\"").as_bytes()) - .ok() - .as_deref(), - Some(upid) - ); - assert!(PveCli::parse_oci_pull_task(b"copy completed without task id\n").is_err()); - } - - #[test] - fn parses_storage_content_and_task_status() { - let content = PveCli::parse_storage_content( - br#"[{"volid":"local:vztmpl/calagopus-oci-abc.tar","content":"vztmpl"}]"#, - ) - .unwrap_or_default(); - assert_eq!( - content.first(), - Some(&StorageContent { - volid: "local:vztmpl/calagopus-oci-abc.tar".to_string(), - content: Some("vztmpl".to_string()), - }) - ); - - assert_eq!( - PveCli::parse_task_status(br#"{"status":"running"}"#).ok(), - Some(TaskStatus { - state: TaskState::Running, - exit_status: None, - }) - ); - assert_eq!( - PveCli::parse_task_status(br#"{"status":"stopped","exitstatus":"OK"}"#).ok(), - Some(TaskStatus { - state: TaskState::Stopped, - exit_status: Some("OK".to_string()), - }) - ); - } - - #[test] - fn attached_process_runs_as_explicit_non_root_user_with_clean_environment() { - let args = PveCli::attach_args(&AttachProcessSpec { - vmid: 1234, - uid: 1000, - gid: 1000, - environment: vec![ - "STARTUP=java -jar server.jar".to_string(), - "SERVER_MEMORY=4096".to_string(), - ], - command: vec!["/bin/bash".to_string(), "/entrypoint.sh".to_string()], - }) - .unwrap_or_default(); - - assert_eq!( - args, - vec![ - "-n", - "1234", - "--clear-env", - "-u", - "1000", - "-g", - "1000", - "-v", - "STARTUP=java -jar server.jar", - "-v", - "SERVER_MEMORY=4096", - "--", - "/bin/bash", - "/entrypoint.sh", - ] - ); - } - - #[test] - fn attached_process_supports_root_and_rejects_invalid_input() { - let spec = AttachProcessSpec { - vmid: 1234, - uid: 0, - gid: 1000, - environment: Vec::new(), - command: vec!["/bin/true".to_string()], - }; - assert!(PveCli::attach_args(&spec).is_ok()); - - let mut spec = spec; - spec.command.clear(); - assert!(PveCli::attach_args(&spec).is_err()); - - spec.command.push("/bin/true".to_string()); - spec.environment.push("BROKEN".to_string()); - assert!(PveCli::attach_args(&spec).is_err()); - } - - #[test] - fn runtime_configuration_uses_stdin_payload_and_validates_line_safety() { - let payload = PveCli::runtime_config_payload(&RuntimeConfigSpec { - vmid: 1234, - environment: vec![ - "STARTUP=java -jar server.jar".to_string(), - "SERVER_MEMORY=4096".to_string(), - ], - halt_signal: Some("SIGINT".to_string()), - init_command: Some("/bin/bash /mnt/install/install.sh".to_string()), - console_logfile: Some("/var/log/calagopus/install.log".to_string()), - cpuset_cpus: Some("0-3,8".to_string()), - pids_limit: Some(5120), - io_weight: Some(4950), - memory_unlimited: true, - swap_unlimited: true, - managed_file_mounts: Some(vec![ManagedFileMountSpec { - source_path: "/var/lib/calagopus wings/vmount/example/hosts".to_string(), - target_path: "/etc/hosts".to_string(), - }]), - }) - .unwrap_or_default(); - let value: serde_json::Value = serde_json::from_slice(&payload).unwrap_or_default(); - - assert_eq!( - value.get("vmid").and_then(serde_json::Value::as_u64), - Some(1234) - ); - assert_eq!( - value - .get("environment") - .and_then(serde_json::Value::as_array) - .map(Vec::len), - Some(2) - ); - assert_eq!( - value.get("halt_signal").and_then(serde_json::Value::as_str), - Some("SIGINT") - ); - assert_eq!( - value - .get("init_command") - .and_then(serde_json::Value::as_str), - Some("/bin/bash /mnt/install/install.sh") - ); - assert_eq!( - value - .get("console_logfile") - .and_then(serde_json::Value::as_str), - Some("/var/log/calagopus/install.log") - ); - assert_eq!( - value.get("cpuset_cpus").and_then(serde_json::Value::as_str), - Some("0-3,8") - ); - assert_eq!( - value.get("pids_limit").and_then(serde_json::Value::as_u64), - Some(5120) - ); - assert_eq!( - value.get("io_weight").and_then(serde_json::Value::as_u64), - Some(4950) - ); - assert_eq!( - value - .get("memory_unlimited") - .and_then(serde_json::Value::as_bool), - Some(true) - ); - assert_eq!( - value - .get("swap_unlimited") - .and_then(serde_json::Value::as_bool), - Some(true) - ); - assert_eq!( - value - .get("managed_file_mounts") - .and_then(serde_json::Value::as_array) - .and_then(|mounts| mounts.first()) - .and_then(|mount| mount.get("entry")) - .and_then(serde_json::Value::as_str), - Some( - "/var/lib/calagopus\\040wings/vmount/example/hosts etc/hosts none bind,ro,create=file 0 0" - ) - ); - assert_eq!( - value - .get("managed_file_targets") - .and_then(serde_json::Value::as_array) - .map(Vec::len), - Some(5) - ); - - let mut invalid = RuntimeConfigSpec { - vmid: 1234, - environment: vec!["BROKEN".to_string()], - halt_signal: None, - init_command: None, - console_logfile: None, - cpuset_cpus: None, - pids_limit: None, - io_weight: None, - memory_unlimited: false, - swap_unlimited: false, - managed_file_mounts: None, - }; - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - invalid.environment = vec!["GOOD=bad\nvalue".to_string()]; - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - invalid.environment.clear(); - invalid.console_logfile = Some("relative.log".to_string()); - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - invalid.console_logfile = None; - invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { - source_path: "/var/lib/calagopus/hosts".to_string(), - target_path: "/etc/shadow".to_string(), - }]); - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - - invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { - source_path: "relative/hosts".to_string(), - target_path: "/etc/hosts".to_string(), - }]); - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - - invalid.managed_file_mounts = Some(vec![ - ManagedFileMountSpec { - source_path: "/var/lib/calagopus/hosts-a".to_string(), - target_path: "/etc/hosts".to_string(), - }, - ManagedFileMountSpec { - source_path: "/var/lib/calagopus/hosts-b".to_string(), - target_path: "/etc/hosts".to_string(), - }, - ]); - assert!(PveCli::runtime_config_payload(&invalid).is_err()); - } - - #[test] - fn init_command_encoding_matches_lxc_whole_word_quoting() { - assert_eq!( - PveCli::encode_init_command(&[ - "/bin/bash".to_string(), - "/mnt/install/install.sh".to_string(), - ]) - .ok() - .as_deref(), - Some("/bin/bash /mnt/install/install.sh") - ); - assert_eq!( - PveCli::encode_init_command(&[ - "/opt/my shell/bash".to_string(), - "/mnt/install/install.sh".to_string(), - ]) - .ok() - .as_deref(), - Some("\"/opt/my shell/bash\" /mnt/install/install.sh") - ); - assert!(PveCli::encode_init_command(&["both'\"quotes here".to_string()]).is_err()); - } - - #[test] - fn lifecycle_arguments_request_nonblocking_graceful_shutdown_and_console() { - assert_eq!( - PveCli::request_shutdown_args(1234), - vec!["-n", "1234", "--nowait", "--nokill"] - ); - assert_eq!( - PveCli::console_args(1234), - vec!["console", "1234", "--escape", "^z"] - ); - } -} +mod tests; diff --git a/application/src/server/executor/pve_lxc/cli/scripts.rs b/application/src/server/executor/pve_lxc/cli/scripts.rs new file mode 100644 index 00000000..375fff69 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli/scripts.rs @@ -0,0 +1,237 @@ +pub(super) const APPLY_RUNTIME_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::LXC::Config; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; + +my $overlay = $payload->{environment} // []; +die "environment must be an array\n" if ref($overlay) ne 'ARRAY'; + +my $managed_mounts = $payload->{managed_file_mounts}; +my $managed_targets = $payload->{managed_file_targets}; +my %managed_target; +if (defined($managed_mounts)) { + die "managed_file_mounts must be an array\n" if ref($managed_mounts) ne 'ARRAY'; + die "managed_file_targets must be an array\n" if ref($managed_targets) ne 'ARRAY'; + for my $target (@$managed_targets) { + die "invalid managed file target\n" if ref($target) || !defined($target) || !length($target); + $managed_target{$target} = 1; + } + for my $mount (@$managed_mounts) { + die "invalid managed file mount\n" if ref($mount) ne 'HASH'; + my $target = $mount->{target}; + my $entry = $mount->{entry}; + die "invalid managed file mount target\n" + if ref($target) || !defined($target) || !exists($managed_target{$target}); + die "invalid managed file mount entry\n" + if ref($entry) || !defined($entry) || !length($entry); + } +} + +sub mount_target { + my ($entry) = @_; + return undef if !defined($entry); + my @fields = grep { length($_) } split(/\s+/, $entry); + return scalar(@fields) >= 2 ? $fields[1] : undef; +} + +PVE::LXC::Config->lock_config($vmid, sub { + my $conf = PVE::LXC::Config->load_config($vmid); + my @lxc = @{$conf->{lxc} // []}; + my @environment; + + if (defined($conf->{env}) && length($conf->{env})) { + push @environment, split(/\0/, $conf->{env}, -1); + } + for my $entry (@lxc) { + next if ref($entry) ne 'ARRAY' || @$entry < 2; + push @environment, $entry->[1] + if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; + } + + # PVE exposes the same values through both `env` and low-level + # lxc.environment records. Collapse those views before applying the panel + # overlay so each synchronization is idempotent. + my @normalized; + my %index; + for my $variable (@environment) { + my ($name) = split(/=/, $variable, 2); + next if !defined($name) || !length($name); + if (exists($index{$name})) { + $normalized[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@normalized); + push @normalized, $variable; + } + } + @environment = @normalized; + for my $variable (@$overlay) { + my ($name) = split(/=/, $variable, 2); + die "invalid environment entry\n" if !defined($name) || !length($name); + if (exists($index{$name})) { + $environment[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@environment); + push @environment, $variable; + } + } + + @lxc = grep { + my $keep = ref($_) ne 'ARRAY' + || @$_ < 2 + || ($_->[0] ne 'lxc.environment' + && $_->[0] ne 'lxc.environment.runtime' + && $_->[0] ne 'lxc.signal.halt' + && $_->[0] ne 'lxc.init.cmd' + && $_->[0] ne 'lxc.console.logfile' + && $_->[0] ne 'lxc.cgroup.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup2.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup.pids.max' + && $_->[0] ne 'lxc.cgroup2.pids.max' + && $_->[0] ne 'lxc.cgroup.blkio.weight' + && $_->[0] ne 'lxc.cgroup2.io.weight' + && $_->[0] ne 'lxc.cgroup.memory.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.max' + && $_->[0] ne 'lxc.cgroup.memory.memsw.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); + if ($keep && defined($managed_mounts) && ref($_) eq 'ARRAY' && @$_ >= 2 + && $_->[0] eq 'lxc.mount.entry') { + my $target = mount_target($_->[1]); + $keep = 0 if defined($target) && exists($managed_target{$target}); + } + $keep; + } @lxc; + # PVE 9.2 serializes its `env` property as lxc.environment.runtime, but + # LXC 7 applies that subkey only to hooks. Mirror the merged environment + # through lxc.environment so PID 1 receives both OCI and panel variables. + $conf->{env} = join("\0", @environment); + push @lxc, map { ['lxc.environment', $_] } @environment; + if (defined($payload->{halt_signal}) && length($payload->{halt_signal})) { + push @lxc, ['lxc.signal.halt', $payload->{halt_signal}]; + } + if (defined($payload->{init_command}) && length($payload->{init_command})) { + push @lxc, ['lxc.init.cmd', $payload->{init_command}]; + } + if (defined($payload->{console_logfile}) && length($payload->{console_logfile})) { + push @lxc, ['lxc.console.logfile', $payload->{console_logfile}]; + } + if (defined($payload->{cpuset_cpus}) && length($payload->{cpuset_cpus})) { + push @lxc, ['lxc.cgroup2.cpuset.cpus', $payload->{cpuset_cpus}]; + } + if (defined($payload->{pids_limit}) && $payload->{pids_limit} > 0) { + push @lxc, ['lxc.cgroup2.pids.max', $payload->{pids_limit}]; + } + if (defined($payload->{io_weight}) && $payload->{io_weight} > 0) { + push @lxc, ['lxc.cgroup2.io.weight', $payload->{io_weight}]; + } + if ($payload->{memory_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.max', 'max']; + } + if ($payload->{swap_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; + } + if (defined($managed_mounts)) { + push @lxc, map { ['lxc.mount.entry', $_->{entry}] } @$managed_mounts; + } + + $conf->{lxc} = \@lxc; + PVE::LXC::Config->write_config($vmid, $conf); +}); +"#; + +pub(super) const APPLY_FIREWALL_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::Firewall; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; +my $prefix = $payload->{managed_prefix} // ''; +die "invalid managed prefix\n" if ref($prefix) || !length($prefix); +my $rules = $payload->{rules} // []; +my $ipsets = $payload->{ipsets} // {}; +die "rules must be an array\n" if ref($rules) ne 'ARRAY'; +die "ipsets must be an object\n" if ref($ipsets) ne 'HASH'; + +for my $rule (@$rules) { + die "invalid firewall rule\n" if ref($rule) ne 'HASH'; + die "invalid firewall rule type\n" + if ref($rule->{type}) || ($rule->{type} // '') ne 'in'; + die "invalid firewall rule action\n" + if ref($rule->{action}) || ($rule->{action} // '') !~ /\A(?:ACCEPT|DROP)\z/; + die "invalid firewall rule interface\n" + if ref($rule->{iface}) || ($rule->{iface} // '') !~ /\A[A-Za-z0-9_.-]+\z/; + if (defined($rule->{proto})) { + die "invalid firewall rule protocol\n" + if ref($rule->{proto}) || $rule->{proto} !~ /\A(?:tcp|udp)\z/; + } + if (defined($rule->{dport})) { + die "invalid firewall rule destination port\n" + if ref($rule->{dport}) || $rule->{dport} !~ /\A[0-9]+(?:,[0-9]+)*\z/; + for my $port (split(/,/, $rule->{dport})) { + die "invalid firewall rule destination port\n" if $port < 1 || $port > 65535; + } + } + if (defined($rule->{source})) { + die "invalid firewall rule source\n" + if ref($rule->{source}) || $rule->{source} =~ /[\x00\r\n]/; + } +} + +# Match the pve-firewall daemon's initialization before loading and rewriting +# existing cluster or guest rules. +PVE::Firewall::init(); + +PVE::Firewall::lock_vmfw_conf($vmid, 10, sub { + my $cluster_conf = PVE::Firewall::load_clusterfw_conf(); + my $conf = PVE::Firewall::load_vmfw_conf($cluster_conf, 'ct', $vmid); + + $conf->{rules} = [grep { + my $comment = $_->{comment} // ''; + index($comment, $prefix) != 0; + } @{$conf->{rules} // []}]; + + for my $name (keys %{$conf->{ipset} // {}}) { + my $comment = $conf->{ipset_comments}->{$name} // ''; + next if index($comment, $prefix) != 0; + delete $conf->{ipset}->{$name}; + delete $conf->{ipset_comments}->{$name}; + } + + for my $name (keys %$ipsets) { + die "invalid ipset name\n" if $name !~ /^[A-Za-z][A-Za-z0-9_-]{0,30}$/; + my $set = $ipsets->{$name}; + die "ipset entries must be an array\n" if ref($set) ne 'ARRAY'; + $conf->{ipset}->{$name} = [map { { cidr => $_ } } @$set]; + $conf->{ipset_comments}->{$name} = "$prefix source file"; + } + + unshift @{$conf->{rules}}, @$rules; + if (@$rules) { + # Wings owns firewall mode for its tagged containers so the guest + # behavior matches the panel's default-allow rule model. + $conf->{options}->{enable} = 1; + $conf->{options}->{policy_in} = 'ACCEPT'; + } elsif (!@{$conf->{rules}}) { + $conf->{options}->{enable} = 0; + } + + PVE::Firewall::save_vmfw_conf($vmid, $conf); +}); +"#; diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs new file mode 100644 index 00000000..145ded35 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -0,0 +1,988 @@ +use super::*; + +#[test] +fn create_arguments_leave_data_mount_for_post_create_configuration() { + let args = PveCli::create_args(&CreateContainerSpec { + vmid: 1234, + template: "local:vztmpl/oci-minecraft.tar".to_string(), + hostname: "server-1234".to_string(), + rootfs_storage: "local-lvm".to_string(), + rootfs_size_gib: 8, + memory_mib: 4096, + swap_mib: 1024, + cpu_limit_percent: Some(250), + cores: Some(2), + network: "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1".to_string(), + tags: vec!["calagopus".to_string(), "calagopus-server".to_string()], + unprivileged: true, + }); + + assert_eq!(args.first().map(String::as_str), Some("create")); + assert!( + args.windows(2) + .any(|pair| pair == ["--rootfs", "local-lvm:8"]) + ); + assert!(args.windows(2).any(|pair| { + pair == [ + "--net0", + "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1", + ] + })); + assert!(!args.iter().any(|arg| arg == "--mp0")); + assert!(args.windows(2).any(|pair| pair == ["--cores", "2"])); + assert!(args.windows(2).any(|pair| pair == ["--cpulimit", "2.5"])); + assert!(args.windows(2).any(|pair| pair == ["--cmode", "console"])); + assert!( + args.windows(2) + .any(|pair| { pair == ["--tags", "calagopus;calagopus-server"] }) + ); +} + +#[test] +fn formats_panel_cpu_percent_for_proxmox_cpu_time() { + assert_eq!(PveCli::format_cpu_limit_percent(1), "0.01"); + assert_eq!(PveCli::format_cpu_limit_percent(50), "0.5"); + assert_eq!(PveCli::format_cpu_limit_percent(100), "1"); + assert_eq!(PveCli::format_cpu_limit_percent(250), "2.5"); + assert_eq!(PveCli::format_cpu_limit_percent(225), "2.25"); +} + +#[test] +fn enables_firewall_on_existing_network_configuration() { + let config = "arch: amd64\nnet0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(config).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let disabled = "net0: name=eth0,bridge=vmbr0,firewall=0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(disabled).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let enabled = "net0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1\n"; + assert_eq!(PveCli::network_with_firewall(enabled).unwrap(), None); + assert!(PveCli::network_with_firewall("arch: amd64\n").is_err()); +} + +#[test] +fn set_tags_replaces_container_tags_atomically() { + assert_eq!( + PveCli::set_tags_args( + 1234, + &[ + "calagopus".to_string(), + "calagopus-server-1234".to_string(), + "calagopus-image-deadbeef".to_string(), + ], + ), + vec![ + "set", + "1234", + "--tags", + "calagopus;calagopus-server-1234;calagopus-image-deadbeef", + ] + ); +} + +#[test] +fn maps_panel_resources_to_native_pve_limits() { + assert_eq!( + PveCli::panel_resources(2048, 256, 1024, 250, None).ok(), + Some(ContainerResources { + memory_mib: 2304, + swap_mib: 1024, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: Some(250), + cores: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: None, + cores: None, + }) + ); +} + +#[test] +fn rejects_invalid_resource_limits() { + assert!(PveCli::panel_resources(-1, 0, 0, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, -2, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("3-1")).is_err()); +} + +#[test] +fn maps_unlimited_memory_and_swap_to_lxc_overrides() { + assert_eq!( + PveCli::panel_resources(0, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 16, + swap_mib: 0, + memory_unlimited: true, + swap_unlimited: true, + cpu_limit_percent: None, + cores: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, -1, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: true, + cpu_limit_percent: None, + cores: None, + }) + ); +} + +#[test] +fn accepts_panel_cpu_pinning_syntax() { + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).is_ok()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0,,2")).is_err()); +} + +#[test] +fn converts_docker_block_io_weight_to_cgroup_v2() { + assert_eq!(PveCli::cgroup2_io_weight(None).ok(), Some(None)); + assert_eq!(PveCli::cgroup2_io_weight(Some(10)).ok(), Some(Some(1))); + assert_eq!(PveCli::cgroup2_io_weight(Some(500)).ok(), Some(Some(4950))); + assert_eq!( + PveCli::cgroup2_io_weight(Some(1000)).ok(), + Some(Some(10_000)) + ); + assert!(PveCli::cgroup2_io_weight(Some(9)).is_err()); +} + +#[test] +fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 4096, + swap_mib: 512, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: Some(250), + cores: Some(4), + }, + ), + vec![ + "set", + "1234", + "--memory", + "4096", + "--swap", + "512", + "--cpulimit", + "2.5", + "--cores", + "4", + ] + ); + + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: None, + cores: None, + }, + ), + vec![ + "set", + "1234", + "--memory", + "2048", + "--swap", + "0", + "--cpulimit", + "0", + ] + ); +} + +#[test] +fn parses_non_root_oci_user_from_pct_config() { + assert_eq!( + PveCli::parse_oci_user( + "arch: amd64\nlxc.init.uid: 1000\nlxc.init.gid: 1001\nostype: unmanaged\n" + ) + .ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); + assert_eq!( + PveCli::parse_oci_user("lxc.init.uid: 0\nlxc.init.gid: 1000\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 1000 }) + ); + assert_eq!( + PveCli::parse_oci_user("arch: amd64\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 0 }) + ); +} + +#[test] +fn pct_config_text_tolerates_nul_bytes() { + let config = PveCli::decode_config_output( + b"arch: amd64\0\nlxc.init.uid: 1000\nlxc.init.gid: 1001\0\n".to_vec(), + ) + .unwrap_or_default(); + + assert_eq!( + PveCli::parse_oci_user(&config).ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); +} + +#[test] +fn data_mount_arguments_map_resolved_image_user_to_host_owner() { + let args = PveCli::set_data_mount_args(&DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 1000, + container_gid: 1001, + host_data_uid: 988, + host_data_gid: 989, + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "set", + "1234", + "--mp0", + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + ] + ); +} + +#[test] +fn data_mount_arguments_support_root_in_an_unprivileged_container() { + let spec = DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 0, + container_gid: 1000, + host_data_uid: 988, + host_data_gid: 988, + }; + + assert_eq!( + PveCli::set_data_mount_args(&spec).ok(), + Some(vec![ + "set".to_string(), + "1234".to_string(), + "--mp0".to_string(), + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:0:988:1;g:1000:988:1".to_string(), + ]) + ); +} + +#[test] +fn bind_mount_arguments_support_multiple_targets_and_reject_duplicates() { + let mounts = vec![ + BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/mnt/server".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + BindMountSpec { + slot: 1, + source_path: "/srv/install".to_string(), + target_path: "/mnt/install".to_string(), + read_only: true, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + ]; + + assert_eq!( + PveCli::set_bind_mount_args(1234, &mounts).unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/mnt/server,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--mp1", + "/srv/install,mp=/mnt/install,backup=0,idmap=u:1000:988:1;g:1001:989:1,ro=1", + ] + ); + + let mut duplicate = mounts.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_bind_mount_args(1234, &duplicate).is_err()); + + let mut unsafe_path = mounts; + unsafe_path[1].target_path = "/mnt/install,bad".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); + unsafe_path[1].target_path = "/mnt/../etc".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); +} + +#[test] +fn device_arguments_set_ownership_mode_and_write_policy() { + let devices = vec![ + DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }, + DevicePassthroughSpec { + slot: 1, + path: "/dev/dri/renderD128".to_string(), + uid: 1000, + gid: 1001, + mode: 0o640, + deny_write: true, + }, + ]; + + assert_eq!( + PveCli::set_devices_args(1234, &devices).unwrap_or_default(), + vec![ + "set", + "1234", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--dev1", + "path=/dev/dri/renderD128,uid=1000,gid=1001,mode=0640,deny-write=1", + ] + ); + + let mut duplicate = devices.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_devices_args(1234, &duplicate).is_err()); + + let mut unsafe_path = devices.clone(); + unsafe_path[1].path = "/dev/dri/renderD128,bad".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + unsafe_path[1].path = "/dev/../etc/shadow".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + + let mut root_owned = devices; + root_owned[0].uid = 0; + root_owned[0].gid = 0; + assert!(PveCli::set_devices_args(1234, &root_owned).is_ok()); +} + +#[test] +fn mount_and_device_reconciliation_removes_stale_slots() { + let mounts = vec![BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/home/container".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }]; + let devices = vec![DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }]; + let current_config = r#" +rootfs: local-lvm:vm-1234-disk-0,size=8G +mp0: /srv/old-server,mp=/home/container +mp2: /srv/removed,mp=/mnt/removed +dev0: path=/dev/kvm +dev3: path=/dev/dri/renderD128 +lxc.init.uid: 1000 +"#; + + assert_eq!( + PveCli::reconcile_mounts_and_devices_args(1234, current_config, &mounts, &devices,) + .unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--delete", + "mp2,dev3", + ] + ); +} + +#[test] +fn parses_pct_status() { + assert_eq!( + ContainerStatus::parse("status: running\n").ok(), + Some(ContainerStatus::Running) + ); + assert_eq!( + ContainerStatus::parse("status: stopped\n").ok(), + Some(ContainerStatus::Stopped) + ); + assert!(ContainerStatus::parse("status: mounted").is_err()); +} + +#[test] +fn parses_local_cluster_node() { + let output = br#"[ + {"type":"cluster","name":"og-cluster"}, + {"type":"node","name":"pve-a","local":0}, + {"type":"node","name":"pve-b","local":1} + ]"#; + + assert_eq!( + PveCli::parse_local_node(output).ok().as_deref(), + Some("pve-b") + ); +} + +#[test] +fn orders_available_active_lxc_storages() { + let output = br#"[ + {"storage":"local-lvm","active":1,"enabled":1,"content":"images,rootdir","avail":100}, + {"storage":"zfs","active":1,"enabled":1,"content":"rootdir,images","avail":200}, + {"storage":"local","active":1,"enabled":1,"content":"iso,vztmpl","avail":300}, + {"storage":"disabled","active":1,"enabled":0,"content":"rootdir","avail":400}, + {"storage":"offline","active":0,"enabled":1,"content":"rootdir","avail":500} + ]"#; + + assert_eq!( + PveCli::parse_available_lxc_storages(output).ok(), + Some(vec![ + LxcStorage { + name: "zfs".to_string(), + available_bytes: 200, + }, + LxcStorage { + name: "local-lvm".to_string(), + available_bytes: 100, + }, + ]) + ); +} + +#[test] +fn parses_lxc_resources_and_ignores_vms() { + let output = br#"[ + {"type":"qemu","vmid":101,"node":"pve-a","status":"running"}, + {"type":"lxc","vmid":102,"node":"pve-b","status":"running","tags":"calagopus;calagopus-server-abc"}, + {"type":"lxc","vmid":"103","node":"pve-b","status":"stopped"} + ]"#; + + let containers = PveCli::parse_containers(output).unwrap_or_default(); + assert_eq!(containers.len(), 2); + assert_eq!( + containers.first().map(|container| container.vmid), + Some(102) + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.first()) + .map(String::as_str), + Some("calagopus") + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.get(1)) + .map(String::as_str), + Some("calagopus-server-abc") + ); + assert_eq!(containers.get(1).map(|container| container.vmid), Some(103)); +} + +#[test] +fn parses_container_interfaces_and_prefers_usable_ipv4() { + let interfaces = PveCli::parse_interfaces( + br#"[ + {"name":"lo","ip-addresses":[ + {"ip-address":"127.0.0.1","ip-address-type":"inet","prefix":8}, + {"ip-address":"::1","ip-address-type":"inet6","prefix":128} + ]}, + {"name":"eth0","ip-addresses":[ + {"ip-address":"fe80::1234","ip-address-type":"inet6","prefix":64}, + {"ip-address":"fd12:3456::20","ip-address-type":"inet6","prefix":64}, + {"ip-address":"10.42.0.20","ip-address-type":"inet","prefix":24} + ]} + ]"#, + ) + .unwrap_or_default(); + + assert_eq!(interfaces.len(), 2); + assert_eq!( + PveCli::primary_container_address(&interfaces), + "10.42.0.20".parse().ok() + ); + assert_eq!( + PveCli::interfaces_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/interfaces", + "--output-format", + "json", + ] + ); +} + +#[test] +fn stopped_container_null_interface_response_is_empty() { + assert_eq!(PveCli::parse_interfaces(b"null").unwrap(), Vec::new()); +} + +#[test] +fn container_addresses_returns_all_usable_addresses_sorted_and_deduplicated() { + let interfaces = vec![ + ContainerInterface { + name: "lo".to_string(), + addresses: vec!["127.0.0.1".parse().unwrap(), "::1".parse().unwrap()], + }, + ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "fe80::1234".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "0.0.0.0".parse().unwrap(), + "ff02::1".parse().unwrap(), + ], + }, + ]; + + assert_eq!( + PveCli::container_addresses(&interfaces), + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); +} + +#[test] +fn parses_runtime_status_for_cgroup_discovery() { + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"running","pid":"4321","uptime":87}"#,).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Running, + pid: Some(4321), + uptime_seconds: 87, + }) + ); + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"stopped","uptime":"0"}"#).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Stopped, + pid: None, + uptime_seconds: 0, + }) + ); + assert!(PveCli::parse_runtime_status(br#"{"status":"mounted"}"#).is_err()); + assert_eq!( + PveCli::runtime_status_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/status/current", + "--output-format", + "json", + ] + ); +} + +#[test] +fn container_address_falls_back_to_non_link_local_ipv6() { + let interfaces = vec![ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fe80::1234" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + "fd12:3456::20" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + ], + }]; + + assert_eq!( + PveCli::primary_container_address(&interfaces), + "fd12:3456::20".parse().ok() + ); +} + +#[test] +fn normalizes_oci_references_for_proxmox() { + assert_eq!( + PveCli::normalize_oci_reference("ghcr.io/example/game").ok(), + Some("ghcr.io/example/game:latest".to_string()) + ); + assert_eq!( + PveCli::normalize_oci_reference("registry.example:5000/example/game:v2").ok(), + Some("registry.example:5000/example/game:v2".to_string()) + ); + assert!(PveCli::normalize_oci_reference("game@sha256:abc").is_err()); +} + +#[test] +fn oci_cache_name_is_stable_and_reference_specific() { + let first = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let again = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let other = PveCli::oci_cache_filename("ghcr.io/example/game:v2"); + + assert_eq!(first, again); + assert_ne!(first, other); + assert!(first.starts_with("calagopus-oci-")); + assert_eq!(first.len(), "calagopus-oci-".len() + 64); +} + +#[test] +fn oci_image_tag_uses_resolved_revision_digest() { + let first = PveCli::oci_revision_tag( + "sha256:052a0ffb2620d9d59dfd81caa280af01ca8e6fe744649b11da481c00a8562d40", + ) + .unwrap_or_default(); + let other = PveCli::oci_revision_tag( + "5c093e071615ce61063007dfb363fd5bb878d4673954445e2a4ee5e7de3fae3b", + ) + .unwrap_or_default(); + + assert_ne!(first, other); + assert_eq!(first, "calagopus-image-052a0ffb2620d9d59dfd81caa280af01"); + assert!(PveCli::oci_revision_tag("sha256:not-a-digest").is_err()); +} + +#[test] +fn oci_pull_arguments_use_proxmox_storage_endpoint() { + let args = PveCli::pull_oci_args(&OciTemplatePullSpec { + node: "pve-a".to_string(), + storage: "local".to_string(), + reference: "ghcr.io/example/game:latest".to_string(), + filename: "calagopus-oci-abc".to_string(), + }); + + assert_eq!( + args, + vec![ + "create", + "/nodes/pve-a/storage/local/oci-registry-pull", + "--reference", + "ghcr.io/example/game:latest", + "--filename", + "calagopus-oci-abc", + "--output-format", + "json", + ] + ); +} + +#[test] +fn parses_oci_pull_task_after_proxmox_progress_output() { + let upid = "UPID:pve4:000348B0:005B00EB:6ABACCF1:ociregistrypull:template.tar:root@pam:"; + let noisy = format!( + "Getting image source signatures\nCopying blob sha256:abc\nWriting manifest to image destination\n\"{upid}\"\n" + ); + + assert_eq!( + PveCli::parse_oci_pull_task(noisy.as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert_eq!( + PveCli::parse_oci_pull_task(format!("\"{upid}\"").as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert!(PveCli::parse_oci_pull_task(b"copy completed without task id\n").is_err()); +} + +#[test] +fn parses_storage_content_and_task_status() { + let content = PveCli::parse_storage_content( + br#"[{"volid":"local:vztmpl/calagopus-oci-abc.tar","content":"vztmpl"}]"#, + ) + .unwrap_or_default(); + assert_eq!( + content.first(), + Some(&StorageContent { + volid: "local:vztmpl/calagopus-oci-abc.tar".to_string(), + content: Some("vztmpl".to_string()), + }) + ); + + assert_eq!( + PveCli::parse_task_status(br#"{"status":"running"}"#).ok(), + Some(TaskStatus { + state: TaskState::Running, + exit_status: None, + }) + ); + assert_eq!( + PveCli::parse_task_status(br#"{"status":"stopped","exitstatus":"OK"}"#).ok(), + Some(TaskStatus { + state: TaskState::Stopped, + exit_status: Some("OK".to_string()), + }) + ); +} + +#[test] +fn attached_process_runs_as_explicit_non_root_user_with_clean_environment() { + let args = PveCli::attach_args(&AttachProcessSpec { + vmid: 1234, + uid: 1000, + gid: 1000, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + command: vec!["/bin/bash".to_string(), "/entrypoint.sh".to_string()], + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "-n", + "1234", + "--clear-env", + "-u", + "1000", + "-g", + "1000", + "-v", + "STARTUP=java -jar server.jar", + "-v", + "SERVER_MEMORY=4096", + "--", + "/bin/bash", + "/entrypoint.sh", + ] + ); +} + +#[test] +fn attached_process_supports_root_and_rejects_invalid_input() { + let spec = AttachProcessSpec { + vmid: 1234, + uid: 0, + gid: 1000, + environment: Vec::new(), + command: vec!["/bin/true".to_string()], + }; + assert!(PveCli::attach_args(&spec).is_ok()); + + let mut spec = spec; + spec.command.clear(); + assert!(PveCli::attach_args(&spec).is_err()); + + spec.command.push("/bin/true".to_string()); + spec.environment.push("BROKEN".to_string()); + assert!(PveCli::attach_args(&spec).is_err()); +} + +#[test] +fn runtime_configuration_uses_stdin_payload_and_validates_line_safety() { + let payload = PveCli::runtime_config_payload(&RuntimeConfigSpec { + vmid: 1234, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + halt_signal: Some("SIGINT".to_string()), + init_command: Some("/bin/bash /mnt/install/install.sh".to_string()), + console_logfile: Some("/var/log/calagopus/install.log".to_string()), + cpuset_cpus: Some("0-3,8".to_string()), + pids_limit: Some(5120), + io_weight: Some(4950), + memory_unlimited: true, + swap_unlimited: true, + managed_file_mounts: Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus wings/vmount/example/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]), + }) + .unwrap_or_default(); + let value: serde_json::Value = serde_json::from_slice(&payload).unwrap_or_default(); + + assert_eq!( + value.get("vmid").and_then(serde_json::Value::as_u64), + Some(1234) + ); + assert_eq!( + value + .get("environment") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(2) + ); + assert_eq!( + value.get("halt_signal").and_then(serde_json::Value::as_str), + Some("SIGINT") + ); + assert_eq!( + value + .get("init_command") + .and_then(serde_json::Value::as_str), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + value + .get("console_logfile") + .and_then(serde_json::Value::as_str), + Some("/var/log/calagopus/install.log") + ); + assert_eq!( + value.get("cpuset_cpus").and_then(serde_json::Value::as_str), + Some("0-3,8") + ); + assert_eq!( + value.get("pids_limit").and_then(serde_json::Value::as_u64), + Some(5120) + ); + assert_eq!( + value.get("io_weight").and_then(serde_json::Value::as_u64), + Some(4950) + ); + assert_eq!( + value + .get("memory_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + value + .get("swap_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + value + .get("managed_file_mounts") + .and_then(serde_json::Value::as_array) + .and_then(|mounts| mounts.first()) + .and_then(|mount| mount.get("entry")) + .and_then(serde_json::Value::as_str), + Some( + "/var/lib/calagopus\\040wings/vmount/example/hosts etc/hosts none bind,ro,create=file 0 0" + ) + ); + assert_eq!( + value + .get("managed_file_targets") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(5) + ); + + let mut invalid = RuntimeConfigSpec { + vmid: 1234, + environment: vec!["BROKEN".to_string()], + halt_signal: None, + init_command: None, + console_logfile: None, + cpuset_cpus: None, + pids_limit: None, + io_weight: None, + memory_unlimited: false, + swap_unlimited: false, + managed_file_mounts: None, + }; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment = vec!["GOOD=bad\nvalue".to_string()]; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment.clear(); + invalid.console_logfile = Some("relative.log".to_string()); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.console_logfile = None; + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts".to_string(), + target_path: "/etc/shadow".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "relative/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-a".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-b".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); +} + +#[test] +fn init_command_encoding_matches_lxc_whole_word_quoting() { + assert_eq!( + PveCli::encode_init_command(&[ + "/bin/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + PveCli::encode_init_command(&[ + "/opt/my shell/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("\"/opt/my shell/bash\" /mnt/install/install.sh") + ); + assert!(PveCli::encode_init_command(&["both'\"quotes here".to_string()]).is_err()); +} + +#[test] +fn lifecycle_arguments_request_nonblocking_graceful_shutdown_and_console() { + assert_eq!( + PveCli::request_shutdown_args(1234), + vec!["-n", "1234", "--nowait", "--nokill"] + ); + assert_eq!( + PveCli::console_args(1234), + vec!["console", "1234", "--escape", "^z"] + ); +} diff --git a/application/src/server/executor/pve_lxc/helper_process.rs b/application/src/server/executor/pve_lxc/helper_process.rs new file mode 100644 index 00000000..389eb2bc --- /dev/null +++ b/application/src/server/executor/pve_lxc/helper_process.rs @@ -0,0 +1,310 @@ +use super::*; + +pub(super) struct PveHelperProcessHandle { + vmid: u32, + cli: PveCli, + log_path: PathBuf, + started: Arc, + cancelled: Arc, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, +} + +impl PveHelperProcessHandle { + #[allow(clippy::too_many_arguments)] + pub(super) async fn new( + vmid: u32, + cli: PveCli, + server: &crate::server::Server, + app_config: Arc, + log_path: PathBuf, + status_tx: tokio::sync::mpsc::Sender, + already_started: bool, + start_at_end: bool, + auto_cleanup: bool, + cleanup_path: Option, + ) -> Result { + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + + let started = Arc::new(AtomicBool::new(already_started)); + let cancelled = Arc::new(AtomicBool::new(false)); + + let status_cli = cli.clone(); + let status_started = Arc::clone(&started); + let status_cancelled = Arc::clone(&cancelled); + let status_server = server.uuid; + tokio::spawn(async move { + let mut sent_running = false; + let mut tick = tokio::time::interval(std::time::Duration::from_millis(500)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + if status_cancelled.load(Ordering::Acquire) { + break; + } + + match status_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => { + if !sent_running { + sent_running = true; + if status_tx.send(super::ProcessStatus::Running).await.is_err() { + break; + } + } + } + Ok(cli::ContainerStatus::Stopped) if status_started.load(Ordering::Acquire) => { + if !sent_running + && status_tx.send(super::ProcessStatus::Running).await.is_err() + { + break; + } + let _ = status_tx + .send(super::ProcessStatus::Stopped { + exit_code: -1, + oom_killed: false, + }) + .await; + break; + } + Ok(cli::ContainerStatus::Stopped) => {} + Err(error) => { + tracing::warn!( + server = %status_server, + vmid, + "failed to read Proxmox helper LXC process state: {error:#}" + ); + } + } + } + }); + + let output_cli = cli.clone(); + let output_started = Arc::clone(&started); + let output_cancelled = Arc::clone(&cancelled); + let output_server = Arc::downgrade(&**server); + let output_log_path = log_path.clone(); + tokio::spawn(async move { + let mut file = match tokio::fs::OpenOptions::new() + .read(true) + .open(&output_log_path) + .await + { + Ok(file) => file, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox helper log: {error:#}"); + return; + } + }; + if start_at_end + && let Ok(metadata) = file.metadata().await + && let Err(error) = file.seek(std::io::SeekFrom::Start(metadata.len())).await + { + tracing::warn!(vmid, "failed to seek Proxmox helper log: {error:#}"); + } + + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + if output_cancelled.load(Ordering::Acquire) { + break; + } + + if let (Ok(position), Ok(metadata)) = + (file.stream_position().await, file.metadata().await) + && metadata.len() < position + { + if let Err(error) = file.seek(std::io::SeekFrom::Start(0)).await { + tracing::warn!( + vmid, + "failed to rewind truncated Proxmox helper log: {error:#}" + ); + } + line_buffer = LineBuffer::new(); + } + + let mut data = Vec::new(); + match file.read_to_end(&mut data).await { + Ok(_) => {} + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox helper log: {error:#}"); + } + } + + if !data.is_empty() { + line_buffer.extend(&data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = output_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + + let stopped = output_started.load(Ordering::Acquire) + && matches!( + output_cli.status(vmid).await, + Ok(cli::ContainerStatus::Stopped) + ); + if stopped { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let mut final_data = Vec::new(); + if file.read_to_end(&mut final_data).await.is_ok() && !final_data.is_empty() { + line_buffer.extend(&final_data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + if let Some(line) = line_buffer.flush() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + break; + } + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + if auto_cleanup { + output_cancelled.store(true, Ordering::Release); + if let Err(error) = output_cli.destroy(vmid).await { + tracing::warn!( + vmid, + "failed to destroy completed Proxmox helper LXC: {error:#}" + ); + } + if let Some(path) = cleanup_path + && let Err(error) = tokio::fs::remove_dir_all(&path).await + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + path = %path.display(), + "failed to remove Proxmox helper staging directory: {error:#}" + ); + } + } + }); + + Ok(Self { + vmid, + cli, + log_path, + started, + cancelled, + stdout_ratelimited_rx, + stdout_rx, + }) + } +} + +impl Drop for PveHelperProcessHandle { + fn drop(&mut self) { + if !self.started.load(Ordering::Acquire) { + self.cancelled.store(true, Ordering::Release); + } + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveHelperProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = tokio::fs::File::open(&self.log_path).await?; + Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)) + } else { + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + } + + async fn send_stdin(&self, _data: Vec) -> Result<(), anyhow::Error> { + Err(anyhow::anyhow!( + "stdin is not supported for Proxmox helper containers" + )) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } +} diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index a27b9e31..c6dad33b 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -6,8 +6,10 @@ pub mod cli; pub mod firewall; +mod helper_process; +mod server_process; -use super::{ProcessHandle, ServerExecutor, StatusReceiver, UsedPort}; +use super::{ProcessHandle, ProcessStatus, ServerExecutor, StatusReceiver, UsedPort, cgroup}; use crate::io::line_buffer::LineBuffer; use anyhow::Context; use compact_str::ToCompactString; @@ -24,6 +26,8 @@ use std::{ use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; pub use cli::PveCli; +use helper_process::PveHelperProcessHandle; +use server_process::PveProcessHandle; pub async fn create_firewall( app_config: &crate::config::Config, @@ -125,1554 +129,6 @@ struct EdgePeer { tunnel_ip: Ipv4Addr, } -struct PveProcessHandle { - vmid: u32, - node: String, - cli: PveCli, - server: Weak, - app_config: Arc, - firewall: Arc, - log_path: PathBuf, - stdin_tx: tokio::sync::mpsc::Sender>, - stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, - stdout_rx: tokio::sync::broadcast::Receiver>, - started: Arc, - status_task: tokio::task::JoinHandle<()>, - stats_task: tokio::task::JoinHandle<()>, - console_task: tokio::task::JoinHandle<()>, -} - -impl PveProcessHandle { - const EXIT_STATUS_FILE: &'static str = ".wings-exit-status"; - - fn observed_status( - status: cli::ContainerStatus, - start_succeeded: bool, - seen_running: &mut bool, - exit_status: Option<(i32, bool)>, - ) -> Option { - match status { - cli::ContainerStatus::Running => { - *seen_running = true; - Some(super::ProcessStatus::Running) - } - cli::ContainerStatus::Stopped if !*seen_running && !start_succeeded => None, - cli::ContainerStatus::Stopped => { - let (exit_code, oom_killed) = exit_status.unwrap_or((-1, false)); - Some(super::ProcessStatus::Stopped { - exit_code, - oom_killed, - }) - } - } - } - - fn parse_exit_status(contents: &str) -> Option<(i32, bool)> { - let mut exit_code = None; - let mut oom_killed = None; - for line in contents.lines() { - let (key, value) = line.split_once('=')?; - match key { - "exit_code" => exit_code = value.parse::().ok(), - "oom_killed" => { - oom_killed = match value { - "0" => Some(false), - "1" => Some(true), - _ => None, - } - } - _ => {} - } - } - Some((exit_code?, oom_killed?)) - } - - async fn read_exit_status(path: &std::path::Path) -> Option<(i32, bool)> { - let contents = tokio::fs::read_to_string(path).await.ok()?; - Self::parse_exit_status(&contents) - } - - fn exit_status_path(server: &crate::server::InnerServer) -> PathBuf { - std::path::Path::new(server.filesystem.base().as_str()).join(Self::EXIT_STATUS_FILE) - } - - fn managed_file_stage_name(target: &str) -> Result<&'static str, anyhow::Error> { - match target { - "/etc/machine-id" => Ok("machine-id"), - "/etc/hosts" => Ok("hosts"), - "/etc/passwd" => Ok("passwd"), - "/etc/group" => Ok("group"), - "/calagopus-entrypoint" => Ok("entrypoint"), - _ => Err(anyhow::anyhow!( - "unsupported managed Proxmox file mount target: {target}" - )), - } - } - - fn managed_file_stage_directory( - app_config: &crate::config::Config, - server_uuid: uuid::Uuid, - ) -> Result { - let config = app_config.load(); - let root = config - .runtime - .pve_lxc - .managed_file_directory - .as_path(&config); - if !root.is_absolute() { - return Err(anyhow::anyhow!( - "runtime.pve_lxc.managed_file_directory must be an absolute path" - )); - } - Ok(root.join(server_uuid.to_compact_string())) - } - - async fn cleanup_managed_file_staging( - app_config: &crate::config::Config, - server_uuid: uuid::Uuid, - ) -> Result<(), anyhow::Error> { - let path = Self::managed_file_stage_directory(app_config, server_uuid)?; - match tokio::fs::remove_dir_all(&path).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error).with_context(|| { - format!( - "failed to remove managed Proxmox file staging directory {}", - path.display() - ) - }), - } - } - - #[allow(clippy::too_many_arguments)] - async fn new( - vmid: u32, - node: String, - cli: PveCli, - stats_sampler: Arc, - server: &crate::server::Server, - app_config: Arc, - firewall: Arc, - status_tx: tokio::sync::mpsc::Sender, - fresh_log: bool, - suppress_initial_stopped: bool, - ) -> Result { - let log_path = Self::runtime_log_path(&app_config, server.uuid); - if let Some(parent) = log_path.parent() { - tokio::fs::create_dir_all(parent).await?; - } - if fresh_log { - tokio::fs::File::create(&log_path).await?; - } else { - tokio::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&log_path) - .await?; - } - - let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); - let websocket_log_count = app_config.load().system.websocket_log_count; - let (stdout_ratelimited_tx, stdout_ratelimited_rx) = - tokio::sync::broadcast::channel::>(websocket_log_count); - let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< - Arc, - >(websocket_log_count * 2); - - let started = Arc::new(AtomicBool::new(!suppress_initial_stopped)); - let state_started = Arc::clone(&started); - let state_cli = cli.clone(); - let state_server_uuid = server.uuid; - let state_exit_status_path = Self::exit_status_path(server); - let status_task = tokio::spawn(async move { - let mut seen_running = state_started.load(Ordering::Acquire); - let mut tick = tokio::time::interval(std::time::Duration::from_secs(2)); - tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - loop { - tick.tick().await; - let process_status = match state_cli.status(vmid).await { - Ok(status) => { - let exit_status = if status == cli::ContainerStatus::Stopped { - Self::read_exit_status(&state_exit_status_path).await - } else { - None - }; - match Self::observed_status( - status, - state_started.load(Ordering::Acquire), - &mut seen_running, - exit_status, - ) { - Some(status) => status, - None => continue, - } - } - Err(error) => { - tracing::warn!( - server = %state_server_uuid, - vmid, - "failed to read Proxmox LXC process state: {error:#}" - ); - continue; - } - }; - - if status_tx.send(process_status).await.is_err() { - break; - } - } - }); - - let stats_cli = cli.clone(); - let stats_node = node.clone(); - let stats_usage = server.resource_usage.clone(); - let stats_server = Arc::downgrade(&**server); - let stats_server_uuid = server.uuid; - let disk_bytes = server.filesystem.limiter_usage().await; - stats_usage.send_modify(|usage| { - usage.wipe(server.state.get_state()); - usage.disk_bytes = disk_bytes; - }); - let stats_task = tokio::spawn(async move { - let mut samples: Option = None; - let mut started_at: Option = None; - let mut previous_cpu_total = 0u64; - let mut previous_sample_at: Option = None; - let mut warned_unresolved_cgroup = false; - let mut resolve_tick = tokio::time::interval(std::time::Duration::from_secs(2)); - resolve_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - loop { - let Some(stats_server) = stats_server.upgrade() else { - break; - }; - let state = stats_server.state.get_state(); - if state == crate::server::state::ServerState::Offline { - let disk_bytes = stats_server.filesystem.limiter_usage().await; - stats_usage.send_modify(|usage| { - usage.wipe(state); - usage.disk_bytes = disk_bytes; - }); - samples = None; - started_at = None; - previous_cpu_total = 0; - previous_sample_at = None; - warned_unresolved_cgroup = false; - resolve_tick.tick().await; - continue; - } - - if samples.is_none() { - resolve_tick.tick().await; - - let runtime_status = match stats_cli.runtime_status(&stats_node, vmid).await { - Ok(status) => status, - Err(error) => { - tracing::warn!( - server = %stats_server_uuid, - vmid, - "failed to resolve Proxmox LXC runtime stats process: {error:#}" - ); - continue; - } - }; - - if runtime_status.status != cli::ContainerStatus::Running { - let state = stats_server.state.get_state(); - let disk_bytes = stats_server.filesystem.limiter_usage().await; - stats_usage.send_modify(|usage| { - usage.wipe(state); - usage.disk_bytes = disk_bytes; - }); - continue; - } - - let Some(pid) = runtime_status.pid else { - if !warned_unresolved_cgroup { - tracing::warn!( - server = %stats_server_uuid, - vmid, - "running Proxmox LXC status did not include an init PID; resource stats are unavailable until it does" - ); - warned_unresolved_cgroup = true; - } - continue; - }; - let Some(files) = super::cgroup::StatFiles::resolve(pid) else { - if !warned_unresolved_cgroup { - tracing::warn!( - server = %stats_server_uuid, - vmid, - pid, - "could not resolve the Proxmox LXC cgroup-v2 files; resource stats are unavailable until they can be resolved" - ); - warned_unresolved_cgroup = true; - } - continue; - }; - - samples = Some(stats_sampler.register(files)); - started_at = std::time::Instant::now().checked_sub( - std::time::Duration::from_secs(runtime_status.uptime_seconds), - ); - previous_cpu_total = 0; - previous_sample_at = None; - warned_unresolved_cgroup = false; - } - - let Some(receiver) = samples.as_mut() else { - continue; - }; - let sample = match tokio::time::timeout( - std::time::Duration::from_secs(3), - receiver.recv(), - ) - .await - { - Ok(Some(Ok(sample))) => sample, - Ok(Some(Err(error))) if error.kind() == std::io::ErrorKind::NotFound => { - samples = None; - started_at = None; - previous_cpu_total = 0; - previous_sample_at = None; - continue; - } - Ok(Some(Err(error))) => { - tracing::warn!( - server = %stats_server_uuid, - vmid, - "failed to read Proxmox LXC cgroup stats: {error}" - ); - samples = None; - continue; - } - Ok(None) | Err(_) => { - tracing::warn!( - server = %stats_server_uuid, - vmid, - "Proxmox LXC cgroup stats sampler stopped delivering; resolving the container again" - ); - samples = None; - continue; - } - }; - - let cpu_absolute = if let Some(previous_at) = previous_sample_at { - let cpu_delta_ns = - sample.cpu_total_ns.saturating_sub(previous_cpu_total) as f64; - let wall_delta_ns = sample.at.duration_since(previous_at).as_nanos() as f64; - - if wall_delta_ns > 0.0 && cpu_delta_ns > 0.0 { - ((cpu_delta_ns / wall_delta_ns) * 100.0 * 1000.0).round() / 1000.0 - } else { - 0.0 - } - } else { - 0.0 - }; - previous_cpu_total = sample.cpu_total_ns; - previous_sample_at = Some(sample.at); - - let disk_bytes = stats_server.filesystem.limiter_usage().await; - let cpu_limit = stats_server.configuration.read().await.build.cpu_limit; - let cpu_limit_absolute = if cpu_limit > 0 { - u32::try_from(cpu_limit).unwrap_or(u32::MAX) - } else { - std::thread::available_parallelism().map_or(1, |threads| threads.get()) as u32 - * 100 - }; - let uptime = started_at - .map(|started_at| { - u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX) - }) - .unwrap_or(0); - let state = stats_server.state.get_state(); - - stats_usage.send_modify(|usage| { - usage.memory_bytes = sample.memory_bytes; - usage.memory_limit_bytes = sample.memory_limit_bytes; - usage.disk_bytes = disk_bytes; - usage.state = state; - usage.cpu_absolute = cpu_absolute; - usage.cpu_limit_absolute = cpu_limit_absolute; - usage.uptime = uptime; - - if let Some((rx_bytes, rx_packets, tx_bytes, tx_packets)) = sample.network { - usage.network.rx_bytes = rx_bytes; - usage.network.rx_packets = rx_packets; - usage.network.tx_bytes = tx_bytes; - usage.network.tx_packets = tx_packets; - } - }); - } - }); - - let console_cli = cli.clone(); - let console_app_config = Arc::clone(&app_config); - let console_log_max_bytes = app_config.load().runtime.pve_lxc.console_log_max_bytes; - let console_server = Arc::downgrade(&**server); - let console_log_path = log_path.clone(); - let console_task = tokio::spawn(async move { - let mut line_buffer = LineBuffer::new(); - let mut ratelimit_counter = 0; - let mut ratelimit_start = std::time::Instant::now(); - - loop { - match console_cli.status(vmid).await { - Ok(cli::ContainerStatus::Running) => {} - Ok(cli::ContainerStatus::Stopped) => { - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - continue; - } - Err(error) => { - tracing::warn!( - vmid, - "failed to read Proxmox LXC state before console attach: {error:#}" - ); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - } - - let session = match console_cli.spawn_console(vmid) { - Ok(session) => session, - Err(error) => { - tracing::warn!(vmid, "failed to attach Proxmox LXC console: {error:#}"); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - }; - // `tokio::fs::File` serializes operations on a single handle. - // A blocking PTY read would therefore prevent stdin writes - // indefinitely if both directions shared one Tokio file. - let cli::ConsoleSession { - mut child, - mut reader, - mut writer, - } = session; - let mut log = match tokio::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&console_log_path) - .await - { - Ok(log) => log, - Err(error) => { - tracing::error!(vmid, "failed to open Proxmox LXC runtime log: {error:#}"); - child.start_kill().ok(); - break; - } - }; - let mut log_size = log.metadata().await.map_or(0, |metadata| metadata.len()); - let mut buffer = vec![0u8; 8192]; - - loop { - tokio::select! { - read = reader.read(&mut buffer) => { - match read { - Ok(0) => break, - Ok(read) => { - let Some(data) = buffer.get(..read) else { - tracing::error!(vmid, read, "Proxmox console returned an invalid read length"); - break; - }; - line_buffer.extend(data); - while let Some(line) = line_buffer.next_line() { - if Self::is_pct_console_banner(line) { - continue; - } - if let Err(error) = Self::write_bounded_log( - &mut log, - &mut log_size, - console_log_max_bytes, - line, - true, - ).await { - tracing::error!(vmid, "failed to append Proxmox LXC runtime log: {error:#}"); - } - let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); - let allow_ratelimit = { - ratelimit_counter += 1; - let config = console_app_config.load(); - if config.throttles.enabled - && config.throttles.line_reset_interval > 0 - && ratelimit_counter >= config.throttles.lines - { - if ratelimit_start.elapsed() - < std::time::Duration::from_millis( - config.throttles.line_reset_interval, - ) - { - if ratelimit_counter == config.throttles.lines - && let Some(server) = console_server.upgrade() - { - server.log_daemon_with_prelude( - "Server is outputting console data too quickly -- throttling...", - ); - } - false - } else { - ratelimit_counter = 0; - ratelimit_start = std::time::Instant::now(); - true - } - } else { - true - } - }; - - if allow_ratelimit { - stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); - } - stdout_tx.send(line).ok(); - } - line_buffer.compact(); - } - Err(error) => { - tracing::warn!(vmid, "failed to read Proxmox LXC console: {error:#}"); - break; - } - } - } - data = stdin_rx.recv() => { - let Some(data) = data else { - child.start_kill().ok(); - return; - }; - if let Err(error) = writer.write_all(&data).await { - tracing::warn!(vmid, "failed to write Proxmox LXC console stdin: {error:#}"); - break; - } - } - result = child.wait() => { - if let Err(error) = result { - tracing::warn!(vmid, "failed waiting for Proxmox LXC console: {error:#}"); - } - break; - } - } - } - - child.start_kill().ok(); - if !matches!( - console_cli.status(vmid).await, - Ok(cli::ContainerStatus::Running) - ) { - if let Some(line) = line_buffer.flush() { - if !Self::is_pct_console_banner(line) - && let Err(error) = Self::write_bounded_log( - &mut log, - &mut log_size, - console_log_max_bytes, - line, - false, - ) - .await - { - tracing::error!( - vmid, - "failed to append final Proxmox LXC runtime log line: {error:#}" - ); - } - let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); - if !Self::is_pct_console_banner(line.as_bytes()) { - stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); - stdout_tx.send(line).ok(); - } - } - line_buffer = LineBuffer::new(); - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } - }); - - Ok(Self { - vmid, - node, - cli, - server: Arc::downgrade(&**server), - app_config, - firewall, - log_path, - stdin_tx, - stdout_ratelimited_rx, - stdout_rx, - started, - status_task, - stats_task, - console_task, - }) - } - - fn runtime_log_path(app_config: &crate::config::Config, server_uuid: uuid::Uuid) -> PathBuf { - app_config - .resolve_as_path(|config| &config.system.log_directory) - .join("pve-lxc") - .join(format!("{server_uuid}.log")) - } - - async fn write_bounded_log( - log: &mut tokio::fs::File, - size: &mut u64, - max_bytes: u64, - line: &[u8], - newline: bool, - ) -> Result<(), std::io::Error> { - let mut line = line; - let mut write_size = u64::try_from(line.len()) - .unwrap_or(u64::MAX) - .saturating_add(u64::from(newline)); - if max_bytes > 0 && write_size > max_bytes { - let newline_size = usize::from(newline); - let keep = usize::try_from(max_bytes) - .unwrap_or(usize::MAX) - .saturating_sub(newline_size); - let start = line.len().saturating_sub(keep); - line = line.get(start..).unwrap_or_default(); - write_size = u64::try_from(line.len()) - .unwrap_or(u64::MAX) - .saturating_add(u64::from(newline)); - } - if max_bytes > 0 && size.saturating_add(write_size) > max_bytes { - log.set_len(0).await?; - log.seek(std::io::SeekFrom::Start(0)).await?; - *size = 0; - } - log.write_all(line).await?; - if newline { - log.write_all(b"\n").await?; - } - *size = size.saturating_add(write_size); - Ok(()) - } - - /// `pct console` forwards terminal key input and expects Enter as a carriage - /// return. Wings command producers use line feeds because Docker stdin is a - /// byte stream, so normalize line endings only at the PVE PTY boundary. - fn normalize_console_input(data: Vec) -> Vec { - let mut normalized = Vec::with_capacity(data.len()); - let mut previous_was_carriage_return = false; - - for byte in data { - if byte == b'\n' { - if !previous_was_carriage_return { - normalized.push(b'\r'); - } - previous_was_carriage_return = false; - } else { - normalized.push(byte); - previous_was_carriage_return = byte == b'\r'; - } - } - - normalized - } - - fn is_pct_console_banner(line: &[u8]) -> bool { - let line = String::from_utf8_lossy(line); - let line = line.trim_matches(|character: char| character.is_whitespace()); - line.starts_with("Connected to tty ") - || line.starts_with("Type to exit the console") - } - - fn get_server(&self) -> Result, anyhow::Error> { - self.server - .upgrade() - .ok_or_else(|| anyhow::anyhow!("server has been dropped")) - } - - fn halt_signal(stop_type: &str, stop_value: Option<&str>) -> Option { - if stop_type != "signal" { - return None; - } - - Some( - match stop_value.map(str::to_uppercase).as_deref() { - Some("SIGABRT") => "SIGABRT", - Some("SIGINT") | Some("C") => "SIGINT", - Some("SIGTERM") => "SIGTERM", - Some("SIGQUIT") => "SIGQUIT", - _ => "SIGKILL", - } - .to_string(), - ) - } - - async fn managed_file_mounts( - server: &crate::server::InnerServer, - app_config: &crate::config::Config, - ) -> Result, anyhow::Error> { - #[cfg(not(unix))] - { - let _ = (server, app_config); - return Err(anyhow::anyhow!( - "managed Proxmox container files require a Unix host" - )); - } - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let server_uuid = { - let configuration = server.configuration.read().await; - configuration.ensure_vmounts(app_config).await?; - configuration.uuid - }; - let (machine_id_enabled, tundra_enabled, passwd_enabled, passwd_directory) = { - let config = app_config.load(); - ( - config.system.machine_id.enabled, - config.tundra.enabled, - config.system.passwd.enabled, - config.system.passwd.directory.as_path(&config), - ) - }; - - let vmount_path = app_config.vmount_path(server_uuid); - let mut files = Vec::with_capacity(4); - if machine_id_enabled { - files.push((vmount_path.join("machine-id"), "/etc/machine-id")); - } - if tundra_enabled { - files.push((vmount_path.join("hosts"), "/etc/hosts")); - } - if passwd_enabled { - files.push((passwd_directory.join("passwd"), "/etc/passwd")); - files.push((passwd_directory.join("group"), "/etc/group")); - } - - let stage_directory = Self::managed_file_stage_directory(app_config, server_uuid)?; - let stage_root = stage_directory - .parent() - .context("managed Proxmox file staging directory did not have a parent")?; - tokio::fs::create_dir_all(stage_root) - .await - .with_context(|| { - format!( - "failed to create managed Proxmox file staging root {}", - stage_root.display() - ) - })?; - tokio::fs::set_permissions(stage_root, std::fs::Permissions::from_mode(0o711)) - .await - .with_context(|| { - format!( - "failed to set managed Proxmox file staging root permissions on {}", - stage_root.display() - ) - })?; - tokio::fs::create_dir_all(&stage_directory) - .await - .with_context(|| { - format!( - "failed to create managed Proxmox file staging directory {}", - stage_directory.display() - ) - })?; - tokio::fs::set_permissions(&stage_directory, std::fs::Permissions::from_mode(0o711)) - .await - .with_context(|| { - format!( - "failed to set managed Proxmox file staging permissions on {}", - stage_directory.display() - ) - })?; - - let mut mounts = Vec::with_capacity(files.len() + 1); - for (source, target) in files { - let source = tokio::fs::canonicalize(&source).await.with_context(|| { - format!( - "failed to resolve managed Proxmox container file {}", - source.display() - ) - })?; - let metadata = tokio::fs::metadata(&source).await.with_context(|| { - format!( - "failed to inspect managed Proxmox container file {}", - source.display() - ) - })?; - if !metadata.is_file() { - return Err(anyhow::anyhow!( - "managed Proxmox container file {} is not a regular file", - source.display() - )); - } - - let staged_source = stage_directory.join(Self::managed_file_stage_name(target)?); - tokio::fs::copy(&source, &staged_source) - .await - .with_context(|| { - format!( - "failed to stage managed Proxmox container file {} at {}", - source.display(), - staged_source.display() - ) - })?; - tokio::fs::set_permissions(&staged_source, std::fs::Permissions::from_mode(0o644)) - .await - .with_context(|| { - format!( - "failed to set managed Proxmox file permissions on {}", - staged_source.display() - ) - })?; - let staged_source = - tokio::fs::canonicalize(&staged_source) - .await - .with_context(|| { - format!( - "failed to resolve staged managed Proxmox container file {}", - staged_source.display() - ) - })?; - let source_path = staged_source.into_os_string().into_string().map_err(|_| { - anyhow::anyhow!("managed Proxmox container file path is not valid UTF-8") - })?; - - mounts.push(cli::ManagedFileMountSpec { - source_path, - target_path: target.to_string(), - }); - } - - let entrypoint_target = "/calagopus-entrypoint"; - let entrypoint_source = - stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); - tokio::fs::write( - &entrypoint_source, - r#"#!/bin/sh -attempt=0 -while [ "$attempt" -lt 120 ]; do - if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then - break - fi - attempt=$((attempt + 1)) - sleep 0.25 -done -if [ "$attempt" -ge 120 ]; then - echo 'timed out waiting for Proxmox DHCP' >&2 - exit 1 -fi - -read_oom_kills() { - value=0 - if [ -r /sys/fs/cgroup/memory.events ]; then - while read -r key count; do - if [ "$key" = oom_kill ]; then - value=$count - break - fi - done < /sys/fs/cgroup/memory.events - fi - printf '%s' "$value" -} - -child=0 -forward_signal() { - if [ "$child" -gt 0 ]; then - kill "-$1" "$child" 2>/dev/null || true - fi -} -trap 'forward_signal HUP' HUP -trap 'forward_signal INT' INT -trap 'forward_signal QUIT' QUIT -trap 'forward_signal ABRT' ABRT -trap 'forward_signal TERM' TERM - -oom_before=$(read_oom_kills) -"$@" <&0 & -child=$! -while :; do - wait "$child" - status=$? - if ! kill -0 "$child" 2>/dev/null; then - break - fi -done -oom_after=$(read_oom_kills) -oom_killed=0 -if [ "$oom_after" -gt "$oom_before" ]; then - oom_killed=1 -fi - -marker=/home/container/.wings-exit-status -temporary="${marker}.tmp.$$" -( - umask 077 - printf 'exit_code=%s\noom_killed=%s\n' "$status" "$oom_killed" > "$temporary" -) -mv -f "$temporary" "$marker" -exit "$status" -"#, - ) - .await - .context("failed to stage Proxmox network-ready entrypoint")?; - tokio::fs::set_permissions(&entrypoint_source, std::fs::Permissions::from_mode(0o755)) - .await - .context("failed to make Proxmox network-ready entrypoint executable")?; - let entrypoint_source = tokio::fs::canonicalize(&entrypoint_source) - .await - .context("failed to resolve staged Proxmox network-ready entrypoint")?; - let source_path = entrypoint_source - .into_os_string() - .into_string() - .map_err(|_| { - anyhow::anyhow!("managed Proxmox entrypoint path is not valid UTF-8") - })?; - mounts.push(cli::ManagedFileMountSpec { - source_path, - target_path: entrypoint_target.to_string(), - }); - - Ok(mounts) - } - } - - async fn runtime_config( - server: &crate::server::InnerServer, - app_config: &crate::config::Config, - cli: &PveCli, - vmid: u32, - ) -> Result { - let ( - environment, - panel_entrypoint, - cpuset_cpus, - io_weight, - memory_unlimited, - swap_unlimited, - ) = { - let configuration = server.configuration.read().await; - let resources = PveCli::panel_resources( - configuration.build.memory_limit, - configuration.build.overhead_memory, - configuration.build.swap, - configuration.build.cpu_limit, - configuration.build.threads.as_deref(), - )?; - ( - configuration.environment(app_config), - configuration.entrypoint.clone(), - configuration - .build - .threads - .as_deref() - .filter(|threads| !threads.trim().is_empty()) - .map(PveCli::normalize_cpuset) - .transpose()?, - PveCli::cgroup2_io_weight(configuration.build.io_weight)?, - resources.memory_unlimited, - resources.swap_unlimited, - ) - }; - let pids_limit = match app_config.load().docker.container_pid_limit { - 0 => None, - limit => Some(limit), - }; - let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; - let process = server.process_configuration.read().await; - let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); - let mut init_arguments = vec!["/calagopus-entrypoint".to_string()]; - if let Some(entrypoint) = panel_entrypoint { - if entrypoint.is_empty() { - return Err(anyhow::anyhow!("panel entrypoint override cannot be empty")); - } - init_arguments.extend(entrypoint); - } else { - let image_entrypoint = cli.oci_entrypoint(vmid).await?; - init_arguments.extend( - shell_words::split(&image_entrypoint) - .context("failed to parse OCI entrypoint for Proxmox LXC")?, - ); - } - let init_command = PveCli::encode_init_command(&init_arguments)?; - - Ok(cli::RuntimeConfigSpec { - vmid, - environment, - halt_signal, - init_command: Some(init_command), - console_logfile: None, - cpuset_cpus, - pids_limit, - io_weight, - memory_unlimited, - swap_unlimited, - managed_file_mounts: Some(managed_file_mounts), - }) - } - - async fn sync_firewall(&self, wait_for_address: bool) -> Result<(), anyhow::Error> { - let server = self.get_server()?; - let has_rules = !server.configuration.read().await.firewall.is_empty(); - let edge_forwarding = PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some(); - let running = self.cli.status(self.vmid).await? == cli::ContainerStatus::Running; - - let container_ips = if running && (has_rules || edge_forwarding) { - let mut last_error = None; - let mut addresses = Vec::new(); - let attempts = if wait_for_address { 40 } else { 1 }; - - for attempt in 0..attempts { - match self.cli.interfaces(&self.node, self.vmid).await { - Ok(interfaces) => { - addresses = PveCli::container_addresses(&interfaces); - if !addresses.is_empty() { - break; - } - last_error = None; - } - Err(error) => last_error = Some(error), - } - - if attempt + 1 < attempts { - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } - } - - if addresses.is_empty() { - if let Some(error) = last_error { - return Err(error.context( - "failed to resolve Proxmox LXC address for firewall and edge forwarding", - )); - } - return Err(anyhow::anyhow!( - "running Proxmox LXC container {} has no usable bridged address; refusing to leave firewall or edge forwarding unapplied", - self.vmid - )); - } - - addresses - } else { - Vec::new() - }; - - let (mut spec, edge_forward) = { - let configuration = server.configuration.read().await; - let destination = container_ips.iter().find_map(|address| match address { - IpAddr::V4(address) => Some(*address), - IpAddr::V6(_) => None, - }); - let edge_forward = if edge_forwarding && running { - Some(PveLxcExecutor::edge_forward_payload( - &configuration, - destination - .context("running Proxmox LXC has no IPv4 address for edge forwarding")?, - )?) - } else { - None - }; - ( - PveLxcExecutor::firewall_spec(&configuration, container_ips, Some(self.vmid)), - edge_forward, - ) - }; - spec.files = Some(PveLxcExecutor::firewall_file_access(&server)); - self.firewall.sync(&spec).await?; - if edge_forwarding { - let (public_ip, payload) = edge_forward.flatten().unzip(); - PveLxcExecutor::sync_edge_forwarding(&self.app_config, server.uuid, public_ip, payload) - .await?; - } - Ok(()) - } - - async fn stop_after_firewall_failure( - &self, - error: anyhow::Error, - context: &'static str, - ) -> anyhow::Error { - match self.cli.stop(self.vmid).await { - Ok(()) => error.context(format!("{context}; container was stopped")), - Err(stop_error) => error.context(format!( - "{context} and failed to stop the exposed container: {stop_error:#}" - )), - } - } -} - -impl Drop for PveProcessHandle { - fn drop(&mut self) { - self.status_task.abort(); - self.stats_task.abort(); - self.console_task.abort(); - } -} - -#[async_trait::async_trait] -impl ProcessHandle for PveProcessHandle { - async fn logs( - &self, - lines: Option, - ) -> Result, anyhow::Error> { - if let Some(lines) = lines { - let file = match tokio::fs::File::open(&self.log_path).await { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(Box::new(Cursor::new(Vec::new()))); - } - Err(error) => return Err(error.into()), - }; - return Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)); - } - - Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) - } - - async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { - self.stdin_tx - .send(Self::normalize_console_input(data)) - .await - .map_err(Into::into) - } - - async fn subscribe_stdout_lines_ratelimited( - &self, - ) -> Result>, anyhow::Error> - { - Ok(self.stdout_ratelimited_rx.resubscribe()) - } - - async fn subscribe_stdout_lines( - &self, - ) -> Result>, anyhow::Error> - { - Ok(self.stdout_rx.resubscribe()) - } - - async fn sync_configuration(&self) -> Result<(), anyhow::Error> { - let server = self.get_server()?; - let resources = { - let configuration = server.configuration.read().await; - PveCli::panel_resources( - configuration.build.memory_limit, - configuration.build.overhead_memory, - configuration.build.swap, - configuration.build.cpu_limit, - configuration.build.threads.as_deref(), - )? - }; - self.cli.set_resources(self.vmid, &resources).await?; - self.cli - .apply_runtime_config( - &Self::runtime_config(&server, &self.app_config, &self.cli, self.vmid).await?, - ) - .await?; - if let Err(error) = self.sync_firewall(true).await { - let configuration = server.configuration.read().await; - let requires_network_policy = !configuration.firewall.is_empty() - || (configuration.allocations.default.is_some() - && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()); - drop(configuration); - if requires_network_policy { - return Err(self - .stop_after_firewall_failure( - error, - "failed to synchronize Proxmox LXC network policy after configuration update", - ) - .await); - } - - tracing::warn!( - vmid = self.vmid, - "failed to clear Proxmox LXC firewall state after configuration update: {error:#}" - ); - } - Ok(()) - } - - async fn start(&self) -> Result<(), anyhow::Error> { - let server = self.get_server()?; - let exit_status_path = Self::exit_status_path(&server); - match tokio::fs::remove_file(&exit_status_path).await { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!( - "failed to clear stale Proxmox LXC exit status {}", - exit_status_path.display() - ) - }); - } - } - self.cli.start(self.vmid).await?; - self.started.store(true, Ordering::Release); - - if let Err(error) = self.sync_firewall(true).await { - let requires_network_policy = match self.get_server() { - Ok(server) => { - let configuration = server.configuration.read().await; - !configuration.firewall.is_empty() - || (configuration.allocations.default.is_some() - && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()) - } - Err(_) => true, - }; - - if requires_network_policy { - return Err(self - .stop_after_firewall_failure( - error, - "failed to apply Proxmox LXC network policy", - ) - .await); - } - - tracing::warn!( - vmid = self.vmid, - "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" - ); - } - - Ok(()) - } - - async fn stop(&self) -> Result<(), anyhow::Error> { - let server = self.get_server()?; - let process = server.process_configuration.read().await; - let stop_type = process.stop.r#type.clone(); - let stop_value = process.stop.value.clone(); - drop(process); - - if stop_type == "command" { - let mut command = stop_value - .map(|value| value.as_bytes().to_vec()) - .unwrap_or_default(); - command.push(b'\n'); - self.stdin_tx.send(command).await.map_err(Into::into) - } else { - self.cli.request_shutdown(self.vmid).await - } - } - - async fn kill(&self) -> Result<(), anyhow::Error> { - let server = self.get_server()?; - let exit_status_path = Self::exit_status_path(&server); - tokio::fs::write(&exit_status_path, "exit_code=137\noom_killed=0\n") - .await - .with_context(|| { - format!( - "failed to record forced Proxmox LXC exit status {}", - exit_status_path.display() - ) - })?; - self.cli.stop(self.vmid).await - } -} - -struct PveHelperProcessHandle { - vmid: u32, - cli: PveCli, - log_path: PathBuf, - started: Arc, - cancelled: Arc, - stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, - stdout_rx: tokio::sync::broadcast::Receiver>, -} - -impl PveHelperProcessHandle { - #[allow(clippy::too_many_arguments)] - async fn new( - vmid: u32, - cli: PveCli, - server: &crate::server::Server, - app_config: Arc, - log_path: PathBuf, - status_tx: tokio::sync::mpsc::Sender, - already_started: bool, - start_at_end: bool, - auto_cleanup: bool, - cleanup_path: Option, - ) -> Result { - if let Some(parent) = log_path.parent() { - tokio::fs::create_dir_all(parent).await?; - } - tokio::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&log_path) - .await?; - - let websocket_log_count = app_config.load().system.websocket_log_count; - let (stdout_ratelimited_tx, stdout_ratelimited_rx) = - tokio::sync::broadcast::channel::>(websocket_log_count); - let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< - Arc, - >(websocket_log_count * 2); - - let started = Arc::new(AtomicBool::new(already_started)); - let cancelled = Arc::new(AtomicBool::new(false)); - - let status_cli = cli.clone(); - let status_started = Arc::clone(&started); - let status_cancelled = Arc::clone(&cancelled); - let status_server = server.uuid; - tokio::spawn(async move { - let mut sent_running = false; - let mut tick = tokio::time::interval(std::time::Duration::from_millis(500)); - tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - loop { - tick.tick().await; - if status_cancelled.load(Ordering::Acquire) { - break; - } - - match status_cli.status(vmid).await { - Ok(cli::ContainerStatus::Running) => { - if !sent_running { - sent_running = true; - if status_tx.send(super::ProcessStatus::Running).await.is_err() { - break; - } - } - } - Ok(cli::ContainerStatus::Stopped) if status_started.load(Ordering::Acquire) => { - if !sent_running - && status_tx.send(super::ProcessStatus::Running).await.is_err() - { - break; - } - let _ = status_tx - .send(super::ProcessStatus::Stopped { - exit_code: -1, - oom_killed: false, - }) - .await; - break; - } - Ok(cli::ContainerStatus::Stopped) => {} - Err(error) => { - tracing::warn!( - server = %status_server, - vmid, - "failed to read Proxmox helper LXC process state: {error:#}" - ); - } - } - } - }); - - let output_cli = cli.clone(); - let output_started = Arc::clone(&started); - let output_cancelled = Arc::clone(&cancelled); - let output_server = Arc::downgrade(&**server); - let output_log_path = log_path.clone(); - tokio::spawn(async move { - let mut file = match tokio::fs::OpenOptions::new() - .read(true) - .open(&output_log_path) - .await - { - Ok(file) => file, - Err(error) => { - tracing::error!(vmid, "failed to open Proxmox helper log: {error:#}"); - return; - } - }; - if start_at_end - && let Ok(metadata) = file.metadata().await - && let Err(error) = file.seek(std::io::SeekFrom::Start(metadata.len())).await - { - tracing::warn!(vmid, "failed to seek Proxmox helper log: {error:#}"); - } - - let mut line_buffer = LineBuffer::new(); - let mut ratelimit_counter = 0; - let mut ratelimit_start = std::time::Instant::now(); - - loop { - if output_cancelled.load(Ordering::Acquire) { - break; - } - - if let (Ok(position), Ok(metadata)) = - (file.stream_position().await, file.metadata().await) - && metadata.len() < position - { - if let Err(error) = file.seek(std::io::SeekFrom::Start(0)).await { - tracing::warn!( - vmid, - "failed to rewind truncated Proxmox helper log: {error:#}" - ); - } - line_buffer = LineBuffer::new(); - } - - let mut data = Vec::new(); - match file.read_to_end(&mut data).await { - Ok(_) => {} - Err(error) => { - tracing::warn!(vmid, "failed to read Proxmox helper log: {error:#}"); - } - } - - if !data.is_empty() { - line_buffer.extend(&data); - while let Some(line) = line_buffer.next_line() { - let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); - let allow_ratelimit = { - ratelimit_counter += 1; - let config = app_config.load(); - if config.throttles.enabled - && config.throttles.line_reset_interval > 0 - && ratelimit_counter >= config.throttles.lines - { - if ratelimit_start.elapsed() - < std::time::Duration::from_millis( - config.throttles.line_reset_interval, - ) - { - if ratelimit_counter == config.throttles.lines - && let Some(server) = output_server.upgrade() - { - server.log_daemon_with_prelude( - "Server is outputting console data too quickly -- throttling...", - ); - } - false - } else { - ratelimit_counter = 0; - ratelimit_start = std::time::Instant::now(); - true - } - } else { - true - } - }; - - if allow_ratelimit { - stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); - } - stdout_tx.send(line).ok(); - } - line_buffer.compact(); - } - - let stopped = output_started.load(Ordering::Acquire) - && matches!( - output_cli.status(vmid).await, - Ok(cli::ContainerStatus::Stopped) - ); - if stopped { - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - let mut final_data = Vec::new(); - if file.read_to_end(&mut final_data).await.is_ok() && !final_data.is_empty() { - line_buffer.extend(&final_data); - while let Some(line) = line_buffer.next_line() { - let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); - stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); - stdout_tx.send(line).ok(); - } - } - if let Some(line) = line_buffer.flush() { - let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); - stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); - stdout_tx.send(line).ok(); - } - break; - } - - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } - - if auto_cleanup { - output_cancelled.store(true, Ordering::Release); - if let Err(error) = output_cli.destroy(vmid).await { - tracing::warn!( - vmid, - "failed to destroy completed Proxmox helper LXC: {error:#}" - ); - } - if let Some(path) = cleanup_path - && let Err(error) = tokio::fs::remove_dir_all(&path).await - && error.kind() != std::io::ErrorKind::NotFound - { - tracing::warn!( - path = %path.display(), - "failed to remove Proxmox helper staging directory: {error:#}" - ); - } - } - }); - - Ok(Self { - vmid, - cli, - log_path, - started, - cancelled, - stdout_ratelimited_rx, - stdout_rx, - }) - } -} - -impl Drop for PveHelperProcessHandle { - fn drop(&mut self) { - if !self.started.load(Ordering::Acquire) { - self.cancelled.store(true, Ordering::Release); - } - } -} - -#[async_trait::async_trait] -impl ProcessHandle for PveHelperProcessHandle { - async fn logs( - &self, - lines: Option, - ) -> Result, anyhow::Error> { - if let Some(lines) = lines { - let file = tokio::fs::File::open(&self.log_path).await?; - Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)) - } else { - Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) - } - } - - async fn send_stdin(&self, _data: Vec) -> Result<(), anyhow::Error> { - Err(anyhow::anyhow!( - "stdin is not supported for Proxmox helper containers" - )) - } - - async fn subscribe_stdout_lines_ratelimited( - &self, - ) -> Result>, anyhow::Error> - { - Ok(self.stdout_ratelimited_rx.resubscribe()) - } - - async fn subscribe_stdout_lines( - &self, - ) -> Result>, anyhow::Error> - { - Ok(self.stdout_rx.resubscribe()) - } - - async fn sync_configuration(&self) -> Result<(), anyhow::Error> { - Ok(()) - } - - async fn start(&self) -> Result<(), anyhow::Error> { - self.cli.start(self.vmid).await?; - self.started.store(true, Ordering::Release); - Ok(()) - } - - async fn stop(&self) -> Result<(), anyhow::Error> { - self.cli.stop(self.vmid).await - } - - async fn kill(&self) -> Result<(), anyhow::Error> { - self.cli.stop(self.vmid).await - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PveVersion { major: u64, @@ -4215,577 +2671,4 @@ impl ServerExecutor for PveLxcExecutor { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn successful_start_followed_by_first_stopped_probe_emits_stopped() { - let mut seen_running = false; - let status = PveProcessHandle::observed_status( - cli::ContainerStatus::Stopped, - true, - &mut seen_running, - Some((42, true)), - ); - assert!(matches!( - status, - Some(crate::server::executor::ProcessStatus::Stopped { - exit_code: 42, - oom_killed: true, - }) - )); - } - - #[test] - fn parses_managed_exit_status_strictly() { - assert_eq!( - PveProcessHandle::parse_exit_status("exit_code=137\noom_killed=1\n"), - Some((137, true)) - ); - assert_eq!( - PveProcessHandle::parse_exit_status("oom_killed=0\nexit_code=0\n"), - Some((0, false)) - ); - assert_eq!(PveProcessHandle::parse_exit_status("exit_code=0\n"), None); - assert_eq!( - PveProcessHandle::parse_exit_status("exit_code=nope\noom_killed=0\n"), - None - ); - } - - #[tokio::test] - async fn runtime_log_never_exceeds_its_retention_limit() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("console.log"); - let mut file = tokio::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&path) - .await - .unwrap(); - let mut size = 0; - - PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"first", true) - .await - .unwrap(); - PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"second", true) - .await - .unwrap(); - PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"oversized-line", true) - .await - .unwrap(); - file.flush().await.unwrap(); - - let data = tokio::fs::read(&path).await.unwrap(); - assert_eq!(data, b"ed-line\n"); - assert_eq!(data.len(), 8); - assert_eq!(size, 8); - } - - #[test] - fn static_allocation_uses_explicit_isolated_vlan_network() { - assert_eq!( - PveLxcExecutor::network_for_allocation( - "vmbr0", - Some(30), - Some(24), - Some("10.0.30.1"), - Some("10.0.30.42"), - ) - .unwrap(), - "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=10.0.30.42/24,gw=10.0.30.1,type=veth" - ); - } - - #[test] - fn static_allocation_never_derives_the_management_network() { - let error = PveLxcExecutor::network_for_allocation( - "vmbr0", - Some(30), - None, - None, - Some("10.0.30.42"), - ) - .unwrap_err(); - assert!(error.to_string().contains("network_prefix")); - - assert_eq!( - PveLxcExecutor::dhcp_network("vmbr0", Some(30)).unwrap(), - "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=dhcp,type=veth,host-managed=1" - ); - } - - #[test] - fn network_bridge_rejects_option_injection() { - assert!(PveLxcExecutor::dhcp_network("vmbr0 firewall=0", None).is_err()); - assert!(PveLxcExecutor::dhcp_network("vmbr0,firewall=0", None).is_err()); - } - - #[test] - fn accepts_pve_9_2_and_newer() { - assert_eq!( - PveVersion::parse("pve-manager/9.2.1/abcdef") - .ok() - .map(PveVersion::is_supported), - Some(true) - ); - assert_eq!( - PveVersion::parse("pve-manager/10.0.0/abcdef") - .ok() - .map(PveVersion::is_supported), - Some(true) - ); - } - - #[test] - fn rejects_pve_older_than_9_2() { - assert_eq!( - PveVersion::parse("pve-manager/9.1.7/abcdef") - .ok() - .map(PveVersion::is_supported), - Some(false) - ); - } - - #[test] - fn pve_runtime_only_targets_the_local_node() { - assert_eq!( - PveLxcExecutor::runtime_node("", "pve-a").ok().as_deref(), - Some("pve-a") - ); - assert_eq!( - PveLxcExecutor::runtime_node("pve-a", "pve-a") - .ok() - .as_deref(), - Some("pve-a") - ); - assert!(PveLxcExecutor::runtime_node("pve-b", "pve-a").is_err()); - } - - #[test] - fn proc_net_parser_reports_listeners_and_udp_reservations() { - fn proc_ipv4(address: std::net::Ipv4Addr) -> String { - format!("{:08X}", u32::from_le_bytes(address.octets())) - } - - let any = proc_ipv4(std::net::Ipv4Addr::UNSPECIFIED); - let loopback = proc_ipv4(std::net::Ipv4Addr::LOCALHOST); - let tcp = format!( - " sl local_address rem_address st\n 0: {any}:63DD 00000000:0000 0A\n 1: {loopback}:C350 0100007F:9C40 01\n" - ); - let udp = format!( - " sl local_address rem_address st\n 0: {any}:4ABC 00000000:0000 07\n 1: {loopback}:D431 0100007F:0035 01\n 2: {any}:0000 00000000:0000 07\n" - ); - - assert_eq!( - PveLxcExecutor::parse_proc_net_ports(&tcp, ProcNetFamily::Ipv4, true).unwrap(), - vec![BoundPort { - family: ProcNetFamily::Ipv4, - address: None, - port: 25565, - }] - ); - assert_eq!( - PveLxcExecutor::parse_proc_net_ports(&udp, ProcNetFamily::Ipv4, false).unwrap(), - vec![ - BoundPort { - family: ProcNetFamily::Ipv4, - address: None, - port: 19132, - }, - BoundPort { - family: ProcNetFamily::Ipv4, - address: Some(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), - port: 54321, - }, - ] - ); - } - - #[test] - fn proc_net_parser_decodes_ipv6_and_matches_ipv4_mapped_bindings() { - fn proc_ipv6(address: std::net::Ipv6Addr) -> String { - address - .octets() - .chunks_exact(4) - .map(|chunk| { - format!( - "{:08X}", - u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) - ) - }) - .collect::() - } - - let address: std::net::Ipv6Addr = "fd12:3456::20".parse().unwrap(); - let raw = proc_ipv6(address); - let table = format!( - " sl local_address rem_address st\n 0: {raw}:63DD 00000000000000000000000000000000:0000 0A\n" - ); - assert_eq!( - PveLxcExecutor::parse_proc_net_ports(&table, ProcNetFamily::Ipv6, true).unwrap(), - vec![BoundPort { - family: ProcNetFamily::Ipv6, - address: Some(IpAddr::V6(address)), - port: 25565, - }] - ); - - let mapped = BoundPort { - family: ProcNetFamily::Ipv6, - address: Some(IpAddr::V6("::ffff:10.42.0.20".parse().unwrap())), - port: 25565, - }; - assert!(PveLxcExecutor::bound_port_matches_address( - mapped, - "10.42.0.20".parse().unwrap() - )); - assert!(!PveLxcExecutor::bound_port_matches_address( - mapped, - "10.42.0.21".parse().unwrap() - )); - assert!(PveLxcExecutor::bound_port_matches_address( - BoundPort { - family: ProcNetFamily::Ipv6, - address: None, - port: 19132, - }, - "fd12:3456::20".parse().unwrap() - )); - assert!(!PveLxcExecutor::bound_port_matches_address( - BoundPort { - family: ProcNetFamily::Ipv4, - address: None, - port: 19132, - }, - "fd12:3456::20".parse().unwrap() - )); - } - - #[test] - fn ownership_tags_require_marker_and_server_uuid() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let server_tag = PveLxcExecutor::server_tag("calagopus", server_uuid); - let tags = vec!["calagopus".to_string(), server_tag]; - - assert_eq!( - PveLxcExecutor::server_from_tags("calagopus", &tags), - Some(server_uuid) - ); - assert_eq!(PveLxcExecutor::server_from_tags("other", &tags), None); - } - - #[test] - fn helper_tags_are_distinct_from_servers_and_roles() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let installer_tag = PveLxcExecutor::helper_tag( - "calagopus", - PveLxcExecutor::INSTALLER_HELPER_ROLE, - server_uuid, - ); - let tags = vec!["calagopus".to_string(), installer_tag]; - - assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); - assert_eq!( - PveLxcExecutor::helper_from_tags( - "calagopus", - PveLxcExecutor::INSTALLER_HELPER_ROLE, - &tags, - ), - Some(server_uuid) - ); - assert_eq!( - PveLxcExecutor::helper_from_tags( - "calagopus", - PveLxcExecutor::SCRIPT_HELPER_ROLE, - &tags, - ), - None - ); - assert_eq!( - PveLxcExecutor::helper_from_tags("other", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), - None - ); - } - - #[test] - fn replacement_tags_are_distinct_and_recoverable() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let replacement_tag = PveLxcExecutor::replacement_tag("calagopus", server_uuid); - let tags = vec!["calagopus".to_string(), replacement_tag]; - - assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); - assert_eq!( - PveLxcExecutor::replacement_from_tags("calagopus", &tags), - Some(server_uuid) - ); - } - - #[test] - fn image_drift_plan_recovers_staged_replacements() { - let desired = "calagopus-image-new"; - let existing = cli::ClusterContainer { - vmid: 101, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags: vec!["calagopus-image-old".to_string()], - }; - let staged = cli::ClusterContainer { - vmid: 102, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags: vec![desired.to_string()], - }; - - assert_eq!( - PveLxcExecutor::plan_server_container(Some(&existing), Some(&staged), desired), - ServerContainerPlan::ReplaceWithStaged { - old_vmid: 101, - staged_vmid: 102, - } - ); - assert_eq!( - PveLxcExecutor::plan_server_container(None, Some(&staged), desired), - ServerContainerPlan::RecoverStaged { staged_vmid: 102 } - ); - } - - #[test] - fn image_drift_plan_discards_stale_staging_without_replacing_good_container() { - let desired = "calagopus-image-current"; - let existing = cli::ClusterContainer { - vmid: 101, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags: vec![desired.to_string()], - }; - let stale = cli::ClusterContainer { - vmid: 102, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags: vec!["calagopus-image-old-attempt".to_string()], - }; - - assert_eq!( - PveLxcExecutor::plan_server_container(Some(&existing), Some(&stale), desired), - ServerContainerPlan::Reuse { - vmid: 101, - stale_replacement: Some(102), - } - ); - assert_eq!( - PveLxcExecutor::plan_server_container(None, Some(&stale), desired), - ServerContainerPlan::CreateFresh { - stale_replacement: Some(102), - } - ); - } - - #[test] - fn owned_container_lookup_rejects_duplicate_claims() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let tags = vec![ - "calagopus".to_string(), - PveLxcExecutor::server_tag("calagopus", server_uuid), - ]; - let containers = vec![ - cli::ClusterContainer { - vmid: 101, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags: tags.clone(), - }, - cli::ClusterContainer { - vmid: 102, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags, - }, - ]; - - assert!( - PveLxcExecutor::owned_server_container(&containers, "pve-a", "calagopus", server_uuid,) - .is_err() - ); - assert_eq!( - PveLxcExecutor::owned_server_container(&containers, "pve-b", "calagopus", server_uuid,) - .ok() - .flatten(), - None - ); - } - - #[test] - fn owned_helper_lookup_rejects_duplicate_claims() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let tags = vec![ - "calagopus".to_string(), - PveLxcExecutor::helper_tag( - "calagopus", - PveLxcExecutor::INSTALLER_HELPER_ROLE, - server_uuid, - ), - ]; - let containers = vec![ - cli::ClusterContainer { - vmid: 201, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Running), - tags: tags.clone(), - }, - cli::ClusterContainer { - vmid: 202, - node: "pve-a".to_string(), - status: Some(cli::ContainerStatus::Stopped), - tags, - }, - ]; - - assert!( - PveLxcExecutor::owned_helper_container( - &containers, - "pve-a", - "calagopus", - PveLxcExecutor::INSTALLER_HELPER_ROLE, - server_uuid, - ) - .is_err() - ); - assert_eq!( - PveLxcExecutor::owned_helper_container( - &containers, - "pve-a", - "calagopus", - PveLxcExecutor::SCRIPT_HELPER_ROLE, - server_uuid, - ) - .ok() - .flatten(), - None - ); - } - - #[test] - fn direct_bridge_firewall_spec_targets_container_addresses_without_published_bindings() { - let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); - let mappings = HashMap::from([ - ( - compact_str::CompactString::from("10.42.0.20"), - vec![25566, 25565, 25565], - ), - (compact_str::CompactString::from("10.42.0.21"), vec![19132]), - ]); - let rules = vec![crate::server::firewall::FirewallRule { - action: crate::server::firewall::FirewallRuleAction::Deny, - protocols: HashSet::from([crate::server::firewall::FirewallRuleProtocol::Tcp]), - sources: Vec::new(), - ports: None, - source_file: None, - }]; - let spec = PveLxcExecutor::direct_bridge_firewall_spec( - server_uuid, - &mappings, - &rules, - vec![ - "fd12:3456::20".parse().unwrap(), - "10.42.0.20".parse().unwrap(), - "10.42.0.20".parse().unwrap(), - ], - None, - ); - - assert!(spec.bindings.is_empty()); - assert_eq!(spec.container_ports, vec![19132, 25565, 25566]); - assert_eq!( - spec.container_ips, - vec![ - "10.42.0.20".parse::().unwrap(), - "fd12:3456::20".parse::().unwrap(), - ] - ); - assert_eq!(spec.rules, rules); - - let concrete = crate::server::firewall::expand_rules(&spec); - assert_eq!(concrete.len(), 6); - assert!( - concrete - .iter() - .all(|rule| matches!(rule.dst, crate::server::firewall::RuleDst::Container { .. })) - ); - } - - #[test] - fn console_input_uses_terminal_enter_without_doubling_crlf() { - assert_eq!( - PveProcessHandle::normalize_console_input(b"list\n".to_vec()), - b"list\r" - ); - assert_eq!( - PveProcessHandle::normalize_console_input(b"first\r\nsecond\n".to_vec()), - b"first\rsecond\r" - ); - assert_eq!( - PveProcessHandle::normalize_console_input(vec![0x03]), - vec![0x03] - ); - assert!(PveProcessHandle::is_pct_console_banner( - b"Connected to tty 0\r" - )); - assert!(PveProcessHandle::is_pct_console_banner( - b"Type to exit the console, to enter Ctrl+z itself\r" - )); - assert!(!PveProcessHandle::is_pct_console_banner( - b"[Server thread/INFO]: Connected to tty 0" - )); - } - - #[test] - fn halt_signal_matches_wings_stop_semantics() { - assert_eq!( - PveProcessHandle::halt_signal("signal", Some("SIGABRT")), - Some("SIGABRT".to_string()) - ); - assert_eq!( - PveProcessHandle::halt_signal("signal", Some("c")), - Some("SIGINT".to_string()) - ); - assert_eq!( - PveProcessHandle::halt_signal("signal", Some("SIGTERM")), - Some("SIGTERM".to_string()) - ); - assert_eq!( - PveProcessHandle::halt_signal("signal", Some("SIGQUIT")), - Some("SIGQUIT".to_string()) - ); - assert_eq!( - PveProcessHandle::halt_signal("signal", Some("unexpected")), - Some("SIGKILL".to_string()) - ); - assert_eq!(PveProcessHandle::halt_signal("command", Some("stop")), None); - } - - #[test] - fn device_permissions_require_exact_pve_devn_semantics() { - let path = std::path::Path::new("/dev/kvm"); - - assert_eq!( - PveLxcExecutor::device_deny_write("r", path).ok(), - Some(true) - ); - assert_eq!( - PveLxcExecutor::device_deny_write("rw", path).ok(), - Some(false) - ); - assert_eq!( - PveLxcExecutor::device_deny_write("wr", path).ok(), - Some(false) - ); - assert!(PveLxcExecutor::device_deny_write("w", path).is_err()); - assert!(PveLxcExecutor::device_deny_write("m", path).is_err()); - assert_eq!( - PveLxcExecutor::device_deny_write("rwm", path).ok(), - Some(false) - ); - assert!(PveLxcExecutor::device_deny_write("rwx", path).is_err()); - } -} +mod tests; diff --git a/application/src/server/executor/pve_lxc/server_process.rs b/application/src/server/executor/pve_lxc/server_process.rs new file mode 100644 index 00000000..d4d24ec2 --- /dev/null +++ b/application/src/server/executor/pve_lxc/server_process.rs @@ -0,0 +1,1240 @@ +use super::*; + +pub(super) struct PveProcessHandle { + vmid: u32, + node: String, + cli: PveCli, + server: Weak, + app_config: Arc, + firewall: Arc, + log_path: PathBuf, + stdin_tx: tokio::sync::mpsc::Sender>, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, + started: Arc, + status_task: tokio::task::JoinHandle<()>, + stats_task: tokio::task::JoinHandle<()>, + console_task: tokio::task::JoinHandle<()>, +} + +impl PveProcessHandle { + const EXIT_STATUS_FILE: &'static str = ".wings-exit-status"; + + pub(super) fn observed_status( + status: cli::ContainerStatus, + start_succeeded: bool, + seen_running: &mut bool, + exit_status: Option<(i32, bool)>, + ) -> Option { + match status { + cli::ContainerStatus::Running => { + *seen_running = true; + Some(super::ProcessStatus::Running) + } + cli::ContainerStatus::Stopped if !*seen_running && !start_succeeded => None, + cli::ContainerStatus::Stopped => { + let (exit_code, oom_killed) = exit_status.unwrap_or((-1, false)); + Some(super::ProcessStatus::Stopped { + exit_code, + oom_killed, + }) + } + } + } + + pub(super) fn parse_exit_status(contents: &str) -> Option<(i32, bool)> { + let mut exit_code = None; + let mut oom_killed = None; + for line in contents.lines() { + let (key, value) = line.split_once('=')?; + match key { + "exit_code" => exit_code = value.parse::().ok(), + "oom_killed" => { + oom_killed = match value { + "0" => Some(false), + "1" => Some(true), + _ => None, + } + } + _ => {} + } + } + Some((exit_code?, oom_killed?)) + } + + async fn read_exit_status(path: &std::path::Path) -> Option<(i32, bool)> { + let contents = tokio::fs::read_to_string(path).await.ok()?; + Self::parse_exit_status(&contents) + } + + fn exit_status_path(server: &crate::server::InnerServer) -> PathBuf { + std::path::Path::new(server.filesystem.base().as_str()).join(Self::EXIT_STATUS_FILE) + } + + fn managed_file_stage_name(target: &str) -> Result<&'static str, anyhow::Error> { + match target { + "/etc/machine-id" => Ok("machine-id"), + "/etc/hosts" => Ok("hosts"), + "/etc/passwd" => Ok("passwd"), + "/etc/group" => Ok("group"), + "/calagopus-entrypoint" => Ok("entrypoint"), + _ => Err(anyhow::anyhow!( + "unsupported managed Proxmox file mount target: {target}" + )), + } + } + + fn managed_file_stage_directory( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result { + let config = app_config.load(); + let root = config + .runtime + .pve_lxc + .managed_file_directory + .as_path(&config); + if !root.is_absolute() { + return Err(anyhow::anyhow!( + "runtime.pve_lxc.managed_file_directory must be an absolute path" + )); + } + Ok(root.join(server_uuid.to_compact_string())) + } + + pub(super) async fn cleanup_managed_file_staging( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result<(), anyhow::Error> { + let path = Self::managed_file_stage_directory(app_config, server_uuid)?; + match tokio::fs::remove_dir_all(&path).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| { + format!( + "failed to remove managed Proxmox file staging directory {}", + path.display() + ) + }), + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn new( + vmid: u32, + node: String, + cli: PveCli, + stats_sampler: Arc, + server: &crate::server::Server, + app_config: Arc, + firewall: Arc, + status_tx: tokio::sync::mpsc::Sender, + fresh_log: bool, + suppress_initial_stopped: bool, + ) -> Result { + let log_path = Self::runtime_log_path(&app_config, server.uuid); + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + if fresh_log { + tokio::fs::File::create(&log_path).await?; + } else { + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + } + + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + + let started = Arc::new(AtomicBool::new(!suppress_initial_stopped)); + let state_started = Arc::clone(&started); + let state_cli = cli.clone(); + let state_server_uuid = server.uuid; + let state_exit_status_path = Self::exit_status_path(server); + let status_task = tokio::spawn(async move { + let mut seen_running = state_started.load(Ordering::Acquire); + let mut tick = tokio::time::interval(std::time::Duration::from_secs(2)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + let process_status = match state_cli.status(vmid).await { + Ok(status) => { + let exit_status = if status == cli::ContainerStatus::Stopped { + Self::read_exit_status(&state_exit_status_path).await + } else { + None + }; + match Self::observed_status( + status, + state_started.load(Ordering::Acquire), + &mut seen_running, + exit_status, + ) { + Some(status) => status, + None => continue, + } + } + Err(error) => { + tracing::warn!( + server = %state_server_uuid, + vmid, + "failed to read Proxmox LXC process state: {error:#}" + ); + continue; + } + }; + + if status_tx.send(process_status).await.is_err() { + break; + } + } + }); + + let stats_cli = cli.clone(); + let stats_node = node.clone(); + let stats_usage = server.resource_usage.clone(); + let stats_server = Arc::downgrade(&**server); + let stats_server_uuid = server.uuid; + let disk_bytes = server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(server.state.get_state()); + usage.disk_bytes = disk_bytes; + }); + let stats_task = tokio::spawn(async move { + let mut samples: Option = None; + let mut started_at: Option = None; + let mut previous_cpu_total = 0u64; + let mut previous_sample_at: Option = None; + let mut warned_unresolved_cgroup = false; + let mut resolve_tick = tokio::time::interval(std::time::Duration::from_secs(2)); + resolve_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + let Some(stats_server) = stats_server.upgrade() else { + break; + }; + let state = stats_server.state.get_state(); + if state == crate::server::state::ServerState::Offline { + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + resolve_tick.tick().await; + continue; + } + + if samples.is_none() { + resolve_tick.tick().await; + + let runtime_status = match stats_cli.runtime_status(&stats_node, vmid).await { + Ok(status) => status, + Err(error) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to resolve Proxmox LXC runtime stats process: {error:#}" + ); + continue; + } + }; + + if runtime_status.status != cli::ContainerStatus::Running { + let state = stats_server.state.get_state(); + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + continue; + } + + let Some(pid) = runtime_status.pid else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "running Proxmox LXC status did not include an init PID; resource stats are unavailable until it does" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + let Some(files) = super::cgroup::StatFiles::resolve(pid) else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + pid, + "could not resolve the Proxmox LXC cgroup-v2 files; resource stats are unavailable until they can be resolved" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + + samples = Some(stats_sampler.register(files)); + started_at = std::time::Instant::now().checked_sub( + std::time::Duration::from_secs(runtime_status.uptime_seconds), + ); + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + } + + let Some(receiver) = samples.as_mut() else { + continue; + }; + let sample = match tokio::time::timeout( + std::time::Duration::from_secs(3), + receiver.recv(), + ) + .await + { + Ok(Some(Ok(sample))) => sample, + Ok(Some(Err(error))) if error.kind() == std::io::ErrorKind::NotFound => { + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + continue; + } + Ok(Some(Err(error))) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to read Proxmox LXC cgroup stats: {error}" + ); + samples = None; + continue; + } + Ok(None) | Err(_) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "Proxmox LXC cgroup stats sampler stopped delivering; resolving the container again" + ); + samples = None; + continue; + } + }; + + let cpu_absolute = if let Some(previous_at) = previous_sample_at { + let cpu_delta_ns = + sample.cpu_total_ns.saturating_sub(previous_cpu_total) as f64; + let wall_delta_ns = sample.at.duration_since(previous_at).as_nanos() as f64; + + if wall_delta_ns > 0.0 && cpu_delta_ns > 0.0 { + ((cpu_delta_ns / wall_delta_ns) * 100.0 * 1000.0).round() / 1000.0 + } else { + 0.0 + } + } else { + 0.0 + }; + previous_cpu_total = sample.cpu_total_ns; + previous_sample_at = Some(sample.at); + + let disk_bytes = stats_server.filesystem.limiter_usage().await; + let cpu_limit = stats_server.configuration.read().await.build.cpu_limit; + let cpu_limit_absolute = if cpu_limit > 0 { + u32::try_from(cpu_limit).unwrap_or(u32::MAX) + } else { + std::thread::available_parallelism().map_or(1, |threads| threads.get()) as u32 + * 100 + }; + let uptime = started_at + .map(|started_at| { + u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX) + }) + .unwrap_or(0); + let state = stats_server.state.get_state(); + + stats_usage.send_modify(|usage| { + usage.memory_bytes = sample.memory_bytes; + usage.memory_limit_bytes = sample.memory_limit_bytes; + usage.disk_bytes = disk_bytes; + usage.state = state; + usage.cpu_absolute = cpu_absolute; + usage.cpu_limit_absolute = cpu_limit_absolute; + usage.uptime = uptime; + + if let Some((rx_bytes, rx_packets, tx_bytes, tx_packets)) = sample.network { + usage.network.rx_bytes = rx_bytes; + usage.network.rx_packets = rx_packets; + usage.network.tx_bytes = tx_bytes; + usage.network.tx_packets = tx_packets; + } + }); + } + }); + + let console_cli = cli.clone(); + let console_app_config = Arc::clone(&app_config); + let console_log_max_bytes = app_config.load().runtime.pve_lxc.console_log_max_bytes; + let console_server = Arc::downgrade(&**server); + let console_log_path = log_path.clone(); + let console_task = tokio::spawn(async move { + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + match console_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => {} + Ok(cli::ContainerStatus::Stopped) => { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + Err(error) => { + tracing::warn!( + vmid, + "failed to read Proxmox LXC state before console attach: {error:#}" + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + } + + let session = match console_cli.spawn_console(vmid) { + Ok(session) => session, + Err(error) => { + tracing::warn!(vmid, "failed to attach Proxmox LXC console: {error:#}"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + // `tokio::fs::File` serializes operations on a single handle. + // A blocking PTY read would therefore prevent stdin writes + // indefinitely if both directions shared one Tokio file. + let cli::ConsoleSession { + mut child, + mut reader, + mut writer, + } = session; + let mut log = match tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&console_log_path) + .await + { + Ok(log) => log, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox LXC runtime log: {error:#}"); + child.start_kill().ok(); + break; + } + }; + let mut log_size = log.metadata().await.map_or(0, |metadata| metadata.len()); + let mut buffer = vec![0u8; 8192]; + + loop { + tokio::select! { + read = reader.read(&mut buffer) => { + match read { + Ok(0) => break, + Ok(read) => { + let Some(data) = buffer.get(..read) else { + tracing::error!(vmid, read, "Proxmox console returned an invalid read length"); + break; + }; + line_buffer.extend(data); + while let Some(line) = line_buffer.next_line() { + if Self::is_pct_console_banner(line) { + continue; + } + if let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + true, + ).await { + tracing::error!(vmid, "failed to append Proxmox LXC runtime log: {error:#}"); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = console_app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = console_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox LXC console: {error:#}"); + break; + } + } + } + data = stdin_rx.recv() => { + let Some(data) = data else { + child.start_kill().ok(); + return; + }; + if let Err(error) = writer.write_all(&data).await { + tracing::warn!(vmid, "failed to write Proxmox LXC console stdin: {error:#}"); + break; + } + } + result = child.wait() => { + if let Err(error) = result { + tracing::warn!(vmid, "failed waiting for Proxmox LXC console: {error:#}"); + } + break; + } + } + } + + child.start_kill().ok(); + if !matches!( + console_cli.status(vmid).await, + Ok(cli::ContainerStatus::Running) + ) { + if let Some(line) = line_buffer.flush() { + if !Self::is_pct_console_banner(line) + && let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + false, + ) + .await + { + tracing::error!( + vmid, + "failed to append final Proxmox LXC runtime log line: {error:#}" + ); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + if !Self::is_pct_console_banner(line.as_bytes()) { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + line_buffer = LineBuffer::new(); + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + }); + + Ok(Self { + vmid, + node, + cli, + server: Arc::downgrade(&**server), + app_config, + firewall, + log_path, + stdin_tx, + stdout_ratelimited_rx, + stdout_rx, + started, + status_task, + stats_task, + console_task, + }) + } + + fn runtime_log_path(app_config: &crate::config::Config, server_uuid: uuid::Uuid) -> PathBuf { + app_config + .resolve_as_path(|config| &config.system.log_directory) + .join("pve-lxc") + .join(format!("{server_uuid}.log")) + } + + pub(super) async fn write_bounded_log( + log: &mut tokio::fs::File, + size: &mut u64, + max_bytes: u64, + line: &[u8], + newline: bool, + ) -> Result<(), std::io::Error> { + let mut line = line; + let mut write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + if max_bytes > 0 && write_size > max_bytes { + let newline_size = usize::from(newline); + let keep = usize::try_from(max_bytes) + .unwrap_or(usize::MAX) + .saturating_sub(newline_size); + let start = line.len().saturating_sub(keep); + line = line.get(start..).unwrap_or_default(); + write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + } + if max_bytes > 0 && size.saturating_add(write_size) > max_bytes { + log.set_len(0).await?; + log.seek(std::io::SeekFrom::Start(0)).await?; + *size = 0; + } + log.write_all(line).await?; + if newline { + log.write_all(b"\n").await?; + } + *size = size.saturating_add(write_size); + Ok(()) + } + + /// `pct console` forwards terminal key input and expects Enter as a carriage + /// return. Wings command producers use line feeds because Docker stdin is a + /// byte stream, so normalize line endings only at the PVE PTY boundary. + pub(super) fn normalize_console_input(data: Vec) -> Vec { + let mut normalized = Vec::with_capacity(data.len()); + let mut previous_was_carriage_return = false; + + for byte in data { + if byte == b'\n' { + if !previous_was_carriage_return { + normalized.push(b'\r'); + } + previous_was_carriage_return = false; + } else { + normalized.push(byte); + previous_was_carriage_return = byte == b'\r'; + } + } + + normalized + } + + pub(super) fn is_pct_console_banner(line: &[u8]) -> bool { + let line = String::from_utf8_lossy(line); + let line = line.trim_matches(|character: char| character.is_whitespace()); + line.starts_with("Connected to tty ") + || line.starts_with("Type to exit the console") + } + + fn get_server(&self) -> Result, anyhow::Error> { + self.server + .upgrade() + .ok_or_else(|| anyhow::anyhow!("server has been dropped")) + } + + pub(super) fn halt_signal(stop_type: &str, stop_value: Option<&str>) -> Option { + if stop_type != "signal" { + return None; + } + + Some( + match stop_value.map(str::to_uppercase).as_deref() { + Some("SIGABRT") => "SIGABRT", + Some("SIGINT") | Some("C") => "SIGINT", + Some("SIGTERM") => "SIGTERM", + Some("SIGQUIT") => "SIGQUIT", + _ => "SIGKILL", + } + .to_string(), + ) + } + + async fn managed_file_mounts( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = (server, app_config); + return Err(anyhow::anyhow!( + "managed Proxmox container files require a Unix host" + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let server_uuid = { + let configuration = server.configuration.read().await; + configuration.ensure_vmounts(app_config).await?; + configuration.uuid + }; + let (machine_id_enabled, tundra_enabled, passwd_enabled, passwd_directory) = { + let config = app_config.load(); + ( + config.system.machine_id.enabled, + config.tundra.enabled, + config.system.passwd.enabled, + config.system.passwd.directory.as_path(&config), + ) + }; + + let vmount_path = app_config.vmount_path(server_uuid); + let mut files = Vec::with_capacity(4); + if machine_id_enabled { + files.push((vmount_path.join("machine-id"), "/etc/machine-id")); + } + if tundra_enabled { + files.push((vmount_path.join("hosts"), "/etc/hosts")); + } + if passwd_enabled { + files.push((passwd_directory.join("passwd"), "/etc/passwd")); + files.push((passwd_directory.join("group"), "/etc/group")); + } + + let stage_directory = Self::managed_file_stage_directory(app_config, server_uuid)?; + let stage_root = stage_directory + .parent() + .context("managed Proxmox file staging directory did not have a parent")?; + tokio::fs::create_dir_all(stage_root) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging root {}", + stage_root.display() + ) + })?; + tokio::fs::set_permissions(stage_root, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging root permissions on {}", + stage_root.display() + ) + })?; + tokio::fs::create_dir_all(&stage_directory) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging directory {}", + stage_directory.display() + ) + })?; + tokio::fs::set_permissions(&stage_directory, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging permissions on {}", + stage_directory.display() + ) + })?; + + let mut mounts = Vec::with_capacity(files.len() + 1); + for (source, target) in files { + let source = tokio::fs::canonicalize(&source).await.with_context(|| { + format!( + "failed to resolve managed Proxmox container file {}", + source.display() + ) + })?; + let metadata = tokio::fs::metadata(&source).await.with_context(|| { + format!( + "failed to inspect managed Proxmox container file {}", + source.display() + ) + })?; + if !metadata.is_file() { + return Err(anyhow::anyhow!( + "managed Proxmox container file {} is not a regular file", + source.display() + )); + } + + let staged_source = stage_directory.join(Self::managed_file_stage_name(target)?); + tokio::fs::copy(&source, &staged_source) + .await + .with_context(|| { + format!( + "failed to stage managed Proxmox container file {} at {}", + source.display(), + staged_source.display() + ) + })?; + tokio::fs::set_permissions(&staged_source, std::fs::Permissions::from_mode(0o644)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file permissions on {}", + staged_source.display() + ) + })?; + let staged_source = + tokio::fs::canonicalize(&staged_source) + .await + .with_context(|| { + format!( + "failed to resolve staged managed Proxmox container file {}", + staged_source.display() + ) + })?; + let source_path = staged_source.into_os_string().into_string().map_err(|_| { + anyhow::anyhow!("managed Proxmox container file path is not valid UTF-8") + })?; + + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: target.to_string(), + }); + } + + let entrypoint_target = "/calagopus-entrypoint"; + let entrypoint_source = + stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); + tokio::fs::write( + &entrypoint_source, + r#"#!/bin/sh +attempt=0 +while [ "$attempt" -lt 120 ]; do + if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then + break + fi + attempt=$((attempt + 1)) + sleep 0.25 +done +if [ "$attempt" -ge 120 ]; then + echo 'timed out waiting for Proxmox DHCP' >&2 + exit 1 +fi + +read_oom_kills() { + value=0 + if [ -r /sys/fs/cgroup/memory.events ]; then + while read -r key count; do + if [ "$key" = oom_kill ]; then + value=$count + break + fi + done < /sys/fs/cgroup/memory.events + fi + printf '%s' "$value" +} + +child=0 +forward_signal() { + if [ "$child" -gt 0 ]; then + kill "-$1" "$child" 2>/dev/null || true + fi +} +trap 'forward_signal HUP' HUP +trap 'forward_signal INT' INT +trap 'forward_signal QUIT' QUIT +trap 'forward_signal ABRT' ABRT +trap 'forward_signal TERM' TERM + +oom_before=$(read_oom_kills) +"$@" <&0 & +child=$! +while :; do + wait "$child" + status=$? + if ! kill -0 "$child" 2>/dev/null; then + break + fi +done +oom_after=$(read_oom_kills) +oom_killed=0 +if [ "$oom_after" -gt "$oom_before" ]; then + oom_killed=1 +fi + +marker=/home/container/.wings-exit-status +temporary="${marker}.tmp.$$" +( + umask 077 + printf 'exit_code=%s\noom_killed=%s\n' "$status" "$oom_killed" > "$temporary" +) +mv -f "$temporary" "$marker" +exit "$status" +"#, + ) + .await + .context("failed to stage Proxmox network-ready entrypoint")?; + tokio::fs::set_permissions(&entrypoint_source, std::fs::Permissions::from_mode(0o755)) + .await + .context("failed to make Proxmox network-ready entrypoint executable")?; + let entrypoint_source = tokio::fs::canonicalize(&entrypoint_source) + .await + .context("failed to resolve staged Proxmox network-ready entrypoint")?; + let source_path = entrypoint_source + .into_os_string() + .into_string() + .map_err(|_| { + anyhow::anyhow!("managed Proxmox entrypoint path is not valid UTF-8") + })?; + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: entrypoint_target.to_string(), + }); + + Ok(mounts) + } + } + + pub(super) async fn runtime_config( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + cli: &PveCli, + vmid: u32, + ) -> Result { + let ( + environment, + panel_entrypoint, + cpuset_cpus, + io_weight, + memory_unlimited, + swap_unlimited, + ) = { + let configuration = server.configuration.read().await; + let resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + ( + configuration.environment(app_config), + configuration.entrypoint.clone(), + configuration + .build + .threads + .as_deref() + .filter(|threads| !threads.trim().is_empty()) + .map(PveCli::normalize_cpuset) + .transpose()?, + PveCli::cgroup2_io_weight(configuration.build.io_weight)?, + resources.memory_unlimited, + resources.swap_unlimited, + ) + }; + let pids_limit = match app_config.load().docker.container_pid_limit { + 0 => None, + limit => Some(limit), + }; + let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; + let process = server.process_configuration.read().await; + let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); + let mut init_arguments = vec!["/calagopus-entrypoint".to_string()]; + if let Some(entrypoint) = panel_entrypoint { + if entrypoint.is_empty() { + return Err(anyhow::anyhow!("panel entrypoint override cannot be empty")); + } + init_arguments.extend(entrypoint); + } else { + let image_entrypoint = cli.oci_entrypoint(vmid).await?; + init_arguments.extend( + shell_words::split(&image_entrypoint) + .context("failed to parse OCI entrypoint for Proxmox LXC")?, + ); + } + let init_command = PveCli::encode_init_command(&init_arguments)?; + + Ok(cli::RuntimeConfigSpec { + vmid, + environment, + halt_signal, + init_command: Some(init_command), + console_logfile: None, + cpuset_cpus, + pids_limit, + io_weight, + memory_unlimited, + swap_unlimited, + managed_file_mounts: Some(managed_file_mounts), + }) + } + + pub(super) async fn sync_firewall(&self, wait_for_address: bool) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let has_rules = !server.configuration.read().await.firewall.is_empty(); + let edge_forwarding = PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some(); + let running = self.cli.status(self.vmid).await? == cli::ContainerStatus::Running; + + let container_ips = if running && (has_rules || edge_forwarding) { + let mut last_error = None; + let mut addresses = Vec::new(); + let attempts = if wait_for_address { 40 } else { 1 }; + + for attempt in 0..attempts { + match self.cli.interfaces(&self.node, self.vmid).await { + Ok(interfaces) => { + addresses = PveCli::container_addresses(&interfaces); + if !addresses.is_empty() { + break; + } + last_error = None; + } + Err(error) => last_error = Some(error), + } + + if attempt + 1 < attempts { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + + if addresses.is_empty() { + if let Some(error) = last_error { + return Err(error.context( + "failed to resolve Proxmox LXC address for firewall and edge forwarding", + )); + } + return Err(anyhow::anyhow!( + "running Proxmox LXC container {} has no usable bridged address; refusing to leave firewall or edge forwarding unapplied", + self.vmid + )); + } + + addresses + } else { + Vec::new() + }; + + let (mut spec, edge_forward) = { + let configuration = server.configuration.read().await; + let destination = container_ips.iter().find_map(|address| match address { + IpAddr::V4(address) => Some(*address), + IpAddr::V6(_) => None, + }); + let edge_forward = if edge_forwarding && running { + Some(PveLxcExecutor::edge_forward_payload( + &configuration, + destination + .context("running Proxmox LXC has no IPv4 address for edge forwarding")?, + )?) + } else { + None + }; + ( + PveLxcExecutor::firewall_spec(&configuration, container_ips, Some(self.vmid)), + edge_forward, + ) + }; + spec.files = Some(PveLxcExecutor::firewall_file_access(&server)); + self.firewall.sync(&spec).await?; + if edge_forwarding { + let (public_ip, payload) = edge_forward.flatten().unzip(); + PveLxcExecutor::sync_edge_forwarding(&self.app_config, server.uuid, public_ip, payload) + .await?; + } + Ok(()) + } + + pub(super) async fn stop_after_firewall_failure( + &self, + error: anyhow::Error, + context: &'static str, + ) -> anyhow::Error { + match self.cli.stop(self.vmid).await { + Ok(()) => error.context(format!("{context}; container was stopped")), + Err(stop_error) => error.context(format!( + "{context} and failed to stop the exposed container: {stop_error:#}" + )), + } + } +} + +impl Drop for PveProcessHandle { + fn drop(&mut self) { + self.status_task.abort(); + self.stats_task.abort(); + self.console_task.abort(); + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = match tokio::fs::File::open(&self.log_path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Box::new(Cursor::new(Vec::new()))); + } + Err(error) => return Err(error.into()), + }; + return Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)); + } + + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + + async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { + self.stdin_tx + .send(Self::normalize_console_input(data)) + .await + .map_err(Into::into) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let resources = { + let configuration = server.configuration.read().await; + PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )? + }; + self.cli.set_resources(self.vmid, &resources).await?; + self.cli + .apply_runtime_config( + &Self::runtime_config(&server, &self.app_config, &self.cli, self.vmid).await?, + ) + .await?; + if let Err(error) = self.sync_firewall(true).await { + let configuration = server.configuration.read().await; + let requires_network_policy = !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()); + drop(configuration); + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to synchronize Proxmox LXC network policy after configuration update", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state after configuration update: {error:#}" + ); + } + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + match tokio::fs::remove_file(&exit_status_path).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to clear stale Proxmox LXC exit status {}", + exit_status_path.display() + ) + }); + } + } + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + + if let Err(error) = self.sync_firewall(true).await { + let requires_network_policy = match self.get_server() { + Ok(server) => { + let configuration = server.configuration.read().await; + !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()) + } + Err(_) => true, + }; + + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to apply Proxmox LXC network policy", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let process = server.process_configuration.read().await; + let stop_type = process.stop.r#type.clone(); + let stop_value = process.stop.value.clone(); + drop(process); + + if stop_type == "command" { + let mut command = stop_value + .map(|value| value.as_bytes().to_vec()) + .unwrap_or_default(); + command.push(b'\n'); + self.stdin_tx.send(command).await.map_err(Into::into) + } else { + self.cli.request_shutdown(self.vmid).await + } + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + tokio::fs::write(&exit_status_path, "exit_code=137\noom_killed=0\n") + .await + .with_context(|| { + format!( + "failed to record forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + })?; + self.cli.stop(self.vmid).await + } +} diff --git a/application/src/server/executor/pve_lxc/tests.rs b/application/src/server/executor/pve_lxc/tests.rs new file mode 100644 index 00000000..756497e3 --- /dev/null +++ b/application/src/server/executor/pve_lxc/tests.rs @@ -0,0 +1,559 @@ +use super::*; + +#[test] +fn successful_start_followed_by_first_stopped_probe_emits_stopped() { + let mut seen_running = false; + let status = PveProcessHandle::observed_status( + cli::ContainerStatus::Stopped, + true, + &mut seen_running, + Some((42, true)), + ); + assert!(matches!( + status, + Some(crate::server::executor::ProcessStatus::Stopped { + exit_code: 42, + oom_killed: true, + }) + )); +} + +#[test] +fn parses_managed_exit_status_strictly() { + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=137\noom_killed=1\n"), + Some((137, true)) + ); + assert_eq!( + PveProcessHandle::parse_exit_status("oom_killed=0\nexit_code=0\n"), + Some((0, false)) + ); + assert_eq!(PveProcessHandle::parse_exit_status("exit_code=0\n"), None); + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=nope\noom_killed=0\n"), + None + ); +} + +#[tokio::test] +async fn runtime_log_never_exceeds_its_retention_limit() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("console.log"); + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await + .unwrap(); + let mut size = 0; + + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"first", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"second", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"oversized-line", true) + .await + .unwrap(); + file.flush().await.unwrap(); + + let data = tokio::fs::read(&path).await.unwrap(); + assert_eq!(data, b"ed-line\n"); + assert_eq!(data.len(), 8); + assert_eq!(size, 8); +} + +#[test] +fn static_allocation_uses_explicit_isolated_vlan_network() { + assert_eq!( + PveLxcExecutor::network_for_allocation( + "vmbr0", + Some(30), + Some(24), + Some("10.0.30.1"), + Some("10.0.30.42"), + ) + .unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=10.0.30.42/24,gw=10.0.30.1,type=veth" + ); +} + +#[test] +fn static_allocation_never_derives_the_management_network() { + let error = + PveLxcExecutor::network_for_allocation("vmbr0", Some(30), None, None, Some("10.0.30.42")) + .unwrap_err(); + assert!(error.to_string().contains("network_prefix")); + + assert_eq!( + PveLxcExecutor::dhcp_network("vmbr0", Some(30)).unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=dhcp,type=veth,host-managed=1" + ); +} + +#[test] +fn network_bridge_rejects_option_injection() { + assert!(PveLxcExecutor::dhcp_network("vmbr0 firewall=0", None).is_err()); + assert!(PveLxcExecutor::dhcp_network("vmbr0,firewall=0", None).is_err()); +} + +#[test] +fn accepts_pve_9_2_and_newer() { + assert_eq!( + PveVersion::parse("pve-manager/9.2.1/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); + assert_eq!( + PveVersion::parse("pve-manager/10.0.0/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); +} + +#[test] +fn rejects_pve_older_than_9_2() { + assert_eq!( + PveVersion::parse("pve-manager/9.1.7/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(false) + ); +} + +#[test] +fn pve_runtime_only_targets_the_local_node() { + assert_eq!( + PveLxcExecutor::runtime_node("", "pve-a").ok().as_deref(), + Some("pve-a") + ); + assert_eq!( + PveLxcExecutor::runtime_node("pve-a", "pve-a") + .ok() + .as_deref(), + Some("pve-a") + ); + assert!(PveLxcExecutor::runtime_node("pve-b", "pve-a").is_err()); +} + +#[test] +fn proc_net_parser_reports_listeners_and_udp_reservations() { + fn proc_ipv4(address: std::net::Ipv4Addr) -> String { + format!("{:08X}", u32::from_le_bytes(address.octets())) + } + + let any = proc_ipv4(std::net::Ipv4Addr::UNSPECIFIED); + let loopback = proc_ipv4(std::net::Ipv4Addr::LOCALHOST); + let tcp = format!( + " sl local_address rem_address st\n 0: {any}:63DD 00000000:0000 0A\n 1: {loopback}:C350 0100007F:9C40 01\n" + ); + let udp = format!( + " sl local_address rem_address st\n 0: {any}:4ABC 00000000:0000 07\n 1: {loopback}:D431 0100007F:0035 01\n 2: {any}:0000 00000000:0000 07\n" + ); + + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&tcp, ProcNetFamily::Ipv4, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 25565, + }] + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&udp, ProcNetFamily::Ipv4, false).unwrap(), + vec![ + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + BoundPort { + family: ProcNetFamily::Ipv4, + address: Some(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + port: 54321, + }, + ] + ); +} + +#[test] +fn proc_net_parser_decodes_ipv6_and_matches_ipv4_mapped_bindings() { + fn proc_ipv6(address: std::net::Ipv6Addr) -> String { + address + .octets() + .chunks_exact(4) + .map(|chunk| { + format!( + "{:08X}", + u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) + ) + }) + .collect::() + } + + let address: std::net::Ipv6Addr = "fd12:3456::20".parse().unwrap(); + let raw = proc_ipv6(address); + let table = format!( + " sl local_address rem_address st\n 0: {raw}:63DD 00000000000000000000000000000000:0000 0A\n" + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&table, ProcNetFamily::Ipv6, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6(address)), + port: 25565, + }] + ); + + let mapped = BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6("::ffff:10.42.0.20".parse().unwrap())), + port: 25565, + }; + assert!(PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.21".parse().unwrap() + )); + assert!(PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv6, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); +} + +#[test] +fn ownership_tags_require_marker_and_server_uuid() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let server_tag = PveLxcExecutor::server_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), server_tag]; + + assert_eq!( + PveLxcExecutor::server_from_tags("calagopus", &tags), + Some(server_uuid) + ); + assert_eq!(PveLxcExecutor::server_from_tags("other", &tags), None); +} + +#[test] +fn helper_tags_are_distinct_from_servers_and_roles() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let installer_tag = PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ); + let tags = vec!["calagopus".to_string(), installer_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::helper_from_tags("calagopus", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), + Some(server_uuid) + ); + assert_eq!( + PveLxcExecutor::helper_from_tags("calagopus", PveLxcExecutor::SCRIPT_HELPER_ROLE, &tags,), + None + ); + assert_eq!( + PveLxcExecutor::helper_from_tags("other", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), + None + ); +} + +#[test] +fn replacement_tags_are_distinct_and_recoverable() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let replacement_tag = PveLxcExecutor::replacement_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), replacement_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::replacement_from_tags("calagopus", &tags), + Some(server_uuid) + ); +} + +#[test] +fn image_drift_plan_recovers_staged_replacements() { + let desired = "calagopus-image-new"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old".to_string()], + }; + let staged = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&staged), desired), + ServerContainerPlan::ReplaceWithStaged { + old_vmid: 101, + staged_vmid: 102, + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&staged), desired), + ServerContainerPlan::RecoverStaged { staged_vmid: 102 } + ); +} + +#[test] +fn image_drift_plan_discards_stale_staging_without_replacing_good_container() { + let desired = "calagopus-image-current"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + let stale = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old-attempt".to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&stale), desired), + ServerContainerPlan::Reuse { + vmid: 101, + stale_replacement: Some(102), + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&stale), desired), + ServerContainerPlan::CreateFresh { + stale_replacement: Some(102), + } + ); +} + +#[test] +fn owned_container_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::server_tag("calagopus", server_uuid), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_server_container(&containers, "pve-a", "calagopus", server_uuid,) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_server_container(&containers, "pve-b", "calagopus", server_uuid,) + .ok() + .flatten(), + None + ); +} + +#[test] +fn owned_helper_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 201, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Running), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 202, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::SCRIPT_HELPER_ROLE, + server_uuid, + ) + .ok() + .flatten(), + None + ); +} + +#[test] +fn direct_bridge_firewall_spec_targets_container_addresses_without_published_bindings() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let mappings = HashMap::from([ + ( + compact_str::CompactString::from("10.42.0.20"), + vec![25566, 25565, 25565], + ), + (compact_str::CompactString::from("10.42.0.21"), vec![19132]), + ]); + let rules = vec![crate::server::firewall::FirewallRule { + action: crate::server::firewall::FirewallRuleAction::Deny, + protocols: HashSet::from([crate::server::firewall::FirewallRuleProtocol::Tcp]), + sources: Vec::new(), + ports: None, + source_file: None, + }]; + let spec = PveLxcExecutor::direct_bridge_firewall_spec( + server_uuid, + &mappings, + &rules, + vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + ], + None, + ); + + assert!(spec.bindings.is_empty()); + assert_eq!(spec.container_ports, vec![19132, 25565, 25566]); + assert_eq!( + spec.container_ips, + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); + assert_eq!(spec.rules, rules); + + let concrete = crate::server::firewall::expand_rules(&spec); + assert_eq!(concrete.len(), 6); + assert!( + concrete + .iter() + .all(|rule| matches!(rule.dst, crate::server::firewall::RuleDst::Container { .. })) + ); +} + +#[test] +fn console_input_uses_terminal_enter_without_doubling_crlf() { + assert_eq!( + PveProcessHandle::normalize_console_input(b"list\n".to_vec()), + b"list\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(b"first\r\nsecond\n".to_vec()), + b"first\rsecond\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(vec![0x03]), + vec![0x03] + ); + assert!(PveProcessHandle::is_pct_console_banner( + b"Connected to tty 0\r" + )); + assert!(PveProcessHandle::is_pct_console_banner( + b"Type to exit the console, to enter Ctrl+z itself\r" + )); + assert!(!PveProcessHandle::is_pct_console_banner( + b"[Server thread/INFO]: Connected to tty 0" + )); +} + +#[test] +fn halt_signal_matches_wings_stop_semantics() { + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGABRT")), + Some("SIGABRT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("c")), + Some("SIGINT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGTERM")), + Some("SIGTERM".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGQUIT")), + Some("SIGQUIT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("unexpected")), + Some("SIGKILL".to_string()) + ); + assert_eq!(PveProcessHandle::halt_signal("command", Some("stop")), None); +} + +#[test] +fn device_permissions_require_exact_pve_devn_semantics() { + let path = std::path::Path::new("/dev/kvm"); + + assert_eq!( + PveLxcExecutor::device_deny_write("r", path).ok(), + Some(true) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("rw", path).ok(), + Some(false) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("wr", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("w", path).is_err()); + assert!(PveLxcExecutor::device_deny_write("m", path).is_err()); + assert_eq!( + PveLxcExecutor::device_deny_write("rwm", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("rwx", path).is_err()); +} From dc3797ad62e8ec0c014683283eeafbed4f3a0726 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 07/13] address PVE runtime review findings --- .github/workflows/test.yml | 5 +- application/src/config.rs | 55 ++++- application/src/server/executor/mod.rs | 71 ++++++- .../src/server/executor/pve_lxc/cli.rs | 184 +++++++++++++++-- .../src/server/executor/pve_lxc/cli/tests.rs | 40 +++- .../server/executor/pve_lxc/helper_process.rs | 81 +++++++- .../src/server/executor/pve_lxc/mod.rs | 189 ++++++++++++------ .../server/executor/pve_lxc/server_process.rs | 83 +++++--- .../src/server/executor/pve_lxc/tests.rs | 47 +++++ application/src/tundra/daemon.rs | 2 +- application/src/tundra/mod.rs | 2 +- scripts/pve-lxc-smoke-test.sh | 29 ++- 12 files changed, 668 insertions(+), 120 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2e25045..df5c1c04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,12 +24,15 @@ jobs: with: toolchain: 1.97.0 target: x86_64-unknown-linux-gnu - components: rustfmt + components: rustfmt, clippy cache-key: x86_64-unknown-linux-gnu-test - name: Check formatting run: cargo fmt --all -- --check + - name: Run Clippy + run: cargo clippy -p wings-rs --bin wings-rs -- -D warnings + - name: Install cross run: cargo install cross --git https://github.com/cross-rs/cross --rev 8c1a8aa4b661711f4b7b6ac07c2e8929ce2f7d27 --locked diff --git a/application/src/config.rs b/application/src/config.rs index 3658cbcc..778fc825 100644 --- a/application/src/config.rs +++ b/application/src/config.rs @@ -11,7 +11,7 @@ use std::{ io::{BufRead, BufWriter, IsTerminal}, path::{Path, PathBuf}, str::FromStr, - sync::Arc, + sync::{Arc, OnceLock}, }; use tracing::level_filters::LevelFilter; use tracing_subscriber::{ @@ -792,6 +792,10 @@ fn pve_lxc_edge_wireguard_interface() -> String { "wg-calagopus".to_string() } +fn pve_lxc_edge_ssh_user() -> String { + "root".to_string() +} + fn pve_lxc_rootfs_size_gib() -> u64 { 8 } @@ -868,6 +872,9 @@ nestify::nest! { /// The panel allocation IP is matched against each peer endpoint. #[serde(default = "pve_lxc_edge_wireguard_interface")] pub edge_wireguard_interface: String, + /// SSH account used by the restricted edge forwarding key. + #[serde(default = "pve_lxc_edge_ssh_user")] + pub edge_ssh_user: String, /// Restricted SSH key used only to synchronize edge port forwards. #[serde(default)] pub edge_ssh_identity_path: String, @@ -888,6 +895,10 @@ nestify::nest! { pub managed_file_directory: SystemPath, #[serde(default = "default_true")] pub unprivileged: bool, + /// Maximum process count for LXC workloads. When omitted, the + /// legacy Docker limit is used for configuration compatibility. + #[serde(default)] + pub pids_limit: Option, #[serde(default)] #[schema(inline)] @@ -1739,6 +1750,7 @@ fn log_filter(debug: bool) -> Targets { pub struct Config { inner: ArcSwap, + active_runtime_backend: OnceLock, log_reload_handle: ReloadHandle, pub path: String, @@ -1876,6 +1888,7 @@ impl Config { let config = Arc::new(Self { inner: ArcSwap::new(Arc::new(inner)), + active_runtime_backend: OnceLock::new(), log_reload_handle, path: path.to_string(), @@ -1896,6 +1909,7 @@ impl Config { Self { inner: ArcSwap::new(Arc::new(inner)), + active_runtime_backend: OnceLock::new(), log_reload_handle: tracing_subscriber::reload::Layer::new(log_filter(false)).1, path: String::new(), ignore_certificate_errors: false, @@ -1910,6 +1924,26 @@ impl Config { self.inner.load() } + /// Runtime selected for this Wings process. Automatic detection is kept in + /// memory so starting Wings never rewrites the operator's configuration. + pub fn active_runtime_backend(&self) -> RuntimeBackend { + self.active_runtime_backend + .get() + .copied() + .unwrap_or_else(|| self.load().runtime.backend) + } + + pub fn set_active_runtime_backend( + &self, + backend: RuntimeBackend, + ) -> Result<(), RuntimeBackend> { + match self.active_runtime_backend.set(backend) { + Ok(()) => Ok(()), + Err(backend) if self.active_runtime_backend.get() == Some(&backend) => Ok(()), + Err(backend) => Err(backend), + } + } + pub fn replace(&self, mut new: InnerConfig) -> Result<(), anyhow::Error> { if Self::migrate_legacy_limits(&mut new) { tracing::warn!("{LEGACY_LIMITS_WARNING}"); @@ -2804,4 +2838,23 @@ mod tests { assert!(Config::migrate_legacy_limits(&mut cfg)); assert_eq!(cfg.limits.server_concurrent_pulls, 5); } + + #[tokio::test] + async fn active_runtime_backend_is_process_local_and_idempotent() { + let config = Config::mock(); + assert_eq!(config.active_runtime_backend(), RuntimeBackend::Auto); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::PveLxc), + Ok(()) + ); + assert_eq!(config.active_runtime_backend(), RuntimeBackend::PveLxc); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::PveLxc), + Ok(()) + ); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::Docker), + Err(RuntimeBackend::Docker) + ); + } } diff --git a/application/src/server/executor/mod.rs b/application/src/server/executor/mod.rs index 37d14be4..ed018f85 100644 --- a/application/src/server/executor/mod.rs +++ b/application/src/server/executor/mod.rs @@ -17,6 +17,11 @@ pub struct Runtime { pub async fn create_runtime(config: Arc) -> Result { let backend = resolve_backend(&config).await?; + config + .set_active_runtime_backend(backend) + .map_err(|active| anyhow::anyhow!( + "runtime backend was already resolved as {active:?}, cannot replace it with {backend:?}" + ))?; match backend { crate::config::RuntimeBackend::Auto => Err(anyhow::anyhow!( @@ -109,6 +114,44 @@ async fn resolve_backend( let pve_lxc = snapshot.runtime.pve_lxc.clone(); drop(snapshot); + let state_path = config + .resolve_as_path(|config| &config.system.root_directory) + .join("runtime-backend"); + match tokio::fs::read_to_string(&state_path).await { + Ok(value) => { + let selected = match value.trim() { + "docker" => crate::config::RuntimeBackend::Docker, + "pve_lxc" => crate::config::RuntimeBackend::PveLxc, + value => anyhow::bail!( + "persisted automatic runtime backend at {} is invalid: {value}", + state_path.display() + ), + }; + let available = match selected { + crate::config::RuntimeBackend::Docker => docker_available(&docker_socket).await, + crate::config::RuntimeBackend::PveLxc => pve_lxc_available(&pve_lxc).await, + crate::config::RuntimeBackend::Auto => false, + }; + if !available { + anyhow::bail!( + "persisted automatic runtime backend {selected:?} is unavailable; remove {} to run detection again", + state_path.display() + ); + } + tracing::info!(runtime_backend = ?selected, "using persisted automatic runtime backend"); + return Ok(selected); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read persisted automatic runtime backend {}", + state_path.display() + ) + }); + } + } + let selected = if docker_available(&docker_socket).await { crate::config::RuntimeBackend::Docker } else if pve_lxc_available(&pve_lxc).await { @@ -119,11 +162,29 @@ async fn resolve_backend( ); }; - let mut persisted: crate::config::InnerConfig = - serde_json::from_value(serde_json::to_value(&**config.load())?)?; - persisted.runtime.backend = selected; - config.replace(persisted)?; - tracing::info!(runtime_backend = ?selected, "automatically selected and persisted runtime backend"); + let temporary = state_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); + let value = match selected { + crate::config::RuntimeBackend::Docker => "docker\n", + crate::config::RuntimeBackend::PveLxc => "pve_lxc\n", + crate::config::RuntimeBackend::Auto => { + anyhow::bail!("automatic runtime selection returned auto") + } + }; + tokio::fs::write(&temporary, value).await.with_context(|| { + format!( + "failed to write automatic runtime backend state {}", + temporary.display() + ) + })?; + tokio::fs::rename(&temporary, &state_path) + .await + .with_context(|| { + format!( + "failed to persist automatic runtime backend state {}", + state_path.display() + ) + })?; + tracing::info!(runtime_backend = ?selected, "automatically selected and persisted runtime backend state"); Ok(selected) } diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index 21d30c2f..49b8ad0d 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -122,7 +122,7 @@ pub struct ContainerOciUser { pub gid: u32, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ContainerResources { pub memory_mib: u64, pub swap_mib: u64, @@ -130,6 +130,7 @@ pub struct ContainerResources { pub swap_unlimited: bool, pub cpu_limit_percent: Option, pub cores: Option, + pub cpuset_cpus: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -212,6 +213,13 @@ pub struct ContainerRuntimeStatus { pub uptime_seconds: u64, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeContainerRuntimeStatus { + pub vmid: u32, + pub status: ContainerStatus, + pub pid: Option, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ContainerInterface { pub name: String, @@ -324,9 +332,9 @@ impl PveCli { value => Some(u64::try_from(value).context("server CPU limit exceeded u64")?), }; - let _cpuset_cpus = threads + let cpuset = threads .filter(|threads| !threads.trim().is_empty()) - .map(Self::normalize_cpuset) + .map(Self::normalize_cpuset_with_count) .transpose()?; Ok(ContainerResources { @@ -335,16 +343,22 @@ impl PveCli { memory_unlimited, swap_unlimited, cpu_limit_percent, - cores: None, + cores: cpuset.as_ref().map(|(_, count)| *count), + cpuset_cpus: cpuset.map(|(cpuset, _)| cpuset), }) } pub(crate) fn normalize_cpuset(value: &str) -> Result { + Self::normalize_cpuset_with_count(value).map(|(cpuset, _)| cpuset) + } + + fn normalize_cpuset_with_count(value: &str) -> Result<(String, u64), anyhow::Error> { let value = value.trim(); if value.is_empty() { return Err(anyhow::anyhow!("CPU set cannot be empty")); } + let mut ranges = Vec::new(); for part in value.split(',') { let part = part.trim(); if part.is_empty() { @@ -364,10 +378,32 @@ impl PveCli { if end < start { return Err(anyhow::anyhow!("invalid descending CPU set range: {part}")); } + ranges.push((start, end)); + } else { + ranges.push((start, start)); + } + } + + ranges.sort_unstable(); + let mut count = 0u64; + let mut merged: Option<(u32, u32)> = None; + for (start, end) in ranges { + match merged { + Some((merged_start, merged_end)) if start <= merged_end.saturating_add(1) => { + merged = Some((merged_start, merged_end.max(end))); + } + Some((merged_start, merged_end)) => { + count = count.saturating_add(u64::from(merged_end - merged_start) + 1); + merged = Some((start, end)); + } + None => merged = Some((start, end)), } } + if let Some((start, end)) = merged { + count = count.saturating_add(u64::from(end - start) + 1); + } - Ok(value.to_string()) + Ok((value.to_string(), count)) } pub(crate) fn cgroup2_io_weight(weight: Option) -> Result, anyhow::Error> { @@ -445,6 +481,60 @@ impl PveCli { )) } + async fn run_perl_helper( + &self, + name: &str, + script: &str, + input: &[u8], + ) -> Result { + let directory = Path::new("/run/calagopus-wings/pve-helpers"); + tokio::fs::create_dir_all(directory) + .await + .context("failed to create Proxmox helper script directory")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(directory, std::fs::Permissions::from_mode(0o700)) + .await + .context("failed to secure Proxmox helper script directory")?; + } + + let path = directory.join(format!("{name}-{}.pl", uuid::Uuid::new_v4())); + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&path) + .await + .context("failed to create Proxmox helper script")?; + file.write_all(script.as_bytes()) + .await + .context("failed to write Proxmox helper script")?; + file.flush() + .await + .context("failed to flush Proxmox helper script")?; + drop(file); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .await + .context("failed to secure Proxmox helper script")?; + } + + let result = Self::run_with_stdin( + &self.perl_path, + &[path.to_string_lossy().into_owned()], + input, + ) + .await; + if let Err(error) = tokio::fs::remove_file(&path).await + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(path = %path.display(), "failed to remove Proxmox helper script: {error}"); + } + result + } + fn runtime_config_payload(spec: &RuntimeConfigSpec) -> Result, anyhow::Error> { for variable in &spec.environment { let Some((name, _)) = variable.split_once('=') else { @@ -630,12 +720,8 @@ impl PveCli { spec: &RuntimeConfigSpec, ) -> Result<(), anyhow::Error> { let payload = Self::runtime_config_payload(spec)?; - Self::run_with_stdin( - &self.perl_path, - &["-e".to_string(), APPLY_RUNTIME_CONFIG_PERL.to_string()], - &payload, - ) - .await?; + self.run_perl_helper("runtime-config", APPLY_RUNTIME_CONFIG_PERL, &payload) + .await?; Ok(()) } @@ -644,12 +730,8 @@ impl PveCli { spec: &FirewallPolicySpec, ) -> Result<(), anyhow::Error> { let payload = serde_json::to_vec(spec).context("failed to serialize Proxmox firewall")?; - Self::run_with_stdin( - &self.perl_path, - &["-e".to_string(), APPLY_FIREWALL_CONFIG_PERL.to_string()], - &payload, - ) - .await?; + self.run_perl_helper("firewall-config", APPLY_FIREWALL_CONFIG_PERL, &payload) + .await?; Ok(()) } @@ -763,6 +845,26 @@ impl PveCli { u32::try_from(raw).context("Proxmox nextid response exceeded the VMID range") } + /// Serialize the advisory `nextid` lookup and the following create on this + /// PVE host. The PVE create operation remains the final cluster-wide + /// authority, so callers still retry a reported VMID collision. + pub async fn lock_vmid_allocation(&self) -> Result { + tokio::task::spawn_blocking(|| { + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open("/run/lock/calagopus-wings-pve-vmid.lock") + .context("failed to open Proxmox VMID allocation lock")?; + lock.lock() + .context("failed to acquire Proxmox VMID allocation lock")?; + Ok(lock) + }) + .await + .context("Proxmox VMID allocation lock task failed")? + } + pub async fn local_node(&self) -> Result { let args = vec![ "get".to_string(), @@ -910,6 +1012,52 @@ impl PveCli { Self::parse_runtime_status(&output.stdout) } + pub async fn node_runtime_statuses( + &self, + node: &str, + ) -> Result, anyhow::Error> { + let output = Self::run( + &self.pvesh_path, + &[ + "get".to_string(), + format!("/nodes/{node}/lxc"), + "--output-format".to_string(), + "json".to_string(), + ], + ) + .await?; + Self::parse_node_runtime_statuses(&output.stdout) + } + + fn parse_node_runtime_statuses( + output: &[u8], + ) -> Result, anyhow::Error> { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox node LXC status response")?; + let entries = value + .as_array() + .context("Proxmox node LXC status response was not an array")?; + entries + .iter() + .map(|entry| { + let vmid = entry + .get("vmid") + .and_then(serde_json::Value::as_u64) + .context("Proxmox node LXC status omitted VMID")?; + Ok(NodeContainerRuntimeStatus { + vmid: u32::try_from(vmid).context("Proxmox LXC VMID exceeded u32")?, + status: ContainerStatus::parse( + entry + .get("status") + .and_then(serde_json::Value::as_str) + .context("Proxmox node LXC status omitted state")?, + )?, + pid: entry.get("pid").and_then(serde_json::Value::as_i64), + }) + }) + .collect() + } + fn parse_runtime_status(output: &[u8]) -> Result { let value: serde_json::Value = serde_json::from_slice(output) .context("failed to parse Proxmox LXC runtime status response")?; @@ -1556,6 +1704,8 @@ impl PveCli { if let Some(cores) = resources.cores { args.extend(["--cores".to_string(), cores.to_string()]); + } else { + args.extend(["--delete".to_string(), "cores".to_string()]); } args diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs index 145ded35..455714a5 100644 --- a/application/src/server/executor/pve_lxc/cli/tests.rs +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -97,6 +97,7 @@ fn maps_panel_resources_to_native_pve_limits() { swap_unlimited: false, cpu_limit_percent: Some(250), cores: None, + cpuset_cpus: None, }) ); assert_eq!( @@ -108,6 +109,7 @@ fn maps_panel_resources_to_native_pve_limits() { swap_unlimited: false, cpu_limit_percent: None, cores: None, + cpuset_cpus: None, }) ); } @@ -130,6 +132,7 @@ fn maps_unlimited_memory_and_swap_to_lxc_overrides() { swap_unlimited: true, cpu_limit_percent: None, cores: None, + cpuset_cpus: None, }) ); assert_eq!( @@ -141,13 +144,22 @@ fn maps_unlimited_memory_and_swap_to_lxc_overrides() { swap_unlimited: true, cpu_limit_percent: None, cores: None, + cpuset_cpus: None, }) ); } #[test] fn accepts_panel_cpu_pinning_syntax() { - assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).is_ok()); + let resources = PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).expect("valid CPU set"); + assert_eq!(resources.cores, Some(5)); + assert_eq!(resources.cpuset_cpus.as_deref(), Some("0-3,8")); + assert_eq!( + PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,2-5")) + .expect("overlapping CPU set") + .cores, + Some(6) + ); assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0,,2")).is_err()); } @@ -175,6 +187,7 @@ fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { swap_unlimited: false, cpu_limit_percent: Some(250), cores: Some(4), + cpuset_cpus: None, }, ), vec![ @@ -201,6 +214,7 @@ fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { swap_unlimited: false, cpu_limit_percent: None, cores: None, + cpuset_cpus: None, }, ), vec![ @@ -212,6 +226,8 @@ fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { "0", "--cpulimit", "0", + "--delete", + "cores", ] ); } @@ -621,6 +637,28 @@ fn parses_runtime_status_for_cgroup_discovery() { ); } +#[test] +fn parses_node_runtime_statuses_in_one_inventory() { + assert_eq!( + PveCli::parse_node_runtime_statuses( + br#"[{"vmid":100,"status":"running","pid":4321},{"vmid":101,"status":"stopped"}]"#, + ) + .ok(), + Some(vec![ + NodeContainerRuntimeStatus { + vmid: 100, + status: ContainerStatus::Running, + pid: Some(4321), + }, + NodeContainerRuntimeStatus { + vmid: 101, + status: ContainerStatus::Stopped, + pid: None, + }, + ]) + ); +} + #[test] fn container_address_falls_back_to_non_link_local_ipv6() { let interfaces = vec![ContainerInterface { diff --git a/application/src/server/executor/pve_lxc/helper_process.rs b/application/src/server/executor/pve_lxc/helper_process.rs index 389eb2bc..0b8e2402 100644 --- a/application/src/server/executor/pve_lxc/helper_process.rs +++ b/application/src/server/executor/pve_lxc/helper_process.rs @@ -6,8 +6,10 @@ pub(super) struct PveHelperProcessHandle { log_path: PathBuf, started: Arc, cancelled: Arc, + stdin_tx: tokio::sync::mpsc::Sender>, stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, stdout_rx: tokio::sync::broadcast::Receiver>, + console_task: tokio::task::JoinHandle<()>, } impl PveHelperProcessHandle { @@ -39,6 +41,7 @@ impl PveHelperProcessHandle { let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< Arc, >(websocket_log_count * 2); + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); let started = Arc::new(AtomicBool::new(already_started)); let cancelled = Arc::new(AtomicBool::new(false)); @@ -236,20 +239,92 @@ impl PveHelperProcessHandle { } }); + let console_cli = cli.clone(); + let console_started = Arc::clone(&started); + let console_cancelled = Arc::clone(&cancelled); + let console_task = tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + if console_cancelled.load(Ordering::Acquire) { + return; + } + if !console_started.load(Ordering::Acquire) + || !matches!( + console_cli.status(vmid).await, + Ok(cli::ContainerStatus::Running) + ) + { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + continue; + } + + let session = match console_cli.spawn_console(vmid) { + Ok(session) => session, + Err(error) => { + tracing::warn!(vmid, "failed to attach Proxmox helper console: {error:#}"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + let cli::ConsoleSession { + mut child, + mut reader, + mut writer, + } = session; + + loop { + tokio::select! { + read = reader.read(&mut buffer) => { + match read { + Ok(0) => break, + Ok(_) => {} + Err(error) => { + tracing::warn!(vmid, "failed to drain Proxmox helper console: {error:#}"); + break; + } + } + } + data = stdin_rx.recv() => { + let Some(data) = data else { + child.start_kill().ok(); + return; + }; + let data = PveProcessHandle::normalize_console_input(data); + if let Err(error) = writer.write_all(&data).await { + tracing::warn!(vmid, "failed to write Proxmox helper stdin: {error:#}"); + break; + } + } + result = child.wait() => { + if let Err(error) = result { + tracing::warn!(vmid, "failed waiting for Proxmox helper console: {error:#}"); + } + break; + } + } + } + child.start_kill().ok(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + }); + Ok(Self { vmid, cli, log_path, started, cancelled, + stdin_tx, stdout_ratelimited_rx, stdout_rx, + console_task, }) } } impl Drop for PveHelperProcessHandle { fn drop(&mut self) { + self.console_task.abort(); if !self.started.load(Ordering::Acquire) { self.cancelled.store(true, Ordering::Release); } @@ -270,10 +345,8 @@ impl ProcessHandle for PveHelperProcessHandle { } } - async fn send_stdin(&self, _data: Vec) -> Result<(), anyhow::Error> { - Err(anyhow::anyhow!( - "stdin is not supported for Proxmox helper containers" - )) + async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { + self.stdin_tx.send(data).await.map_err(anyhow::Error::from) } async fn subscribe_stdout_lines_ratelimited( diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index c6dad33b..2841ba82 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -193,10 +193,25 @@ pub struct PveLxcExecutor { firewall: Arc, stats_sampler: Arc, node: OnceLock, - template_provisioning: tokio::sync::Mutex<()>, + template_provisioning: tokio::sync::Mutex>>>, } impl PveLxcExecutor { + const DATA_MOUNT_TARGET: &'static str = "/home/container"; + + pub(super) fn pids_limit(config: &crate::config::Config) -> Option { + let config = config.load(); + match config + .runtime + .pve_lxc + .pids_limit + .unwrap_or(config.docker.container_pid_limit) + { + 0 => None, + limit => Some(limit), + } + } + fn image_cache_max_age(config: &crate::config::InnerConfig) -> std::time::Duration { let cache = config.docker.registry_image_fetch_cache; if cache.enabled { @@ -206,6 +221,23 @@ impl PveLxcExecutor { } } + async fn template_provisioning_lock( + &self, + node: &str, + storage: &str, + image: &str, + ) -> Arc> { + let key = format!("{node}\0{storage}\0{image}"); + let mut locks = self.template_provisioning.lock().await; + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(key, Arc::downgrade(&lock)); + lock + } + fn network_option( bridge: &str, vlan_tag: Option, @@ -321,7 +353,7 @@ impl PveLxcExecutor { firewall, stats_sampler: Arc::new(super::cgroup::StatsSampler::default()), node: OnceLock::new(), - template_provisioning: tokio::sync::Mutex::new(()), + template_provisioning: tokio::sync::Mutex::new(HashMap::new()), } } @@ -862,27 +894,38 @@ impl PveLxcExecutor { fn edge_forwarding_config( app_config: &crate::config::Config, - ) -> Result, anyhow::Error> { + ) -> Result, anyhow::Error> { let config = app_config.load(); let pve = &config.runtime.pve_lxc; let interface = pve.edge_wireguard_interface.trim().to_string(); + let user = pve.edge_ssh_user.trim().to_string(); let identity_path = pve.edge_ssh_identity_path.trim().to_string(); let known_hosts_path = pve.edge_known_hosts_path.trim().to_string(); if identity_path.is_empty() && known_hosts_path.is_empty() { return Ok(None); } - if interface.is_empty() || identity_path.is_empty() || known_hosts_path.is_empty() { + if interface.is_empty() + || user.is_empty() + || identity_path.is_empty() + || known_hosts_path.is_empty() + { return Err(anyhow::anyhow!( - "Proxmox edge forwarding requires edge_wireguard_interface, edge_ssh_identity_path, and edge_known_hosts_path" + "Proxmox edge forwarding requires edge_wireguard_interface, edge_ssh_user, edge_ssh_identity_path, and edge_known_hosts_path" )); } - Ok(Some((interface, identity_path, known_hosts_path))) + if !user + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + anyhow::bail!("Proxmox edge SSH user contains unsupported characters: {user}"); + } + Ok(Some((interface, user, identity_path, known_hosts_path))) } async fn validate_edge_forwarding_config( app_config: &crate::config::Config, ) -> Result<(), anyhow::Error> { - let Some((_, identity_path, _)) = Self::edge_forwarding_config(app_config)? else { + let Some((_, _, identity_path, _)) = Self::edge_forwarding_config(app_config)? else { return Ok(()); }; #[cfg(unix)] @@ -959,6 +1002,7 @@ impl PveLxcExecutor { async fn edge_ssh( host: Ipv4Addr, + user: &str, identity_path: &str, known_hosts_path: &str, server: uuid::Uuid, @@ -980,7 +1024,7 @@ impl PveLxcExecutor { "StrictHostKeyChecking=yes", "-o", &format!("UserKnownHostsFile={known_hosts_path}"), - &format!("root@{host}"), + &format!("{user}@{host}"), action, &server.to_string(), ]) @@ -1025,7 +1069,7 @@ impl PveLxcExecutor { public_ip: Option, payload: Option, ) -> Result<(), anyhow::Error> { - let Some((interface, identity_path, known_hosts_path)) = + let Some((interface, user, identity_path, known_hosts_path)) = Self::edge_forwarding_config(app_config)? else { return Ok(()); @@ -1034,6 +1078,7 @@ impl PveLxcExecutor { for peer in &peers { Self::edge_ssh( peer.tunnel_ip, + &user, &identity_path, &known_hosts_path, server, @@ -1053,6 +1098,7 @@ impl PveLxcExecutor { })?; Self::edge_ssh( peer.tunnel_ip, + &user, &identity_path, &known_hosts_path, server, @@ -1158,7 +1204,7 @@ impl PveLxcExecutor { let mut mounts = Vec::with_capacity(configured_mounts.len() + 2); mounts.push(( server.filesystem.get_base_fs_mount_path().await, - "/home/container".to_string(), + Self::DATA_MOUNT_TARGET.to_string(), false, )); @@ -1395,7 +1441,10 @@ impl PveLxcExecutor { }; let template = { - let _template = self.template_provisioning.lock().await; + let template_lock = self + .template_provisioning_lock(&node, &template_storage, &image) + .await; + let _template = template_lock.lock().await; self.cli .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) .await? @@ -1503,10 +1552,7 @@ exit 1 let (pids_limit, io_weight) = { let configuration = server.configuration.read().await; ( - match self.app_config.load().docker.container_pid_limit { - 0 => None, - limit => Some(limit), - }, + Self::pids_limit(&self.app_config), PveCli::cgroup2_io_weight(configuration.build.io_weight)?, ) }; @@ -1516,7 +1562,7 @@ exit 1 halt_signal: None, init_command: Some(init_command), console_logfile: Some(log_path.to_string()), - cpuset_cpus: None, + cpuset_cpus: resources.cpuset_cpus.clone(), pids_limit, io_weight, memory_unlimited: resources.memory_unlimited, @@ -1619,7 +1665,10 @@ exit 1 }; let template = { - let _template = self.template_provisioning.lock().await; + let template_lock = self + .template_provisioning_lock(&node, &template_storage, &image) + .await; + let _template = template_lock.lock().await; self.cli .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) .await? @@ -1770,6 +1819,7 @@ exit 1 }; let mut created = None; for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; let vmid = self.cli.next_vmid().await?; let create = cli::CreateContainerSpec { vmid, @@ -1876,37 +1926,46 @@ impl ServerExecutor for PveLxcExecutor { &self, servers: &[crate::remote::servers::RawServer], ) -> Result<(), anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let containers = self.cli.list_containers().await?; let mut specs = Vec::with_capacity(servers.len()); for server in servers { - let addresses = self - .server_addresses(server.settings.uuid) - .await - .with_context(|| { - format!( - "failed to resolve Proxmox LXC addresses while reconciling firewall for {}", - server.settings.uuid - ) - })?; - let container = self.server_container(server.settings.uuid).await?; + let container = + Self::owned_server_container(&containers, node, &tag_prefix, server.settings.uuid)?; + let running = container + .as_ref() + .is_some_and(|container| container.status == Some(cli::ContainerStatus::Running)); + let addresses = if let Some(container) = container.as_ref().filter(|_| running) { + PveCli::container_addresses( + &self + .cli + .interfaces(&container.node, container.vmid) + .await + .with_context(|| { + format!( + "failed to resolve Proxmox LXC addresses while reconciling firewall for {}", + server.settings.uuid + ) + })?, + ) + } else { + Vec::new() + }; let mut spec = Self::firewall_spec( &server.settings, addresses, container.as_ref().map(|container| container.vmid), ); - if !server.settings.firewall.is_empty() && spec.container_ips.is_empty() { - let running = self - .server_container(server.settings.uuid) - .await? - .is_some_and(|container| { - container.status == Some(cli::ContainerStatus::Running) - }); - if running { - return Err(anyhow::anyhow!( - "running Proxmox LXC server {} has no usable bridged address; refusing to reconcile configured firewall rules without a destination", - server.settings.uuid - )); - } + if !server.settings.firewall.is_empty() && spec.container_ips.is_empty() && running { + return Err(anyhow::anyhow!( + "running Proxmox LXC server {} has no usable bridged address; refusing to reconcile configured firewall rules without a destination", + server.settings.uuid + )); } if spec.references_files() { @@ -2236,6 +2295,7 @@ impl ServerExecutor for PveLxcExecutor { let mut created_vmid = None; for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; let vmid = self.cli.next_vmid().await?; match self .create_helper_container( @@ -2243,7 +2303,7 @@ impl ServerExecutor for PveLxcExecutor { script, Self::INSTALLER_HELPER_ROLE, vmid, - resources, + resources.clone(), environment.clone(), &staging_path, "/mnt/install", @@ -2407,6 +2467,7 @@ impl ServerExecutor for PveLxcExecutor { let mut created_vmid = None; for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; let vmid = self.cli.next_vmid().await?; match self .create_helper_container( @@ -2414,7 +2475,7 @@ impl ServerExecutor for PveLxcExecutor { script, Self::SCRIPT_HELPER_ROLE, vmid, - resources, + resources.clone(), environment.clone(), &staging_path, "/mnt/script", @@ -2522,17 +2583,24 @@ impl ServerExecutor for PveLxcExecutor { }) .collect::>(); + let runtime_statuses = match self.cli.node_runtime_statuses(node).await { + Ok(statuses) => statuses + .into_iter() + .map(|status| (status.vmid, status)) + .collect::>(), + Err(error) => { + tracing::error!("failed to list Proxmox LXC runtime status: {error:#}"); + return HashMap::new(); + } + }; + let mut references = HashMap::new(); for (server_uuid, container) in owned { if container.status != Some(cli::ContainerStatus::Running) { continue; } - match self - .cli - .runtime_status(&container.node, container.vmid) - .await - { - Ok(status) if status.status == cli::ContainerStatus::Running => { + match runtime_statuses.get(&container.vmid) { + Some(status) if status.status == cli::ContainerStatus::Running => { if let Some(pid) = status.pid.filter(|pid| *pid > 0) { references.insert(server_uuid, format!("pid:{pid}")); } else { @@ -2543,11 +2611,11 @@ impl ServerExecutor for PveLxcExecutor { ); } } - Ok(_) => {} - Err(error) => tracing::warn!( + Some(_) => {} + None => tracing::warn!( server = %server_uuid, vmid = container.vmid, - "failed to resolve Proxmox LXC init PID for private networking: {error:#}" + "Proxmox node inventory omitted an LXC needed for private networking" ), } } @@ -2577,6 +2645,13 @@ impl ServerExecutor for PveLxcExecutor { let mut containers = self.cli.list_containers().await?; containers.sort_unstable_by_key(|container| container.vmid); + let runtime_statuses = self + .cli + .node_runtime_statuses(node) + .await? + .into_iter() + .map(|status| (status.vmid, status)) + .collect::>(); for container in containers.into_iter().filter(|container| { container.node == *node && container.status == Some(cli::ContainerStatus::Running) @@ -2599,16 +2674,12 @@ impl ServerExecutor for PveLxcExecutor { continue; } - let runtime = match self - .cli - .runtime_status(&container.node, container.vmid) - .await - { - Ok(runtime) => runtime, - Err(error) => { + let runtime = match runtime_statuses.get(&container.vmid) { + Some(runtime) => runtime, + None => { tracing::warn!( vmid = container.vmid, - "skipping Proxmox LXC during used-port discovery after runtime lookup failed: {error:#}" + "skipping Proxmox LXC omitted from the node runtime inventory" ); continue; } diff --git a/application/src/server/executor/pve_lxc/server_process.rs b/application/src/server/executor/pve_lxc/server_process.rs index d4d24ec2..59061463 100644 --- a/application/src/server/executor/pve_lxc/server_process.rs +++ b/application/src/server/executor/pve_lxc/server_process.rs @@ -816,9 +816,7 @@ impl PveProcessHandle { let entrypoint_target = "/calagopus-entrypoint"; let entrypoint_source = stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); - tokio::fs::write( - &entrypoint_source, - r#"#!/bin/sh + let entrypoint_script = r#"#!/bin/sh attempt=0 while [ "$attempt" -lt 120 ]; do if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then @@ -846,9 +844,14 @@ read_oom_kills() { } child=0 +child_group=0 forward_signal() { if [ "$child" -gt 0 ]; then - kill "-$1" "$child" 2>/dev/null || true + if [ "$child_group" -eq 1 ]; then + kill "-$1" "-$child" 2>/dev/null || true + else + kill "-$1" "$child" 2>/dev/null || true + fi fi } trap 'forward_signal HUP' HUP @@ -858,7 +861,12 @@ trap 'forward_signal ABRT' ABRT trap 'forward_signal TERM' TERM oom_before=$(read_oom_kills) -"$@" <&0 & +if command -v setsid >/dev/null 2>&1; then + setsid "$@" <&0 & + child_group=1 +else + "$@" <&0 & +fi child=$! while :; do wait "$child" @@ -873,7 +881,7 @@ if [ "$oom_after" -gt "$oom_before" ]; then oom_killed=1 fi -marker=/home/container/.wings-exit-status +marker=__WINGS_EXIT_STATUS_PATH__ temporary="${marker}.tmp.$$" ( umask 077 @@ -881,10 +889,18 @@ temporary="${marker}.tmp.$$" ) mv -f "$temporary" "$marker" exit "$status" -"#, - ) - .await - .context("failed to stage Proxmox network-ready entrypoint")?; +"# + .replace( + "__WINGS_EXIT_STATUS_PATH__", + &format!( + "{}/{}", + PveLxcExecutor::DATA_MOUNT_TARGET, + Self::EXIT_STATUS_FILE + ), + ); + tokio::fs::write(&entrypoint_source, entrypoint_script) + .await + .context("failed to stage Proxmox network-ready entrypoint")?; tokio::fs::set_permissions(&entrypoint_source, std::fs::Permissions::from_mode(0o755)) .await .context("failed to make Proxmox network-ready entrypoint executable")?; @@ -931,22 +947,13 @@ exit "$status" ( configuration.environment(app_config), configuration.entrypoint.clone(), - configuration - .build - .threads - .as_deref() - .filter(|threads| !threads.trim().is_empty()) - .map(PveCli::normalize_cpuset) - .transpose()?, + resources.cpuset_cpus.clone(), PveCli::cgroup2_io_weight(configuration.build.io_weight)?, resources.memory_unlimited, resources.swap_unlimited, ) }; - let pids_limit = match app_config.load().docker.container_pid_limit { - 0 => None, - limit => Some(limit), - }; + let pids_limit = PveLxcExecutor::pids_limit(app_config); let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; let process = server.process_configuration.read().await; let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); @@ -1227,14 +1234,34 @@ impl ProcessHandle for PveProcessHandle { async fn kill(&self) -> Result<(), anyhow::Error> { let server = self.get_server()?; let exit_status_path = Self::exit_status_path(&server); - tokio::fs::write(&exit_status_path, "exit_code=137\noom_killed=0\n") + if self.cli.status(self.vmid).await? == cli::ContainerStatus::Stopped { + return Ok(()); + } + match tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&exit_status_path) .await - .with_context(|| { - format!( - "failed to record forced Proxmox LXC exit status {}", - exit_status_path.display() - ) - })?; + { + Ok(mut marker) => marker + .write_all(b"exit_code=137\noom_killed=0\n") + .await + .with_context(|| { + format!( + "failed to record forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + })?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to create forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + }); + } + } self.cli.stop(self.vmid).await } } diff --git a/application/src/server/executor/pve_lxc/tests.rs b/application/src/server/executor/pve_lxc/tests.rs index 756497e3..63126300 100644 --- a/application/src/server/executor/pve_lxc/tests.rs +++ b/application/src/server/executor/pve_lxc/tests.rs @@ -557,3 +557,50 @@ fn device_permissions_require_exact_pve_devn_semantics() { ); assert!(PveLxcExecutor::device_deny_write("rwx", path).is_err()); } + +#[tokio::test] +async fn lxc_pid_limit_overrides_the_legacy_docker_setting() { + let config = crate::config::Config::mock(); + config + .mutate_in_place_for_testing() + .docker + .container_pid_limit = 256; + assert_eq!(PveLxcExecutor::pids_limit(&config), Some(256)); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .pids_limit = Some(1024); + assert_eq!(PveLxcExecutor::pids_limit(&config), Some(1024)); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .pids_limit = Some(0); + assert_eq!(PveLxcExecutor::pids_limit(&config), None); +} + +#[tokio::test] +async fn edge_ssh_user_is_configurable_and_validated() { + let config = crate::config::Config::mock(); + { + let inner = config.mutate_in_place_for_testing(); + inner.runtime.pve_lxc.edge_ssh_identity_path = "/root/edge-key".to_string(); + inner.runtime.pve_lxc.edge_known_hosts_path = "/root/known-hosts".to_string(); + inner.runtime.pve_lxc.edge_ssh_user = "edge-forward".to_string(); + } + + let resolved = PveLxcExecutor::edge_forwarding_config(&config) + .expect("valid edge configuration") + .expect("configured edge forwarding"); + assert_eq!(resolved.1, "edge-forward"); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .edge_ssh_user = "root@host".to_string(); + assert!(PveLxcExecutor::edge_forwarding_config(&config).is_err()); +} diff --git a/application/src/tundra/daemon.rs b/application/src/tundra/daemon.rs index 9349c547..6e868aa0 100644 --- a/application/src/tundra/daemon.rs +++ b/application/src/tundra/daemon.rs @@ -615,7 +615,7 @@ pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow let source = config.tundra.binary.as_path(&config); let source_image = config.tundra.source_image.clone(); let metrics_port = config.tundra.metrics_port; - let hosts_path = match config.runtime.backend { + let hosts_path = match state.config.active_runtime_backend() { crate::config::RuntimeBackend::PveLxc => config .runtime .pve_lxc diff --git a/application/src/tundra/mod.rs b/application/src/tundra/mod.rs index 77824e59..a4d98a58 100644 --- a/application/src/tundra/mod.rs +++ b/application/src/tundra/mod.rs @@ -219,7 +219,7 @@ impl TundraManager { return; }; - if state.config.load().runtime.backend == crate::config::RuntimeBackend::PveLxc { + if state.config.active_runtime_backend() == crate::config::RuntimeBackend::PveLxc { let supported = self .hub .request_metrics() diff --git a/scripts/pve-lxc-smoke-test.sh b/scripts/pve-lxc-smoke-test.sh index 00ec73f2..225c3132 100755 --- a/scripts/pve-lxc-smoke-test.sh +++ b/scripts/pve-lxc-smoke-test.sh @@ -357,7 +357,9 @@ PVE::LXC::Config->lock_config($vmid, sub { && $_->[0] ne 'lxc.environment.runtime' && $_->[0] ne 'lxc.signal.halt' && $_->[0] ne 'lxc.init.cmd' - && $_->[0] ne 'lxc.console.logfile'); + && $_->[0] ne 'lxc.console.logfile' + && $_->[0] ne 'lxc.cgroup2.memory.max' + && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); if ($keep && ref($_) eq 'ARRAY' && @$_ >= 2 && $_->[0] eq 'lxc.mount.entry') { my $target = mount_target($_->[1]); $keep = 0 if defined($target) && $target eq 'etc/hosts'; @@ -370,6 +372,8 @@ PVE::LXC::Config->lock_config($vmid, sub { push @lxc, ['lxc.signal.halt', 'SIGTERM']; push @lxc, ['lxc.init.cmd', '/bin/sh /home/container/runtime-wrapper.sh']; push @lxc, ['lxc.console.logfile', $console_log]; + push @lxc, ['lxc.cgroup2.memory.max', 'max']; + push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; push @lxc, [ 'lxc.mount.entry', "$managed_hosts etc/hosts none bind,ro,create=file 0 0", @@ -430,7 +434,7 @@ start_ct() { } verify_runtime_contracts() { - local attempt + local attempt runtime_json pid cgroup_path memory_max swap_max configure_runtime_contracts start_ct @@ -460,6 +464,27 @@ verify_runtime_contracts() { fi echo "managed file bind: /etc/hosts is visible and read-only" + runtime_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json)" + pid="$(printf '%s' "$runtime_json" | json_field pid)" + cgroup_path="$(awk -F: '$1 == "0" {print $3; exit}' "/proc/${pid}/cgroup")" + if [[ -z "$cgroup_path" ]]; then + echo "error: the live PVE smoke test requires a cgroup v2 host" >&2 + exit 1 + fi + memory_max="$(cat "/sys/fs/cgroup${cgroup_path}/memory.max")" + swap_max="$(cat "/sys/fs/cgroup${cgroup_path}/memory.swap.max")" + if [[ "$memory_max" != "max" || "$swap_max" != "max" ]]; then + echo "error: PVE did not preserve unlimited cgroup2 memory settings on start" >&2 + echo "memory.max=$memory_max memory.swap.max=$swap_max" >&2 + exit 1 + fi + if ! pct config "$vmid" | grep -Fq 'lxc.cgroup2.memory.max: max' \ + || ! pct config "$vmid" | grep -Fq 'lxc.cgroup2.memory.swap.max: max'; then + echo "error: PVE dropped unlimited cgroup2 memory settings from CT config" >&2 + exit 1 + fi + echo "cgroup2: PVE preserved memory.max=max and memory.swap.max=max after start" + verify_running_contracts lxc-stop -n "$vmid" --nowait --nokill From 13e35c54bbb7fb7ec8c9de25e26489a50f225e71 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 08/13] allow root OCI users in PVE smoke test --- scripts/pve-lxc-smoke-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pve-lxc-smoke-test.sh b/scripts/pve-lxc-smoke-test.sh index 225c3132..27088b1e 100755 --- a/scripts/pve-lxc-smoke-test.sh +++ b/scripts/pve-lxc-smoke-test.sh @@ -188,8 +188,8 @@ configure_data_mount() { config="$(pct config "$vmid" | tr -d '\000')" uid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.uid" {print $2; exit}')" gid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.gid" {print $2; exit}')" - if [[ -z "$uid" || -z "$gid" || "$uid" == "0" || "$gid" == "0" ]]; then - echo "error: OCI image did not resolve to a non-root lxc.init.uid/gid" >&2 + if [[ ! "$uid" =~ ^[0-9]+$ || ! "$gid" =~ ^[0-9]+$ ]]; then + echo "error: OCI image did not resolve to numeric lxc.init.uid/gid values" >&2 printf '%s\n' "$config" >&2 exit 1 fi From bb1fabaea7de01f751657c598248af4ab58bd2fa Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 09/13] drop redundant Tundra revision pin --- Cargo.lock | 2 +- application/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6164bef3..74c8b948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5416,7 +5416,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tundra-common" version = "1.0.1" -source = "git+https://github.com/calagopus/tundra.git?rev=ccb05e1406232f8c39f05a984133277026cd79d6#ccb05e1406232f8c39f05a984133277026cd79d6" +source = "git+https://github.com/calagopus/tundra.git#ccb05e1406232f8c39f05a984133277026cd79d6" dependencies = [ "base64 0.23.1", "bytes", diff --git a/application/Cargo.toml b/application/Cargo.toml index 6284adc2..f598fc9f 100644 --- a/application/Cargo.toml +++ b/application/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" tempfile = "3.27.0" unrar = { path = "../unrar-rs" } pbs-client = { path = "../pbs-client" } -tundra-common = { git = "https://github.com/calagopus/tundra.git", rev = "ccb05e1406232f8c39f05a984133277026cd79d6" } +tundra-common = { git = "https://github.com/calagopus/tundra.git" } tracing = { version = "0.1.41", features = ["max_level_debug"] } tracing-subscriber = { version = "0.3.19", features = ["chrono"] } tracing-appender = "0.2.3" From 8e0760dc0df9b5ce17a4063f55cca4b9e1a11de6 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 10/13] add minimum VMID for PVE LXC auto-allocation Wings now allocates Proxmox container IDs at or above runtime.pve_lxc.min_vmid (default 200) so auto-created guests stay clear of manually created infrastructure containers below the floor. When /cluster/nextid points below the floor, the allocator scans the cluster inventory across all guest types for the first free ID. --- application/src/config.rs | 11 +++ .../src/server/executor/pve_lxc/cli.rs | 79 ++++++++++++++++++- .../src/server/executor/pve_lxc/cli/tests.rs | 31 ++++++++ .../src/server/executor/pve_lxc/mod.rs | 13 ++- scripts/pve-lxc-smoke-test.sh | 15 ++++ 5 files changed, 143 insertions(+), 6 deletions(-) diff --git a/application/src/config.rs b/application/src/config.rs index 778fc825..fe48da76 100644 --- a/application/src/config.rs +++ b/application/src/config.rs @@ -799,6 +799,11 @@ fn pve_lxc_edge_ssh_user() -> String { fn pve_lxc_rootfs_size_gib() -> u64 { 8 } + +fn pve_lxc_min_vmid() -> u32 { + 200 +} + fn pve_lxc_console_log_max_bytes() -> u64 { 5 * 1024 * 1024 } @@ -899,6 +904,12 @@ nestify::nest! { /// legacy Docker limit is used for configuration compatibility. #[serde(default)] pub pids_limit: Option, + /// Minimum VMID used when Wings auto-allocates Proxmox + /// containers. PVE itself starts at 100; raising the floor + /// (for example to 200) keeps Wings-owned guests clear of + /// manually created infrastructure containers. + #[serde(default = "pve_lxc_min_vmid")] + pub min_vmid: u32, #[serde(default)] #[schema(inline)] diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index 49b8ad0d..28d24cc8 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -828,7 +828,12 @@ impl PveCli { .to_string()) } - pub async fn next_vmid(&self) -> Result { + pub async fn next_vmid(&self, min_vmid: u32) -> Result { + if !(100..=999_999_999).contains(&min_vmid) { + anyhow::bail!( + "Proxmox minimum VMID must be between 100 and 999999999, got {min_vmid}" + ); + } let args = vec![ "get".to_string(), "/cluster/nextid".to_string(), @@ -842,7 +847,77 @@ impl PveCli { .as_u64() .or_else(|| value.as_str().and_then(|value| value.parse().ok())) .context("Proxmox nextid response was not a VMID")?; - u32::try_from(raw).context("Proxmox nextid response exceeded the VMID range") + let raw = u32::try_from(raw).context("Proxmox nextid response exceeded the VMID range")?; + if raw >= min_vmid { + return Ok(raw); + } + + // `/cluster/nextid` tracks the lowest free ID cluster-wide, which can + // sit below the Wings floor. Scan for the first free ID at or above + // the floor instead of squatting a low ID reserved for infrastructure. + // VMIDs are shared across QEMU and LXC guests, so every `vmid` in the + // inventory counts, not just LXC rows. + let used = self.allocated_vmids().await?; + Self::first_free_vmid_at_or_above(min_vmid, &used).with_context(|| { + format!("Proxmox VMID range above the minimum {min_vmid} is exhausted") + }) + } + + fn first_free_vmid_at_or_above( + min_vmid: u32, + used: &std::collections::HashSet, + ) -> Option { + let mut sorted: Vec = used.iter().copied().filter(|id| *id >= min_vmid).collect(); + sorted.sort_unstable(); + let mut candidate = min_vmid; + for id in sorted { + if id == candidate { + candidate = candidate.checked_add(1)?; + } else if id > candidate { + break; + } + if candidate > 999_999_999 { + return None; + } + } + (candidate <= 999_999_999).then_some(candidate) + } + + /// Every VMID in use cluster-wide, across all guest types. VMIDs are a + /// single shared namespace in Proxmox, so a QEMU VM squats the ID for LXC + /// too. + async fn allocated_vmids(&self) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + "/cluster/resources".to_string(), + "--type".to_string(), + "vm".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_allocated_vmids(&output.stdout) + } + + fn parse_allocated_vmids( + output: &[u8], + ) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster resources response")?; + let mut used = std::collections::HashSet::new(); + for row in rows { + let Some(raw) = row.get("vmid").and_then(|value| { + value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + }) else { + continue; + }; + if let Ok(vmid) = u32::try_from(raw) { + used.insert(vmid); + } + } + Ok(used) } /// Serialize the advisory `nextid` lookup and the following create on this diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs index 455714a5..f750f2b5 100644 --- a/application/src/server/executor/pve_lxc/cli/tests.rs +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -1024,3 +1024,34 @@ fn lifecycle_arguments_request_nonblocking_graceful_shutdown_and_console() { vec!["console", "1234", "--escape", "^z"] ); } + +#[test] +fn vmid_floor_skips_ids_below_the_minimum() { + let used: std::collections::HashSet = [100, 101, 200].into_iter().collect(); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(201)); + assert_eq!( + PveCli::first_free_vmid_at_or_above(200, &std::collections::HashSet::new()), + Some(200) + ); +} + +#[test] +fn vmid_floor_finds_gaps_and_ignores_lower_ids() { + let used: std::collections::HashSet = [100, 200, 201, 203].into_iter().collect(); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(202)); +} + +#[test] +fn allocated_vmids_include_every_guest_type() { + let output = br#"[ + {"vmid": 200, "type": "lxc", "node": "pve1"}, + {"vmid": "201", "type": "qemu", "node": "pve1"}, + {"type": "lxc", "node": "pve1"}, + {"vmid": 202, "type": "storage", "node": "pve1"} + ]"#; + let used = PveCli::parse_allocated_vmids(output).expect("valid inventory"); + assert!(used.contains(&200)); + assert!(used.contains(&201)); + assert!(used.contains(&202)); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(203)); +} diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index 2841ba82..2a63dc79 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -1818,9 +1818,10 @@ exit 1 final_tags.clone() }; let mut created = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; for _ in 0..3 { let _vmid_lock = self.cli.lock_vmid_allocation().await?; - let vmid = self.cli.next_vmid().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; let create = cli::CreateContainerSpec { vmid, template: template.volid.clone(), @@ -1911,11 +1912,13 @@ impl ServerExecutor for PveLxcExecutor { .await .context("failed to initialize Proxmox LXC host firewall")?; - let next_vmid = self.cli.next_vmid().await?; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; + let next_vmid = self.cli.next_vmid(min_vmid).await?; tracing::info!( pve_version = %raw_version, node = %node, next_vmid, + min_vmid, "Proxmox VE LXC runtime is available" ); @@ -2294,9 +2297,10 @@ impl ServerExecutor for PveLxcExecutor { })?; let mut created_vmid = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; for _ in 0..3 { let _vmid_lock = self.cli.lock_vmid_allocation().await?; - let vmid = self.cli.next_vmid().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; match self .create_helper_container( server, @@ -2466,9 +2470,10 @@ impl ServerExecutor for PveLxcExecutor { } let mut created_vmid = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; for _ in 0..3 { let _vmid_lock = self.cli.lock_vmid_allocation().await?; - let vmid = self.cli.next_vmid().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; match self .create_helper_container( server, diff --git a/scripts/pve-lxc-smoke-test.sh b/scripts/pve-lxc-smoke-test.sh index 27088b1e..6455654e 100755 --- a/scripts/pve-lxc-smoke-test.sh +++ b/scripts/pve-lxc-smoke-test.sh @@ -54,6 +54,21 @@ if [[ ! "$vmid" =~ ^[0-9]+$ || "$vmid" -le 0 ]]; then echo "error: Proxmox returned an invalid next VMID: ${vmid:-empty}" >&2 exit 1 fi +# Mirror runtime.pve_lxc.min_vmid so the disposable CT does not squat a low +# ID reserved for infrastructure. Override with MIN_VMID=100 to test the floor. +min_vmid="${MIN_VMID:-200}" +if [[ ! "$min_vmid" =~ ^[0-9]+$ || "$min_vmid" -lt 100 ]]; then + echo "error: MIN_VMID must be a VMID at or above 100, got ${min_vmid:-empty}" >&2 + exit 1 +fi +if [[ "$vmid" -lt "$min_vmid" ]]; then + used_vmids="$(pvesh get /cluster/resources --type vm --output-format json | perl -ne 'while (/\"vmid\"\s*:\s*\"?(\d+)\"?/g) { print "$1\n" }' | sort -n -u)" + candidate="$min_vmid" + while printf '%s\n' "$used_vmids" | grep -qx "$candidate"; do + candidate=$((candidate + 1)) + done + vmid="$candidate" +fi server_uuid="$(cat /proc/sys/kernel/random/uuid)" run_tag="calagopus-smoke" server_tag="calagopus-server-${server_uuid}" From 821f87637c546aab873752e29a52f98a088e2858 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 11/13] fix(pve-lxc): normalize gzip OCI templates for Proxmox --- .../src/server/executor/pve_lxc/cli.rs | 248 +++++++++++++++++- .../src/server/executor/pve_lxc/cli/tests.rs | 111 ++++++++ 2 files changed, 356 insertions(+), 3 deletions(-) diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index 28d24cc8..f414c292 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -1,6 +1,9 @@ use anyhow::Context; +use flate2::read::GzDecoder; use sha2::{Digest, Sha256}; use std::{ + collections::HashMap, + io::{Read, Write}, net::IpAddr, path::{Component, Path, PathBuf}, process::Output, @@ -830,9 +833,7 @@ impl PveCli { pub async fn next_vmid(&self, min_vmid: u32) -> Result { if !(100..=999_999_999).contains(&min_vmid) { - anyhow::bail!( - "Proxmox minimum VMID must be between 100 and 999999999, got {min_vmid}" - ); + anyhow::bail!("Proxmox minimum VMID must be between 100 and 999999999, got {min_vmid}"); } let args = vec![ "get".to_string(), @@ -1328,6 +1329,243 @@ impl PveCli { Ok(path.into()) } + /// Rewrites OCI gzip layers to the uncompressed representation expected by + /// Proxmox's LXC image importer. OCI `rootfs.diff_ids` are hashes of the + /// uncompressed layer tar streams, whereas some registry pulls retain gzip + /// blobs and PVE currently resolves `diff_ids` as blob names. + fn normalize_oci_template(path: PathBuf) -> Result<(), anyhow::Error> { + let source = std::fs::File::open(&path) + .with_context(|| format!("failed to open OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(source); + let mut index = None; + let mut manifests = HashMap::new(); + let mut configs = HashMap::new(); + for entry in archive + .entries() + .context("failed to enumerate OCI template")? + { + let mut entry = entry.context("failed to read OCI template entry")?; + let name = entry.path()?.to_string_lossy().into_owned(); + // Index, manifest, and config blobs are tiny JSON documents. Never + // buffer the multi-gigabyte layer blobs during metadata discovery. + if name != "index.json" + && (!name.starts_with("blobs/sha256/") || entry.size() > 4 * 1024 * 1024) + { + continue; + } + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + if name == "index.json" { + index = Some( + serde_json::from_slice::(&bytes) + .context("failed to parse OCI template index.json")?, + ); + } else if name.starts_with("blobs/sha256/") { + if let Ok(value) = serde_json::from_slice::(&bytes) { + if value.get("schemaVersion").is_some() && value.get("layers").is_some() { + manifests.insert(name, value); + } else if value.get("rootfs").is_some() { + configs.insert(name, value); + } + } + } + } + let mut index = index.context("OCI template did not contain index.json")?; + let manifest_digest = index + .pointer("/manifests/0/digest") + .and_then(serde_json::Value::as_str) + .context("OCI template index did not include a manifest digest")? + .strip_prefix("sha256:") + .context("OCI template manifest digest was not sha256")?; + let old_manifest_path = format!("blobs/sha256/{manifest_digest}"); + let mut manifest = manifests + .remove(&old_manifest_path) + .context("OCI template did not contain its indexed manifest")?; + let config_digest = manifest + .pointer("/config/digest") + .and_then(serde_json::Value::as_str) + .context("OCI manifest did not include a config digest")? + .strip_prefix("sha256:") + .context("OCI config digest was not sha256")?; + let config = configs + .remove(&format!("blobs/sha256/{config_digest}")) + .context("OCI template did not contain its config blob")?; + let diff_ids = config + .pointer("/rootfs/diff_ids") + .and_then(serde_json::Value::as_array) + .context("OCI config did not include rootfs.diff_ids")?; + let layer_specs = manifest + .get("layers") + .and_then(serde_json::Value::as_array) + .context("OCI manifest did not include layers")?; + if layer_specs.len() != diff_ids.len() { + return Err(anyhow::anyhow!( + "OCI manifest layer count did not match rootfs.diff_ids" + )); + } + let layer_specs: Vec<(String, String, bool)> = layer_specs + .iter() + .zip(diff_ids) + .map(|(layer, diff_id)| { + let digest = layer + .get("digest") + .and_then(serde_json::Value::as_str) + .context("OCI layer did not include a digest")? + .strip_prefix("sha256:") + .context("OCI layer digest was not sha256")?; + let expected = diff_id + .as_str() + .context("OCI diff_id was not a string")? + .strip_prefix("sha256:") + .context("OCI diff_id was not sha256")?; + let media_type = layer + .get("mediaType") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + Ok(( + format!("blobs/sha256/{digest}"), + expected.to_string(), + media_type.ends_with("+gzip") || media_type.ends_with(".gzip"), + )) + }) + .collect::>()?; + + let scratch = + tempfile::tempdir().context("failed to create OCI normalization scratch directory")?; + let replacements: HashMap = layer_specs + .into_iter() + .map(|(name, expected, gzip)| { + ( + name, + (expected.clone(), scratch.path().join(&expected), gzip), + ) + }) + .collect(); + + let source = std::fs::File::open(&path) + .with_context(|| format!("failed to reopen OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(source); + for entry in archive + .entries() + .context("failed to enumerate OCI layers")? + { + let entry = entry.context("failed to read OCI layer entry")?; + let name = entry.path()?.to_string_lossy().into_owned(); + let Some((expected, replacement, gzip)) = replacements.get(&name) else { + continue; + }; + let mut hasher = Sha256::new(); + let mut output = std::fs::File::create(replacement).with_context(|| { + format!( + "failed to create normalized OCI layer {}", + replacement.display() + ) + })?; + let mut reader: Box = if *gzip { + Box::new(GzDecoder::new(entry)) + } else { + Box::new(entry) + }; + let mut buffer = [0u8; 1024 * 1024]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + output.write_all(&buffer[..count])?; + } + let actual = hex::encode(hasher.finalize()); + if actual != *expected { + return Err(anyhow::anyhow!( + "OCI layer diff_id mismatch: expected sha256:{expected}, got sha256:{actual}" + )); + } + } + if replacements + .values() + .any(|(_, replacement, _)| !replacement.is_file()) + { + return Err(anyhow::anyhow!( + "OCI template was missing one or more layer blobs" + )); + } + let layers = manifest + .get_mut("layers") + .and_then(serde_json::Value::as_array_mut) + .context("OCI manifest did not include layers")?; + for (position, layer) in layers.iter_mut().enumerate() { + let expected = diff_ids[position].as_str().unwrap_or_default(); + let (_, replacement, _) = replacements + .values() + .find(|(digest, _, _)| expected == format!("sha256:{digest}")) + .context("normalized OCI layer replacement was missing")?; + layer["digest"] = serde_json::Value::String(expected.to_string()); + layer["size"] = serde_json::Value::from(std::fs::metadata(replacement)?.len()); + layer["mediaType"] = + serde_json::Value::String("application/vnd.oci.image.layer.v1.tar".to_string()); + } + let manifest_bytes = serde_json::to_vec(&manifest)?; + let new_manifest_digest = hex::encode(Sha256::digest(&manifest_bytes)); + index["manifests"][0]["digest"] = + serde_json::Value::String(format!("sha256:{new_manifest_digest}")); + index["manifests"][0]["size"] = serde_json::Value::from(manifest_bytes.len()); + let index_bytes = serde_json::to_vec(&index)?; + let temporary = path.with_extension("tar.normalize-tmp"); + let output = std::fs::File::create(&temporary).with_context(|| { + format!( + "failed to create normalized OCI template {}", + temporary.display() + ) + })?; + let mut builder = tar::Builder::new(output); + let source = std::fs::File::open(&path)?; + let mut archive = tar::Archive::new(source); + for entry in archive.entries()? { + let mut entry = entry?; + let name = entry.path()?.to_string_lossy().into_owned(); + if name == "index.json" || name == old_manifest_path || replacements.contains_key(&name) + { + continue; + } + if !entry.header().entry_type().is_file() { + continue; + } + let mut header = entry.header().clone(); + builder.append_data(&mut header, name, &mut entry)?; + } + for (expected, replacement, _) in replacements.values() { + let mut layer = std::fs::File::open(replacement)?; + let mut header = tar::Header::new_gnu(); + header.set_size(std::fs::metadata(replacement)?.len()); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, format!("blobs/sha256/{expected}"), &mut layer)?; + } + let mut header = tar::Header::new_gnu(); + header.set_size(manifest_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data( + &mut header, + format!("blobs/sha256/{new_manifest_digest}"), + manifest_bytes.as_slice(), + )?; + let mut header = tar::Header::new_gnu(); + header.set_size(index_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, "index.json", index_bytes.as_slice())?; + builder.finish()?; + std::fs::rename(&temporary, &path).with_context(|| { + format!( + "failed to atomically install normalized OCI template {}", + path.display() + ) + })?; + Ok(()) + } + async fn template_revision(&self, volid: &str) -> Result { let path = self.template_path(volid).await?; tokio::task::spawn_blocking(move || { @@ -1551,6 +1789,10 @@ impl PveCli { }) .await?; self.wait_for_task(node, &upid).await?; + let path = self.template_path(&volid).await?; + tokio::task::spawn_blocking(move || Self::normalize_oci_template(path)) + .await + .context("OCI template normalization task failed")??; let exists = self .list_storage_content(node, storage) diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs index f750f2b5..3467962e 100644 --- a/application/src/server/executor/pve_lxc/cli/tests.rs +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -1055,3 +1055,114 @@ fn allocated_vmids_include_every_guest_type() { assert!(used.contains(&202)); assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(203)); } + +#[test] +fn normalizes_gzip_oci_layers_for_proxmox_diff_ids() { + use flate2::{Compression, write::GzEncoder}; + use sha2::{Digest, Sha256}; + use std::io::{Cursor, Write}; + + fn append(builder: &mut tar::Builder, path: &str, bytes: &[u8]) { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(bytes)) + .expect("append tar entry"); + } + + let directory = tempfile::tempdir().expect("temp directory"); + let path = directory.path().join("image.tar"); + let layer = b"uncompressed OCI layer contents"; + let diff_id = hex::encode(Sha256::digest(layer)); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(layer).expect("compress layer"); + let compressed = encoder.finish().expect("finish compression"); + let compressed_digest = hex::encode(Sha256::digest(&compressed)); + let config = serde_json::json!({"rootfs": {"type": "layers", "diff_ids": [format!("sha256:{diff_id}")]}}); + let config = serde_json::to_vec(&config).expect("encode config"); + let config_digest = hex::encode(Sha256::digest(&config)); + let manifest = serde_json::json!({ + "schemaVersion": 2, + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": format!("sha256:{config_digest}"), "size": config.len()}, + "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": format!("sha256:{compressed_digest}"), "size": compressed.len()}] + }); + let manifest = serde_json::to_vec(&manifest).expect("encode manifest"); + let manifest_digest = hex::encode(Sha256::digest(&manifest)); + let index = serde_json::to_vec(&serde_json::json!({"schemaVersion": 2, "manifests": [{"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": format!("sha256:{manifest_digest}"), "size": manifest.len()}]})).expect("encode index"); + let file = std::fs::File::create(&path).expect("create archive"); + let mut builder = tar::Builder::new(file); + append( + &mut builder, + "oci-layout", + br#"{"imageLayoutVersion":"1.0.0"}"#, + ); + append( + &mut builder, + &format!("blobs/sha256/{config_digest}"), + &config, + ); + append( + &mut builder, + &format!("blobs/sha256/{compressed_digest}"), + &compressed, + ); + append( + &mut builder, + &format!("blobs/sha256/{manifest_digest}"), + &manifest, + ); + append(&mut builder, "index.json", &index); + builder.finish().expect("finish archive"); + + PveCli::normalize_oci_template(path.clone()).expect("normalize OCI archive"); + let mut archive = + tar::Archive::new(std::fs::File::open(path).expect("open normalized archive")); + let mut normalized_index = None; + let mut blobs = std::collections::HashMap::new(); + for entry in archive.entries().expect("entries") { + let mut entry = entry.expect("entry"); + let name = entry.path().expect("path").to_string_lossy().into_owned(); + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read entry"); + if name == "index.json" { + normalized_index = Some(bytes); + } else { + blobs.insert(name, bytes); + } + } + let index: serde_json::Value = + serde_json::from_slice(&normalized_index.expect("index")).expect("parse index"); + let manifest_digest = index + .pointer("/manifests/0/digest") + .and_then(serde_json::Value::as_str) + .expect("manifest digest") + .strip_prefix("sha256:") + .expect("sha digest"); + let manifest: serde_json::Value = serde_json::from_slice( + blobs + .get(&format!("blobs/sha256/{manifest_digest}")) + .expect("manifest blob"), + ) + .expect("parse manifest"); + let expected_digest = format!("sha256:{diff_id}"); + assert_eq!( + manifest + .pointer("/layers/0/digest") + .and_then(serde_json::Value::as_str), + Some(expected_digest.as_str()) + ); + assert_eq!( + manifest + .pointer("/layers/0/mediaType") + .and_then(serde_json::Value::as_str), + Some("application/vnd.oci.image.layer.v1.tar") + ); + assert_eq!( + blobs + .get(&format!("blobs/sha256/{diff_id}")) + .map(Vec::as_slice), + Some(layer.as_slice()) + ); +} From bc7a2be1247f54a2dc19485904a78ea8ffa07e0d Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 22:52:23 +0000 Subject: [PATCH 12/13] fix(pve-lxc): wait longer for DHCP readiness --- application/src/server/executor/pve_lxc/mod.rs | 2 +- application/src/server/executor/pve_lxc/server_process.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs index 2a63dc79..ec6882d0 100644 --- a/application/src/server/executor/pve_lxc/mod.rs +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -1503,7 +1503,7 @@ impl PveLxcExecutor { &helper_wrapper_path, r#"#!/bin/sh attempt=0 -while [ "$attempt" -lt 120 ]; do +while [ "$attempt" -lt 480 ]; do if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then "$@" status=$? diff --git a/application/src/server/executor/pve_lxc/server_process.rs b/application/src/server/executor/pve_lxc/server_process.rs index 59061463..1e77648a 100644 --- a/application/src/server/executor/pve_lxc/server_process.rs +++ b/application/src/server/executor/pve_lxc/server_process.rs @@ -818,14 +818,14 @@ impl PveProcessHandle { stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); let entrypoint_script = r#"#!/bin/sh attempt=0 -while [ "$attempt" -lt 120 ]; do +while [ "$attempt" -lt 480 ]; do if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then break fi attempt=$((attempt + 1)) sleep 0.25 done -if [ "$attempt" -ge 120 ]; then +if [ "$attempt" -ge 480 ]; then echo 'timed out waiting for Proxmox DHCP' >&2 exit 1 fi From 8622db3bb0c9ee7c000522a0ea197a57e4bc25b7 Mon Sep 17 00:00:00 2001 From: Philip Ulrich Date: Sun, 11 Oct 2026 23:12:32 +0000 Subject: [PATCH 13/13] fix(pve-lxc): wait for usable DHCP readiness --- .../src/server/executor/pve_lxc/cli.rs | 61 +++++++++++++++++-- .../src/server/executor/pve_lxc/cli/tests.rs | 16 +++++ .../server/executor/pve_lxc/server_process.rs | 12 +++- .../src/server/executor/pve_lxc/tests.rs | 11 ++++ 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs index f414c292..cf10fbc8 100644 --- a/application/src/server/executor/pve_lxc/cli.rs +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -424,13 +424,66 @@ impl PveCli { Ok(Some(1 + (u64::from(weight) - 10) * 9999 / 990)) } + #[cfg(target_os = "linux")] + fn isolate_command_process_group(command: &mut tokio::process::Command) { + use std::os::unix::process::CommandExt; + + // Proxmox's `pct` client starts a task subprocess. `kill_on_drop` only + // kills the direct client, which used to leave its task child holding + // pve-config-.lock after a Wings timeout or restart. A dedicated + // session makes the client's PID the process-group leader so timeout + // cleanup can terminate the complete task tree. + unsafe { + command.as_std_mut().pre_exec(|| { + rustix::process::setsid()?; + Ok(()) + }); + } + } + + #[cfg(target_os = "linux")] + async fn terminate_command_process_group(pid: u32) { + use rustix::process::{Pid, Signal, kill_process_group}; + + let Some(group) = Pid::from_raw(pid as i32) else { + return; + }; + let _ = kill_process_group(group, Signal::TERM); + tokio::time::sleep(Duration::from_secs(1)).await; + let _ = kill_process_group(group, Signal::KILL); + } + async fn run(program: &std::path::Path, args: &[String]) -> Result { let mut command = tokio::process::Command::new(program); - command.args(args).kill_on_drop(true); - let output = tokio::time::timeout(COMMAND_TIMEOUT, command.output()) - .await - .with_context(|| format!("timed out executing {}", program.display()))? + // `wait_with_output` only captures streams explicitly piped before + // `spawn`; unlike `Command::output`, bare `spawn` inherits them. + // PVE commands emit JSON on stdout, which must not leak to journald + // or be lost before the caller parses it. + command + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + Self::isolate_command_process_group(&mut command); + + let child = command + .spawn() .with_context(|| format!("failed to execute {}", program.display()))?; + let pid = child.id(); + let output = match tokio::time::timeout(COMMAND_TIMEOUT, child.wait_with_output()).await { + Ok(result) => { + result.with_context(|| format!("failed to execute {}", program.display()))? + } + Err(_) => { + #[cfg(target_os = "linux")] + if let Some(pid) = pid { + Self::terminate_command_process_group(pid).await; + } + return Err(anyhow::anyhow!("timed out executing {}", program.display())); + } + }; if output.status.success() { return Ok(output); diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs index 3467962e..dab5db73 100644 --- a/application/src/server/executor/pve_lxc/cli/tests.rs +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -1,5 +1,21 @@ use super::*; +#[cfg(target_os = "linux")] +#[tokio::test] +async fn command_runner_captures_stdout_and_stderr_after_spawn() { + let args = vec![ + "-c".to_string(), + "printf stdout-value; printf stderr-value >&2".to_string(), + ]; + + let output = PveCli::run(std::path::Path::new("/bin/sh"), &args) + .await + .expect("command should succeed"); + + assert_eq!(output.stdout, b"stdout-value"); + assert_eq!(output.stderr, b"stderr-value"); +} + #[test] fn create_arguments_leave_data_mount_for_post_create_configuration() { let args = PveCli::create_args(&CreateContainerSpec { diff --git a/application/src/server/executor/pve_lxc/server_process.rs b/application/src/server/executor/pve_lxc/server_process.rs index 1e77648a..279df831 100644 --- a/application/src/server/executor/pve_lxc/server_process.rs +++ b/application/src/server/executor/pve_lxc/server_process.rs @@ -19,6 +19,10 @@ pub(super) struct PveProcessHandle { impl PveProcessHandle { const EXIT_STATUS_FILE: &'static str = ".wings-exit-status"; + // PVE's host-managed DHCP can lag CT init by several minutes during + // replacement starts. Keep the entrypoint alive for four real minutes. + pub(super) const DHCP_ROUTE_WAIT_INTERVAL_MILLIS: u32 = 250; + pub(super) const DHCP_ROUTE_WAIT_ATTEMPTS: u32 = 1920; pub(super) fn observed_status( status: cli::ContainerStatus, @@ -818,14 +822,14 @@ impl PveProcessHandle { stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); let entrypoint_script = r#"#!/bin/sh attempt=0 -while [ "$attempt" -lt 480 ]; do +while [ "$attempt" -lt __WINGS_DHCP_WAIT_ATTEMPTS__ ]; do if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then break fi attempt=$((attempt + 1)) sleep 0.25 done -if [ "$attempt" -ge 480 ]; then +if [ "$attempt" -ge __WINGS_DHCP_WAIT_ATTEMPTS__ ]; then echo 'timed out waiting for Proxmox DHCP' >&2 exit 1 fi @@ -890,6 +894,10 @@ temporary="${marker}.tmp.$$" mv -f "$temporary" "$marker" exit "$status" "# + .replace( + "__WINGS_DHCP_WAIT_ATTEMPTS__", + &Self::DHCP_ROUTE_WAIT_ATTEMPTS.to_string(), + ) .replace( "__WINGS_EXIT_STATUS_PATH__", &format!( diff --git a/application/src/server/executor/pve_lxc/tests.rs b/application/src/server/executor/pve_lxc/tests.rs index 63126300..c51b2d50 100644 --- a/application/src/server/executor/pve_lxc/tests.rs +++ b/application/src/server/executor/pve_lxc/tests.rs @@ -1,5 +1,16 @@ use super::*; +#[test] +fn dhcp_route_wait_allows_the_configured_four_minutes() { + assert_eq!(PveProcessHandle::DHCP_ROUTE_WAIT_INTERVAL_MILLIS, 250); + assert_eq!(PveProcessHandle::DHCP_ROUTE_WAIT_ATTEMPTS, 1920); + assert_eq!( + u64::from(PveProcessHandle::DHCP_ROUTE_WAIT_INTERVAL_MILLIS) + * u64::from(PveProcessHandle::DHCP_ROUTE_WAIT_ATTEMPTS), + 480_000 + ); +} + #[test] fn successful_start_followed_by_first_stopped_probe_emits_stopped() { let mut seen_running = false;