diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..df5c1c04 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,42 @@ +name: Test + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + linux: + name: Linux tests + runs-on: ubuntu-24.04 + + steps: + - name: Checkout sources + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v2 + with: + toolchain: 1.97.0 + target: x86_64-unknown-linux-gnu + components: rustfmt, clippy + cache-key: x86_64-unknown-linux-gnu-test + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Run Clippy + run: cargo clippy -p wings-rs --bin wings-rs -- -D warnings + + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross --rev 8c1a8aa4b661711f4b7b6ac07c2e8929ce2f7d27 --locked + + - name: Run Linux tests + run: cross test --release --target x86_64-unknown-linux-gnu + env: + JEMALLOC_SYS_WITH_LG_PAGE: 12 diff --git a/Cargo.lock b/Cargo.lock index 29df9d38..74c8b948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6204,6 +6204,7 @@ dependencies = [ "sevenz-rust2", "sha1 0.11.0", "sha2", + "shell-words", "sntpc", "sntpc-net-tokio", "sysinfo", diff --git a/application/Cargo.toml b/application/Cargo.toml index f079f240..f598fc9f 100644 --- a/application/Cargo.toml +++ b/application/Cargo.toml @@ -24,7 +24,7 @@ utoipa = { version = "5.4.0", features = ["axum_extras", "preserve_order", "chro utoipa-axum = "0.2.0" reqwest = { workspace = true } url = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["io-util"] } chrono = { version = "0.4.40", features = ["serde"] } colored = "3.0.0" jsonwebtoken = { version = "11.0.0", features = ["aws_lc_rs"] } @@ -59,7 +59,7 @@ dialoguer = { version = "0.12.0", default-features = false } sysinfo = { version = "0.39.0", default-features = false, features = ["disk", "network", "system", "user"] } crc32fast = { workspace = true } cap-std = "4.0.0" -rustix = { version = "1.0.7", features = ["fs", "process", "net", "thread"] } +rustix = { version = "1.0.7", features = ["fs", "process", "net", "thread", "pty"] } cidr = { version = "0.3.1", features = ["serde"] } hickory-resolver = { workspace = true } constant_time_eq = "0.6.0" @@ -73,6 +73,7 @@ zstd = { workspace = true } base64 = "0.23.0" chunked_vec = "=0.5.1" async-channel = "2.5.0" +shell-words = "1.1.0" lzzzz = "2.0.0" xmltree = { version = "0.12.0", features = ["attribute-order"] } xml = "1.3.0" diff --git a/application/build.rs b/application/build.rs index 75b00df1..7bd6b35b 100644 --- a/application/build.rs +++ b/application/build.rs @@ -191,6 +191,8 @@ fn handle_git_info() { { git_hash = String::from_utf8_lossy(&output.stdout).trim().to_string(); } + } else { + println!("cargo:rustc-env=CARGO_GIT_BRANCH=unknown"); } println!("cargo:rustc-env=CARGO_GIT_COMMIT={git_hash}"); } diff --git a/application/src/config.rs b/application/src/config.rs index 7a2bae45..fe48da76 100644 --- a/application/src/config.rs +++ b/application/src/config.rs @@ -11,7 +11,7 @@ use std::{ io::{BufRead, BufWriter, IsTerminal}, path::{Path, PathBuf}, str::FromStr, - sync::Arc, + sync::{Arc, OnceLock}, }; use tracing::level_filters::LevelFilter; use tracing_subscriber::{ @@ -25,6 +25,9 @@ use utoipa::ToSchema; fn app_name() -> String { "Calagopus".to_string() } +fn default_true() -> bool { + true +} fn api_host() -> String { "0.0.0.0".to_string() } @@ -726,6 +729,93 @@ impl From for SystemPath { } } +#[derive(ToSchema, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Debug, Default)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeBackend { + #[default] + Auto, + Docker, + PveLxc, +} + +#[derive(ToSchema, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Debug, Default)] +#[serde(rename_all = "snake_case")] +pub enum PveLxcFirewallBackend { + #[default] + Auto, + Proxmox, + Nftables, + Iptables, + Disabled, +} + +fn pve_lxc_pct_path() -> String { + "/usr/sbin/pct".to_string() +} + +fn pve_lxc_pvesh_path() -> String { + "/usr/bin/pvesh".to_string() +} + +fn pve_lxc_pveversion_path() -> String { + "/usr/bin/pveversion".to_string() +} +fn pve_lxc_pvesm_path() -> String { + "/usr/sbin/pvesm".to_string() +} + +fn pve_lxc_lxc_attach_path() -> String { + "/usr/bin/lxc-attach".to_string() +} + +fn pve_lxc_lxc_stop_path() -> String { + "/usr/bin/lxc-stop".to_string() +} + +fn pve_lxc_perl_path() -> String { + "/usr/bin/perl".to_string() +} + +fn pve_lxc_template_storage() -> String { + "local".to_string() +} + +fn pve_lxc_rootfs_storage() -> String { + "auto".to_string() +} + +fn pve_lxc_bridge() -> String { + "vmbr0".to_string() +} + +fn pve_lxc_edge_wireguard_interface() -> String { + "wg-calagopus".to_string() +} + +fn pve_lxc_edge_ssh_user() -> String { + "root".to_string() +} + +fn pve_lxc_rootfs_size_gib() -> u64 { + 8 +} + +fn pve_lxc_min_vmid() -> u32 { + 200 +} + +fn pve_lxc_console_log_max_bytes() -> u64 { + 5 * 1024 * 1024 +} + +fn pve_lxc_tag_prefix() -> String { + "calagopus".to_string() +} + +fn pve_lxc_managed_file_directory() -> SystemPath { + SystemPath::new("/var/lib/lxc/calagopus-wings-managed") +} + nestify::nest! { #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] pub struct InnerConfig { @@ -741,6 +831,99 @@ nestify::nest! { #[serde(default)] pub token: String, + #[serde(default)] + #[schema(inline)] + pub runtime: #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct Runtime { + #[serde(default)] + pub backend: RuntimeBackend, + + #[serde(default)] + #[schema(inline)] + pub pve_lxc: #[derive(Clone, ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct PveLxcRuntime { + #[serde(default = "pve_lxc_pct_path")] + pub pct_path: String, + #[serde(default = "pve_lxc_pvesh_path")] + pub pvesh_path: String, + #[serde(default = "pve_lxc_pveversion_path")] + pub pveversion_path: String, + #[serde(default = "pve_lxc_pvesm_path")] + pub pvesm_path: String, + #[serde(default = "pve_lxc_lxc_attach_path")] + pub lxc_attach_path: String, + #[serde(default = "pve_lxc_lxc_stop_path")] + pub lxc_stop_path: String, + #[serde(default = "pve_lxc_perl_path")] + pub perl_path: String, + + /// Empty means discover the local Proxmox node name at runtime. + #[serde(default)] + pub node: String, + #[serde(default = "pve_lxc_template_storage")] + pub template_storage: String, + #[serde(default = "pve_lxc_rootfs_storage")] + pub rootfs_storage: String, + #[serde(default = "pve_lxc_bridge")] + pub bridge: String, + /// Optional VLAN tag applied to each LXC veth on `bridge`. + #[serde(default)] + pub vlan_tag: Option, + /// IPv4 CIDR prefix used by static panel allocations (for example, 24). + #[serde(default)] + pub network_prefix: Option, + /// IPv4 gateway used by static panel allocations on the LXC network. + #[serde(default)] + pub gateway: Option, + /// WireGuard interface whose peers publish private LXC allocations. + /// The panel allocation IP is matched against each peer endpoint. + #[serde(default = "pve_lxc_edge_wireguard_interface")] + pub edge_wireguard_interface: String, + /// SSH account used by the restricted edge forwarding key. + #[serde(default = "pve_lxc_edge_ssh_user")] + pub edge_ssh_user: String, + /// Restricted SSH key used only to synchronize edge port forwards. + #[serde(default)] + pub edge_ssh_identity_path: String, + /// Pinned SSH known-hosts file for the edge host. + #[serde(default)] + pub edge_known_hosts_path: String, + #[serde(default = "pve_lxc_rootfs_size_gib")] + pub rootfs_size_gib: u64, + /// Maximum retained console bytes for each LXC server. When the + /// limit is reached Wings starts a fresh bounded log. + #[serde(default = "pve_lxc_console_log_max_bytes")] + pub console_log_max_bytes: u64, + #[serde(default = "pve_lxc_tag_prefix")] + pub tag_prefix: String, + /// Host-visible staging directory used for read-only managed file binds. + /// Its parent path must be traversable by unprivileged LXC mount setup. + #[serde(default = "pve_lxc_managed_file_directory")] + pub managed_file_directory: SystemPath, + #[serde(default = "default_true")] + pub unprivileged: bool, + /// Maximum process count for LXC workloads. When omitted, the + /// legacy Docker limit is used for configuration compatibility. + #[serde(default)] + pub pids_limit: Option, + /// Minimum VMID used when Wings auto-allocates Proxmox + /// containers. PVE itself starts at 100; raising the floor + /// (for example to 200) keeps Wings-owned guests clear of + /// manually created infrastructure containers. + #[serde(default = "pve_lxc_min_vmid")] + pub min_vmid: u32, + + #[serde(default)] + #[schema(inline)] + pub firewall: #[derive(Clone, ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct PveLxcFirewall { + #[serde(default)] + pub backend: PveLxcFirewallBackend, + #[serde(default = "docker_firewall_source_file_max_entries")] + pub source_file_max_entries: u64, + #[serde(default = "docker_firewall_source_file_max_bytes")] + pub source_file_max_bytes: u64, + }, + }, + }, + #[serde(default)] #[schema(inline)] pub api: #[derive(ToSchema, Deserialize, Serialize, DefaultFromSerde)] #[serde(default)] pub struct Api { @@ -1578,6 +1761,7 @@ fn log_filter(debug: bool) -> Targets { pub struct Config { inner: ArcSwap, + active_runtime_backend: OnceLock, log_reload_handle: ReloadHandle, pub path: String, @@ -1715,6 +1899,7 @@ impl Config { let config = Arc::new(Self { inner: ArcSwap::new(Arc::new(inner)), + active_runtime_backend: OnceLock::new(), log_reload_handle, path: path.to_string(), @@ -1735,6 +1920,7 @@ impl Config { Self { inner: ArcSwap::new(Arc::new(inner)), + active_runtime_backend: OnceLock::new(), log_reload_handle: tracing_subscriber::reload::Layer::new(log_filter(false)).1, path: String::new(), ignore_certificate_errors: false, @@ -1749,6 +1935,26 @@ impl Config { self.inner.load() } + /// Runtime selected for this Wings process. Automatic detection is kept in + /// memory so starting Wings never rewrites the operator's configuration. + pub fn active_runtime_backend(&self) -> RuntimeBackend { + self.active_runtime_backend + .get() + .copied() + .unwrap_or_else(|| self.load().runtime.backend) + } + + pub fn set_active_runtime_backend( + &self, + backend: RuntimeBackend, + ) -> Result<(), RuntimeBackend> { + match self.active_runtime_backend.set(backend) { + Ok(()) => Ok(()), + Err(backend) if self.active_runtime_backend.get() == Some(&backend) => Ok(()), + Err(backend) => Err(backend), + } + } + pub fn replace(&self, mut new: InnerConfig) -> Result<(), anyhow::Error> { if Self::migrate_legacy_limits(&mut new) { tracing::warn!("{LEGACY_LIMITS_WARNING}"); @@ -2643,4 +2849,23 @@ mod tests { assert!(Config::migrate_legacy_limits(&mut cfg)); assert_eq!(cfg.limits.server_concurrent_pulls, 5); } + + #[tokio::test] + async fn active_runtime_backend_is_process_local_and_idempotent() { + let config = Config::mock(); + assert_eq!(config.active_runtime_backend(), RuntimeBackend::Auto); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::PveLxc), + Ok(()) + ); + assert_eq!(config.active_runtime_backend(), RuntimeBackend::PveLxc); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::PveLxc), + Ok(()) + ); + assert_eq!( + config.set_active_runtime_backend(RuntimeBackend::Docker), + Err(RuntimeBackend::Docker) + ); + } } diff --git a/application/src/main.rs b/application/src/main.rs index 3bfbd046..f82b1153 100644 --- a/application/src/main.rs +++ b/application/src/main.rs @@ -1,4 +1,4 @@ -use crate::{response::ApiResponse, routes::GetState, server::executor::ServerExecutor}; +use crate::{response::ApiResponse, routes::GetState}; use anyhow::Context; use axum::{ body::Body, @@ -493,44 +493,12 @@ async fn main_rt() { Ok::<_, anyhow::Error>(()) }); - tracing::info!("connecting to docker"); - let (executor, docker) = { - let config_ref = config.load(); - let docker = Arc::new( - match if config_ref.docker.socket.starts_with("http://") - || config_ref.docker.socket.starts_with("tcp://") - { - bollard::Docker::connect_with_http( - &config_ref.docker.socket, - 120, - bollard::API_DEFAULT_VERSION, - ) - } else { - bollard::Docker::connect_with_local( - &config_ref.docker.socket, - 120, - bollard::API_DEFAULT_VERSION, - ) - } { - Ok(docker) => docker, - Err(err) => exit_error!("failed to connect to docker: {:?}", err), - }, - ); - - let own_container = - crate::server::executor::docker::DockerExecutor::own_container(&docker).await; - let firewall = - crate::server::firewall::create(&config, &docker, own_container.as_ref()).await; - - ( - Arc::new(crate::server::executor::docker::DockerExecutor::new( - Arc::clone(&docker), - config.clone(), - firewall, - )), - docker, - ) + let runtime = match crate::server::executor::create_runtime(Arc::clone(&config)).await { + Ok(runtime) => runtime, + Err(err) => exit_error!("failed to initialize server runtime: {:#}", err), }; + let executor = runtime.executor; + let docker = runtime.docker; tracing::info!("running server executor boot tasks"); if let Err(err) = executor.boot().await { @@ -579,7 +547,7 @@ async fn main_rt() { #[cfg(unix)] let tundra = if config.load().tundra.enabled { - match crate::tundra::TundraManager::create(&config, Arc::clone(&docker)) { + match crate::tundra::TundraManager::create(&config, docker.as_ref().map(Arc::clone)) { Ok(tundra) => Some(tundra), Err(err) => exit_error!("failed to set up the tundra control plane: {:?}", err), } @@ -603,7 +571,7 @@ async fn main_rt() { config: Arc::clone(&config), executor, - stats_manager: Arc::new(crate::stats::StatsManager::default()), + stats_manager: Arc::new(crate::stats::StatsManager::new(Arc::clone(&config))), server_manager: Arc::new(crate::server::manager::ServerManager::new(&servers)), backup_manager: Arc::new(crate::server::backup::manager::BackupManager::default()), inotify_manager: Arc::new(crate::server::filesystem::inotify::InotifyManager::new()), diff --git a/application/src/server/configuration/mod.rs b/application/src/server/configuration/mod.rs index 41735976..7b76eb0e 100644 --- a/application/src/server/configuration/mod.rs +++ b/application/src/server/configuration/mod.rs @@ -746,7 +746,7 @@ impl ServerConfiguration { ), }; - if self.build.oom_disabled && crate::server::executor::docker::cgroup::is_unified() { + if self.build.oom_disabled && crate::server::executor::cgroup::is_unified() { tracing::warn!( server = %self.uuid, "oom_disabled is set, but the container engine discards it on cgroup v2 hosts" diff --git a/application/src/server/executor/docker/cgroup.rs b/application/src/server/executor/cgroup.rs similarity index 100% rename from application/src/server/executor/docker/cgroup.rs rename to application/src/server/executor/cgroup.rs diff --git a/application/src/server/executor/docker/mod.rs b/application/src/server/executor/docker/mod.rs index 142c6560..05fb0433 100644 --- a/application/src/server/executor/docker/mod.rs +++ b/application/src/server/executor/docker/mod.rs @@ -17,7 +17,7 @@ use std::{ }; use tokio::io::{AsyncWriteExt, ReadBuf}; -pub mod cgroup; +use super::cgroup; pub mod host_mounts; #[cfg(unix)] pub mod lxcfs; @@ -488,6 +488,7 @@ impl DockerServerConfigurationExt for crate::server::configuration::ServerConfig crate::server::firewall::FirewallServerSpec { server: self.uuid, + target: None, bindings: bindings.into_iter().collect(), container_ports: container_ports.into_iter().collect(), container_ips: Vec::new(), @@ -2918,6 +2919,13 @@ impl super::ServerExecutor for DockerExecutor { Ok(()) } + async fn reconcile_firewall( + &self, + servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + DockerExecutor::reconcile_firewall(self, servers).await + } + async fn setup_server_process( &self, server: &super::super::Server, diff --git a/application/src/server/executor/mod.rs b/application/src/server/executor/mod.rs index 3e52145f..ed018f85 100644 --- a/application/src/server/executor/mod.rs +++ b/application/src/server/executor/mod.rs @@ -1,9 +1,193 @@ +use anyhow::Context; use serde::Serialize; -use std::{collections::HashMap, net::IpAddr, sync::Arc}; +use std::{collections::HashMap, net::IpAddr, path::Path, process::Stdio, sync::Arc}; use utoipa::ToSchema; +pub mod cgroup; pub mod docker; pub mod noop; +pub mod pve_lxc; + +pub struct Runtime { + pub executor: Arc, + /// Transitional escape hatch for Docker-only subsystems such as Tundra. + /// New runtime-neutral code should use `executor` instead. + pub docker: Option>, +} + +pub async fn create_runtime(config: Arc) -> Result { + let backend = resolve_backend(&config).await?; + config + .set_active_runtime_backend(backend) + .map_err(|active| anyhow::anyhow!( + "runtime backend was already resolved as {active:?}, cannot replace it with {backend:?}" + ))?; + + match backend { + crate::config::RuntimeBackend::Auto => Err(anyhow::anyhow!( + "automatic runtime selection was not resolved" + )), + crate::config::RuntimeBackend::Docker => { + tracing::info!("connecting to docker"); + let docker = Arc::new(docker_client(&config.load().docker.socket)?); + + let own_container = docker::DockerExecutor::own_container(&docker).await; + let firewall = + crate::server::firewall::create(&config, &docker, own_container.as_ref()).await; + let executor: Arc = Arc::new(docker::DockerExecutor::new( + Arc::clone(&docker), + config, + firewall, + )); + + Ok(Runtime { + executor, + docker: Some(docker), + }) + } + crate::config::RuntimeBackend::PveLxc => { + tracing::info!("initializing Proxmox VE LXC runtime"); + let cli = { + let snapshot = config.load(); + pve_lxc::PveCli::from_config(&snapshot.runtime.pve_lxc) + }; + let firewall = pve_lxc::create_firewall(&config, &cli).await?; + let executor: Arc = + Arc::new(pve_lxc::PveLxcExecutor::new(config, cli, firewall)); + + Ok(Runtime { + executor, + docker: None, + }) + } + } +} + +fn docker_client(socket: &str) -> Result { + if socket.starts_with("http://") || socket.starts_with("tcp://") { + bollard::Docker::connect_with_http(socket, 120, bollard::API_DEFAULT_VERSION) + } else { + bollard::Docker::connect_with_local(socket, 120, bollard::API_DEFAULT_VERSION) + } + .context("failed to connect to docker") +} + +async fn docker_available(socket: &str) -> bool { + match docker_client(socket) { + Ok(docker) => tokio::time::timeout(std::time::Duration::from_secs(3), docker.version()) + .await + .is_ok_and(|result| result.is_ok()), + Err(_) => false, + } +} + +async fn pve_lxc_available(runtime: &crate::config::PveLxcRuntime) -> bool { + if !Path::new(&runtime.pct_path).is_file() || !Path::new(&runtime.pveversion_path).is_file() { + return false; + } + + tokio::time::timeout( + std::time::Duration::from_secs(3), + tokio::process::Command::new(&runtime.pveversion_path) + .arg("--verbose") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .status(), + ) + .await + .ok() + .and_then(Result::ok) + .is_some_and(|status| status.success()) +} + +async fn resolve_backend( + config: &Arc, +) -> Result { + let snapshot = config.load(); + if snapshot.runtime.backend != crate::config::RuntimeBackend::Auto { + return Ok(snapshot.runtime.backend); + } + + let docker_socket = snapshot.docker.socket.clone(); + let pve_lxc = snapshot.runtime.pve_lxc.clone(); + drop(snapshot); + + let state_path = config + .resolve_as_path(|config| &config.system.root_directory) + .join("runtime-backend"); + match tokio::fs::read_to_string(&state_path).await { + Ok(value) => { + let selected = match value.trim() { + "docker" => crate::config::RuntimeBackend::Docker, + "pve_lxc" => crate::config::RuntimeBackend::PveLxc, + value => anyhow::bail!( + "persisted automatic runtime backend at {} is invalid: {value}", + state_path.display() + ), + }; + let available = match selected { + crate::config::RuntimeBackend::Docker => docker_available(&docker_socket).await, + crate::config::RuntimeBackend::PveLxc => pve_lxc_available(&pve_lxc).await, + crate::config::RuntimeBackend::Auto => false, + }; + if !available { + anyhow::bail!( + "persisted automatic runtime backend {selected:?} is unavailable; remove {} to run detection again", + state_path.display() + ); + } + tracing::info!(runtime_backend = ?selected, "using persisted automatic runtime backend"); + return Ok(selected); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read persisted automatic runtime backend {}", + state_path.display() + ) + }); + } + } + + let selected = if docker_available(&docker_socket).await { + crate::config::RuntimeBackend::Docker + } else if pve_lxc_available(&pve_lxc).await { + crate::config::RuntimeBackend::PveLxc + } else { + anyhow::bail!( + "runtime.backend is auto but neither Docker nor a local Proxmox VE LXC runtime is available" + ); + }; + + let temporary = state_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); + let value = match selected { + crate::config::RuntimeBackend::Docker => "docker\n", + crate::config::RuntimeBackend::PveLxc => "pve_lxc\n", + crate::config::RuntimeBackend::Auto => { + anyhow::bail!("automatic runtime selection returned auto") + } + }; + tokio::fs::write(&temporary, value).await.with_context(|| { + format!( + "failed to write automatic runtime backend state {}", + temporary.display() + ) + })?; + tokio::fs::rename(&temporary, &state_path) + .await + .with_context(|| { + format!( + "failed to persist automatic runtime backend state {}", + state_path.display() + ) + })?; + tracing::info!(runtime_backend = ?selected, "automatically selected and persisted runtime backend state"); + + Ok(selected) +} type StatusReceiver = tokio::sync::mpsc::Receiver; @@ -46,6 +230,13 @@ pub trait ProcessHandle: Send + Sync { pub trait ServerExecutor: Send + Sync { async fn boot(&self) -> Result<(), anyhow::Error>; + async fn reconcile_firewall( + &self, + _servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + Ok(()) + } + async fn setup_server_process( &self, server: &super::Server, diff --git a/application/src/server/executor/pve_lxc/cli.rs b/application/src/server/executor/pve_lxc/cli.rs new file mode 100644 index 00000000..cf10fbc8 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli.rs @@ -0,0 +1,2562 @@ +use anyhow::Context; +use flate2::read::GzDecoder; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + io::{Read, Write}, + net::IpAddr, + path::{Component, Path, PathBuf}, + process::Output, + process::Stdio, + time::Duration, +}; +use tokio::io::AsyncWriteExt; + +const TASK_POLL_INTERVAL: Duration = Duration::from_millis(500); +const TASK_TIMEOUT: Duration = Duration::from_secs(15 * 60); +const CONFIG_HELPER_TIMEOUT: Duration = Duration::from_secs(30); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(15 * 60); +const START_SETTLE_ATTEMPTS: usize = 20; +const START_SETTLE_DELAY: Duration = Duration::from_millis(500); +const MANAGED_FILE_TARGETS: [&str; 5] = [ + "/etc/machine-id", + "/etc/hosts", + "/etc/passwd", + "/etc/group", + "/calagopus-entrypoint", +]; + +mod scripts; + +use scripts::{APPLY_FIREWALL_CONFIG_PERL, APPLY_RUNTIME_CONFIG_PERL}; + +#[derive(Debug, Clone)] +pub struct PveCli { + pct_path: PathBuf, + pvesh_path: PathBuf, + pveversion_path: PathBuf, + pvesm_path: PathBuf, + #[allow(dead_code)] + lxc_attach_path: PathBuf, + lxc_stop_path: PathBuf, + perl_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateContainerSpec { + pub vmid: u32, + pub template: String, + pub hostname: String, + pub rootfs_storage: String, + pub rootfs_size_gib: u64, + pub memory_mib: u64, + pub swap_mib: u64, + /// Panel CPU limit expressed as a percentage where 100 means one CPU of time. + pub cpu_limit_percent: Option, + pub cores: Option, + /// Complete Proxmox `net0` value, including static allocation details when + /// a server has a primary panel allocation. + pub network: String, + pub tags: Vec, + pub unprivileged: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataMountSpec { + pub vmid: u32, + pub data_path: String, + /// UID used by the game process inside the container. + pub container_uid: u32, + /// GID used by the game process inside the container. + pub container_gid: u32, + /// UID that owns the server data on the Proxmox host. + pub host_data_uid: u32, + /// GID that owns the server data on the Proxmox host. + pub host_data_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostBindMountSpec { + pub slot: u8, + pub source_path: String, + pub target_path: String, + pub read_only: bool, + /// UID that owns the bind source on the Proxmox host. + pub host_uid: u32, + /// GID that owns the bind source on the Proxmox host. + pub host_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BindMountSpec { + pub slot: u8, + pub source_path: String, + pub target_path: String, + pub read_only: bool, + pub container_uid: u32, + pub container_gid: u32, + pub host_uid: u32, + pub host_gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostDevicePassthroughSpec { + pub slot: u8, + pub path: String, + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub deny_write: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DevicePassthroughSpec { + pub slot: u8, + pub path: String, + pub uid: u32, + pub gid: u32, + pub mode: u32, + pub deny_write: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerOciUser { + pub uid: u32, + pub gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContainerResources { + pub memory_mib: u64, + pub swap_mib: u64, + pub memory_unlimited: bool, + pub swap_unlimited: bool, + pub cpu_limit_percent: Option, + pub cores: Option, + pub cpuset_cpus: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AttachProcessSpec { + pub vmid: u32, + pub uid: u32, + pub gid: u32, + pub environment: Vec, + pub command: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedFileMountSpec { + pub source_path: String, + pub target_path: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeConfigSpec { + pub vmid: u32, + pub environment: Vec, + pub halt_signal: Option, + pub init_command: Option, + pub console_logfile: Option, + /// Exact host CPUs assigned through the panel's `build.threads` setting. + /// `None` removes a previously managed CPU set. + pub cpuset_cpus: Option, + /// Maximum process count for the container. `None` removes the managed + /// cgroup limit. + pub pids_limit: Option, + /// cgroup v2 I/O weight in the kernel's 1..=10000 range. + pub io_weight: Option, + pub memory_unlimited: bool, + pub swap_unlimited: bool, + /// `None` leaves low-level LXC file mounts untouched. `Some`, including an + /// empty vector, reconciles all Wings-reserved managed-file targets. + pub managed_file_mounts: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct FirewallRuleSpec { + pub r#type: String, + pub action: String, + pub iface: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub proto: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dport: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + pub comment: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct FirewallPolicySpec { + pub vmid: u32, + pub managed_prefix: String, + pub rules: Vec, + pub ipsets: std::collections::BTreeMap>, +} + +pub struct ConsoleSession { + pub child: tokio::process::Child, + pub reader: tokio::fs::File, + pub writer: tokio::fs::File, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClusterContainer { + pub vmid: u32, + pub node: String, + pub status: Option, + pub tags: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerRuntimeStatus { + pub status: ContainerStatus, + pub pid: Option, + pub uptime_seconds: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeContainerRuntimeStatus { + pub vmid: u32, + pub status: ContainerStatus, + pub pid: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContainerInterface { + pub name: String, + pub addresses: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageContent { + pub volid: String, + pub content: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LxcStorage { + pub name: String, + pub available_bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OciTemplatePullSpec { + pub node: String, + pub storage: String, + pub reference: String, + pub filename: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OciTemplate { + pub volid: String, + pub revision: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskState { + Running, + Stopped, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TaskStatus { + pub state: TaskState, + pub exit_status: Option, +} + +impl PveCli { + pub fn from_config(config: &crate::config::PveLxcRuntime) -> Self { + Self { + pct_path: config.pct_path.clone().into(), + pvesh_path: config.pvesh_path.clone().into(), + pveversion_path: config.pveversion_path.clone().into(), + pvesm_path: config.pvesm_path.clone().into(), + lxc_attach_path: config.lxc_attach_path.clone().into(), + lxc_stop_path: config.lxc_stop_path.clone().into(), + perl_path: config.perl_path.clone().into(), + } + } + + pub fn panel_resources( + memory_limit: i64, + overhead_memory: i64, + swap: i64, + cpu_limit: i64, + threads: Option<&str>, + ) -> Result { + if memory_limit < 0 { + return Err(anyhow::anyhow!( + "invalid negative memory limit for Proxmox LXC: {memory_limit}" + )); + } + if overhead_memory < 0 { + return Err(anyhow::anyhow!( + "server overhead memory cannot be negative for the Proxmox LXC runtime" + )); + } + let memory_unlimited = memory_limit == 0; + let memory_mib = if memory_unlimited { + // PVE requires a regular memory property even when a low-level LXC + // cgroup override removes the limit before the first start. + 16 + } else { + let memory_mib = memory_limit + .checked_add(overhead_memory) + .context("server memory plus overhead overflowed")?; + if memory_mib < 16 { + return Err(anyhow::anyhow!( + "Proxmox LXC requires at least 16 MiB of memory, resolved server limit was {memory_mib} MiB" + )); + } + u64::try_from(memory_mib).context("server memory limit was negative")? + }; + + let swap_unlimited = memory_unlimited || swap == -1; + let swap_mib = match swap { + -1 => 0, + value if value < 0 => { + return Err(anyhow::anyhow!( + "invalid negative swap limit for Proxmox LXC: {value}" + )); + } + value => u64::try_from(value).context("server swap limit exceeded u64")?, + }; + + let cpu_limit_percent = match cpu_limit { + value if value < 0 => { + return Err(anyhow::anyhow!( + "invalid negative CPU limit for Proxmox LXC: {value}" + )); + } + 0 => None, + value => Some(u64::try_from(value).context("server CPU limit exceeded u64")?), + }; + + let cpuset = threads + .filter(|threads| !threads.trim().is_empty()) + .map(Self::normalize_cpuset_with_count) + .transpose()?; + + Ok(ContainerResources { + memory_mib, + swap_mib, + memory_unlimited, + swap_unlimited, + cpu_limit_percent, + cores: cpuset.as_ref().map(|(_, count)| *count), + cpuset_cpus: cpuset.map(|(cpuset, _)| cpuset), + }) + } + + pub(crate) fn normalize_cpuset(value: &str) -> Result { + Self::normalize_cpuset_with_count(value).map(|(cpuset, _)| cpuset) + } + + fn normalize_cpuset_with_count(value: &str) -> Result<(String, u64), anyhow::Error> { + let value = value.trim(); + if value.is_empty() { + return Err(anyhow::anyhow!("CPU set cannot be empty")); + } + + let mut ranges = Vec::new(); + for part in value.split(',') { + let part = part.trim(); + if part.is_empty() { + return Err(anyhow::anyhow!("CPU set contains an empty item")); + } + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start, Some(end)), + None => (part, None), + }; + let start = start + .parse::() + .with_context(|| format!("invalid CPU set item: {part}"))?; + if let Some(end) = end { + let end = end + .parse::() + .with_context(|| format!("invalid CPU set item: {part}"))?; + if end < start { + return Err(anyhow::anyhow!("invalid descending CPU set range: {part}")); + } + ranges.push((start, end)); + } else { + ranges.push((start, start)); + } + } + + ranges.sort_unstable(); + let mut count = 0u64; + let mut merged: Option<(u32, u32)> = None; + for (start, end) in ranges { + match merged { + Some((merged_start, merged_end)) if start <= merged_end.saturating_add(1) => { + merged = Some((merged_start, merged_end.max(end))); + } + Some((merged_start, merged_end)) => { + count = count.saturating_add(u64::from(merged_end - merged_start) + 1); + merged = Some((start, end)); + } + None => merged = Some((start, end)), + } + } + if let Some((start, end)) = merged { + count = count.saturating_add(u64::from(end - start) + 1); + } + + Ok((value.to_string(), count)) + } + + pub(crate) fn cgroup2_io_weight(weight: Option) -> Result, anyhow::Error> { + let Some(weight) = weight else { + return Ok(None); + }; + if !(10..=1000).contains(&weight) { + return Err(anyhow::anyhow!( + "block I/O weight must be between 10 and 1000, got {weight}" + )); + } + + // Match opencontainers/cgroups' conversion from the cgroup v1 API + // range used by Docker to the cgroup v2 io.weight range. + Ok(Some(1 + (u64::from(weight) - 10) * 9999 / 990)) + } + + #[cfg(target_os = "linux")] + fn isolate_command_process_group(command: &mut tokio::process::Command) { + use std::os::unix::process::CommandExt; + + // Proxmox's `pct` client starts a task subprocess. `kill_on_drop` only + // kills the direct client, which used to leave its task child holding + // pve-config-.lock after a Wings timeout or restart. A dedicated + // session makes the client's PID the process-group leader so timeout + // cleanup can terminate the complete task tree. + unsafe { + command.as_std_mut().pre_exec(|| { + rustix::process::setsid()?; + Ok(()) + }); + } + } + + #[cfg(target_os = "linux")] + async fn terminate_command_process_group(pid: u32) { + use rustix::process::{Pid, Signal, kill_process_group}; + + let Some(group) = Pid::from_raw(pid as i32) else { + return; + }; + let _ = kill_process_group(group, Signal::TERM); + tokio::time::sleep(Duration::from_secs(1)).await; + let _ = kill_process_group(group, Signal::KILL); + } + + async fn run(program: &std::path::Path, args: &[String]) -> Result { + let mut command = tokio::process::Command::new(program); + // `wait_with_output` only captures streams explicitly piped before + // `spawn`; unlike `Command::output`, bare `spawn` inherits them. + // PVE commands emit JSON on stdout, which must not leak to journald + // or be lost before the caller parses it. + command + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + Self::isolate_command_process_group(&mut command); + + let child = command + .spawn() + .with_context(|| format!("failed to execute {}", program.display()))?; + let pid = child.id(); + let output = match tokio::time::timeout(COMMAND_TIMEOUT, child.wait_with_output()).await { + Ok(result) => { + result.with_context(|| format!("failed to execute {}", program.display()))? + } + Err(_) => { + #[cfg(target_os = "linux")] + if let Some(pid) = pid { + Self::terminate_command_process_group(pid).await; + } + return Err(anyhow::anyhow!("timed out executing {}", program.display())); + } + }; + + if output.status.success() { + return Ok(output); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "{} exited with {}: {}", + program.display(), + output.status, + stderr.trim() + )) + } + + async fn run_with_stdin( + program: &std::path::Path, + args: &[String], + input: &[u8], + ) -> Result { + let mut child = tokio::process::Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("failed to execute {}", program.display()))?; + + let mut stdin = child + .stdin + .take() + .context("spawned Proxmox helper did not expose stdin")?; + let operation = async move { + stdin.write_all(input).await?; + drop(stdin); + child.wait_with_output().await + }; + let output = tokio::time::timeout(CONFIG_HELPER_TIMEOUT, operation) + .await + .with_context(|| format!("timed out executing {}", program.display()))??; + if output.status.success() { + return Ok(output); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + Err(anyhow::anyhow!( + "{} exited with {}: {}", + program.display(), + output.status, + stderr.trim() + )) + } + + async fn run_perl_helper( + &self, + name: &str, + script: &str, + input: &[u8], + ) -> Result { + let directory = Path::new("/run/calagopus-wings/pve-helpers"); + tokio::fs::create_dir_all(directory) + .await + .context("failed to create Proxmox helper script directory")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(directory, std::fs::Permissions::from_mode(0o700)) + .await + .context("failed to secure Proxmox helper script directory")?; + } + + let path = directory.join(format!("{name}-{}.pl", uuid::Uuid::new_v4())); + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&path) + .await + .context("failed to create Proxmox helper script")?; + file.write_all(script.as_bytes()) + .await + .context("failed to write Proxmox helper script")?; + file.flush() + .await + .context("failed to flush Proxmox helper script")?; + drop(file); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .await + .context("failed to secure Proxmox helper script")?; + } + + let result = Self::run_with_stdin( + &self.perl_path, + &[path.to_string_lossy().into_owned()], + input, + ) + .await; + if let Err(error) = tokio::fs::remove_file(&path).await + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(path = %path.display(), "failed to remove Proxmox helper script: {error}"); + } + result + } + + fn runtime_config_payload(spec: &RuntimeConfigSpec) -> Result, anyhow::Error> { + for variable in &spec.environment { + let Some((name, _)) = variable.split_once('=') else { + return Err(anyhow::anyhow!( + "runtime environment entry must be KEY=VALUE: {variable}" + )); + }; + if name.is_empty() + || variable.contains('\0') + || variable.contains('\n') + || variable.contains('\r') + { + return Err(anyhow::anyhow!( + "runtime environment entry cannot be represented safely in an LXC config: {variable}" + )); + } + } + if spec + .halt_signal + .as_deref() + .is_some_and(|signal| signal.is_empty() || signal.chars().any(char::is_control)) + { + return Err(anyhow::anyhow!("invalid LXC halt signal")); + } + if spec.init_command.as_deref().is_some_and(|command| { + command.is_empty() + || command.contains('\0') + || command.contains('\n') + || command.contains('\r') + }) { + return Err(anyhow::anyhow!("invalid LXC init command")); + } + if spec.console_logfile.as_deref().is_some_and(|path| { + !path.starts_with('/') + || path.contains('\0') + || path.contains('\n') + || path.contains('\r') + }) { + return Err(anyhow::anyhow!("invalid LXC console logfile path")); + } + if let Some(cpuset) = spec.cpuset_cpus.as_deref() { + Self::normalize_cpuset(cpuset)?; + } + if spec.pids_limit == Some(0) { + return Err(anyhow::anyhow!("LXC PID limit must be positive")); + } + if spec + .io_weight + .is_some_and(|weight| !(1..=10_000).contains(&weight)) + { + return Err(anyhow::anyhow!( + "cgroup v2 I/O weight must be between 1 and 10000" + )); + } + + let (managed_file_mounts, managed_file_targets) = + if let Some(mounts) = &spec.managed_file_mounts { + let mut seen_targets = std::collections::HashSet::with_capacity(mounts.len()); + let mut encoded = Vec::with_capacity(mounts.len()); + + for mount in mounts { + if !MANAGED_FILE_TARGETS.contains(&mount.target_path.as_str()) { + return Err(anyhow::anyhow!( + "unsupported managed Proxmox file mount target: {}", + mount.target_path + )); + } + if !seen_targets.insert(mount.target_path.as_str()) { + return Err(anyhow::anyhow!( + "duplicate managed Proxmox file mount target: {}", + mount.target_path + )); + } + + let source = Self::encode_lxc_mount_field(&mount.source_path, false)?; + let target = Self::encode_lxc_mount_field(&mount.target_path, true)?; + encoded.push(serde_json::json!({ + "target": target, + "entry": format!("{source} {target} none bind,ro,create=file 0 0"), + })); + } + + let targets = MANAGED_FILE_TARGETS + .iter() + .map(|target| Self::encode_lxc_mount_field(target, true)) + .collect::, _>>()?; + (Some(encoded), Some(targets)) + } else { + (None, None) + }; + + serde_json::to_vec(&serde_json::json!({ + "vmid": spec.vmid, + "environment": spec.environment, + "halt_signal": spec.halt_signal, + "init_command": spec.init_command, + "console_logfile": spec.console_logfile, + "cpuset_cpus": spec.cpuset_cpus, + "pids_limit": spec.pids_limit, + "io_weight": spec.io_weight, + "memory_unlimited": spec.memory_unlimited, + "swap_unlimited": spec.swap_unlimited, + "managed_file_mounts": managed_file_mounts, + "managed_file_targets": managed_file_targets, + })) + .context("failed to serialize Proxmox runtime configuration") + } + + fn encode_lxc_mount_field(path: &str, relative_target: bool) -> Result { + if !path.starts_with('/') + || path.contains('\0') + || path.contains('\n') + || path.contains('\r') + { + return Err(anyhow::anyhow!( + "invalid LXC managed file mount path: {path}" + )); + } + + let value = if relative_target { + path.strip_prefix('/').unwrap_or(path) + } else { + path + }; + if value.is_empty() { + return Err(anyhow::anyhow!( + "invalid empty LXC managed file mount target" + )); + } + + let mut encoded = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\\' => encoded.push_str("\\134"), + ' ' => encoded.push_str("\\040"), + '\t' => encoded.push_str("\\011"), + _ => encoded.push(character), + } + } + Ok(encoded) + } + + /// LXC's `lxc.init.cmd` parser supports whole-word single or double quotes, + /// but does not support escape sequences inside a quoted word. + pub fn encode_init_command(arguments: &[String]) -> Result { + if arguments.is_empty() { + return Err(anyhow::anyhow!("LXC init command cannot be empty")); + } + + let mut encoded = Vec::with_capacity(arguments.len()); + for argument in arguments { + if argument.contains('\0') || argument.contains('\n') || argument.contains('\r') { + return Err(anyhow::anyhow!( + "LXC init command argument contains an unsupported control character" + )); + } + + if !argument.is_empty() + && !argument.chars().any(char::is_whitespace) + && !argument.starts_with('\'') + && !argument.starts_with('"') + { + encoded.push(argument.clone()); + } else if !argument.contains('"') { + encoded.push(format!("\"{argument}\"")); + } else if !argument.contains('\'') { + encoded.push(format!("'{argument}'")); + } else { + return Err(anyhow::anyhow!( + "LXC init command argument cannot be represented safely: {argument}" + )); + } + } + + Ok(encoded.join(" ")) + } + + /// Applies dynamic Wings runtime settings without putting the NUL-separated + /// PVE `env` value on a Unix argv. The helper edits the regular PVE LXC + /// config under Proxmox's own config lock, preserving all unrelated options. + pub async fn apply_runtime_config( + &self, + spec: &RuntimeConfigSpec, + ) -> Result<(), anyhow::Error> { + let payload = Self::runtime_config_payload(spec)?; + self.run_perl_helper("runtime-config", APPLY_RUNTIME_CONFIG_PERL, &payload) + .await?; + Ok(()) + } + + pub async fn apply_firewall_config( + &self, + spec: &FirewallPolicySpec, + ) -> Result<(), anyhow::Error> { + let payload = serde_json::to_vec(spec).context("failed to serialize Proxmox firewall")?; + self.run_perl_helper("firewall-config", APPLY_FIREWALL_CONFIG_PERL, &payload) + .await?; + Ok(()) + } + + pub fn request_shutdown_args(vmid: u32) -> Vec { + vec![ + "-n".to_string(), + vmid.to_string(), + "--nowait".to_string(), + "--nokill".to_string(), + ] + } + + /// Requests the container init process to stop and returns immediately. + /// `lxc.signal.halt` determines which signal init receives. + pub async fn request_shutdown(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run(&self.lxc_stop_path, &Self::request_shutdown_args(vmid)).await?; + Ok(()) + } + + pub fn console_args(vmid: u32) -> Vec { + vec![ + "console".to_string(), + vmid.to_string(), + "--escape".to_string(), + "^z".to_string(), + ] + } + + /// Starts `pct console` on a real pseudo-terminal. `pct console` is an + /// interactive terminal client and does not behave correctly when wired to + /// ordinary pipes. + #[cfg(target_os = "linux")] + pub fn spawn_console(&self, vmid: u32) -> Result { + use rustix::pty::{OpenptFlags, grantpt, ioctl_tiocgptpeer, openpt, unlockpt}; + use std::os::unix::process::CommandExt; + + let flags = OpenptFlags::RDWR | OpenptFlags::NOCTTY | OpenptFlags::CLOEXEC; + let controller = openpt(flags).context("failed to allocate Proxmox console PTY")?; + grantpt(&controller).context("failed to grant Proxmox console PTY")?; + unlockpt(&controller).context("failed to unlock Proxmox console PTY")?; + let user = ioctl_tiocgptpeer(&controller, flags) + .context("failed to open Proxmox console PTY peer")?; + + let stdin = std::fs::File::from(user); + let stdout = stdin.try_clone()?; + let stderr = stdin.try_clone()?; + let controller = std::fs::File::from(controller); + let controller_writer = controller + .try_clone() + .context("failed to clone Proxmox console PTY controller")?; + + let mut command = tokio::process::Command::new(&self.pct_path); + command + .args(Self::console_args(vmid)) + .stdin(Stdio::from(stdin)) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .kill_on_drop(true); + + // `pct console` is an interactive terminal client. Merely connecting + // its stdio to a PTY is enough to receive output, but keyboard input is + // ignored unless the slave is also the child's controlling terminal. + unsafe { + command.as_std_mut().pre_exec(|| { + rustix::process::setsid()?; + rustix::process::ioctl_tiocsctty(std::io::stdin())?; + Ok(()) + }); + } + + let child = command + .spawn() + .with_context(|| format!("failed to execute {} console", self.pct_path.display()))?; + + Ok(ConsoleSession { + child, + reader: tokio::fs::File::from_std(controller), + writer: tokio::fs::File::from_std(controller_writer), + }) + } + + #[cfg(not(target_os = "linux"))] + pub fn spawn_console(&self, _vmid: u32) -> Result { + Err(anyhow::anyhow!( + "Proxmox LXC console attachment is only supported on Linux" + )) + } + + pub async fn version(&self) -> Result { + let output = Self::run(&self.pveversion_path, &[]).await?; + Ok(String::from_utf8(output.stdout) + .context("pveversion returned non-UTF8 output")? + .trim() + .to_string()) + } + + pub async fn next_vmid(&self, min_vmid: u32) -> Result { + if !(100..=999_999_999).contains(&min_vmid) { + anyhow::bail!("Proxmox minimum VMID must be between 100 and 999999999, got {min_vmid}"); + } + let args = vec![ + "get".to_string(), + "/cluster/nextid".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + let value: serde_json::Value = serde_json::from_slice(&output.stdout) + .context("failed to parse Proxmox nextid response")?; + let raw = value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + .context("Proxmox nextid response was not a VMID")?; + let raw = u32::try_from(raw).context("Proxmox nextid response exceeded the VMID range")?; + if raw >= min_vmid { + return Ok(raw); + } + + // `/cluster/nextid` tracks the lowest free ID cluster-wide, which can + // sit below the Wings floor. Scan for the first free ID at or above + // the floor instead of squatting a low ID reserved for infrastructure. + // VMIDs are shared across QEMU and LXC guests, so every `vmid` in the + // inventory counts, not just LXC rows. + let used = self.allocated_vmids().await?; + Self::first_free_vmid_at_or_above(min_vmid, &used).with_context(|| { + format!("Proxmox VMID range above the minimum {min_vmid} is exhausted") + }) + } + + fn first_free_vmid_at_or_above( + min_vmid: u32, + used: &std::collections::HashSet, + ) -> Option { + let mut sorted: Vec = used.iter().copied().filter(|id| *id >= min_vmid).collect(); + sorted.sort_unstable(); + let mut candidate = min_vmid; + for id in sorted { + if id == candidate { + candidate = candidate.checked_add(1)?; + } else if id > candidate { + break; + } + if candidate > 999_999_999 { + return None; + } + } + (candidate <= 999_999_999).then_some(candidate) + } + + /// Every VMID in use cluster-wide, across all guest types. VMIDs are a + /// single shared namespace in Proxmox, so a QEMU VM squats the ID for LXC + /// too. + async fn allocated_vmids(&self) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + "/cluster/resources".to_string(), + "--type".to_string(), + "vm".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_allocated_vmids(&output.stdout) + } + + fn parse_allocated_vmids( + output: &[u8], + ) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster resources response")?; + let mut used = std::collections::HashSet::new(); + for row in rows { + let Some(raw) = row.get("vmid").and_then(|value| { + value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + }) else { + continue; + }; + if let Ok(vmid) = u32::try_from(raw) { + used.insert(vmid); + } + } + Ok(used) + } + + /// Serialize the advisory `nextid` lookup and the following create on this + /// PVE host. The PVE create operation remains the final cluster-wide + /// authority, so callers still retry a reported VMID collision. + pub async fn lock_vmid_allocation(&self) -> Result { + tokio::task::spawn_blocking(|| { + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open("/run/lock/calagopus-wings-pve-vmid.lock") + .context("failed to open Proxmox VMID allocation lock")?; + lock.lock() + .context("failed to acquire Proxmox VMID allocation lock")?; + Ok(lock) + }) + .await + .context("Proxmox VMID allocation lock task failed")? + } + + pub async fn local_node(&self) -> Result { + let args = vec![ + "get".to_string(), + "/cluster/status".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_local_node(&output.stdout) + } + + pub async fn cluster_firewall_enabled(&self) -> Result { + let args = vec![ + "get".to_string(), + "/cluster/firewall/options".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + let value: serde_json::Value = serde_json::from_slice(&output.stdout) + .context("failed to parse Proxmox cluster firewall options")?; + Ok(value + .get("enable") + .and_then(|value| value.as_u64()) + .is_some_and(|value| value != 0) + || value + .get("enable") + .and_then(|value| value.as_bool()) + .unwrap_or(false)) + } + + /// Lists active LXC rootfs storages ordered by free capacity, so callers can + /// place a container on the largest pool that can satisfy its requested size. + pub async fn available_lxc_storages( + &self, + node: &str, + ) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/storage"), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_available_lxc_storages(&output.stdout) + } + + fn parse_local_node(output: &[u8]) -> Result { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster status response")?; + + rows.iter() + .find(|row| { + row.get("type").and_then(serde_json::Value::as_str) == Some("node") + && row.get("local").is_some_and(|local| { + local.as_bool() == Some(true) || local.as_u64() == Some(1) + }) + }) + .and_then(|row| row.get("name")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .context("Proxmox cluster status did not identify the local node") + } + + fn parse_available_lxc_storages(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox node storage response")?; + + let mut storages = rows + .iter() + .filter(|row| { + row.get("active").and_then(serde_json::Value::as_u64) == Some(1) + && row.get("enabled").and_then(serde_json::Value::as_u64) == Some(1) + && row + .get("content") + .and_then(serde_json::Value::as_str) + .is_some_and(|content| content.split(',').any(|kind| kind == "rootdir")) + }) + .map(|row| { + Ok(LxcStorage { + name: row + .get("storage") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC storage response did not include a name")? + .to_string(), + available_bytes: row + .get("avail") + .and_then(serde_json::Value::as_u64) + .context( + "Proxmox LXC storage response did not include available capacity", + )?, + }) + }) + .collect::, anyhow::Error>>()?; + storages.sort_unstable_by_key(|storage| std::cmp::Reverse(storage.available_bytes)); + Ok(storages) + } + + pub async fn list_containers(&self) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + "/cluster/resources".to_string(), + "--type".to_string(), + "vm".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_containers(&output.stdout) + } + + pub fn interfaces_args(node: &str, vmid: u32) -> Vec { + vec![ + "get".to_string(), + format!("/nodes/{node}/lxc/{vmid}/interfaces"), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn interfaces( + &self, + node: &str, + vmid: u32, + ) -> Result, anyhow::Error> { + let output = Self::run(&self.pvesh_path, &Self::interfaces_args(node, vmid)).await?; + Self::parse_interfaces(&output.stdout) + } + + pub fn runtime_status_args(node: &str, vmid: u32) -> Vec { + vec![ + "get".to_string(), + format!("/nodes/{node}/lxc/{vmid}/status/current"), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn runtime_status( + &self, + node: &str, + vmid: u32, + ) -> Result { + let output = Self::run(&self.pvesh_path, &Self::runtime_status_args(node, vmid)).await?; + Self::parse_runtime_status(&output.stdout) + } + + pub async fn node_runtime_statuses( + &self, + node: &str, + ) -> Result, anyhow::Error> { + let output = Self::run( + &self.pvesh_path, + &[ + "get".to_string(), + format!("/nodes/{node}/lxc"), + "--output-format".to_string(), + "json".to_string(), + ], + ) + .await?; + Self::parse_node_runtime_statuses(&output.stdout) + } + + fn parse_node_runtime_statuses( + output: &[u8], + ) -> Result, anyhow::Error> { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox node LXC status response")?; + let entries = value + .as_array() + .context("Proxmox node LXC status response was not an array")?; + entries + .iter() + .map(|entry| { + let vmid = entry + .get("vmid") + .and_then(serde_json::Value::as_u64) + .context("Proxmox node LXC status omitted VMID")?; + Ok(NodeContainerRuntimeStatus { + vmid: u32::try_from(vmid).context("Proxmox LXC VMID exceeded u32")?, + status: ContainerStatus::parse( + entry + .get("status") + .and_then(serde_json::Value::as_str) + .context("Proxmox node LXC status omitted state")?, + )?, + pid: entry.get("pid").and_then(serde_json::Value::as_i64), + }) + }) + .collect() + } + + fn parse_runtime_status(output: &[u8]) -> Result { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox LXC runtime status response")?; + let status = value + .get("status") + .and_then(serde_json::Value::as_str) + .and_then(ContainerStatus::from_word) + .context("Proxmox LXC runtime status did not include a known status")?; + let pid = value.get("pid").and_then(|pid| { + pid.as_i64() + .or_else(|| pid.as_str().and_then(|pid| pid.parse().ok())) + }); + let uptime_seconds = value + .get("uptime") + .and_then(|uptime| { + uptime + .as_u64() + .or_else(|| uptime.as_str().and_then(|uptime| uptime.parse().ok())) + }) + .unwrap_or(0); + + Ok(ContainerRuntimeStatus { + status, + pid, + uptime_seconds, + }) + } + + fn parse_interfaces(output: &[u8]) -> Result, anyhow::Error> { + let rows: Option> = serde_json::from_slice(output) + .context("failed to parse Proxmox LXC interface response")?; + + rows.unwrap_or_default() + .into_iter() + .map(|row| { + let name = row + .get("name") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC interface row did not include a name")? + .to_string(); + let addresses = row + .get("ip-addresses") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|address| address.get("ip-address")) + .filter_map(serde_json::Value::as_str) + .filter_map(|address| address.parse::().ok()) + .collect(); + + Ok(ContainerInterface { name, addresses }) + }) + .collect() + } + + pub fn primary_container_address(interfaces: &[ContainerInterface]) -> Option { + let mut addresses = Self::container_addresses(interfaces).into_iter(); + + addresses + .clone() + .find(IpAddr::is_ipv4) + .or_else(|| addresses.find(IpAddr::is_ipv6)) + } + + pub fn container_addresses(interfaces: &[ContainerInterface]) -> Vec { + fn usable(address: IpAddr) -> bool { + match address { + IpAddr::V4(address) => { + !address.is_loopback() + && !address.is_unspecified() + && !address.is_multicast() + && !address.is_link_local() + } + IpAddr::V6(address) => { + !address.is_loopback() + && !address.is_unspecified() + && !address.is_multicast() + && !address.is_unicast_link_local() + } + } + } + + let mut addresses = interfaces + .iter() + .filter(|interface| interface.name != "lo") + .flat_map(|interface| interface.addresses.iter().copied()) + .filter(|address| usable(*address)) + .collect::>(); + addresses.sort_unstable(); + addresses.dedup(); + addresses + } + + pub async fn list_storage_content( + &self, + node: &str, + storage: &str, + ) -> Result, anyhow::Error> { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/storage/{storage}/content"), + "--content".to_string(), + "vztmpl".to_string(), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_storage_content(&output.stdout) + } + + fn parse_storage_content(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox storage content response")?; + + rows.into_iter() + .map(|row| { + let volid = row + .get("volid") + .and_then(serde_json::Value::as_str) + .context("Proxmox storage content row did not include a volid")? + .to_string(); + let content = row + .get("content") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + + Ok(StorageContent { volid, content }) + }) + .collect() + } + + pub fn normalize_oci_reference(reference: &str) -> Result { + let reference = reference.trim(); + if reference.is_empty() { + return Err(anyhow::anyhow!("OCI image reference cannot be empty")); + } + if reference.contains('@') { + return Err(anyhow::anyhow!( + "Proxmox OCI registry pull currently requires a tagged image reference, not a digest reference: {reference}" + )); + } + + let last_slash = reference.rfind('/'); + let last_colon = reference.rfind(':'); + let has_tag = match (last_slash, last_colon) { + (_, None) => false, + (None, Some(_)) => true, + (Some(slash), Some(colon)) => colon > slash, + }; + + if has_tag { + Ok(reference.to_string()) + } else { + Ok(format!("{reference}:latest")) + } + } + + pub fn oci_cache_filename(reference: &str) -> String { + let digest = Sha256::digest(reference.as_bytes()); + format!("calagopus-oci-{}", hex::encode(digest)) + } + + pub fn oci_template_volid(storage: &str, reference: &str) -> String { + format!( + "{storage}:vztmpl/{}.tar", + Self::oci_cache_filename(reference) + ) + } + + pub fn oci_revision_tag(revision: &str) -> Result { + let revision = revision.strip_prefix("sha256:").unwrap_or(revision); + let short = revision + .get(..32) + .context("OCI image revision digest was unexpectedly short")?; + if !short.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(anyhow::anyhow!( + "OCI image revision was not a SHA-256 digest" + )); + } + Ok(format!("calagopus-image-{short}")) + } + + async fn template_path(&self, volid: &str) -> Result { + let output = Self::run(&self.pvesm_path, &["path".to_string(), volid.to_string()]).await?; + let path = + String::from_utf8(output.stdout).context("pvesm path returned non-UTF8 output")?; + let path = path.trim(); + if path.is_empty() { + return Err(anyhow::anyhow!( + "pvesm path returned an empty path for {volid}" + )); + } + Ok(path.into()) + } + + /// Rewrites OCI gzip layers to the uncompressed representation expected by + /// Proxmox's LXC image importer. OCI `rootfs.diff_ids` are hashes of the + /// uncompressed layer tar streams, whereas some registry pulls retain gzip + /// blobs and PVE currently resolves `diff_ids` as blob names. + fn normalize_oci_template(path: PathBuf) -> Result<(), anyhow::Error> { + let source = std::fs::File::open(&path) + .with_context(|| format!("failed to open OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(source); + let mut index = None; + let mut manifests = HashMap::new(); + let mut configs = HashMap::new(); + for entry in archive + .entries() + .context("failed to enumerate OCI template")? + { + let mut entry = entry.context("failed to read OCI template entry")?; + let name = entry.path()?.to_string_lossy().into_owned(); + // Index, manifest, and config blobs are tiny JSON documents. Never + // buffer the multi-gigabyte layer blobs during metadata discovery. + if name != "index.json" + && (!name.starts_with("blobs/sha256/") || entry.size() > 4 * 1024 * 1024) + { + continue; + } + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + if name == "index.json" { + index = Some( + serde_json::from_slice::(&bytes) + .context("failed to parse OCI template index.json")?, + ); + } else if name.starts_with("blobs/sha256/") { + if let Ok(value) = serde_json::from_slice::(&bytes) { + if value.get("schemaVersion").is_some() && value.get("layers").is_some() { + manifests.insert(name, value); + } else if value.get("rootfs").is_some() { + configs.insert(name, value); + } + } + } + } + let mut index = index.context("OCI template did not contain index.json")?; + let manifest_digest = index + .pointer("/manifests/0/digest") + .and_then(serde_json::Value::as_str) + .context("OCI template index did not include a manifest digest")? + .strip_prefix("sha256:") + .context("OCI template manifest digest was not sha256")?; + let old_manifest_path = format!("blobs/sha256/{manifest_digest}"); + let mut manifest = manifests + .remove(&old_manifest_path) + .context("OCI template did not contain its indexed manifest")?; + let config_digest = manifest + .pointer("/config/digest") + .and_then(serde_json::Value::as_str) + .context("OCI manifest did not include a config digest")? + .strip_prefix("sha256:") + .context("OCI config digest was not sha256")?; + let config = configs + .remove(&format!("blobs/sha256/{config_digest}")) + .context("OCI template did not contain its config blob")?; + let diff_ids = config + .pointer("/rootfs/diff_ids") + .and_then(serde_json::Value::as_array) + .context("OCI config did not include rootfs.diff_ids")?; + let layer_specs = manifest + .get("layers") + .and_then(serde_json::Value::as_array) + .context("OCI manifest did not include layers")?; + if layer_specs.len() != diff_ids.len() { + return Err(anyhow::anyhow!( + "OCI manifest layer count did not match rootfs.diff_ids" + )); + } + let layer_specs: Vec<(String, String, bool)> = layer_specs + .iter() + .zip(diff_ids) + .map(|(layer, diff_id)| { + let digest = layer + .get("digest") + .and_then(serde_json::Value::as_str) + .context("OCI layer did not include a digest")? + .strip_prefix("sha256:") + .context("OCI layer digest was not sha256")?; + let expected = diff_id + .as_str() + .context("OCI diff_id was not a string")? + .strip_prefix("sha256:") + .context("OCI diff_id was not sha256")?; + let media_type = layer + .get("mediaType") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + Ok(( + format!("blobs/sha256/{digest}"), + expected.to_string(), + media_type.ends_with("+gzip") || media_type.ends_with(".gzip"), + )) + }) + .collect::>()?; + + let scratch = + tempfile::tempdir().context("failed to create OCI normalization scratch directory")?; + let replacements: HashMap = layer_specs + .into_iter() + .map(|(name, expected, gzip)| { + ( + name, + (expected.clone(), scratch.path().join(&expected), gzip), + ) + }) + .collect(); + + let source = std::fs::File::open(&path) + .with_context(|| format!("failed to reopen OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(source); + for entry in archive + .entries() + .context("failed to enumerate OCI layers")? + { + let entry = entry.context("failed to read OCI layer entry")?; + let name = entry.path()?.to_string_lossy().into_owned(); + let Some((expected, replacement, gzip)) = replacements.get(&name) else { + continue; + }; + let mut hasher = Sha256::new(); + let mut output = std::fs::File::create(replacement).with_context(|| { + format!( + "failed to create normalized OCI layer {}", + replacement.display() + ) + })?; + let mut reader: Box = if *gzip { + Box::new(GzDecoder::new(entry)) + } else { + Box::new(entry) + }; + let mut buffer = [0u8; 1024 * 1024]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + output.write_all(&buffer[..count])?; + } + let actual = hex::encode(hasher.finalize()); + if actual != *expected { + return Err(anyhow::anyhow!( + "OCI layer diff_id mismatch: expected sha256:{expected}, got sha256:{actual}" + )); + } + } + if replacements + .values() + .any(|(_, replacement, _)| !replacement.is_file()) + { + return Err(anyhow::anyhow!( + "OCI template was missing one or more layer blobs" + )); + } + let layers = manifest + .get_mut("layers") + .and_then(serde_json::Value::as_array_mut) + .context("OCI manifest did not include layers")?; + for (position, layer) in layers.iter_mut().enumerate() { + let expected = diff_ids[position].as_str().unwrap_or_default(); + let (_, replacement, _) = replacements + .values() + .find(|(digest, _, _)| expected == format!("sha256:{digest}")) + .context("normalized OCI layer replacement was missing")?; + layer["digest"] = serde_json::Value::String(expected.to_string()); + layer["size"] = serde_json::Value::from(std::fs::metadata(replacement)?.len()); + layer["mediaType"] = + serde_json::Value::String("application/vnd.oci.image.layer.v1.tar".to_string()); + } + let manifest_bytes = serde_json::to_vec(&manifest)?; + let new_manifest_digest = hex::encode(Sha256::digest(&manifest_bytes)); + index["manifests"][0]["digest"] = + serde_json::Value::String(format!("sha256:{new_manifest_digest}")); + index["manifests"][0]["size"] = serde_json::Value::from(manifest_bytes.len()); + let index_bytes = serde_json::to_vec(&index)?; + let temporary = path.with_extension("tar.normalize-tmp"); + let output = std::fs::File::create(&temporary).with_context(|| { + format!( + "failed to create normalized OCI template {}", + temporary.display() + ) + })?; + let mut builder = tar::Builder::new(output); + let source = std::fs::File::open(&path)?; + let mut archive = tar::Archive::new(source); + for entry in archive.entries()? { + let mut entry = entry?; + let name = entry.path()?.to_string_lossy().into_owned(); + if name == "index.json" || name == old_manifest_path || replacements.contains_key(&name) + { + continue; + } + if !entry.header().entry_type().is_file() { + continue; + } + let mut header = entry.header().clone(); + builder.append_data(&mut header, name, &mut entry)?; + } + for (expected, replacement, _) in replacements.values() { + let mut layer = std::fs::File::open(replacement)?; + let mut header = tar::Header::new_gnu(); + header.set_size(std::fs::metadata(replacement)?.len()); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, format!("blobs/sha256/{expected}"), &mut layer)?; + } + let mut header = tar::Header::new_gnu(); + header.set_size(manifest_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data( + &mut header, + format!("blobs/sha256/{new_manifest_digest}"), + manifest_bytes.as_slice(), + )?; + let mut header = tar::Header::new_gnu(); + header.set_size(index_bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, "index.json", index_bytes.as_slice())?; + builder.finish()?; + std::fs::rename(&temporary, &path).with_context(|| { + format!( + "failed to atomically install normalized OCI template {}", + path.display() + ) + })?; + Ok(()) + } + + async fn template_revision(&self, volid: &str) -> Result { + let path = self.template_path(volid).await?; + tokio::task::spawn_blocking(move || { + use std::io::Read; + + let file = std::fs::File::open(&path) + .with_context(|| format!("failed to open OCI template {}", path.display()))?; + let mut archive = tar::Archive::new(file); + for entry in archive + .entries() + .context("failed to read OCI template archive")? + { + let mut entry = entry.context("failed to read OCI template entry")?; + if entry.path()?.as_ref() != std::path::Path::new("index.json") { + continue; + } + let mut index = String::new(); + entry.read_to_string(&mut index)?; + let value: serde_json::Value = serde_json::from_str(&index) + .context("failed to parse OCI template index.json")?; + return value + .get("manifests") + .and_then(serde_json::Value::as_array) + .and_then(|manifests| manifests.first()) + .and_then(|manifest| manifest.get("digest")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .context("OCI template index did not include a manifest digest"); + } + Err(anyhow::anyhow!("OCI template did not contain index.json")) + }) + .await + .context("OCI template inspection task failed")? + } + + async fn template_is_fresh( + &self, + volid: &str, + max_age: Duration, + ) -> Result { + if max_age.is_zero() { + return Ok(false); + } + let path = self.template_path(volid).await?; + let modified = tokio::fs::metadata(&path).await?.modified()?; + Ok(modified.elapsed().is_ok_and(|age| age < max_age)) + } + + async fn remove_template(&self, volid: &str) -> Result<(), anyhow::Error> { + Self::run(&self.pvesm_path, &["free".to_string(), volid.to_string()]).await?; + Ok(()) + } + + pub fn pull_oci_args(spec: &OciTemplatePullSpec) -> Vec { + vec![ + "create".to_string(), + format!( + "/nodes/{}/storage/{}/oci-registry-pull", + spec.node, spec.storage + ), + "--reference".to_string(), + spec.reference.clone(), + "--filename".to_string(), + spec.filename.clone(), + "--output-format".to_string(), + "json".to_string(), + ] + } + + pub async fn pull_oci_template( + &self, + spec: &OciTemplatePullSpec, + ) -> Result { + let output = Self::run(&self.pvesh_path, &Self::pull_oci_args(spec)).await?; + Self::parse_oci_pull_task(&output.stdout) + } + + fn parse_oci_pull_task(output: &[u8]) -> Result { + if let Ok(value) = serde_json::from_slice::(output) + && let Some(upid) = value.as_str().filter(|upid| upid.starts_with("UPID:")) + { + return Ok(upid.to_string()); + } + + let output = std::str::from_utf8(output) + .context("Proxmox OCI pull response contained non-UTF8 output")?; + for line in output + .lines() + .rev() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if let Ok(value) = serde_json::from_str::(line) + && let Some(upid) = value.as_str().filter(|upid| upid.starts_with("UPID:")) + { + return Ok(upid.to_string()); + } + if line.starts_with("UPID:") { + return Ok(line.to_string()); + } + } + + Err(anyhow::anyhow!( + "Proxmox OCI pull response did not contain a task UPID" + )) + } + + pub async fn task_status(&self, node: &str, upid: &str) -> Result { + let args = vec![ + "get".to_string(), + format!("/nodes/{node}/tasks/{upid}/status"), + "--output-format".to_string(), + "json".to_string(), + ]; + let output = Self::run(&self.pvesh_path, &args).await?; + Self::parse_task_status(&output.stdout) + } + + fn parse_task_status(output: &[u8]) -> Result { + let value: serde_json::Value = serde_json::from_slice(output) + .context("failed to parse Proxmox task status response")?; + let state = match value.get("status").and_then(serde_json::Value::as_str) { + Some("running") => TaskState::Running, + Some("stopped") => TaskState::Stopped, + Some(status) => { + return Err(anyhow::anyhow!( + "unknown Proxmox task status response: {status}" + )); + } + None => { + return Err(anyhow::anyhow!( + "Proxmox task status response did not include status" + )); + } + }; + let exit_status = value + .get("exitstatus") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + + Ok(TaskStatus { state, exit_status }) + } + + pub async fn wait_for_task(&self, node: &str, upid: &str) -> Result<(), anyhow::Error> { + tokio::time::timeout(TASK_TIMEOUT, async { + loop { + let status = self.task_status(node, upid).await?; + match status.state { + TaskState::Running => tokio::time::sleep(TASK_POLL_INTERVAL).await, + TaskState::Stopped => { + return match status.exit_status.as_deref() { + Some("OK") => Ok(()), + Some(exit_status) => Err(anyhow::anyhow!( + "Proxmox task {upid} failed with exit status {exit_status}" + )), + None => Err(anyhow::anyhow!( + "Proxmox task {upid} stopped without an exit status" + )), + }; + } + } + } + }) + .await + .with_context(|| format!("timed out waiting for Proxmox task {upid}"))? + } + + fn validate_absolute_path(path: &str, label: &str) -> Result<(), anyhow::Error> { + let path = Path::new(path); + if !path.is_absolute() + || path + .components() + .any(|component| matches!(component, Component::ParentDir | Component::CurDir)) + || path + .as_os_str() + .as_encoded_bytes() + .iter() + .any(|byte| matches!(byte, 0 | b',' | b'\n' | b'\r')) + { + return Err(anyhow::anyhow!( + "invalid Proxmox {label} path: {}", + path.display() + )); + } + Ok(()) + } + + pub async fn ensure_oci_template( + &self, + node: &str, + storage: &str, + reference: &str, + max_age: Duration, + ) -> Result { + let reference = Self::normalize_oci_reference(reference)?; + let filename = Self::oci_cache_filename(&reference); + let volid = Self::oci_template_volid(storage, &reference); + let exists = self + .list_storage_content(node, storage) + .await? + .into_iter() + .any(|entry| entry.volid == volid); + if exists && self.template_is_fresh(&volid, max_age).await? { + return Ok(OciTemplate { + revision: self.template_revision(&volid).await?, + volid, + }); + } + if exists { + self.remove_template(&volid) + .await + .with_context(|| format!("failed to remove stale Proxmox OCI template {volid}"))?; + } + + let upid = self + .pull_oci_template(&OciTemplatePullSpec { + node: node.to_string(), + storage: storage.to_string(), + reference, + filename, + }) + .await?; + self.wait_for_task(node, &upid).await?; + let path = self.template_path(&volid).await?; + tokio::task::spawn_blocking(move || Self::normalize_oci_template(path)) + .await + .context("OCI template normalization task failed")??; + + let exists = self + .list_storage_content(node, storage) + .await? + .into_iter() + .any(|entry| entry.volid == volid); + if exists { + Ok(OciTemplate { + revision: self.template_revision(&volid).await?, + volid, + }) + } else { + Err(anyhow::anyhow!( + "Proxmox OCI pull task completed but template {volid} was not present in storage" + )) + } + } + + fn parse_containers(output: &[u8]) -> Result, anyhow::Error> { + let rows: Vec = serde_json::from_slice(output) + .context("failed to parse Proxmox cluster resources response")?; + let mut containers = Vec::new(); + + for row in rows { + if row.get("type").and_then(serde_json::Value::as_str) != Some("lxc") { + continue; + } + + let raw_vmid = row + .get("vmid") + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + }) + .context("Proxmox LXC resource did not include a VMID")?; + let vmid = u32::try_from(raw_vmid).context("Proxmox LXC VMID exceeded u32")?; + let node = row + .get("node") + .and_then(serde_json::Value::as_str) + .context("Proxmox LXC resource did not include a node")? + .to_string(); + let status = row + .get("status") + .and_then(serde_json::Value::as_str) + .and_then(ContainerStatus::from_word); + let tags = row + .get("tags") + .and_then(serde_json::Value::as_str) + .map(|tags| { + tags.split(';') + .map(str::trim) + .filter(|tag| !tag.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + + containers.push(ClusterContainer { + vmid, + node, + status, + tags, + }); + } + + Ok(containers) + } + + pub async fn status(&self, vmid: u32) -> Result { + let args = vec!["status".to_string(), vmid.to_string()]; + let output = Self::run(&self.pct_path, &args).await?; + ContainerStatus::parse(&String::from_utf8_lossy(&output.stdout)) + } + + pub async fn start(&self, vmid: u32) -> Result<(), anyhow::Error> { + let args = ["start".to_string(), vmid.to_string()]; + let mut last_error = None; + + for attempt in 0..START_SETTLE_ATTEMPTS { + match Self::run(&self.pct_path, &args).await { + Ok(_) => return Ok(()), + Err(error) => { + let message = error.to_string(); + if !message.contains("monitor socket") + && !message.contains("got timeout") + && !message.contains("unable to get PID for CT") + { + return Err(error); + } + tokio::time::sleep(START_SETTLE_DELAY).await; + // A failed pct client can still have started the container. + // Reissuing pct start in that case would produce "already running". + if matches!(self.status(vmid).await, Ok(ContainerStatus::Running)) { + return Ok(()); + } + last_error = Some(error); + if attempt + 1 < START_SETTLE_ATTEMPTS { + tracing::warn!( + vmid, + attempt = attempt + 1, + "retrying Proxmox LXC start after monitor teardown" + ); + } + } + } + } + + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("failed to start LXC container"))) + } + + #[allow(dead_code)] + pub async fn shutdown(&self, vmid: u32, timeout_seconds: u64) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "shutdown".to_string(), + vmid.to_string(), + "--timeout".to_string(), + timeout_seconds.to_string(), + ], + ) + .await?; + Ok(()) + } + + pub async fn stop(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &["stop".to_string(), vmid.to_string()]).await?; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if self.status(vmid).await? == ContainerStatus::Stopped { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .with_context(|| format!("timed out waiting for Proxmox LXC {vmid} to stop"))? + } + + pub async fn destroy(&self, vmid: u32) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "destroy".to_string(), + vmid.to_string(), + "--purge".to_string(), + "1".to_string(), + ], + ) + .await?; + Ok(()) + } + + pub fn create_args(spec: &CreateContainerSpec) -> Vec { + let mut args = vec![ + "create".to_string(), + spec.vmid.to_string(), + spec.template.clone(), + "--hostname".to_string(), + spec.hostname.clone(), + "--rootfs".to_string(), + format!("{}:{}", spec.rootfs_storage, spec.rootfs_size_gib), + "--memory".to_string(), + spec.memory_mib.to_string(), + "--swap".to_string(), + spec.swap_mib.to_string(), + "--net0".to_string(), + spec.network.clone(), + "--unprivileged".to_string(), + u8::from(spec.unprivileged).to_string(), + "--cmode".to_string(), + "console".to_string(), + "--onboot".to_string(), + "0".to_string(), + ]; + + if let Some(cores) = spec.cores { + args.extend(["--cores".to_string(), cores.to_string()]); + } + if let Some(cpu_limit_percent) = spec.cpu_limit_percent { + args.extend([ + "--cpulimit".to_string(), + Self::format_cpu_limit_percent(cpu_limit_percent), + ]); + } + if !spec.tags.is_empty() { + args.extend(["--tags".to_string(), spec.tags.join(";")]); + } + + args + } + + fn format_cpu_limit_percent(percent: u64) -> String { + let whole = percent / 100; + let fractional = percent % 100; + if fractional == 0 { + return whole.to_string(); + } + if fractional.is_multiple_of(10) { + format!("{whole}.{}", fractional / 10) + } else { + format!("{whole}.{fractional:02}") + } + } + + pub async fn create(&self, spec: &CreateContainerSpec) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::create_args(spec)).await?; + Ok(()) + } + + pub fn set_resources_args(vmid: u32, resources: &ContainerResources) -> Vec { + let mut args = vec![ + "set".to_string(), + vmid.to_string(), + "--memory".to_string(), + resources.memory_mib.to_string(), + "--swap".to_string(), + resources.swap_mib.to_string(), + "--cpulimit".to_string(), + resources + .cpu_limit_percent + .map(Self::format_cpu_limit_percent) + .unwrap_or_else(|| "0".to_string()), + ]; + + if let Some(cores) = resources.cores { + args.extend(["--cores".to_string(), cores.to_string()]); + } else { + args.extend(["--delete".to_string(), "cores".to_string()]); + } + + args + } + + pub async fn set_resources( + &self, + vmid: u32, + resources: &ContainerResources, + ) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_resources_args(vmid, resources)).await?; + Ok(()) + } + + pub async fn set_network(&self, vmid: u32, network: &str) -> Result<(), anyhow::Error> { + Self::run( + &self.pct_path, + &[ + "set".to_string(), + vmid.to_string(), + "--net0".to_string(), + network.to_string(), + ], + ) + .await?; + Ok(()) + } + + pub fn network_with_firewall(config: &str) -> Result, anyhow::Error> { + let network = config + .lines() + .find_map(|line| line.strip_prefix("net0:")) + .map(str::trim) + .context("Proxmox LXC config has no net0 interface")?; + let mut options: Vec<&str> = network + .split(',') + .filter(|option| !option.starts_with("firewall=")) + .collect(); + options.push("firewall=1"); + let wanted = options.join(","); + + Ok((wanted != network).then_some(wanted)) + } + + pub async fn enable_network_firewall(&self, vmid: u32) -> Result<(), anyhow::Error> { + let config = self.config(vmid).await?; + if let Some(network) = Self::network_with_firewall(&config)? { + self.set_network(vmid, &network).await?; + } + Ok(()) + } + + pub fn set_tags_args(vmid: u32, tags: &[String]) -> Vec { + vec![ + "set".to_string(), + vmid.to_string(), + "--tags".to_string(), + tags.join(";"), + ] + } + + pub async fn set_tags(&self, vmid: u32, tags: &[String]) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_tags_args(vmid, tags)).await?; + Ok(()) + } + + pub async fn config(&self, vmid: u32) -> Result { + let output = Self::run(&self.pct_path, &["config".to_string(), vmid.to_string()]).await?; + Self::decode_config_output(output.stdout) + } + + fn decode_config_output(stdout: Vec) -> Result { + let mut config = + String::from_utf8(stdout).context("pct config returned non-UTF8 output")?; + config.retain(|character| character != '\0'); + Ok(config) + } + + fn parse_oci_user(config: &str) -> Result { + let mut uid = None; + let mut gid = None; + + for line in config.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim(); + match key.trim() { + "lxc.init.uid" => { + uid = Some( + value + .parse::() + .with_context(|| format!("invalid lxc.init.uid value: {value}"))?, + ); + } + "lxc.init.gid" => { + gid = Some( + value + .parse::() + .with_context(|| format!("invalid lxc.init.gid value: {value}"))?, + ); + } + _ => {} + } + } + + Ok(ContainerOciUser { + uid: uid.unwrap_or(0), + gid: gid.unwrap_or(0), + }) + } + + pub async fn oci_user(&self, vmid: u32) -> Result { + let config = self.config(vmid).await?; + Self::parse_oci_user(&config) + } + + pub async fn oci_entrypoint(&self, vmid: u32) -> Result { + let config = self.config(vmid).await?; + let entrypoint = config + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + (key.trim() == "entrypoint").then(|| value.trim().to_string()) + }) + .filter(|value| !value.is_empty()) + .context("OCI-backed Proxmox container does not define an entrypoint")?; + if entrypoint.contains('\0') || entrypoint.contains('\n') || entrypoint.contains('\r') { + return Err(anyhow::anyhow!( + "OCI-backed Proxmox container entrypoint contains an unsupported control character" + )); + } + Ok(entrypoint) + } + + pub fn set_data_mount_args(spec: &DataMountSpec) -> Result, anyhow::Error> { + Self::set_bind_mount_args( + spec.vmid, + &[BindMountSpec { + slot: 0, + source_path: spec.data_path.clone(), + target_path: "/home/container".to_string(), + read_only: false, + container_uid: spec.container_uid, + container_gid: spec.container_gid, + host_uid: spec.host_data_uid, + host_gid: spec.host_data_gid, + }], + ) + } + + #[allow(dead_code)] + pub async fn set_data_mount(&self, spec: &DataMountSpec) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_data_mount_args(spec)?).await?; + Ok(()) + } + + pub fn set_bind_mount_args( + vmid: u32, + mounts: &[BindMountSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + let mut slots = std::collections::HashSet::new(); + + for mount in mounts { + if !slots.insert(mount.slot) { + return Err(anyhow::anyhow!( + "duplicate Proxmox bind mount slot mp{}", + mount.slot + )); + } + for (label, path) in [ + ("source", mount.source_path.as_str()), + ("target", mount.target_path.as_str()), + ] { + Self::validate_absolute_path(path, &format!("bind mount {label}"))?; + } + + let idmap = format!( + "u:{}:{}:1;g:{}:{}:1", + mount.container_uid, mount.host_uid, mount.container_gid, mount.host_gid + ); + let read_only = if mount.read_only { ",ro=1" } else { "" }; + args.extend([ + format!("--mp{}", mount.slot), + format!( + "{},mp={},backup=0,idmap={idmap}{read_only}", + mount.source_path, mount.target_path, + ), + ]); + } + + Ok(args) + } + + #[allow(dead_code)] + pub async fn set_bind_mounts( + &self, + vmid: u32, + mounts: &[BindMountSpec], + ) -> Result<(), anyhow::Error> { + Self::run(&self.pct_path, &Self::set_bind_mount_args(vmid, mounts)?).await?; + Ok(()) + } + + pub fn set_devices_args( + vmid: u32, + devices: &[DevicePassthroughSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + let mut slots = std::collections::HashSet::new(); + + for device in devices { + if !slots.insert(device.slot) { + return Err(anyhow::anyhow!( + "duplicate Proxmox device passthrough slot dev{}", + device.slot + )); + } + Self::validate_absolute_path(&device.path, "device passthrough")?; + if device.mode > 0o777 { + return Err(anyhow::anyhow!( + "invalid Proxmox device passthrough mode {:o}; only permission bits are supported", + device.mode + )); + } + + let mut value = format!( + "path={},uid={},gid={},mode={:04o}", + device.path, device.uid, device.gid, device.mode + ); + if device.deny_write { + value.push_str(",deny-write=1"); + } + args.extend([format!("--dev{}", device.slot), value]); + } + + Ok(args) + } + + #[allow(dead_code)] + pub async fn set_devices( + &self, + vmid: u32, + devices: &[DevicePassthroughSpec], + ) -> Result<(), anyhow::Error> { + if devices.is_empty() { + return Ok(()); + } + Self::run(&self.pct_path, &Self::set_devices_args(vmid, devices)?).await?; + Ok(()) + } + + fn configured_slots(config: &str, prefix: &str) -> std::collections::BTreeSet { + config + .lines() + .filter_map(|line| line.split_once(':').map(|(key, _)| key.trim())) + .filter_map(|key| key.strip_prefix(prefix)) + .filter_map(|slot| slot.parse::().ok()) + .collect() + } + + pub fn reconcile_mounts_and_devices_args( + vmid: u32, + current_config: &str, + mounts: &[BindMountSpec], + devices: &[DevicePassthroughSpec], + ) -> Result, anyhow::Error> { + let mut args = vec!["set".to_string(), vmid.to_string()]; + + let mount_args = Self::set_bind_mount_args(vmid, mounts)?; + args.extend(mount_args.into_iter().skip(2)); + let device_args = Self::set_devices_args(vmid, devices)?; + args.extend(device_args.into_iter().skip(2)); + + let desired_mounts = mounts + .iter() + .map(|mount| mount.slot) + .collect::>(); + let desired_devices = devices + .iter() + .map(|device| device.slot) + .collect::>(); + + let mut stale = Self::configured_slots(current_config, "mp") + .difference(&desired_mounts) + .map(|slot| format!("mp{slot}")) + .collect::>(); + stale.extend( + Self::configured_slots(current_config, "dev") + .difference(&desired_devices) + .map(|slot| format!("dev{slot}")), + ); + if !stale.is_empty() { + args.extend(["--delete".to_string(), stale.join(",")]); + } + + Ok(args) + } + + pub async fn create_with_bind_mounts( + &self, + spec: &CreateContainerSpec, + mounts: &[HostBindMountSpec], + ) -> Result { + self.create_with_bind_mounts_and_devices(spec, mounts, &[]) + .await + } + + pub async fn create_with_bind_mounts_and_devices( + &self, + spec: &CreateContainerSpec, + mounts: &[HostBindMountSpec], + devices: &[HostDevicePassthroughSpec], + ) -> Result { + self.create(spec).await?; + + let result = async { + let user = self.oci_user(spec.vmid).await?; + self.configure_mounts_and_devices(spec.vmid, user, mounts, devices) + .await?; + Ok(user) + } + .await; + + match result { + Ok(user) => Ok(user), + Err(error) => match self.destroy(spec.vmid).await { + Ok(()) => Err(error), + Err(cleanup_error) => Err(anyhow::anyhow!( + "{error:#}; additionally failed to destroy newly created container {}: {cleanup_error:#}", + spec.vmid + )), + }, + } + } + + pub async fn configure_mounts_and_devices( + &self, + vmid: u32, + user: ContainerOciUser, + mounts: &[HostBindMountSpec], + devices: &[HostDevicePassthroughSpec], + ) -> Result<(), anyhow::Error> { + let mounts = mounts + .iter() + .map(|mount| BindMountSpec { + slot: mount.slot, + source_path: mount.source_path.clone(), + target_path: mount.target_path.clone(), + read_only: mount.read_only, + container_uid: user.uid, + container_gid: user.gid, + host_uid: mount.host_uid, + host_gid: mount.host_gid, + }) + .collect::>(); + + let devices = devices + .iter() + .map(|device| DevicePassthroughSpec { + slot: device.slot, + path: device.path.clone(), + uid: device.uid, + gid: device.gid, + mode: device.mode, + deny_write: device.deny_write, + }) + .collect::>(); + + let current_config = self.config(vmid).await?; + let args = + Self::reconcile_mounts_and_devices_args(vmid, ¤t_config, &mounts, &devices)?; + if args.len() > 2 { + Self::run(&self.pct_path, &args).await?; + } + + Ok(()) + } + + #[allow(dead_code)] + pub async fn create_with_data_mount( + &self, + spec: &CreateContainerSpec, + data_path: &str, + host_data_uid: u32, + host_data_gid: u32, + ) -> Result { + self.create_with_bind_mounts( + spec, + &[HostBindMountSpec { + slot: 0, + source_path: data_path.to_string(), + target_path: "/home/container".to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }], + ) + .await + } + + pub fn attach_args(spec: &AttachProcessSpec) -> Result, anyhow::Error> { + if spec.command.is_empty() { + return Err(anyhow::anyhow!("attached process command cannot be empty")); + } + + let mut args = vec![ + "-n".to_string(), + spec.vmid.to_string(), + "--clear-env".to_string(), + "-u".to_string(), + spec.uid.to_string(), + "-g".to_string(), + spec.gid.to_string(), + ]; + + for variable in &spec.environment { + let Some((name, _)) = variable.split_once('=') else { + return Err(anyhow::anyhow!( + "attached process environment entry must be KEY=VALUE: {variable}" + )); + }; + if name.is_empty() || name.contains('\0') || variable.contains('\0') { + return Err(anyhow::anyhow!( + "attached process environment entry has an invalid name: {variable}" + )); + } + args.extend(["-v".to_string(), variable.clone()]); + } + + if spec.command.iter().any(|argument| argument.contains('\0')) { + return Err(anyhow::anyhow!( + "attached process command contains a NUL byte" + )); + } + + args.push("--".to_string()); + args.extend(spec.command.iter().cloned()); + + Ok(args) + } + + #[allow(dead_code)] + pub fn spawn_attached_process( + &self, + spec: &AttachProcessSpec, + ) -> Result { + let args = Self::attach_args(spec)?; + tokio::process::Command::new(&self.lxc_attach_path) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {}", self.lxc_attach_path.display())) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContainerStatus { + Running, + Stopped, +} + +impl ContainerStatus { + fn from_word(value: &str) -> Option { + match value { + "running" => Some(Self::Running), + "stopped" => Some(Self::Stopped), + _ => None, + } + } + + fn parse(value: &str) -> Result { + let value = value.trim(); + let status = value.strip_prefix("status: ").unwrap_or(value); + Self::from_word(status) + .ok_or_else(|| anyhow::anyhow!("unknown pct status response: {value}")) + } +} + +#[cfg(test)] +mod tests; diff --git a/application/src/server/executor/pve_lxc/cli/scripts.rs b/application/src/server/executor/pve_lxc/cli/scripts.rs new file mode 100644 index 00000000..375fff69 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli/scripts.rs @@ -0,0 +1,237 @@ +pub(super) const APPLY_RUNTIME_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::LXC::Config; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; + +my $overlay = $payload->{environment} // []; +die "environment must be an array\n" if ref($overlay) ne 'ARRAY'; + +my $managed_mounts = $payload->{managed_file_mounts}; +my $managed_targets = $payload->{managed_file_targets}; +my %managed_target; +if (defined($managed_mounts)) { + die "managed_file_mounts must be an array\n" if ref($managed_mounts) ne 'ARRAY'; + die "managed_file_targets must be an array\n" if ref($managed_targets) ne 'ARRAY'; + for my $target (@$managed_targets) { + die "invalid managed file target\n" if ref($target) || !defined($target) || !length($target); + $managed_target{$target} = 1; + } + for my $mount (@$managed_mounts) { + die "invalid managed file mount\n" if ref($mount) ne 'HASH'; + my $target = $mount->{target}; + my $entry = $mount->{entry}; + die "invalid managed file mount target\n" + if ref($target) || !defined($target) || !exists($managed_target{$target}); + die "invalid managed file mount entry\n" + if ref($entry) || !defined($entry) || !length($entry); + } +} + +sub mount_target { + my ($entry) = @_; + return undef if !defined($entry); + my @fields = grep { length($_) } split(/\s+/, $entry); + return scalar(@fields) >= 2 ? $fields[1] : undef; +} + +PVE::LXC::Config->lock_config($vmid, sub { + my $conf = PVE::LXC::Config->load_config($vmid); + my @lxc = @{$conf->{lxc} // []}; + my @environment; + + if (defined($conf->{env}) && length($conf->{env})) { + push @environment, split(/\0/, $conf->{env}, -1); + } + for my $entry (@lxc) { + next if ref($entry) ne 'ARRAY' || @$entry < 2; + push @environment, $entry->[1] + if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; + } + + # PVE exposes the same values through both `env` and low-level + # lxc.environment records. Collapse those views before applying the panel + # overlay so each synchronization is idempotent. + my @normalized; + my %index; + for my $variable (@environment) { + my ($name) = split(/=/, $variable, 2); + next if !defined($name) || !length($name); + if (exists($index{$name})) { + $normalized[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@normalized); + push @normalized, $variable; + } + } + @environment = @normalized; + for my $variable (@$overlay) { + my ($name) = split(/=/, $variable, 2); + die "invalid environment entry\n" if !defined($name) || !length($name); + if (exists($index{$name})) { + $environment[$index{$name}] = $variable; + } else { + $index{$name} = scalar(@environment); + push @environment, $variable; + } + } + + @lxc = grep { + my $keep = ref($_) ne 'ARRAY' + || @$_ < 2 + || ($_->[0] ne 'lxc.environment' + && $_->[0] ne 'lxc.environment.runtime' + && $_->[0] ne 'lxc.signal.halt' + && $_->[0] ne 'lxc.init.cmd' + && $_->[0] ne 'lxc.console.logfile' + && $_->[0] ne 'lxc.cgroup.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup2.cpuset.cpus' + && $_->[0] ne 'lxc.cgroup.pids.max' + && $_->[0] ne 'lxc.cgroup2.pids.max' + && $_->[0] ne 'lxc.cgroup.blkio.weight' + && $_->[0] ne 'lxc.cgroup2.io.weight' + && $_->[0] ne 'lxc.cgroup.memory.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.max' + && $_->[0] ne 'lxc.cgroup.memory.memsw.limit_in_bytes' + && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); + if ($keep && defined($managed_mounts) && ref($_) eq 'ARRAY' && @$_ >= 2 + && $_->[0] eq 'lxc.mount.entry') { + my $target = mount_target($_->[1]); + $keep = 0 if defined($target) && exists($managed_target{$target}); + } + $keep; + } @lxc; + # PVE 9.2 serializes its `env` property as lxc.environment.runtime, but + # LXC 7 applies that subkey only to hooks. Mirror the merged environment + # through lxc.environment so PID 1 receives both OCI and panel variables. + $conf->{env} = join("\0", @environment); + push @lxc, map { ['lxc.environment', $_] } @environment; + if (defined($payload->{halt_signal}) && length($payload->{halt_signal})) { + push @lxc, ['lxc.signal.halt', $payload->{halt_signal}]; + } + if (defined($payload->{init_command}) && length($payload->{init_command})) { + push @lxc, ['lxc.init.cmd', $payload->{init_command}]; + } + if (defined($payload->{console_logfile}) && length($payload->{console_logfile})) { + push @lxc, ['lxc.console.logfile', $payload->{console_logfile}]; + } + if (defined($payload->{cpuset_cpus}) && length($payload->{cpuset_cpus})) { + push @lxc, ['lxc.cgroup2.cpuset.cpus', $payload->{cpuset_cpus}]; + } + if (defined($payload->{pids_limit}) && $payload->{pids_limit} > 0) { + push @lxc, ['lxc.cgroup2.pids.max', $payload->{pids_limit}]; + } + if (defined($payload->{io_weight}) && $payload->{io_weight} > 0) { + push @lxc, ['lxc.cgroup2.io.weight', $payload->{io_weight}]; + } + if ($payload->{memory_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.max', 'max']; + } + if ($payload->{swap_unlimited}) { + push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; + } + if (defined($managed_mounts)) { + push @lxc, map { ['lxc.mount.entry', $_->{entry}] } @$managed_mounts; + } + + $conf->{lxc} = \@lxc; + PVE::LXC::Config->write_config($vmid, $conf); +}); +"#; + +pub(super) const APPLY_FIREWALL_CONFIG_PERL: &str = r#" +use strict; +use warnings; +use JSON::PP qw(decode_json); +use PVE::Firewall; + +my $input; +{ + local $/; + $input = ; +} +my $payload = decode_json($input // '{}'); +my $vmid = int($payload->{vmid} // 0); +die "invalid VMID\n" if $vmid <= 0; +my $prefix = $payload->{managed_prefix} // ''; +die "invalid managed prefix\n" if ref($prefix) || !length($prefix); +my $rules = $payload->{rules} // []; +my $ipsets = $payload->{ipsets} // {}; +die "rules must be an array\n" if ref($rules) ne 'ARRAY'; +die "ipsets must be an object\n" if ref($ipsets) ne 'HASH'; + +for my $rule (@$rules) { + die "invalid firewall rule\n" if ref($rule) ne 'HASH'; + die "invalid firewall rule type\n" + if ref($rule->{type}) || ($rule->{type} // '') ne 'in'; + die "invalid firewall rule action\n" + if ref($rule->{action}) || ($rule->{action} // '') !~ /\A(?:ACCEPT|DROP)\z/; + die "invalid firewall rule interface\n" + if ref($rule->{iface}) || ($rule->{iface} // '') !~ /\A[A-Za-z0-9_.-]+\z/; + if (defined($rule->{proto})) { + die "invalid firewall rule protocol\n" + if ref($rule->{proto}) || $rule->{proto} !~ /\A(?:tcp|udp)\z/; + } + if (defined($rule->{dport})) { + die "invalid firewall rule destination port\n" + if ref($rule->{dport}) || $rule->{dport} !~ /\A[0-9]+(?:,[0-9]+)*\z/; + for my $port (split(/,/, $rule->{dport})) { + die "invalid firewall rule destination port\n" if $port < 1 || $port > 65535; + } + } + if (defined($rule->{source})) { + die "invalid firewall rule source\n" + if ref($rule->{source}) || $rule->{source} =~ /[\x00\r\n]/; + } +} + +# Match the pve-firewall daemon's initialization before loading and rewriting +# existing cluster or guest rules. +PVE::Firewall::init(); + +PVE::Firewall::lock_vmfw_conf($vmid, 10, sub { + my $cluster_conf = PVE::Firewall::load_clusterfw_conf(); + my $conf = PVE::Firewall::load_vmfw_conf($cluster_conf, 'ct', $vmid); + + $conf->{rules} = [grep { + my $comment = $_->{comment} // ''; + index($comment, $prefix) != 0; + } @{$conf->{rules} // []}]; + + for my $name (keys %{$conf->{ipset} // {}}) { + my $comment = $conf->{ipset_comments}->{$name} // ''; + next if index($comment, $prefix) != 0; + delete $conf->{ipset}->{$name}; + delete $conf->{ipset_comments}->{$name}; + } + + for my $name (keys %$ipsets) { + die "invalid ipset name\n" if $name !~ /^[A-Za-z][A-Za-z0-9_-]{0,30}$/; + my $set = $ipsets->{$name}; + die "ipset entries must be an array\n" if ref($set) ne 'ARRAY'; + $conf->{ipset}->{$name} = [map { { cidr => $_ } } @$set]; + $conf->{ipset_comments}->{$name} = "$prefix source file"; + } + + unshift @{$conf->{rules}}, @$rules; + if (@$rules) { + # Wings owns firewall mode for its tagged containers so the guest + # behavior matches the panel's default-allow rule model. + $conf->{options}->{enable} = 1; + $conf->{options}->{policy_in} = 'ACCEPT'; + } elsif (!@{$conf->{rules}}) { + $conf->{options}->{enable} = 0; + } + + PVE::Firewall::save_vmfw_conf($vmid, $conf); +}); +"#; diff --git a/application/src/server/executor/pve_lxc/cli/tests.rs b/application/src/server/executor/pve_lxc/cli/tests.rs new file mode 100644 index 00000000..dab5db73 --- /dev/null +++ b/application/src/server/executor/pve_lxc/cli/tests.rs @@ -0,0 +1,1184 @@ +use super::*; + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn command_runner_captures_stdout_and_stderr_after_spawn() { + let args = vec![ + "-c".to_string(), + "printf stdout-value; printf stderr-value >&2".to_string(), + ]; + + let output = PveCli::run(std::path::Path::new("/bin/sh"), &args) + .await + .expect("command should succeed"); + + assert_eq!(output.stdout, b"stdout-value"); + assert_eq!(output.stderr, b"stderr-value"); +} + +#[test] +fn create_arguments_leave_data_mount_for_post_create_configuration() { + let args = PveCli::create_args(&CreateContainerSpec { + vmid: 1234, + template: "local:vztmpl/oci-minecraft.tar".to_string(), + hostname: "server-1234".to_string(), + rootfs_storage: "local-lvm".to_string(), + rootfs_size_gib: 8, + memory_mib: 4096, + swap_mib: 1024, + cpu_limit_percent: Some(250), + cores: Some(2), + network: "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1".to_string(), + tags: vec!["calagopus".to_string(), "calagopus-server".to_string()], + unprivileged: true, + }); + + assert_eq!(args.first().map(String::as_str), Some("create")); + assert!( + args.windows(2) + .any(|pair| pair == ["--rootfs", "local-lvm:8"]) + ); + assert!(args.windows(2).any(|pair| { + pair == [ + "--net0", + "name=eth0,bridge=vmbr0,ip=dhcp,type=veth,host-managed=1", + ] + })); + assert!(!args.iter().any(|arg| arg == "--mp0")); + assert!(args.windows(2).any(|pair| pair == ["--cores", "2"])); + assert!(args.windows(2).any(|pair| pair == ["--cpulimit", "2.5"])); + assert!(args.windows(2).any(|pair| pair == ["--cmode", "console"])); + assert!( + args.windows(2) + .any(|pair| { pair == ["--tags", "calagopus;calagopus-server"] }) + ); +} + +#[test] +fn formats_panel_cpu_percent_for_proxmox_cpu_time() { + assert_eq!(PveCli::format_cpu_limit_percent(1), "0.01"); + assert_eq!(PveCli::format_cpu_limit_percent(50), "0.5"); + assert_eq!(PveCli::format_cpu_limit_percent(100), "1"); + assert_eq!(PveCli::format_cpu_limit_percent(250), "2.5"); + assert_eq!(PveCli::format_cpu_limit_percent(225), "2.25"); +} + +#[test] +fn enables_firewall_on_existing_network_configuration() { + let config = "arch: amd64\nnet0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(config).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let disabled = "net0: name=eth0,bridge=vmbr0,firewall=0,ip=dhcp,type=veth\n"; + assert_eq!( + PveCli::network_with_firewall(disabled).unwrap().as_deref(), + Some("name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1") + ); + + let enabled = "net0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth,firewall=1\n"; + assert_eq!(PveCli::network_with_firewall(enabled).unwrap(), None); + assert!(PveCli::network_with_firewall("arch: amd64\n").is_err()); +} + +#[test] +fn set_tags_replaces_container_tags_atomically() { + assert_eq!( + PveCli::set_tags_args( + 1234, + &[ + "calagopus".to_string(), + "calagopus-server-1234".to_string(), + "calagopus-image-deadbeef".to_string(), + ], + ), + vec![ + "set", + "1234", + "--tags", + "calagopus;calagopus-server-1234;calagopus-image-deadbeef", + ] + ); +} + +#[test] +fn maps_panel_resources_to_native_pve_limits() { + assert_eq!( + PveCli::panel_resources(2048, 256, 1024, 250, None).ok(), + Some(ContainerResources { + memory_mib: 2304, + swap_mib: 1024, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: Some(250), + cores: None, + cpuset_cpus: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: None, + cores: None, + cpuset_cpus: None, + }) + ); +} + +#[test] +fn rejects_invalid_resource_limits() { + assert!(PveCli::panel_resources(-1, 0, 0, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, -2, 0, None).is_err()); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("3-1")).is_err()); +} + +#[test] +fn maps_unlimited_memory_and_swap_to_lxc_overrides() { + assert_eq!( + PveCli::panel_resources(0, 0, 0, 0, None).ok(), + Some(ContainerResources { + memory_mib: 16, + swap_mib: 0, + memory_unlimited: true, + swap_unlimited: true, + cpu_limit_percent: None, + cores: None, + cpuset_cpus: None, + }) + ); + assert_eq!( + PveCli::panel_resources(2048, 0, -1, 0, None).ok(), + Some(ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: true, + cpu_limit_percent: None, + cores: None, + cpuset_cpus: None, + }) + ); +} + +#[test] +fn accepts_panel_cpu_pinning_syntax() { + let resources = PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,8")).expect("valid CPU set"); + assert_eq!(resources.cores, Some(5)); + assert_eq!(resources.cpuset_cpus.as_deref(), Some("0-3,8")); + assert_eq!( + PveCli::panel_resources(2048, 0, 0, 0, Some("0-3,2-5")) + .expect("overlapping CPU set") + .cores, + Some(6) + ); + assert!(PveCli::panel_resources(2048, 0, 0, 0, Some("0,,2")).is_err()); +} + +#[test] +fn converts_docker_block_io_weight_to_cgroup_v2() { + assert_eq!(PveCli::cgroup2_io_weight(None).ok(), Some(None)); + assert_eq!(PveCli::cgroup2_io_weight(Some(10)).ok(), Some(Some(1))); + assert_eq!(PveCli::cgroup2_io_weight(Some(500)).ok(), Some(Some(4950))); + assert_eq!( + PveCli::cgroup2_io_weight(Some(1000)).ok(), + Some(Some(10_000)) + ); + assert!(PveCli::cgroup2_io_weight(Some(9)).is_err()); +} + +#[test] +fn resource_update_arguments_set_all_mutable_limits_and_clear_cpu_limit() { + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 4096, + swap_mib: 512, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: Some(250), + cores: Some(4), + cpuset_cpus: None, + }, + ), + vec![ + "set", + "1234", + "--memory", + "4096", + "--swap", + "512", + "--cpulimit", + "2.5", + "--cores", + "4", + ] + ); + + assert_eq!( + PveCli::set_resources_args( + 1234, + &ContainerResources { + memory_mib: 2048, + swap_mib: 0, + memory_unlimited: false, + swap_unlimited: false, + cpu_limit_percent: None, + cores: None, + cpuset_cpus: None, + }, + ), + vec![ + "set", + "1234", + "--memory", + "2048", + "--swap", + "0", + "--cpulimit", + "0", + "--delete", + "cores", + ] + ); +} + +#[test] +fn parses_non_root_oci_user_from_pct_config() { + assert_eq!( + PveCli::parse_oci_user( + "arch: amd64\nlxc.init.uid: 1000\nlxc.init.gid: 1001\nostype: unmanaged\n" + ) + .ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); + assert_eq!( + PveCli::parse_oci_user("lxc.init.uid: 0\nlxc.init.gid: 1000\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 1000 }) + ); + assert_eq!( + PveCli::parse_oci_user("arch: amd64\n").ok(), + Some(ContainerOciUser { uid: 0, gid: 0 }) + ); +} + +#[test] +fn pct_config_text_tolerates_nul_bytes() { + let config = PveCli::decode_config_output( + b"arch: amd64\0\nlxc.init.uid: 1000\nlxc.init.gid: 1001\0\n".to_vec(), + ) + .unwrap_or_default(); + + assert_eq!( + PveCli::parse_oci_user(&config).ok(), + Some(ContainerOciUser { + uid: 1000, + gid: 1001, + }) + ); +} + +#[test] +fn data_mount_arguments_map_resolved_image_user_to_host_owner() { + let args = PveCli::set_data_mount_args(&DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 1000, + container_gid: 1001, + host_data_uid: 988, + host_data_gid: 989, + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "set", + "1234", + "--mp0", + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + ] + ); +} + +#[test] +fn data_mount_arguments_support_root_in_an_unprivileged_container() { + let spec = DataMountSpec { + vmid: 1234, + data_path: "/var/lib/calagopus-wings/volumes/example".to_string(), + container_uid: 0, + container_gid: 1000, + host_data_uid: 988, + host_data_gid: 988, + }; + + assert_eq!( + PveCli::set_data_mount_args(&spec).ok(), + Some(vec![ + "set".to_string(), + "1234".to_string(), + "--mp0".to_string(), + "/var/lib/calagopus-wings/volumes/example,mp=/home/container,backup=0,idmap=u:0:988:1;g:1000:988:1".to_string(), + ]) + ); +} + +#[test] +fn bind_mount_arguments_support_multiple_targets_and_reject_duplicates() { + let mounts = vec![ + BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/mnt/server".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + BindMountSpec { + slot: 1, + source_path: "/srv/install".to_string(), + target_path: "/mnt/install".to_string(), + read_only: true, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }, + ]; + + assert_eq!( + PveCli::set_bind_mount_args(1234, &mounts).unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/mnt/server,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--mp1", + "/srv/install,mp=/mnt/install,backup=0,idmap=u:1000:988:1;g:1001:989:1,ro=1", + ] + ); + + let mut duplicate = mounts.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_bind_mount_args(1234, &duplicate).is_err()); + + let mut unsafe_path = mounts; + unsafe_path[1].target_path = "/mnt/install,bad".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); + unsafe_path[1].target_path = "/mnt/../etc".to_string(); + assert!(PveCli::set_bind_mount_args(1234, &unsafe_path).is_err()); +} + +#[test] +fn device_arguments_set_ownership_mode_and_write_policy() { + let devices = vec![ + DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }, + DevicePassthroughSpec { + slot: 1, + path: "/dev/dri/renderD128".to_string(), + uid: 1000, + gid: 1001, + mode: 0o640, + deny_write: true, + }, + ]; + + assert_eq!( + PveCli::set_devices_args(1234, &devices).unwrap_or_default(), + vec![ + "set", + "1234", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--dev1", + "path=/dev/dri/renderD128,uid=1000,gid=1001,mode=0640,deny-write=1", + ] + ); + + let mut duplicate = devices.clone(); + duplicate[1].slot = 0; + assert!(PveCli::set_devices_args(1234, &duplicate).is_err()); + + let mut unsafe_path = devices.clone(); + unsafe_path[1].path = "/dev/dri/renderD128,bad".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + unsafe_path[1].path = "/dev/../etc/shadow".to_string(); + assert!(PveCli::set_devices_args(1234, &unsafe_path).is_err()); + + let mut root_owned = devices; + root_owned[0].uid = 0; + root_owned[0].gid = 0; + assert!(PveCli::set_devices_args(1234, &root_owned).is_ok()); +} + +#[test] +fn mount_and_device_reconciliation_removes_stale_slots() { + let mounts = vec![BindMountSpec { + slot: 0, + source_path: "/srv/server".to_string(), + target_path: "/home/container".to_string(), + read_only: false, + container_uid: 1000, + container_gid: 1001, + host_uid: 988, + host_gid: 989, + }]; + let devices = vec![DevicePassthroughSpec { + slot: 0, + path: "/dev/kvm".to_string(), + uid: 1000, + gid: 1001, + mode: 0o660, + deny_write: false, + }]; + let current_config = r#" +rootfs: local-lvm:vm-1234-disk-0,size=8G +mp0: /srv/old-server,mp=/home/container +mp2: /srv/removed,mp=/mnt/removed +dev0: path=/dev/kvm +dev3: path=/dev/dri/renderD128 +lxc.init.uid: 1000 +"#; + + assert_eq!( + PveCli::reconcile_mounts_and_devices_args(1234, current_config, &mounts, &devices,) + .unwrap_or_default(), + vec![ + "set", + "1234", + "--mp0", + "/srv/server,mp=/home/container,backup=0,idmap=u:1000:988:1;g:1001:989:1", + "--dev0", + "path=/dev/kvm,uid=1000,gid=1001,mode=0660", + "--delete", + "mp2,dev3", + ] + ); +} + +#[test] +fn parses_pct_status() { + assert_eq!( + ContainerStatus::parse("status: running\n").ok(), + Some(ContainerStatus::Running) + ); + assert_eq!( + ContainerStatus::parse("status: stopped\n").ok(), + Some(ContainerStatus::Stopped) + ); + assert!(ContainerStatus::parse("status: mounted").is_err()); +} + +#[test] +fn parses_local_cluster_node() { + let output = br#"[ + {"type":"cluster","name":"og-cluster"}, + {"type":"node","name":"pve-a","local":0}, + {"type":"node","name":"pve-b","local":1} + ]"#; + + assert_eq!( + PveCli::parse_local_node(output).ok().as_deref(), + Some("pve-b") + ); +} + +#[test] +fn orders_available_active_lxc_storages() { + let output = br#"[ + {"storage":"local-lvm","active":1,"enabled":1,"content":"images,rootdir","avail":100}, + {"storage":"zfs","active":1,"enabled":1,"content":"rootdir,images","avail":200}, + {"storage":"local","active":1,"enabled":1,"content":"iso,vztmpl","avail":300}, + {"storage":"disabled","active":1,"enabled":0,"content":"rootdir","avail":400}, + {"storage":"offline","active":0,"enabled":1,"content":"rootdir","avail":500} + ]"#; + + assert_eq!( + PveCli::parse_available_lxc_storages(output).ok(), + Some(vec![ + LxcStorage { + name: "zfs".to_string(), + available_bytes: 200, + }, + LxcStorage { + name: "local-lvm".to_string(), + available_bytes: 100, + }, + ]) + ); +} + +#[test] +fn parses_lxc_resources_and_ignores_vms() { + let output = br#"[ + {"type":"qemu","vmid":101,"node":"pve-a","status":"running"}, + {"type":"lxc","vmid":102,"node":"pve-b","status":"running","tags":"calagopus;calagopus-server-abc"}, + {"type":"lxc","vmid":"103","node":"pve-b","status":"stopped"} + ]"#; + + let containers = PveCli::parse_containers(output).unwrap_or_default(); + assert_eq!(containers.len(), 2); + assert_eq!( + containers.first().map(|container| container.vmid), + Some(102) + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.first()) + .map(String::as_str), + Some("calagopus") + ); + assert_eq!( + containers + .first() + .and_then(|container| container.tags.get(1)) + .map(String::as_str), + Some("calagopus-server-abc") + ); + assert_eq!(containers.get(1).map(|container| container.vmid), Some(103)); +} + +#[test] +fn parses_container_interfaces_and_prefers_usable_ipv4() { + let interfaces = PveCli::parse_interfaces( + br#"[ + {"name":"lo","ip-addresses":[ + {"ip-address":"127.0.0.1","ip-address-type":"inet","prefix":8}, + {"ip-address":"::1","ip-address-type":"inet6","prefix":128} + ]}, + {"name":"eth0","ip-addresses":[ + {"ip-address":"fe80::1234","ip-address-type":"inet6","prefix":64}, + {"ip-address":"fd12:3456::20","ip-address-type":"inet6","prefix":64}, + {"ip-address":"10.42.0.20","ip-address-type":"inet","prefix":24} + ]} + ]"#, + ) + .unwrap_or_default(); + + assert_eq!(interfaces.len(), 2); + assert_eq!( + PveCli::primary_container_address(&interfaces), + "10.42.0.20".parse().ok() + ); + assert_eq!( + PveCli::interfaces_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/interfaces", + "--output-format", + "json", + ] + ); +} + +#[test] +fn stopped_container_null_interface_response_is_empty() { + assert_eq!(PveCli::parse_interfaces(b"null").unwrap(), Vec::new()); +} + +#[test] +fn container_addresses_returns_all_usable_addresses_sorted_and_deduplicated() { + let interfaces = vec![ + ContainerInterface { + name: "lo".to_string(), + addresses: vec!["127.0.0.1".parse().unwrap(), "::1".parse().unwrap()], + }, + ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "fe80::1234".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "0.0.0.0".parse().unwrap(), + "ff02::1".parse().unwrap(), + ], + }, + ]; + + assert_eq!( + PveCli::container_addresses(&interfaces), + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); +} + +#[test] +fn parses_runtime_status_for_cgroup_discovery() { + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"running","pid":"4321","uptime":87}"#,).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Running, + pid: Some(4321), + uptime_seconds: 87, + }) + ); + assert_eq!( + PveCli::parse_runtime_status(br#"{"status":"stopped","uptime":"0"}"#).ok(), + Some(ContainerRuntimeStatus { + status: ContainerStatus::Stopped, + pid: None, + uptime_seconds: 0, + }) + ); + assert!(PveCli::parse_runtime_status(br#"{"status":"mounted"}"#).is_err()); + assert_eq!( + PveCli::runtime_status_args("pve-a", 1234), + vec![ + "get", + "/nodes/pve-a/lxc/1234/status/current", + "--output-format", + "json", + ] + ); +} + +#[test] +fn parses_node_runtime_statuses_in_one_inventory() { + assert_eq!( + PveCli::parse_node_runtime_statuses( + br#"[{"vmid":100,"status":"running","pid":4321},{"vmid":101,"status":"stopped"}]"#, + ) + .ok(), + Some(vec![ + NodeContainerRuntimeStatus { + vmid: 100, + status: ContainerStatus::Running, + pid: Some(4321), + }, + NodeContainerRuntimeStatus { + vmid: 101, + status: ContainerStatus::Stopped, + pid: None, + }, + ]) + ); +} + +#[test] +fn container_address_falls_back_to_non_link_local_ipv6() { + let interfaces = vec![ContainerInterface { + name: "eth0".to_string(), + addresses: vec![ + "fe80::1234" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + "fd12:3456::20" + .parse() + .unwrap_or(IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)), + ], + }]; + + assert_eq!( + PveCli::primary_container_address(&interfaces), + "fd12:3456::20".parse().ok() + ); +} + +#[test] +fn normalizes_oci_references_for_proxmox() { + assert_eq!( + PveCli::normalize_oci_reference("ghcr.io/example/game").ok(), + Some("ghcr.io/example/game:latest".to_string()) + ); + assert_eq!( + PveCli::normalize_oci_reference("registry.example:5000/example/game:v2").ok(), + Some("registry.example:5000/example/game:v2".to_string()) + ); + assert!(PveCli::normalize_oci_reference("game@sha256:abc").is_err()); +} + +#[test] +fn oci_cache_name_is_stable_and_reference_specific() { + let first = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let again = PveCli::oci_cache_filename("ghcr.io/example/game:latest"); + let other = PveCli::oci_cache_filename("ghcr.io/example/game:v2"); + + assert_eq!(first, again); + assert_ne!(first, other); + assert!(first.starts_with("calagopus-oci-")); + assert_eq!(first.len(), "calagopus-oci-".len() + 64); +} + +#[test] +fn oci_image_tag_uses_resolved_revision_digest() { + let first = PveCli::oci_revision_tag( + "sha256:052a0ffb2620d9d59dfd81caa280af01ca8e6fe744649b11da481c00a8562d40", + ) + .unwrap_or_default(); + let other = PveCli::oci_revision_tag( + "5c093e071615ce61063007dfb363fd5bb878d4673954445e2a4ee5e7de3fae3b", + ) + .unwrap_or_default(); + + assert_ne!(first, other); + assert_eq!(first, "calagopus-image-052a0ffb2620d9d59dfd81caa280af01"); + assert!(PveCli::oci_revision_tag("sha256:not-a-digest").is_err()); +} + +#[test] +fn oci_pull_arguments_use_proxmox_storage_endpoint() { + let args = PveCli::pull_oci_args(&OciTemplatePullSpec { + node: "pve-a".to_string(), + storage: "local".to_string(), + reference: "ghcr.io/example/game:latest".to_string(), + filename: "calagopus-oci-abc".to_string(), + }); + + assert_eq!( + args, + vec![ + "create", + "/nodes/pve-a/storage/local/oci-registry-pull", + "--reference", + "ghcr.io/example/game:latest", + "--filename", + "calagopus-oci-abc", + "--output-format", + "json", + ] + ); +} + +#[test] +fn parses_oci_pull_task_after_proxmox_progress_output() { + let upid = "UPID:pve4:000348B0:005B00EB:6ABACCF1:ociregistrypull:template.tar:root@pam:"; + let noisy = format!( + "Getting image source signatures\nCopying blob sha256:abc\nWriting manifest to image destination\n\"{upid}\"\n" + ); + + assert_eq!( + PveCli::parse_oci_pull_task(noisy.as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert_eq!( + PveCli::parse_oci_pull_task(format!("\"{upid}\"").as_bytes()) + .ok() + .as_deref(), + Some(upid) + ); + assert!(PveCli::parse_oci_pull_task(b"copy completed without task id\n").is_err()); +} + +#[test] +fn parses_storage_content_and_task_status() { + let content = PveCli::parse_storage_content( + br#"[{"volid":"local:vztmpl/calagopus-oci-abc.tar","content":"vztmpl"}]"#, + ) + .unwrap_or_default(); + assert_eq!( + content.first(), + Some(&StorageContent { + volid: "local:vztmpl/calagopus-oci-abc.tar".to_string(), + content: Some("vztmpl".to_string()), + }) + ); + + assert_eq!( + PveCli::parse_task_status(br#"{"status":"running"}"#).ok(), + Some(TaskStatus { + state: TaskState::Running, + exit_status: None, + }) + ); + assert_eq!( + PveCli::parse_task_status(br#"{"status":"stopped","exitstatus":"OK"}"#).ok(), + Some(TaskStatus { + state: TaskState::Stopped, + exit_status: Some("OK".to_string()), + }) + ); +} + +#[test] +fn attached_process_runs_as_explicit_non_root_user_with_clean_environment() { + let args = PveCli::attach_args(&AttachProcessSpec { + vmid: 1234, + uid: 1000, + gid: 1000, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + command: vec!["/bin/bash".to_string(), "/entrypoint.sh".to_string()], + }) + .unwrap_or_default(); + + assert_eq!( + args, + vec![ + "-n", + "1234", + "--clear-env", + "-u", + "1000", + "-g", + "1000", + "-v", + "STARTUP=java -jar server.jar", + "-v", + "SERVER_MEMORY=4096", + "--", + "/bin/bash", + "/entrypoint.sh", + ] + ); +} + +#[test] +fn attached_process_supports_root_and_rejects_invalid_input() { + let spec = AttachProcessSpec { + vmid: 1234, + uid: 0, + gid: 1000, + environment: Vec::new(), + command: vec!["/bin/true".to_string()], + }; + assert!(PveCli::attach_args(&spec).is_ok()); + + let mut spec = spec; + spec.command.clear(); + assert!(PveCli::attach_args(&spec).is_err()); + + spec.command.push("/bin/true".to_string()); + spec.environment.push("BROKEN".to_string()); + assert!(PveCli::attach_args(&spec).is_err()); +} + +#[test] +fn runtime_configuration_uses_stdin_payload_and_validates_line_safety() { + let payload = PveCli::runtime_config_payload(&RuntimeConfigSpec { + vmid: 1234, + environment: vec![ + "STARTUP=java -jar server.jar".to_string(), + "SERVER_MEMORY=4096".to_string(), + ], + halt_signal: Some("SIGINT".to_string()), + init_command: Some("/bin/bash /mnt/install/install.sh".to_string()), + console_logfile: Some("/var/log/calagopus/install.log".to_string()), + cpuset_cpus: Some("0-3,8".to_string()), + pids_limit: Some(5120), + io_weight: Some(4950), + memory_unlimited: true, + swap_unlimited: true, + managed_file_mounts: Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus wings/vmount/example/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]), + }) + .unwrap_or_default(); + let value: serde_json::Value = serde_json::from_slice(&payload).unwrap_or_default(); + + assert_eq!( + value.get("vmid").and_then(serde_json::Value::as_u64), + Some(1234) + ); + assert_eq!( + value + .get("environment") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(2) + ); + assert_eq!( + value.get("halt_signal").and_then(serde_json::Value::as_str), + Some("SIGINT") + ); + assert_eq!( + value + .get("init_command") + .and_then(serde_json::Value::as_str), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + value + .get("console_logfile") + .and_then(serde_json::Value::as_str), + Some("/var/log/calagopus/install.log") + ); + assert_eq!( + value.get("cpuset_cpus").and_then(serde_json::Value::as_str), + Some("0-3,8") + ); + assert_eq!( + value.get("pids_limit").and_then(serde_json::Value::as_u64), + Some(5120) + ); + assert_eq!( + value.get("io_weight").and_then(serde_json::Value::as_u64), + Some(4950) + ); + assert_eq!( + value + .get("memory_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + value + .get("swap_unlimited") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + value + .get("managed_file_mounts") + .and_then(serde_json::Value::as_array) + .and_then(|mounts| mounts.first()) + .and_then(|mount| mount.get("entry")) + .and_then(serde_json::Value::as_str), + Some( + "/var/lib/calagopus\\040wings/vmount/example/hosts etc/hosts none bind,ro,create=file 0 0" + ) + ); + assert_eq!( + value + .get("managed_file_targets") + .and_then(serde_json::Value::as_array) + .map(Vec::len), + Some(5) + ); + + let mut invalid = RuntimeConfigSpec { + vmid: 1234, + environment: vec!["BROKEN".to_string()], + halt_signal: None, + init_command: None, + console_logfile: None, + cpuset_cpus: None, + pids_limit: None, + io_weight: None, + memory_unlimited: false, + swap_unlimited: false, + managed_file_mounts: None, + }; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment = vec!["GOOD=bad\nvalue".to_string()]; + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.environment.clear(); + invalid.console_logfile = Some("relative.log".to_string()); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + invalid.console_logfile = None; + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts".to_string(), + target_path: "/etc/shadow".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ManagedFileMountSpec { + source_path: "relative/hosts".to_string(), + target_path: "/etc/hosts".to_string(), + }]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); + + invalid.managed_file_mounts = Some(vec![ + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-a".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ManagedFileMountSpec { + source_path: "/var/lib/calagopus/hosts-b".to_string(), + target_path: "/etc/hosts".to_string(), + }, + ]); + assert!(PveCli::runtime_config_payload(&invalid).is_err()); +} + +#[test] +fn init_command_encoding_matches_lxc_whole_word_quoting() { + assert_eq!( + PveCli::encode_init_command(&[ + "/bin/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("/bin/bash /mnt/install/install.sh") + ); + assert_eq!( + PveCli::encode_init_command(&[ + "/opt/my shell/bash".to_string(), + "/mnt/install/install.sh".to_string(), + ]) + .ok() + .as_deref(), + Some("\"/opt/my shell/bash\" /mnt/install/install.sh") + ); + assert!(PveCli::encode_init_command(&["both'\"quotes here".to_string()]).is_err()); +} + +#[test] +fn lifecycle_arguments_request_nonblocking_graceful_shutdown_and_console() { + assert_eq!( + PveCli::request_shutdown_args(1234), + vec!["-n", "1234", "--nowait", "--nokill"] + ); + assert_eq!( + PveCli::console_args(1234), + vec!["console", "1234", "--escape", "^z"] + ); +} + +#[test] +fn vmid_floor_skips_ids_below_the_minimum() { + let used: std::collections::HashSet = [100, 101, 200].into_iter().collect(); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(201)); + assert_eq!( + PveCli::first_free_vmid_at_or_above(200, &std::collections::HashSet::new()), + Some(200) + ); +} + +#[test] +fn vmid_floor_finds_gaps_and_ignores_lower_ids() { + let used: std::collections::HashSet = [100, 200, 201, 203].into_iter().collect(); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(202)); +} + +#[test] +fn allocated_vmids_include_every_guest_type() { + let output = br#"[ + {"vmid": 200, "type": "lxc", "node": "pve1"}, + {"vmid": "201", "type": "qemu", "node": "pve1"}, + {"type": "lxc", "node": "pve1"}, + {"vmid": 202, "type": "storage", "node": "pve1"} + ]"#; + let used = PveCli::parse_allocated_vmids(output).expect("valid inventory"); + assert!(used.contains(&200)); + assert!(used.contains(&201)); + assert!(used.contains(&202)); + assert_eq!(PveCli::first_free_vmid_at_or_above(200, &used), Some(203)); +} + +#[test] +fn normalizes_gzip_oci_layers_for_proxmox_diff_ids() { + use flate2::{Compression, write::GzEncoder}; + use sha2::{Digest, Sha256}; + use std::io::{Cursor, Write}; + + fn append(builder: &mut tar::Builder, path: &str, bytes: &[u8]) { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, path, Cursor::new(bytes)) + .expect("append tar entry"); + } + + let directory = tempfile::tempdir().expect("temp directory"); + let path = directory.path().join("image.tar"); + let layer = b"uncompressed OCI layer contents"; + let diff_id = hex::encode(Sha256::digest(layer)); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(layer).expect("compress layer"); + let compressed = encoder.finish().expect("finish compression"); + let compressed_digest = hex::encode(Sha256::digest(&compressed)); + let config = serde_json::json!({"rootfs": {"type": "layers", "diff_ids": [format!("sha256:{diff_id}")]}}); + let config = serde_json::to_vec(&config).expect("encode config"); + let config_digest = hex::encode(Sha256::digest(&config)); + let manifest = serde_json::json!({ + "schemaVersion": 2, + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": format!("sha256:{config_digest}"), "size": config.len()}, + "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": format!("sha256:{compressed_digest}"), "size": compressed.len()}] + }); + let manifest = serde_json::to_vec(&manifest).expect("encode manifest"); + let manifest_digest = hex::encode(Sha256::digest(&manifest)); + let index = serde_json::to_vec(&serde_json::json!({"schemaVersion": 2, "manifests": [{"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": format!("sha256:{manifest_digest}"), "size": manifest.len()}]})).expect("encode index"); + let file = std::fs::File::create(&path).expect("create archive"); + let mut builder = tar::Builder::new(file); + append( + &mut builder, + "oci-layout", + br#"{"imageLayoutVersion":"1.0.0"}"#, + ); + append( + &mut builder, + &format!("blobs/sha256/{config_digest}"), + &config, + ); + append( + &mut builder, + &format!("blobs/sha256/{compressed_digest}"), + &compressed, + ); + append( + &mut builder, + &format!("blobs/sha256/{manifest_digest}"), + &manifest, + ); + append(&mut builder, "index.json", &index); + builder.finish().expect("finish archive"); + + PveCli::normalize_oci_template(path.clone()).expect("normalize OCI archive"); + let mut archive = + tar::Archive::new(std::fs::File::open(path).expect("open normalized archive")); + let mut normalized_index = None; + let mut blobs = std::collections::HashMap::new(); + for entry in archive.entries().expect("entries") { + let mut entry = entry.expect("entry"); + let name = entry.path().expect("path").to_string_lossy().into_owned(); + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read entry"); + if name == "index.json" { + normalized_index = Some(bytes); + } else { + blobs.insert(name, bytes); + } + } + let index: serde_json::Value = + serde_json::from_slice(&normalized_index.expect("index")).expect("parse index"); + let manifest_digest = index + .pointer("/manifests/0/digest") + .and_then(serde_json::Value::as_str) + .expect("manifest digest") + .strip_prefix("sha256:") + .expect("sha digest"); + let manifest: serde_json::Value = serde_json::from_slice( + blobs + .get(&format!("blobs/sha256/{manifest_digest}")) + .expect("manifest blob"), + ) + .expect("parse manifest"); + let expected_digest = format!("sha256:{diff_id}"); + assert_eq!( + manifest + .pointer("/layers/0/digest") + .and_then(serde_json::Value::as_str), + Some(expected_digest.as_str()) + ); + assert_eq!( + manifest + .pointer("/layers/0/mediaType") + .and_then(serde_json::Value::as_str), + Some("application/vnd.oci.image.layer.v1.tar") + ); + assert_eq!( + blobs + .get(&format!("blobs/sha256/{diff_id}")) + .map(Vec::as_slice), + Some(layer.as_slice()) + ); +} diff --git a/application/src/server/executor/pve_lxc/firewall.rs b/application/src/server/executor/pve_lxc/firewall.rs new file mode 100644 index 00000000..75718469 --- /dev/null +++ b/application/src/server/executor/pve_lxc/firewall.rs @@ -0,0 +1,651 @@ +use super::cli::{FirewallPolicySpec, FirewallRuleSpec, PveCli}; +use crate::server::firewall::{ + FirewallBackend, FirewallRuleAction, FirewallRuleProtocol, FirewallServerSpec, FirewallTarget, + sets, +}; +use anyhow::Context; +use std::{ + collections::{BTreeMap, BTreeSet, HashSet}, + sync::Arc, +}; + +const MANAGED_COMMENT_PREFIX: &str = "calagopus-wings:"; + +#[derive(Clone)] +struct StoredPolicy { + vmid: u32, + server: uuid::Uuid, + ports: Vec, + rules: Vec, + files: Option, + ipsets: BTreeMap>, +} + +struct Watch { + changed: Arc, + task: tokio::task::AbortHandle, + notifier: crate::server::filesystem::inotify::InotifyServerNotifier, +} + +impl Drop for Watch { + fn drop(&mut self) { + self.task.abort(); + self.notifier + .watch_firewall_files(Vec::new(), Arc::clone(&self.changed)); + } +} + +#[derive(Default)] +struct State { + policies: BTreeMap, + watches: BTreeMap, +} + +struct Inner { + cli: PveCli, + node: String, + tag_prefix: String, + limits: sets::SourceFileLimits, + state: tokio::sync::Mutex, + operation: tokio::sync::Mutex<()>, +} + +pub struct ProxmoxFirewall { + inner: Arc, +} + +impl ProxmoxFirewall { + pub fn new( + cli: PveCli, + node: String, + tag_prefix: String, + limits: sets::SourceFileLimits, + ) -> Self { + Self { + inner: Arc::new(Inner { + cli, + node, + tag_prefix, + limits, + state: tokio::sync::Mutex::new(State::default()), + operation: tokio::sync::Mutex::new(()), + }), + } + } + + fn stored(spec: &FirewallServerSpec) -> Result { + let FirewallTarget::ProxmoxLxc { vmid, interface } = spec + .target + .as_ref() + .context("Proxmox firewall requires an LXC target")?; + if interface != "net0" { + anyhow::bail!("unsupported Proxmox firewall interface {interface}"); + } + + let mut ports = spec.container_ports.clone(); + ports.sort_unstable(); + ports.dedup(); + Ok(StoredPolicy { + vmid: *vmid, + server: spec.server, + ports, + rules: spec.rules.clone(), + files: spec.files.clone(), + ipsets: BTreeMap::new(), + }) + } + + fn managed_prefix(server: uuid::Uuid) -> String { + format!("{MANAGED_COMMENT_PREFIX}{server}:") + } + + fn rule_sources( + server: uuid::Uuid, + rule: &crate::server::firewall::FirewallRule, + ) -> Vec> { + let mut sources: Vec> = rule + .sources + .iter() + .map(|source| Some(format!("{source:#}"))) + .collect(); + sources.sort(); + if let Some(file) = &rule.source_file { + let path = sets::source_file_path(file); + sources.push(Some(format!("+{}", sets::set_base_name(server, &path)))); + } + if sources.is_empty() { + sources.push(None); + } + sources + } + + fn translate(policy: &StoredPolicy) -> Vec { + let mut translated = Vec::new(); + let prefix = Self::managed_prefix(policy.server); + + for (rule_index, rule) in policy.rules.iter().enumerate() { + if rule.action == FirewallRuleAction::Deny + && rule.protocols.is_empty() + && rule.sources.is_empty() + && rule.ports.is_none() + && rule.source_file.is_none() + { + let sequence = translated.len(); + translated.push(FirewallRuleSpec { + r#type: "in".to_string(), + action: "DROP".to_string(), + iface: "net0".to_string(), + proto: None, + dport: None, + source: None, + comment: format!("{prefix}{rule_index}:{sequence} managed by Wings"), + }); + continue; + } + + let allowed_ports: BTreeSet = match &rule.ports { + Some(restricted) => policy + .ports + .iter() + .copied() + .filter(|port| restricted.contains(port)) + .collect(), + None => policy.ports.iter().copied().collect(), + }; + if allowed_ports.is_empty() { + continue; + } + let dport = allowed_ports + .into_iter() + .map(|port| port.to_string()) + .collect::>() + .join(","); + let protocols: Vec = + [FirewallRuleProtocol::Tcp, FirewallRuleProtocol::Udp] + .into_iter() + .filter(|protocol| { + rule.protocols.is_empty() || rule.protocols.contains(protocol) + }) + .collect(); + + for protocol in protocols { + for source in Self::rule_sources(policy.server, rule) { + let sequence = translated.len(); + translated.push(FirewallRuleSpec { + r#type: "in".to_string(), + action: match rule.action { + FirewallRuleAction::Allow => "ACCEPT", + FirewallRuleAction::Deny => "DROP", + } + .to_string(), + iface: "net0".to_string(), + proto: Some(protocol.as_str().to_string()), + dport: Some(dport.clone()), + source, + comment: format!("{prefix}{rule_index}:{sequence} managed by Wings"), + }); + } + } + } + + translated + } + + async fn load_ipsets(&self, policy: &mut StoredPolicy) { + let Some(access) = &policy.files else { + policy.ipsets.clear(); + return; + }; + + let referenced = policy.rules.iter().filter_map(|rule| { + rule.source_file.as_ref().map(|file| { + let path = sets::source_file_path(file); + (sets::set_base_name(policy.server, &path), path) + }) + }); + let mut wanted = BTreeSet::new(); + for (name, path) in referenced { + wanted.insert(name.clone()); + policy.ipsets.entry(name.clone()).or_default(); + match sets::read_source_file(&access.filesystem, &path, self.inner.limits).await { + Ok((entries, stats)) => { + policy + .ipsets + .insert(name, entries.into_iter().map(|entry| format!("{entry:#}")).collect()); + let message = format!( + "Firewall source file {} loaded with {} entries{}.", + path.display(), + stats.entries, + if stats.invalid == 0 { + String::new() + } else { + format!(", {} invalid lines skipped", stats.invalid) + } + ); + if stats.entries == 0 || stats.invalid > 0 { + access.log_error(&message); + } else { + access.log(&message); + } + } + Err(error) => access.log_error(&format!( + "Firewall source file {} could not be loaded, the previous entries are retained: {error}", + path.display() + )), + } + } + policy.ipsets.retain(|name, _| wanted.contains(name)); + } + + async fn apply_policy(&self, policy: &mut StoredPolicy) -> Result<(), anyhow::Error> { + self.load_ipsets(policy).await; + // Enable filtering first. Applying the managed rules is then the + // committing operation, so a crash cannot leave new rules ineffective. + self.inner.cli.enable_network_firewall(policy.vmid).await?; + self.inner + .cli + .apply_firewall_config(&FirewallPolicySpec { + vmid: policy.vmid, + managed_prefix: Self::managed_prefix(policy.server), + rules: Self::translate(policy), + ipsets: policy.ipsets.clone(), + }) + .await + } + + async fn apply_empty(&self, vmid: u32, server: uuid::Uuid) -> Result<(), anyhow::Error> { + self.inner + .cli + .apply_firewall_config(&FirewallPolicySpec { + vmid, + managed_prefix: Self::managed_prefix(server), + rules: Vec::new(), + ipsets: BTreeMap::new(), + }) + .await + } + + async fn reload(inner: Arc, server: uuid::Uuid) { + let firewall = Self { inner }; + let _operation = firewall.inner.operation.lock().await; + let policy = firewall + .inner + .state + .lock() + .await + .policies + .get(&server) + .cloned(); + let Some(mut policy) = policy else { + return; + }; + if let Err(error) = firewall.apply_policy(&mut policy).await { + tracing::error!(server = %server, "failed to reload Proxmox firewall source files: {error:#}"); + return; + } + firewall + .inner + .state + .lock() + .await + .policies + .insert(server, policy); + } + + async fn update_watch(&self, policy: &StoredPolicy) { + let mut state = self.inner.state.lock().await; + state.watches.remove(&policy.server); + let Some(access) = &policy.files else { + return; + }; + let paths: Vec<_> = policy + .rules + .iter() + .filter_map(|rule| rule.source_file.as_ref()) + .map(|file| { + access + .filesystem + .base_path + .join(sets::source_file_path(file)) + }) + .collect(); + if paths.is_empty() { + return; + } + + let changed = Arc::new(tokio::sync::Notify::new()); + let Some(notifier) = access.notifier.clone() else { + return; + }; + notifier.watch_firewall_files(paths, Arc::clone(&changed)); + let inner = Arc::clone(&self.inner); + let server = policy.server; + let task = tokio::spawn({ + let changed = Arc::clone(&changed); + async move { + loop { + changed.notified().await; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + Self::reload(Arc::clone(&inner), server).await; + } + } + }) + .abort_handle(); + state.watches.insert( + policy.server, + Watch { + changed, + task, + notifier, + }, + ); + } + + async fn owned_vmids(&self, server: uuid::Uuid) -> Result, anyhow::Error> { + let server_tag = format!("{}-server-{server}", self.inner.tag_prefix); + let replacement_tag = format!("{}-repl-{server}", self.inner.tag_prefix); + Ok(self + .inner + .cli + .list_containers() + .await? + .into_iter() + .filter(|container| { + container.node == self.inner.node + && container + .tags + .iter() + .any(|tag| tag == &server_tag || tag == &replacement_tag) + }) + .map(|container| container.vmid) + .collect()) + } + + fn managed_vmids_for( + containers: Vec, + node: &str, + tag_prefix: &str, + ) -> BTreeMap> { + let server_prefix = format!("{tag_prefix}-server-"); + let replacement_prefix = format!("{tag_prefix}-repl-"); + let mut managed = BTreeMap::>::new(); + for container in containers { + if container.node != node { + continue; + } + for tag in &container.tags { + let value = tag + .strip_prefix(&server_prefix) + .or_else(|| tag.strip_prefix(&replacement_prefix)); + let Some(server) = value.and_then(|value| uuid::Uuid::parse_str(value).ok()) else { + continue; + }; + managed.entry(server).or_default().push(container.vmid); + break; + } + } + managed + } + + async fn managed_vmids(&self) -> Result>, anyhow::Error> { + Ok(Self::managed_vmids_for( + self.inner.cli.list_containers().await?, + &self.inner.node, + &self.inner.tag_prefix, + )) + } + + /// Removes Wings-owned guest rules before switching to a host-local + /// backend. Administrator rules and containers on other cluster nodes are + /// left untouched. + pub async fn clear_managed(&self) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + for (server, vmids) in self.managed_vmids().await? { + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + } + self.inner.state.lock().await.policies.clear(); + self.inner.state.lock().await.watches.clear(); + Ok(()) + } +} + +#[async_trait::async_trait] +impl FirewallBackend for ProxmoxFirewall { + async fn boot(&self) -> Result<(), anyhow::Error> { + if !self.inner.cli.cluster_firewall_enabled().await? { + anyhow::bail!( + "the Proxmox datacenter firewall is disabled; enable it before using the Proxmox LXC firewall backend" + ); + } + tracing::info!("using Proxmox guest firewall backend"); + Ok(()) + } + + async fn sync(&self, spec: &FirewallServerSpec) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + let mut policy = Self::stored(spec)?; + if let Some(previous) = self.inner.state.lock().await.policies.get(&spec.server) { + policy.ipsets = previous.ipsets.clone(); + } + self.apply_policy(&mut policy).await?; + self.inner + .state + .lock() + .await + .policies + .insert(spec.server, policy.clone()); + drop(_operation); + self.update_watch(&policy).await; + Ok(()) + } + + async fn clear(&self, server: uuid::Uuid) -> Result<(), anyhow::Error> { + let _operation = self.inner.operation.lock().await; + let remembered = self + .inner + .state + .lock() + .await + .policies + .remove(&server) + .map(|policy| policy.vmid); + self.inner.state.lock().await.watches.remove(&server); + let mut vmids: HashSet = self.owned_vmids(server).await?.into_iter().collect(); + if let Some(vmid) = remembered { + vmids.insert(vmid); + } + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + Ok(()) + } + + async fn reconcile(&self, specs: &[FirewallServerSpec]) -> Result<(), anyhow::Error> { + let wanted: HashSet = specs.iter().map(|spec| spec.server).collect(); + let mut stale: HashSet = self + .inner + .state + .lock() + .await + .policies + .keys() + .filter(|server| !wanted.contains(server)) + .copied() + .collect(); + let discovered = self.managed_vmids().await?; + stale.extend( + discovered + .keys() + .filter(|server| !wanted.contains(server)) + .copied(), + ); + for server in stale { + let _operation = self.inner.operation.lock().await; + let remembered = self + .inner + .state + .lock() + .await + .policies + .remove(&server) + .map(|policy| policy.vmid); + self.inner.state.lock().await.watches.remove(&server); + let mut vmids: HashSet = discovered + .get(&server) + .into_iter() + .flatten() + .copied() + .collect(); + if let Some(vmid) = remembered { + vmids.insert(vmid); + } + for vmid in vmids { + self.apply_empty(vmid, server).await?; + } + } + for spec in specs { + self.sync(spec).await?; + } + crate::server::firewall::clear_host_local(self.inner.limits).await; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn managed_inventory_excludes_other_cluster_nodes() { + let local = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let remote = uuid::Uuid::parse_str("12345678-3456-7890-abcd-ef1234567890").unwrap(); + let containers = vec![ + super::super::cli::ClusterContainer { + vmid: 100, + node: "pve-a".to_string(), + status: None, + tags: vec![format!("calagopus-server-{local}")], + }, + super::super::cli::ClusterContainer { + vmid: 200, + node: "pve-b".to_string(), + status: None, + tags: vec![format!("calagopus-server-{remote}")], + }, + ]; + + let managed = ProxmoxFirewall::managed_vmids_for(containers, "pve-a", "calagopus"); + assert_eq!(managed.get(&local), Some(&vec![100])); + assert!(!managed.contains_key(&remote)); + } + + #[test] + fn translates_panel_rules_to_vm_scoped_proxmox_rules() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565, 25566], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::from([FirewallRuleProtocol::Tcp]), + sources: vec!["10.0.0.0/8".parse().unwrap()], + ports: Some(vec![25565]), + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].iface, "net0"); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[0].dport.as_deref(), Some("25565")); + assert_eq!(rules[0].source.as_deref(), Some("10.0.0.0/8")); + } + + #[test] + fn translates_empty_protocols_and_source_files() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let source_file = "blocked.txt".to_string(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Allow, + protocols: HashSet::new(), + sources: Vec::new(), + ports: None, + source_file: Some(source_file.clone().into()), + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 2); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[1].proto.as_deref(), Some("udp")); + let path = sets::source_file_path(&source_file); + let expected = format!("+{}", sets::set_base_name(server, &path)); + assert_eq!(rules[0].source.as_deref(), Some(expected.as_str())); + assert_eq!(rules[1].source.as_deref(), Some(expected.as_str())); + } + + #[test] + fn does_not_emit_synthetic_fallback_rules() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565, 25566], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::from([FirewallRuleProtocol::Tcp]), + sources: Vec::new(), + ports: Some(vec![25565]), + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].proto.as_deref(), Some("tcp")); + assert_eq!(rules[0].dport.as_deref(), Some("25565")); + } + + #[test] + fn translates_terminal_deny_everything_as_an_unrestricted_drop() { + let server = uuid::Uuid::parse_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(); + let policy = StoredPolicy { + vmid: 100, + server, + ports: vec![25565], + rules: vec![crate::server::firewall::FirewallRule { + action: FirewallRuleAction::Deny, + protocols: HashSet::new(), + sources: Vec::new(), + ports: None, + source_file: None, + }], + files: None, + ipsets: BTreeMap::new(), + }; + + let rules = ProxmoxFirewall::translate(&policy); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].action, "DROP"); + assert_eq!(rules[0].iface, "net0"); + assert_eq!(rules[0].proto, None); + assert_eq!(rules[0].dport, None); + assert_eq!(rules[0].source, None); + } +} diff --git a/application/src/server/executor/pve_lxc/helper_process.rs b/application/src/server/executor/pve_lxc/helper_process.rs new file mode 100644 index 00000000..0b8e2402 --- /dev/null +++ b/application/src/server/executor/pve_lxc/helper_process.rs @@ -0,0 +1,383 @@ +use super::*; + +pub(super) struct PveHelperProcessHandle { + vmid: u32, + cli: PveCli, + log_path: PathBuf, + started: Arc, + cancelled: Arc, + stdin_tx: tokio::sync::mpsc::Sender>, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, + console_task: tokio::task::JoinHandle<()>, +} + +impl PveHelperProcessHandle { + #[allow(clippy::too_many_arguments)] + pub(super) async fn new( + vmid: u32, + cli: PveCli, + server: &crate::server::Server, + app_config: Arc, + log_path: PathBuf, + status_tx: tokio::sync::mpsc::Sender, + already_started: bool, + start_at_end: bool, + auto_cleanup: bool, + cleanup_path: Option, + ) -> Result { + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); + + let started = Arc::new(AtomicBool::new(already_started)); + let cancelled = Arc::new(AtomicBool::new(false)); + + let status_cli = cli.clone(); + let status_started = Arc::clone(&started); + let status_cancelled = Arc::clone(&cancelled); + let status_server = server.uuid; + tokio::spawn(async move { + let mut sent_running = false; + let mut tick = tokio::time::interval(std::time::Duration::from_millis(500)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + if status_cancelled.load(Ordering::Acquire) { + break; + } + + match status_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => { + if !sent_running { + sent_running = true; + if status_tx.send(super::ProcessStatus::Running).await.is_err() { + break; + } + } + } + Ok(cli::ContainerStatus::Stopped) if status_started.load(Ordering::Acquire) => { + if !sent_running + && status_tx.send(super::ProcessStatus::Running).await.is_err() + { + break; + } + let _ = status_tx + .send(super::ProcessStatus::Stopped { + exit_code: -1, + oom_killed: false, + }) + .await; + break; + } + Ok(cli::ContainerStatus::Stopped) => {} + Err(error) => { + tracing::warn!( + server = %status_server, + vmid, + "failed to read Proxmox helper LXC process state: {error:#}" + ); + } + } + } + }); + + let output_cli = cli.clone(); + let output_started = Arc::clone(&started); + let output_cancelled = Arc::clone(&cancelled); + let output_server = Arc::downgrade(&**server); + let output_log_path = log_path.clone(); + tokio::spawn(async move { + let mut file = match tokio::fs::OpenOptions::new() + .read(true) + .open(&output_log_path) + .await + { + Ok(file) => file, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox helper log: {error:#}"); + return; + } + }; + if start_at_end + && let Ok(metadata) = file.metadata().await + && let Err(error) = file.seek(std::io::SeekFrom::Start(metadata.len())).await + { + tracing::warn!(vmid, "failed to seek Proxmox helper log: {error:#}"); + } + + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + if output_cancelled.load(Ordering::Acquire) { + break; + } + + if let (Ok(position), Ok(metadata)) = + (file.stream_position().await, file.metadata().await) + && metadata.len() < position + { + if let Err(error) = file.seek(std::io::SeekFrom::Start(0)).await { + tracing::warn!( + vmid, + "failed to rewind truncated Proxmox helper log: {error:#}" + ); + } + line_buffer = LineBuffer::new(); + } + + let mut data = Vec::new(); + match file.read_to_end(&mut data).await { + Ok(_) => {} + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox helper log: {error:#}"); + } + } + + if !data.is_empty() { + line_buffer.extend(&data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = output_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + + let stopped = output_started.load(Ordering::Acquire) + && matches!( + output_cli.status(vmid).await, + Ok(cli::ContainerStatus::Stopped) + ); + if stopped { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let mut final_data = Vec::new(); + if file.read_to_end(&mut final_data).await.is_ok() && !final_data.is_empty() { + line_buffer.extend(&final_data); + while let Some(line) = line_buffer.next_line() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + if let Some(line) = line_buffer.flush() { + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + break; + } + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + if auto_cleanup { + output_cancelled.store(true, Ordering::Release); + if let Err(error) = output_cli.destroy(vmid).await { + tracing::warn!( + vmid, + "failed to destroy completed Proxmox helper LXC: {error:#}" + ); + } + if let Some(path) = cleanup_path + && let Err(error) = tokio::fs::remove_dir_all(&path).await + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + path = %path.display(), + "failed to remove Proxmox helper staging directory: {error:#}" + ); + } + } + }); + + let console_cli = cli.clone(); + let console_started = Arc::clone(&started); + let console_cancelled = Arc::clone(&cancelled); + let console_task = tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + if console_cancelled.load(Ordering::Acquire) { + return; + } + if !console_started.load(Ordering::Acquire) + || !matches!( + console_cli.status(vmid).await, + Ok(cli::ContainerStatus::Running) + ) + { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + continue; + } + + let session = match console_cli.spawn_console(vmid) { + Ok(session) => session, + Err(error) => { + tracing::warn!(vmid, "failed to attach Proxmox helper console: {error:#}"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + let cli::ConsoleSession { + mut child, + mut reader, + mut writer, + } = session; + + loop { + tokio::select! { + read = reader.read(&mut buffer) => { + match read { + Ok(0) => break, + Ok(_) => {} + Err(error) => { + tracing::warn!(vmid, "failed to drain Proxmox helper console: {error:#}"); + break; + } + } + } + data = stdin_rx.recv() => { + let Some(data) = data else { + child.start_kill().ok(); + return; + }; + let data = PveProcessHandle::normalize_console_input(data); + if let Err(error) = writer.write_all(&data).await { + tracing::warn!(vmid, "failed to write Proxmox helper stdin: {error:#}"); + break; + } + } + result = child.wait() => { + if let Err(error) = result { + tracing::warn!(vmid, "failed waiting for Proxmox helper console: {error:#}"); + } + break; + } + } + } + child.start_kill().ok(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + }); + + Ok(Self { + vmid, + cli, + log_path, + started, + cancelled, + stdin_tx, + stdout_ratelimited_rx, + stdout_rx, + console_task, + }) + } +} + +impl Drop for PveHelperProcessHandle { + fn drop(&mut self) { + self.console_task.abort(); + if !self.started.load(Ordering::Acquire) { + self.cancelled.store(true, Ordering::Release); + } + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveHelperProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = tokio::fs::File::open(&self.log_path).await?; + Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)) + } else { + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + } + + async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { + self.stdin_tx.send(data).await.map_err(anyhow::Error::from) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + self.cli.stop(self.vmid).await + } +} diff --git a/application/src/server/executor/pve_lxc/mod.rs b/application/src/server/executor/pve_lxc/mod.rs new file mode 100644 index 00000000..ec6882d0 --- /dev/null +++ b/application/src/server/executor/pve_lxc/mod.rs @@ -0,0 +1,2750 @@ +//! Proxmox VE LXC runtime support. +//! +//! This module intentionally sits behind the existing `ServerExecutor` boundary so +//! Docker remains the default runtime and upstream changes to the Docker executor +//! do not have to be carried into the PVE implementation. + +pub mod cli; +pub mod firewall; +mod helper_process; +mod server_process; + +use super::{ProcessHandle, ProcessStatus, ServerExecutor, StatusReceiver, UsedPort, cgroup}; +use crate::io::line_buffer::LineBuffer; +use anyhow::Context; +use compact_str::ToCompactString; +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + io::Cursor, + net::{IpAddr, Ipv4Addr}, + path::PathBuf, + sync::{ + Arc, OnceLock, Weak, + atomic::{AtomicBool, Ordering}, + }, +}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +pub use cli::PveCli; +use helper_process::PveHelperProcessHandle; +use server_process::PveProcessHandle; + +pub async fn create_firewall( + app_config: &crate::config::Config, + cli: &PveCli, +) -> Result, anyhow::Error> { + let (backend, limits, tag_prefix, configured_node) = { + let config = app_config.load(); + let firewall = &config.runtime.pve_lxc.firewall; + ( + firewall.backend, + crate::server::firewall::sets::SourceFileLimits { + max_entries: usize::try_from(firewall.source_file_max_entries) + .context("Proxmox firewall source entry limit exceeds usize")?, + max_bytes: firewall.source_file_max_bytes, + }, + config.runtime.pve_lxc.tag_prefix.clone(), + config.runtime.pve_lxc.node.clone(), + ) + }; + let local_node = cli.local_node().await?; + let node = PveLxcExecutor::runtime_node(&configured_node, &local_node)?; + + let proxmox_enabled = match cli.cluster_firewall_enabled().await { + Ok(enabled) => enabled, + Err(error) if backend == crate::config::PveLxcFirewallBackend::Auto => { + tracing::warn!( + "failed to probe the Proxmox datacenter firewall, falling back to host firewalling: {error:#}" + ); + false + } + Err(error) => return Err(error.context("failed to probe the Proxmox datacenter firewall")), + }; + + let firewall: Arc = match backend { + crate::config::PveLxcFirewallBackend::Auto if proxmox_enabled => Arc::new( + firewall::ProxmoxFirewall::new(cli.clone(), node.clone(), tag_prefix, limits), + ), + crate::config::PveLxcFirewallBackend::Proxmox => Arc::new(firewall::ProxmoxFirewall::new( + cli.clone(), + node, + tag_prefix, + limits, + )), + crate::config::PveLxcFirewallBackend::Auto => { + tracing::info!( + "Proxmox datacenter firewall is disabled; using the host-local firewall backend" + ); + firewall::ProxmoxFirewall::new(cli.clone(), node, tag_prefix, limits) + .clear_managed() + .await + .context("failed to clear stale Proxmox guest firewall rules before selecting the host-local backend")?; + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Auto, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Nftables => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Nftables, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Iptables => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Iptables, + limits, + ) + .await + } + crate::config::PveLxcFirewallBackend::Disabled => { + crate::server::firewall::create_host_local_with( + crate::server::firewall::FirewallBackendKind::Disabled, + limits, + ) + .await + } + }; + Ok(firewall) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProcNetFamily { + Ipv4, + Ipv6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BoundPort { + family: ProcNetFamily, + address: Option, + port: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct EdgePeer { + public_ip: Ipv4Addr, + tunnel_ip: Ipv4Addr, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PveVersion { + major: u64, + minor: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ServerContainerPlan { + Reuse { + vmid: u32, + stale_replacement: Option, + }, + ReplaceWithStaged { + old_vmid: u32, + staged_vmid: u32, + }, + ReplaceFresh { + old_vmid: u32, + stale_replacement: Option, + }, + RecoverStaged { + staged_vmid: u32, + }, + CreateFresh { + stale_replacement: Option, + }, +} + +impl PveVersion { + fn parse(value: &str) -> Result { + let (_, version_and_rest) = value + .trim() + .split_once('/') + .context("pveversion output did not include a version")?; + let version = version_and_rest + .split('/') + .next() + .context("pveversion output did not include a version number")?; + let mut parts = version.split('.'); + let major = parts + .next() + .context("pveversion output did not include a major version")? + .parse() + .context("failed to parse Proxmox major version")?; + let minor = parts + .next() + .context("pveversion output did not include a minor version")? + .parse() + .context("failed to parse Proxmox minor version")?; + + Ok(Self { major, minor }) + } + + fn is_supported(self) -> bool { + self.major > 9 || (self.major == 9 && self.minor >= 2) + } +} + +pub struct PveLxcExecutor { + cli: PveCli, + app_config: Arc, + firewall: Arc, + stats_sampler: Arc, + node: OnceLock, + template_provisioning: tokio::sync::Mutex>>>, +} + +impl PveLxcExecutor { + const DATA_MOUNT_TARGET: &'static str = "/home/container"; + + pub(super) fn pids_limit(config: &crate::config::Config) -> Option { + let config = config.load(); + match config + .runtime + .pve_lxc + .pids_limit + .unwrap_or(config.docker.container_pid_limit) + { + 0 => None, + limit => Some(limit), + } + } + + fn image_cache_max_age(config: &crate::config::InnerConfig) -> std::time::Duration { + let cache = config.docker.registry_image_fetch_cache; + if cache.enabled { + std::time::Duration::from_secs(cache.duration) + } else { + std::time::Duration::ZERO + } + } + + async fn template_provisioning_lock( + &self, + node: &str, + storage: &str, + image: &str, + ) -> Arc> { + let key = format!("{node}\0{storage}\0{image}"); + let mut locks = self.template_provisioning.lock().await; + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(key, Arc::downgrade(&lock)); + lock + } + + fn network_option( + bridge: &str, + vlan_tag: Option, + address: Option<(Ipv4Addr, u8, Ipv4Addr)>, + ) -> Result { + if bridge.is_empty() + || !bridge + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) + { + return Err(anyhow::anyhow!("invalid Proxmox LXC bridge: {bridge}")); + } + if let Some(tag) = vlan_tag + && !(1..=4094).contains(&tag) + { + return Err(anyhow::anyhow!( + "Proxmox LXC VLAN tag must be between 1 and 4094, got {tag}" + )); + } + + let mut options = vec![ + format!("name=eth0,bridge={bridge}"), + "firewall=1".to_string(), + ]; + if let Some(tag) = vlan_tag { + options.push(format!("tag={tag}")); + } + match address { + Some((ip, prefix, gateway)) => { + if prefix > 32 { + return Err(anyhow::anyhow!( + "Proxmox LXC network prefix must be between 0 and 32, got {prefix}" + )); + } + options.push(format!("ip={ip}/{prefix}")); + options.push(format!("gw={gateway}")); + } + None => options.push("ip=dhcp".to_string()), + } + options.push("type=veth".to_string()); + if address.is_none() { + options.push("host-managed=1".to_string()); + } + Ok(options.join(",")) + } + + fn dhcp_network(bridge: &str, vlan_tag: Option) -> Result { + Self::network_option(bridge, vlan_tag, None) + } + + fn network_for_allocation( + bridge: &str, + vlan_tag: Option, + network_prefix: Option, + gateway: Option<&str>, + allocation: Option<&str>, + ) -> Result { + let Some(allocation) = allocation else { + return Self::dhcp_network(bridge, vlan_tag); + }; + let allocation = allocation.parse::().with_context(|| { + format!("LXC primary allocation {allocation} is not an IPv4 address") + })?; + let prefix = network_prefix.context( + "Proxmox LXC static allocations require runtime.pve_lxc.network_prefix; configure the isolated LXC subnet before assigning panel allocations", + )?; + let gateway = gateway + .filter(|gateway| !gateway.trim().is_empty()) + .context( + "Proxmox LXC static allocations require runtime.pve_lxc.gateway; configure the isolated LXC subnet before assigning panel allocations", + )? + .parse::() + .context("Proxmox LXC gateway is not an IPv4 address")?; + Self::network_option(bridge, vlan_tag, Some((allocation, prefix, gateway))) + } + + async fn rootfs_storage( + &self, + node: &str, + configured_storage: String, + rootfs_size_gib: u64, + ) -> Result { + if !configured_storage.is_empty() && configured_storage != "auto" { + return Ok(configured_storage); + } + + let required_bytes = rootfs_size_gib + .checked_mul(1024 * 1024 * 1024) + .context("Proxmox LXC rootfs size overflowed u64")?; + self.cli + .available_lxc_storages(node) + .await? + .into_iter() + .find(|storage| storage.available_bytes >= required_bytes) + .map(|storage| storage.name) + .context(format!( + "no active Proxmox LXC storage has {} GiB free for the requested rootfs", + rootfs_size_gib + )) + } + + const INSTALLER_HELPER_ROLE: &'static str = "installer"; + const SCRIPT_HELPER_ROLE: &'static str = "script"; + + pub fn new( + app_config: Arc, + cli: PveCli, + firewall: Arc, + ) -> Self { + Self { + cli, + app_config, + firewall, + stats_sampler: Arc::new(super::cgroup::StatsSampler::default()), + node: OnceLock::new(), + template_provisioning: tokio::sync::Mutex::new(HashMap::new()), + } + } + + #[cfg(not(target_os = "linux"))] + fn unsupported(operation: &str) -> Result { + Err(anyhow::anyhow!( + "Proxmox VE LXC runtime does not implement {operation} yet" + )) + } + + fn runtime_node(configured_node: &str, local_node: &str) -> Result { + if configured_node.trim().is_empty() { + return Ok(local_node.to_string()); + } + if configured_node != local_node { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime must target the local node ({local_node}); configured node was {configured_node}" + )); + } + + Ok(configured_node.to_string()) + } + + fn vmid_conflict(error: &anyhow::Error) -> bool { + let message = error.to_string().to_ascii_lowercase(); + message.contains("already exists") || message.contains("already in use") + } + + fn parse_proc_net_address(raw: &str, family: ProcNetFamily) -> Result { + match family { + ProcNetFamily::Ipv4 => { + if raw.len() != 8 { + return Err(anyhow::anyhow!( + "invalid IPv4 address in Linux socket table: {raw}" + )); + } + let word = u32::from_str_radix(raw, 16) + .context("failed to parse IPv4 address from Linux socket table")?; + Ok(std::net::Ipv4Addr::from(word.to_le_bytes()).into()) + } + ProcNetFamily::Ipv6 => { + if raw.len() != 32 { + return Err(anyhow::anyhow!( + "invalid IPv6 address in Linux socket table: {raw}" + )); + } + + let mut bytes = [0u8; 16]; + for (index, chunk) in raw.as_bytes().chunks_exact(8).enumerate() { + let chunk = std::str::from_utf8(chunk) + .context("Linux IPv6 socket table contained non-UTF8 data")?; + let word = u32::from_str_radix(chunk, 16) + .context("failed to parse IPv6 address from Linux socket table")?; + bytes + .get_mut(index * 4..index * 4 + 4) + .context("IPv6 socket-table word exceeded its destination buffer")? + .copy_from_slice(&word.to_le_bytes()); + } + Ok(std::net::Ipv6Addr::from(bytes).into()) + } + } + } + + fn parse_proc_net_ports( + contents: &str, + family: ProcNetFamily, + tcp: bool, + ) -> Result, anyhow::Error> { + let mut ports = Vec::new(); + + for line in contents.lines().skip(1) { + let fields = line.split_whitespace().collect::>(); + if fields.len() < 4 { + continue; + } + let state = fields + .get(3) + .context("Linux socket table row did not include a state")?; + if tcp && *state != "0A" { + continue; + } + + let endpoint = fields + .get(1) + .context("Linux socket table row did not include a local endpoint")?; + let (raw_address, raw_port) = endpoint + .rsplit_once(':') + .context("Linux socket table local address did not include a port")?; + let port = u16::from_str_radix(raw_port, 16) + .context("failed to parse port from Linux socket table")?; + if port == 0 { + continue; + } + + let address = Self::parse_proc_net_address(raw_address, family)?; + ports.push(BoundPort { + family, + address: (!address.is_unspecified()).then_some(address), + port, + }); + } + + Ok(ports) + } + + async fn container_bound_ports(pid: i64) -> Result, anyhow::Error> { + if pid <= 0 { + return Err(anyhow::anyhow!( + "Proxmox LXC runtime returned invalid PID {pid}" + )); + } + + let tables = [ + ("tcp", ProcNetFamily::Ipv4, true), + ("tcp6", ProcNetFamily::Ipv6, true), + ("udp", ProcNetFamily::Ipv4, false), + ("udp6", ProcNetFamily::Ipv6, false), + ]; + let mut ports = Vec::new(); + let mut tables_read = 0usize; + + for (table, family, tcp) in tables { + let path = format!("/proc/{pid}/net/{table}"); + match tokio::fs::read_to_string(&path).await { + Ok(contents) => { + tables_read += 1; + ports.extend( + Self::parse_proc_net_ports(&contents, family, tcp).with_context(|| { + format!("failed to parse Proxmox LXC socket table {path}") + })?, + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("failed to read Proxmox LXC socket table {path}") + }); + } + } + } + + if tables_read == 0 { + return Err(anyhow::anyhow!( + "Proxmox LXC process {pid} disappeared before its socket tables could be read" + )); + } + + Ok(ports) + } + + fn bound_port_matches_address(bound: BoundPort, address: IpAddr) -> bool { + match bound.address { + None => matches!( + (bound.family, address), + (ProcNetFamily::Ipv4, IpAddr::V4(_)) | (ProcNetFamily::Ipv6, IpAddr::V6(_)) + ), + Some(IpAddr::V6(bound_v6)) => { + bound_v6 + .to_ipv4_mapped() + .map(IpAddr::V4) + .is_some_and(|mapped| mapped == address) + || IpAddr::V6(bound_v6) == address + } + Some(bound_address) => bound_address == address, + } + } + + fn server_tag(prefix: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-server-{uuid}") + } + + fn replacement_tag(prefix: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-repl-{uuid}") + } + + fn helper_tag(prefix: &str, role: &str, uuid: uuid::Uuid) -> String { + format!("{prefix}-helper-{role}-{uuid}") + } + + fn server_from_tags(prefix: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let server_tag_prefix = format!("{prefix}-server-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&server_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn replacement_from_tags(prefix: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let replacement_tag_prefix = format!("{prefix}-repl-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&replacement_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn has_image_tag(container: &cli::ClusterContainer, image_tag: &str) -> bool { + container.tags.iter().any(|tag| tag == image_tag) + } + + fn plan_server_container( + existing: Option<&cli::ClusterContainer>, + replacement: Option<&cli::ClusterContainer>, + image_tag: &str, + ) -> ServerContainerPlan { + match (existing, replacement) { + (Some(existing), replacement) if Self::has_image_tag(existing, image_tag) => { + ServerContainerPlan::Reuse { + vmid: existing.vmid, + stale_replacement: replacement.map(|container| container.vmid), + } + } + (Some(existing), Some(replacement)) if Self::has_image_tag(replacement, image_tag) => { + ServerContainerPlan::ReplaceWithStaged { + old_vmid: existing.vmid, + staged_vmid: replacement.vmid, + } + } + (Some(existing), replacement) => ServerContainerPlan::ReplaceFresh { + old_vmid: existing.vmid, + stale_replacement: replacement.map(|container| container.vmid), + }, + (None, Some(replacement)) if Self::has_image_tag(replacement, image_tag) => { + ServerContainerPlan::RecoverStaged { + staged_vmid: replacement.vmid, + } + } + (None, replacement) => ServerContainerPlan::CreateFresh { + stale_replacement: replacement.map(|container| container.vmid), + }, + } + } + + fn helper_from_tags(prefix: &str, role: &str, tags: &[String]) -> Option { + if !tags.iter().any(|tag| tag == prefix) { + return None; + } + + let helper_tag_prefix = format!("{prefix}-helper-{role}-"); + tags.iter().find_map(|tag| { + tag.strip_prefix(&helper_tag_prefix) + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + }) + } + + fn owned_server_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::server_from_tags(prefix, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple Proxmox LXC containers claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + fn owned_replacement_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::replacement_from_tags(prefix, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple staged Proxmox LXC replacements claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + async fn ensure_container_stopped( + &self, + container: &cli::ClusterContainer, + purpose: &str, + ) -> Result<(), anyhow::Error> { + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + return Err(anyhow::anyhow!( + "Proxmox LXC container {} must be stopped before {purpose}", + container.vmid + )); + } + Ok(()) + } + + async fn remove_staged_replacement( + &self, + replacement: &cli::ClusterContainer, + server_uuid: uuid::Uuid, + ) -> Result<(), anyhow::Error> { + self.ensure_container_stopped(replacement, "discarding a staged image replacement") + .await?; + self.cli.destroy(replacement.vmid).await.with_context(|| { + format!( + "failed to remove stale staged Proxmox LXC replacement {} for server {server_uuid}", + replacement.vmid + ) + }) + } + + fn owned_helper_container( + containers: &[cli::ClusterContainer], + node: &str, + prefix: &str, + role: &str, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let mut owned = containers + .iter() + .filter(|container| container.node == node) + .filter(|container| { + Self::helper_from_tags(prefix, role, &container.tags) == Some(server_uuid) + }) + .cloned() + .collect::>(); + + match owned.len() { + 0 => Ok(None), + 1 => Ok(owned.pop()), + _ => { + let mut vmids = owned + .iter() + .map(|container| container.vmid) + .collect::>(); + vmids.sort_unstable(); + Err(anyhow::anyhow!( + "multiple Proxmox LXC {role} helpers claim server {server_uuid}: VMIDs {}", + vmids + .into_iter() + .map(|vmid| vmid.to_string()) + .collect::>() + .join(", ") + )) + } + } + } + + async fn server_container( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + Self::owned_server_container( + &self.cli.list_containers().await?, + node, + &tag_prefix, + server_uuid, + ) + } + + async fn helper_container( + &self, + server_uuid: uuid::Uuid, + role: &str, + ) -> Result, anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + Self::owned_helper_container( + &self.cli.list_containers().await?, + node, + &tag_prefix, + role, + server_uuid, + ) + } + + async fn server_address( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let Some(container) = self.server_container(server_uuid).await? else { + return Ok(None); + }; + if container.status != Some(cli::ContainerStatus::Running) { + return Ok(None); + } + + let interfaces = self.cli.interfaces(&container.node, container.vmid).await?; + Ok(PveCli::primary_container_address(&interfaces)) + } + + async fn server_addresses( + &self, + server_uuid: uuid::Uuid, + ) -> Result, anyhow::Error> { + let Some(container) = self.server_container(server_uuid).await? else { + return Ok(Vec::new()); + }; + if container.status != Some(cli::ContainerStatus::Running) { + return Ok(Vec::new()); + } + + let interfaces = self.cli.interfaces(&container.node, container.vmid).await?; + Ok(PveCli::container_addresses(&interfaces)) + } + + fn direct_bridge_firewall_spec( + server: uuid::Uuid, + mappings: &HashMap>, + rules: &[crate::server::firewall::FirewallRule], + mut container_ips: Vec, + vmid: Option, + ) -> crate::server::firewall::FirewallServerSpec { + let mut ports = BTreeSet::new(); + for allocation_ports in mappings.values() { + ports.extend(allocation_ports.iter().copied()); + } + container_ips.sort_unstable(); + container_ips.dedup(); + + crate::server::firewall::FirewallServerSpec { + server, + target: vmid.map(|vmid| crate::server::firewall::FirewallTarget::ProxmoxLxc { + vmid, + interface: "net0".to_string(), + }), + // Bridged PVE containers do not publish Docker-style host bindings. + bindings: Vec::new(), + container_ports: ports.into_iter().collect(), + container_ips, + rules: rules.to_vec(), + files: None, + } + } + + fn firewall_spec( + configuration: &crate::server::configuration::ServerConfiguration, + container_ips: Vec, + vmid: Option, + ) -> crate::server::firewall::FirewallServerSpec { + Self::direct_bridge_firewall_spec( + configuration.uuid, + &configuration.allocations.mappings, + &configuration.firewall, + container_ips, + vmid, + ) + } + + fn edge_forward_payload( + configuration: &crate::server::configuration::ServerConfiguration, + destination: Ipv4Addr, + ) -> Result, anyhow::Error> { + let Some(allocation) = configuration.allocations.default.as_ref() else { + return Ok(None); + }; + let public_ip = allocation + .ip + .to_string() + .parse::() + .with_context(|| { + format!( + "Proxmox edge forwarding requires an IPv4 primary allocation, got {}", + allocation.ip + ) + })?; + let ports = configuration + .allocations + .mappings + .get(&allocation.ip.to_compact_string()) + .cloned() + .unwrap_or_default(); + Ok(Some(( + public_ip, + serde_json::json!({ + "ip": destination.to_string(), + // Panel allocations are protocol-neutral. Publish both transports so + // TCP games and UDP games preserve the same allocation contract. + "tcp": ports, + "udp": ports, + }), + ))) + } + + fn edge_forwarding_config( + app_config: &crate::config::Config, + ) -> Result, anyhow::Error> { + let config = app_config.load(); + let pve = &config.runtime.pve_lxc; + let interface = pve.edge_wireguard_interface.trim().to_string(); + let user = pve.edge_ssh_user.trim().to_string(); + let identity_path = pve.edge_ssh_identity_path.trim().to_string(); + let known_hosts_path = pve.edge_known_hosts_path.trim().to_string(); + if identity_path.is_empty() && known_hosts_path.is_empty() { + return Ok(None); + } + if interface.is_empty() + || user.is_empty() + || identity_path.is_empty() + || known_hosts_path.is_empty() + { + return Err(anyhow::anyhow!( + "Proxmox edge forwarding requires edge_wireguard_interface, edge_ssh_user, edge_ssh_identity_path, and edge_known_hosts_path" + )); + } + if !user + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + anyhow::bail!("Proxmox edge SSH user contains unsupported characters: {user}"); + } + Ok(Some((interface, user, identity_path, known_hosts_path))) + } + + async fn validate_edge_forwarding_config( + app_config: &crate::config::Config, + ) -> Result<(), anyhow::Error> { + let Some((_, _, identity_path, _)) = Self::edge_forwarding_config(app_config)? else { + return Ok(()); + }; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let metadata = tokio::fs::metadata(&identity_path).await.with_context(|| { + format!("failed to inspect Proxmox edge SSH identity {identity_path}") + })?; + if !metadata.is_file() { + anyhow::bail!("Proxmox edge SSH identity is not a regular file: {identity_path}"); + } + if metadata.uid() != 0 || metadata.permissions().mode() & 0o077 != 0 { + anyhow::bail!( + "Proxmox edge SSH identity must be owned by root and inaccessible to group and other users: {identity_path}" + ); + } + } + Ok(()) + } + + async fn edge_peers(interface: &str) -> Result, anyhow::Error> { + let mut command = tokio::process::Command::new("/usr/bin/wg"); + command.args(["show", interface, "dump"]).kill_on_drop(true); + let output = tokio::time::timeout(std::time::Duration::from_secs(5), command.output()) + .await + .context("timed out inspecting WireGuard peers for Proxmox edge forwarding")? + .context("failed to inspect WireGuard peers for Proxmox edge forwarding")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to inspect WireGuard interface {interface}: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + + let dump = String::from_utf8(output.stdout) + .context("WireGuard peer output was not valid UTF-8")?; + let mut peers = Vec::new(); + for line in dump.lines().skip(1) { + let fields = line.split('\t').collect::>(); + let Some([_, _, endpoint, allowed_ips, ..]) = fields.get(..) else { + continue; + }; + if *endpoint == "(none)" { + continue; + } + let endpoint = endpoint + .parse::() + .with_context(|| format!("WireGuard peer endpoint was invalid: {endpoint}"))?; + let IpAddr::V4(public_ip) = endpoint.ip() else { + tracing::warn!(endpoint = %endpoint, "ignoring IPv6 WireGuard edge endpoint because panel allocations require IPv4"); + continue; + }; + let Some(tunnel_ip) = allowed_ips + .split(',') + .filter_map(|cidr| cidr.split_once('/')) + .find_map(|(ip, prefix)| { + (prefix == "32") + .then(|| ip.parse::().ok()) + .flatten() + }) + else { + continue; + }; + peers.push(EdgePeer { + public_ip, + tunnel_ip, + }); + } + peers.sort_unstable_by_key(|peer| (peer.public_ip, peer.tunnel_ip)); + peers.dedup(); + Ok(peers) + } + + async fn edge_ssh( + host: Ipv4Addr, + user: &str, + identity_path: &str, + known_hosts_path: &str, + server: uuid::Uuid, + payload: Option, + ) -> Result<(), anyhow::Error> { + let action = if payload.is_some() { "sync" } else { "clear" }; + let mut command = tokio::process::Command::new("/usr/bin/ssh"); + command + .args([ + "-i", + identity_path, + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=5", + "-o", + "StrictHostKeyChecking=yes", + "-o", + &format!("UserKnownHostsFile={known_hosts_path}"), + &format!("{user}@{host}"), + action, + &server.to_string(), + ]) + .stdin(if payload.is_some() { + std::process::Stdio::piped() + } else { + std::process::Stdio::null() + }) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let mut child = command + .spawn() + .context("failed to start Proxmox edge forwarding SSH command")?; + if let Some(payload) = payload { + let bytes = serde_json::to_vec(&payload) + .context("failed to serialize Proxmox edge forwarding payload")?; + child + .stdin + .as_mut() + .context("Proxmox edge forwarding SSH stdin was unavailable")? + .write_all(&bytes) + .await?; + } + let output = + tokio::time::timeout(std::time::Duration::from_secs(15), child.wait_with_output()) + .await + .context("timed out synchronizing Proxmox edge forwarding")? + .context("failed while synchronizing Proxmox edge forwarding")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "Proxmox edge forwarding {action} for {server} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(()) + } + + async fn sync_edge_forwarding( + app_config: &crate::config::Config, + server: uuid::Uuid, + public_ip: Option, + payload: Option, + ) -> Result<(), anyhow::Error> { + let Some((interface, user, identity_path, known_hosts_path)) = + Self::edge_forwarding_config(app_config)? + else { + return Ok(()); + }; + let peers = Self::edge_peers(&interface).await?; + for peer in &peers { + Self::edge_ssh( + peer.tunnel_ip, + &user, + &identity_path, + &known_hosts_path, + server, + None, + ) + .await?; + } + if let Some(payload) = payload { + let public_ip = public_ip.context( + "Proxmox edge forwarding payload did not include a public allocation IP", + )?; + let peer = peers + .iter() + .find(|peer| peer.public_ip == public_ip) + .with_context(|| { + format!("no WireGuard peer on {interface} has endpoint address {public_ip}") + })?; + Self::edge_ssh( + peer.tunnel_ip, + &user, + &identity_path, + &known_hosts_path, + server, + Some(payload), + ) + .await?; + } + Ok(()) + } + + fn firewall_file_access( + server: &Arc, + ) -> crate::server::firewall::sets::FirewallFileAccess { + crate::server::firewall::sets::FirewallFileAccess { + filesystem: (*server.filesystem).clone(), + notifier: Some(server.filesystem.server_notifier().clone()), + server: Some(Arc::downgrade(server)), + } + } + + async fn sync_server_firewall( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + let container = self.server_container(server.uuid).await?; + let addresses = self.server_addresses(server.uuid).await?; + let (has_rules, mut spec) = { + let configuration = server.configuration.read().await; + ( + !configuration.firewall.is_empty(), + Self::firewall_spec( + &configuration, + addresses, + container.as_ref().map(|container| container.vmid), + ), + ) + }; + if has_rules && spec.container_ips.is_empty() { + let running = self + .server_container(server.uuid) + .await? + .is_some_and(|container| container.status == Some(cli::ContainerStatus::Running)); + if running { + return Err(anyhow::anyhow!( + "running Proxmox LXC server {} has no usable bridged address; refusing to leave configured firewall rules unapplied", + server.uuid + )); + } + } + spec.files = Some(Self::firewall_file_access(server)); + self.firewall.sync(&spec).await + } + + fn helper_resources( + configuration: &crate::server::configuration::ServerConfiguration, + app_config: &crate::config::Config, + ) -> Result { + let mut resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + let config = app_config.load(); + let limits = &config.docker.installer_limits; + + resources.memory_mib = resources.memory_mib.max(limits.memory.as_mib()); + if let Some(cpu_limit_percent) = resources.cpu_limit_percent + && cpu_limit_percent < limits.cpu + { + resources.cpu_limit_percent = Some(limits.cpu); + } + + Ok(resources) + } + + async fn server_mount_specs( + &self, + server: &crate::server::Server, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = server; + return Self::unsupported("bind mounts on this host platform"); + } + + #[cfg(unix)] + { + let (server_uuid, hugepages_passthrough_enabled, configured_mounts) = { + let configuration = server.configuration.read().await; + ( + configuration.uuid, + configuration.container.hugepages_passthrough_enabled, + configuration.mounts.clone(), + ) + }; + let (host_data_uid, host_data_gid) = { + let config = self.app_config.load(); + (config.system.user.uid, config.system.user.gid) + }; + + let mut mounts = Vec::with_capacity(configured_mounts.len() + 2); + mounts.push(( + server.filesystem.get_base_fs_mount_path().await, + Self::DATA_MOUNT_TARGET.to_string(), + false, + )); + + if hugepages_passthrough_enabled { + mounts.push(( + PathBuf::from("/dev/hugepages"), + "/dev/hugepages".to_string(), + false, + )); + } + + if !configured_mounts.is_empty() { + let allowed = + crate::server::configuration::AllowedMounts::load(&self.app_config).await; + for mount in configured_mounts { + let source = match mount.resolve_allowed_source(&allowed).await { + Ok(source) => source, + Err(error) => { + tracing::warn!( + server = %server_uuid, + "not mounting {} -> {} in Proxmox LXC: {error:#}", + mount.source, + mount.target, + ); + continue; + } + }; + mounts.push((source, mount.target.to_string(), mount.read_only)); + } + } + + let mut specs = Vec::with_capacity(mounts.len()); + for (slot, (source, target, read_only)) in mounts.into_iter().enumerate() { + let slot = u8::try_from(slot).map_err(|_| { + anyhow::anyhow!("Proxmox LXC supports at most 256 bind mount slots") + })?; + let source = tokio::fs::canonicalize(&source).await.with_context(|| { + format!( + "failed to resolve Proxmox bind mount source {}", + source.display() + ) + })?; + let metadata = tokio::fs::metadata(&source).await.with_context(|| { + format!( + "failed to inspect Proxmox bind mount source {}", + source.display() + ) + })?; + if !metadata.is_dir() { + return Err(anyhow::anyhow!( + "Proxmox LXC native mpN bind mounts require a directory source; {} -> {target} is not a directory", + source.display() + )); + } + let source_path = source + .into_os_string() + .into_string() + .map_err(|_| anyhow::anyhow!("Proxmox bind mount source is not valid UTF-8"))?; + + specs.push(cli::HostBindMountSpec { + slot, + source_path, + target_path: target, + read_only, + host_uid: host_data_uid, + host_gid: host_data_gid, + }); + } + + Ok(specs) + } + } + + fn device_deny_write(permissions: &str, path: &std::path::Path) -> Result { + let readable = permissions.contains('r'); + let writable = permissions.contains('w'); + let invalid = permissions + .chars() + .any(|permission| !matches!(permission, 'r' | 'w' | 'm')); + + if !readable || invalid { + return Err(anyhow::anyhow!( + "Proxmox LXC devN passthrough for {} requires read access and accepts r, w, and m permission flags; requested {permissions}", + path.display() + )); + } + + Ok(!writable) + } + + async fn server_device_specs( + &self, + server: &crate::server::Server, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = server; + return Self::unsupported("device passthrough on this host platform"); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + + let (server_uuid, kvm_passthrough_enabled, configured_devices) = { + let configuration = server.configuration.read().await; + ( + configuration.uuid, + configuration.container.kvm_passthrough_enabled, + configuration.devices.clone(), + ) + }; + + let mut devices = Vec::<(PathBuf, String)>::with_capacity( + configured_devices.len() + usize::from(kvm_passthrough_enabled), + ); + if kvm_passthrough_enabled { + devices.push((PathBuf::from("/dev/kvm"), "rw".to_string())); + } + + if !configured_devices.is_empty() { + let allowed = + crate::server::configuration::AllowedDevices::load(&self.app_config).await; + for device in configured_devices { + let source = match device.resolve_allowed_source(&allowed).await { + Ok(source) => source, + Err(error) => { + tracing::warn!( + server = %server_uuid, + "not passing device {} -> {} into Proxmox LXC: {error:#}", + device.source, + device.target, + ); + continue; + } + }; + let source_string = source.to_string_lossy(); + if device.target.as_str() != source_string.as_ref() { + return Err(anyhow::anyhow!( + "Proxmox LXC devN passthrough cannot remap device {} to {}; source and target must be identical", + source.display(), + device.target + )); + } + devices.push((source, device.permissions.to_string())); + } + } + + let mut specs = Vec::with_capacity(devices.len()); + for (slot, (path, permissions)) in devices.into_iter().enumerate() { + let slot = u8::try_from(slot).map_err(|_| { + anyhow::anyhow!("Proxmox LXC supports at most 256 device passthrough slots") + })?; + let deny_write = Self::device_deny_write(&permissions, &path)?; + + let metadata = tokio::fs::metadata(&path).await.with_context(|| { + format!( + "failed to inspect Proxmox device passthrough source {}", + path.display() + ) + })?; + let file_type = metadata.file_type(); + if !file_type.is_char_device() && !file_type.is_block_device() { + return Err(anyhow::anyhow!( + "Proxmox device passthrough source {} is not a character or block device", + path.display() + )); + } + let path = path + .into_os_string() + .into_string() + .map_err(|_| anyhow::anyhow!("Proxmox device path is not valid UTF-8"))?; + + specs.push(cli::HostDevicePassthroughSpec { + slot, + path, + uid: metadata.uid(), + gid: metadata.gid(), + mode: metadata.mode() & 0o777, + deny_write, + }); + } + + Ok(specs) + } + } + + #[allow(clippy::too_many_arguments)] + async fn create_helper_container( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + role: &str, + vmid: u32, + resources: cli::ContainerResources, + environment: Vec, + staging_path: &std::path::Path, + staging_target: &str, + script_target: &str, + log_path: &std::path::Path, + ) -> Result<(), anyhow::Error> { + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + let image = PveCli::normalize_oci_reference(script.container_image.trim_end_matches('~'))?; + let ( + template_storage, + rootfs_storage, + bridge, + vlan_tag, + rootfs_size_gib, + tag_prefix, + unprivileged, + host_data_uid, + host_data_gid, + image_cache_max_age, + ) = { + let config = self.app_config.load(); + let pve = &config.runtime.pve_lxc; + ( + pve.template_storage.clone(), + pve.rootfs_storage.clone(), + pve.bridge.clone(), + pve.vlan_tag, + pve.rootfs_size_gib, + pve.tag_prefix.clone(), + pve.unprivileged, + config.system.user.uid, + config.system.user.gid, + Self::image_cache_max_age(&config), + ) + }; + + let template = { + let template_lock = self + .template_provisioning_lock(&node, &template_storage, &image) + .await; + let _template = template_lock.lock().await; + self.cli + .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) + .await? + }; + let image_tag = PveCli::oci_revision_tag(&template.revision)?; + let rootfs_storage = self + .rootfs_storage(&node, rootfs_storage, rootfs_size_gib) + .await?; + let network = Self::dhcp_network(&bridge, vlan_tag)?; + let staging_path_utf8 = staging_path + .to_str() + .context("Proxmox helper staging path is not valid UTF-8")?; + let log_path = log_path + .to_str() + .context("Proxmox helper log path is not valid UTF-8")?; + let create = cli::CreateContainerSpec { + vmid, + template: template.volid, + hostname: format!("{role}-{vmid}"), + rootfs_storage, + rootfs_size_gib, + memory_mib: resources.memory_mib, + swap_mib: resources.swap_mib, + cpu_limit_percent: resources.cpu_limit_percent, + cores: resources.cores, + network, + tags: vec![ + tag_prefix.clone(), + Self::helper_tag(&tag_prefix, role, server.uuid), + image_tag, + ], + unprivileged, + }; + let mounts = [ + cli::HostBindMountSpec { + slot: 0, + source_path: server.filesystem.base().to_string(), + target_path: "/mnt/server".to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }, + cli::HostBindMountSpec { + slot: 1, + source_path: staging_path_utf8.to_string(), + target_path: staging_target.to_string(), + read_only: false, + host_uid: host_data_uid, + host_gid: host_data_gid, + }, + ]; + + let helper_wrapper_name = "calagopus-helper-entrypoint"; + let helper_wrapper_path = staging_path.join(helper_wrapper_name); + tokio::fs::write( + &helper_wrapper_path, + r#"#!/bin/sh +attempt=0 +while [ "$attempt" -lt 480 ]; do + if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then + "$@" + status=$? + if [ -n "${INSTALL_STATUS_FILE:-}" ]; then + if [ "$status" -eq 0 ]; then + : > "$INSTALL_STATUS_FILE" + else + printf 'installer exited with status %s\n' "$status" > "$INSTALL_STATUS_FILE" + fi + fi + exit "$status" + fi + attempt=$((attempt + 1)) + sleep 0.25 +done +echo 'timed out waiting for Proxmox DHCP' >&2 +if [ -n "${INSTALL_STATUS_FILE:-}" ]; then + printf '%s\n' 'timed out waiting for Proxmox DHCP' > "$INSTALL_STATUS_FILE" +fi +exit 1 +"#, + ) + .await + .context("failed to write Proxmox helper entrypoint wrapper")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions( + &helper_wrapper_path, + std::fs::Permissions::from_mode(0o755), + ) + .await + .context("failed to make Proxmox helper entrypoint wrapper executable")?; + } + + self.cli.create_with_bind_mounts(&create, &mounts).await?; + // PVE applies a host-managed DHCP lease immediately after CT init starts. + // Keep PID 1 alive until that lease installs a default route; otherwise a + // fast installer can attempt its first download and exit before `pct` + // has a chance to run its DHCP hook. + let init_command = PveCli::encode_init_command(&[ + format!("{staging_target}/{helper_wrapper_name}"), + script.entrypoint.to_string(), + script_target.to_string(), + ])?; + let (pids_limit, io_weight) = { + let configuration = server.configuration.read().await; + ( + Self::pids_limit(&self.app_config), + PveCli::cgroup2_io_weight(configuration.build.io_weight)?, + ) + }; + let runtime_config = cli::RuntimeConfigSpec { + vmid, + environment, + halt_signal: None, + init_command: Some(init_command), + console_logfile: Some(log_path.to_string()), + cpuset_cpus: resources.cpuset_cpus.clone(), + pids_limit, + io_weight, + memory_unlimited: resources.memory_unlimited, + swap_unlimited: resources.swap_unlimited, + managed_file_mounts: None, + }; + + if let Err(error) = self.cli.apply_runtime_config(&runtime_config).await { + if let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox helper after runtime configuration failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to configure Proxmox helper LXC")); + } + + Ok(()) + } + + async fn prepare_server_container( + &self, + server: &crate::server::Server, + ) -> Result<(u32, cli::ContainerOciUser, bool), anyhow::Error> { + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + + let (server_uuid, image, resources, primary_allocation) = { + let configuration = server.configuration.read().await; + let resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + + ( + configuration.uuid, + configuration + .container + .image + .trim_end_matches('~') + .to_string(), + resources, + configuration + .allocations + .default + .as_ref() + .map(|allocation| allocation.ip.to_string()), + ) + }; + let image = PveCli::normalize_oci_reference(&image)?; + let mounts = self.server_mount_specs(server).await?; + let devices = self.server_device_specs(server).await?; + + let ( + template_storage, + rootfs_storage, + bridge, + vlan_tag, + network_prefix, + gateway, + rootfs_size_gib, + tag_prefix, + unprivileged, + edge_forwarding, + image_cache_max_age, + ) = { + let config = self.app_config.load(); + let pve = &config.runtime.pve_lxc; + ( + pve.template_storage.clone(), + pve.rootfs_storage.clone(), + pve.bridge.clone(), + pve.vlan_tag, + pve.network_prefix, + pve.gateway.clone(), + pve.rootfs_size_gib, + pve.tag_prefix.clone(), + pve.unprivileged, + Self::edge_forwarding_config(&self.app_config)?.is_some(), + Self::image_cache_max_age(&config), + ) + }; + let network = if edge_forwarding { + Self::dhcp_network(&bridge, vlan_tag)? + } else { + Self::network_for_allocation( + &bridge, + vlan_tag, + network_prefix, + gateway.as_deref(), + primary_allocation.as_deref(), + )? + }; + + let template = { + let template_lock = self + .template_provisioning_lock(&node, &template_storage, &image) + .await; + let _template = template_lock.lock().await; + self.cli + .ensure_oci_template(&node, &template_storage, &image, image_cache_max_age) + .await? + }; + let image_tag = PveCli::oci_revision_tag(&template.revision)?; + let containers = self.cli.list_containers().await?; + let existing = Self::owned_server_container(&containers, &node, &tag_prefix, server_uuid)?; + let replacement = + Self::owned_replacement_container(&containers, &node, &tag_prefix, server_uuid)?; + let plan = Self::plan_server_container(existing.as_ref(), replacement.as_ref(), &image_tag); + + let final_tags = vec![ + tag_prefix.clone(), + Self::server_tag(&tag_prefix, server_uuid), + image_tag.clone(), + ]; + + match plan { + ServerContainerPlan::Reuse { + vmid, + stale_replacement, + } => { + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + + self.cli.set_resources(vmid, &resources).await?; + self.cli.set_network(vmid, &network).await?; + let user = self.cli.oci_user(vmid).await?; + self.cli + .configure_mounts_and_devices(vmid, user, &mounts, &devices) + .await?; + return Ok((vmid, user, false)); + } + ServerContainerPlan::RecoverStaged { staged_vmid } => { + let staged = replacement + .as_ref() + .filter(|container| container.vmid == staged_vmid) + .context("staged Proxmox replacement disappeared from the inventory plan")?; + self.ensure_container_stopped(staged, "recovering a staged image replacement") + .await?; + self.cli.set_resources(staged_vmid, &resources).await?; + self.cli.set_network(staged_vmid, &network).await?; + let user = self.cli.oci_user(staged_vmid).await?; + self.cli + .configure_mounts_and_devices(staged_vmid, user, &mounts, &devices) + .await?; + self.cli + .set_tags(staged_vmid, &final_tags) + .await + .with_context(|| { + format!( + "failed to promote staged Proxmox LXC replacement {staged_vmid} for server {server_uuid}" + ) + })?; + return Ok((staged_vmid, user, true)); + } + ServerContainerPlan::CreateFresh { stale_replacement } => { + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + } + ServerContainerPlan::ReplaceFresh { + old_vmid, + stale_replacement, + } => { + let old = existing + .as_ref() + .filter(|container| container.vmid == old_vmid) + .context("owned Proxmox container disappeared from the inventory plan")?; + self.ensure_container_stopped(old, "replacing its OCI image") + .await?; + if let Some(stale_vmid) = stale_replacement { + let stale = replacement + .as_ref() + .filter(|container| container.vmid == stale_vmid) + .context( + "staged Proxmox replacement disappeared from the inventory plan", + )?; + self.remove_staged_replacement(stale, server_uuid).await?; + } + } + ServerContainerPlan::ReplaceWithStaged { + old_vmid, + staged_vmid, + } => { + let old = existing + .as_ref() + .filter(|container| container.vmid == old_vmid) + .context("owned Proxmox container disappeared from the inventory plan")?; + let staged = replacement + .as_ref() + .filter(|container| container.vmid == staged_vmid) + .context("staged Proxmox replacement disappeared from the inventory plan")?; + self.ensure_container_stopped(old, "replacing its OCI image") + .await?; + self.ensure_container_stopped(staged, "promoting a staged image replacement") + .await?; + + self.cli.set_resources(staged_vmid, &resources).await?; + self.cli.set_network(staged_vmid, &network).await?; + let user = self.cli.oci_user(staged_vmid).await?; + self.cli + .configure_mounts_and_devices(staged_vmid, user, &mounts, &devices) + .await?; + + if let Err(error) = self.cli.destroy(old_vmid).await { + return Err(error.context(format!( + "failed to remove old Proxmox LXC container {old_vmid} while staged replacement {staged_vmid} remains recoverable" + ))); + } + self.cli + .set_tags(staged_vmid, &final_tags) + .await + .with_context(|| { + format!( + "old Proxmox LXC container {old_vmid} was removed, but failed to promote staged replacement {staged_vmid}; the staged replacement remains recoverable on the next setup" + ) + })?; + return Ok((staged_vmid, user, true)); + } + } + + let rootfs_storage = self + .rootfs_storage(&node, rootfs_storage, rootfs_size_gib) + .await?; + let replacing = matches!(plan, ServerContainerPlan::ReplaceFresh { .. }); + let create_tags = if replacing { + vec![ + tag_prefix.clone(), + Self::replacement_tag(&tag_prefix, server_uuid), + image_tag, + ] + } else { + final_tags.clone() + }; + let mut created = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; + for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; + let create = cli::CreateContainerSpec { + vmid, + template: template.volid.clone(), + hostname: server_uuid.to_string(), + rootfs_storage: rootfs_storage.clone(), + rootfs_size_gib, + memory_mib: resources.memory_mib, + swap_mib: resources.swap_mib, + cpu_limit_percent: resources.cpu_limit_percent, + cores: resources.cores, + network: network.clone(), + tags: create_tags.clone(), + unprivileged, + }; + match self + .cli + .create_with_bind_mounts_and_devices(&create, &mounts, &devices) + .await + { + Ok(user) => { + created = Some((vmid, user)); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!(vmid, "retrying Proxmox LXC create after VMID collision"); + } + Err(error) => return Err(error), + } + } + let (vmid, user) = created + .context("failed to create Proxmox LXC after three VMID allocation collisions")?; + + if let ServerContainerPlan::ReplaceFresh { old_vmid, .. } = plan { + if let Err(error) = self.cli.destroy(old_vmid).await { + return match self.cli.destroy(vmid).await { + Ok(()) => Err(error.context(format!( + "failed to remove old Proxmox LXC container {old_vmid}; discarded prepared replacement {vmid}" + ))), + Err(cleanup_error) => Err(anyhow::anyhow!( + "failed to remove old Proxmox LXC container {old_vmid}: {error:#}; additionally failed to discard staged replacement {vmid}: {cleanup_error:#}" + )), + }; + } + + self.cli.set_tags(vmid, &final_tags).await.with_context(|| { + format!( + "old Proxmox LXC container {old_vmid} was removed, but failed to promote staged replacement {vmid}; the staged replacement remains recoverable on the next setup" + ) + })?; + } + + Ok((vmid, user, true)) + } +} + +#[async_trait::async_trait] +impl ServerExecutor for PveLxcExecutor { + async fn boot(&self) -> Result<(), anyhow::Error> { + if !cfg!(target_os = "linux") { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime requires a Linux host running Proxmox VE" + )); + } + if !self.app_config.load().runtime.pve_lxc.unprivileged { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime currently requires unprivileged containers because per-mount idmap is required for Wings-owned server data" + )); + } + Self::validate_edge_forwarding_config(&self.app_config).await?; + + let raw_version = self.cli.version().await?; + let version = PveVersion::parse(&raw_version)?; + if !version.is_supported() { + return Err(anyhow::anyhow!( + "Proxmox VE LXC runtime requires Proxmox VE 9.2 or newer, found {raw_version}" + )); + } + + let configured_node = self.app_config.load().runtime.pve_lxc.node.clone(); + let local_node = self.cli.local_node().await?; + let node = Self::runtime_node(&configured_node, &local_node)?; + self.node + .set(node.clone()) + .map_err(|_| anyhow::anyhow!("Proxmox VE LXC runtime was booted more than once"))?; + + self.firewall + .boot() + .await + .context("failed to initialize Proxmox LXC host firewall")?; + + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; + let next_vmid = self.cli.next_vmid(min_vmid).await?; + tracing::info!( + pve_version = %raw_version, + node = %node, + next_vmid, + min_vmid, + "Proxmox VE LXC runtime is available" + ); + + Ok(()) + } + + async fn reconcile_firewall( + &self, + servers: &[crate::remote::servers::RawServer], + ) -> Result<(), anyhow::Error> { + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let containers = self.cli.list_containers().await?; + let mut specs = Vec::with_capacity(servers.len()); + for server in servers { + let container = + Self::owned_server_container(&containers, node, &tag_prefix, server.settings.uuid)?; + let running = container + .as_ref() + .is_some_and(|container| container.status == Some(cli::ContainerStatus::Running)); + let addresses = if let Some(container) = container.as_ref().filter(|_| running) { + PveCli::container_addresses( + &self + .cli + .interfaces(&container.node, container.vmid) + .await + .with_context(|| { + format!( + "failed to resolve Proxmox LXC addresses while reconciling firewall for {}", + server.settings.uuid + ) + })?, + ) + } else { + Vec::new() + }; + let mut spec = Self::firewall_spec( + &server.settings, + addresses, + container.as_ref().map(|container| container.vmid), + ); + + if !server.settings.firewall.is_empty() && spec.container_ips.is_empty() && running { + return Err(anyhow::anyhow!( + "running Proxmox LXC server {} has no usable bridged address; refusing to reconcile configured firewall rules without a destination", + server.settings.uuid + )); + } + + if spec.references_files() { + match crate::server::filesystem::cap::CapFilesystem::new( + &self.app_config.data_path(spec.server), + ) + .await + { + Ok(filesystem) => { + spec.files = Some(crate::server::firewall::sets::FirewallFileAccess { + filesystem, + notifier: None, + server: None, + }); + } + Err(error) => { + tracing::warn!( + server = %spec.server, + "failed to open the server directory for its firewall source files: {error}" + ); + } + } + } + + specs.push(spec); + } + + self.firewall.reconcile(&specs).await + } + + async fn setup_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let (vmid, _oci_user, created) = self.prepare_server_container(server).await?; + let runtime_config = match PveProcessHandle::runtime_config( + server, + &self.app_config, + &self.cli, + vmid, + ) + .await + { + Ok(runtime_config) => runtime_config, + Err(error) => { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after managed file preparation failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to prepare managed Proxmox container files")); + } + }; + + if let Err(error) = self.cli.apply_runtime_config(&runtime_config).await { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after runtime configuration failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to apply Proxmox LXC runtime configuration")); + } + + if let Err(error) = self.sync_server_firewall(server).await { + let has_rules = !server.configuration.read().await.firewall.is_empty(); + if has_rules { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after firewall setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to prepare Proxmox LXC server firewall")); + } + + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let node = self + .node + .get() + .cloned() + .context("Proxmox VE LXC runtime has not been booted")?; + let handle = match PveProcessHandle::new( + vmid, + node, + self.cli.clone(), + Arc::clone(&self.stats_sampler), + server, + Arc::clone(&self.app_config), + Arc::clone(&self.firewall), + status_tx, + true, + true, + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + if created && let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove newly created Proxmox LXC container after process setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to create Proxmox LXC process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn attach_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let container = self + .server_container(server.uuid) + .await? + .context("no Proxmox LXC container found for server")?; + if container.status != Some(cli::ContainerStatus::Running) { + return Err(anyhow::anyhow!( + "Proxmox LXC container {} for server {} is not running", + container.vmid, + server.uuid + )); + } + + // Reconcile runtime-owned files and mount entries when Wings adopts an + // already-running container. This covers features enabled after the + // container was created, including Tundra's managed /etc/hosts file. + // LXC applies a newly-added mount entry on the container's next start, + // while creating the backing file immediately lets Tundra reconcile + // its state without racing a missing host path. + self.cli + .apply_runtime_config( + &PveProcessHandle::runtime_config( + server, + &self.app_config, + &self.cli, + container.vmid, + ) + .await?, + ) + .await + .context("failed to reconcile an adopted Proxmox LXC runtime configuration")?; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = Arc::new( + PveProcessHandle::new( + container.vmid, + container.node.clone(), + self.cli.clone(), + Arc::clone(&self.stats_sampler), + server, + Arc::clone(&self.app_config), + Arc::clone(&self.firewall), + status_tx, + false, + false, + ) + .await?, + ); + + if let Err(error) = handle.sync_firewall(true).await { + let configuration = server.configuration.read().await; + let requires_network_policy = !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && Self::edge_forwarding_config(&self.app_config)?.is_some()); + drop(configuration); + if requires_network_policy { + return Err(handle + .stop_after_firewall_failure( + error, + "failed to attach Proxmox LXC network policy", + ) + .await); + } + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + Ok((handle, status_rx)) + } + + async fn cleanup_server_process( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + if let Err(error) = self.firewall.clear(server.uuid).await { + tracing::warn!( + server = %server.uuid, + "failed to clear Proxmox LXC firewall rules during cleanup: {error:#}" + ); + } + Self::sync_edge_forwarding(&self.app_config, server.uuid, None, None).await?; + + let Some(container) = self.server_container(server.uuid).await? else { + PveProcessHandle::cleanup_managed_file_staging(&self.app_config, server.uuid).await?; + return Ok(()); + }; + + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + self.cli.stop(container.vmid).await?; + } + self.cli.destroy(container.vmid).await?; + PveProcessHandle::cleanup_managed_file_staging(&self.app_config, server.uuid).await?; + + Ok(()) + } + + async fn setup_installation_process( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + if let Some(container) = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + { + return Err(anyhow::anyhow!( + "Proxmox installer helper already exists for server {} as VMID {}", + server.uuid, + container.vmid + )); + } + + let (resources, mut environment, host_data_uid, host_data_gid) = { + let configuration = server.configuration.read().await; + let config = self.app_config.load(); + ( + Self::helper_resources(&configuration, &self.app_config)?, + configuration.environment(&self.app_config), + config.system.user.uid, + config.system.user.gid, + ) + }; + for (key, value) in &script.environment { + environment.push(format!( + "{key}={}", + match value { + serde_json::Value::String(value) => value.clone(), + other => other.to_string(), + } + )); + } + environment.push(format!( + "INSTALL_STATUS_FILE=/mnt/install/{}", + crate::server::installation::INSTALL_STATUS_FILE_NAME + )); + environment.push(format!( + "INSTALL_PROGRESS_FILE=/mnt/install/{}", + crate::server::installation::INSTALL_PROGRESS_FILE_NAME + )); + + let staging_path = self.app_config.tmp_data_path(server.uuid); + let status_path = staging_path.join(crate::server::installation::INSTALL_STATUS_FILE_NAME); + let progress_path = + staging_path.join(crate::server::installation::INSTALL_PROGRESS_FILE_NAME); + let log_path = staging_path.join("pve-installer-console.log"); + async { + tokio::fs::create_dir_all(&staging_path).await?; + tokio::fs::write( + staging_path.join("install.sh"), + script.script.replace("\r\n", "\n"), + ) + .await?; + // The helper clears this marker only after the installer exits + // successfully. A killed helper therefore cannot be mistaken for a + // completed installation. + tokio::fs::write(&status_path, "installation process did not complete\n").await?; + tokio::fs::write(&progress_path, "").await?; + tokio::fs::File::create(&log_path).await?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o755)) + .await?; + tokio::fs::set_permissions(&status_path, std::fs::Permissions::from_mode(0o644)) + .await?; + tokio::fs::set_permissions(&progress_path, std::fs::Permissions::from_mode(0o644)) + .await?; + let status_path = status_path.clone(); + let progress_path = progress_path.clone(); + tokio::task::spawn_blocking(move || { + std::os::unix::fs::chown( + &status_path, + Some(host_data_uid), + Some(host_data_gid), + )?; + std::os::unix::fs::chown( + &progress_path, + Some(host_data_uid), + Some(host_data_gid), + ) + }) + .await + .map_err(std::io::Error::other)??; + } + + Ok::<(), std::io::Error>(()) + } + .await + .with_context(|| { + format!( + "failed to prepare Proxmox installer staging directory {}", + staging_path.display() + ) + })?; + + let mut created_vmid = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; + for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; + match self + .create_helper_container( + server, + script, + Self::INSTALLER_HELPER_ROLE, + vmid, + resources.clone(), + environment.clone(), + &staging_path, + "/mnt/install", + "/mnt/install/install.sh", + &log_path, + ) + .await + { + Ok(()) => { + created_vmid = Some(vmid); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!( + vmid, + "retrying Proxmox installer create after VMID collision" + ); + } + Err(error) => return Err(error), + } + } + let vmid = created_vmid + .context("failed to create Proxmox installer after three VMID allocation collisions")?; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = match PveHelperProcessHandle::new( + vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + false, + false, + false, + None, + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + if let Err(cleanup_error) = self.cli.destroy(vmid).await { + tracing::warn!( + server = %server.uuid, + vmid, + "failed to remove Proxmox installer helper after handle setup failed: {cleanup_error:#}" + ); + } + return Err(error.context("failed to create Proxmox installer process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn attach_installation_process( + &self, + server: &crate::server::Server, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let container = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + .context("no Proxmox installer helper found")?; + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status != cli::ContainerStatus::Running { + return Err(anyhow::anyhow!( + "Proxmox installer helper {} for server {} is not running", + container.vmid, + server.uuid + )); + } + + let log_path = self + .app_config + .tmp_data_path(server.uuid) + .join("pve-installer-console.log"); + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = Arc::new( + PveHelperProcessHandle::new( + container.vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + true, + true, + false, + None, + ) + .await?, + ); + + Ok((handle, status_rx)) + } + + async fn cleanup_installation_process( + &self, + server: &crate::server::Server, + ) -> Result<(), anyhow::Error> { + let Some(container) = self + .helper_container(server.uuid, Self::INSTALLER_HELPER_ROLE) + .await? + else { + return Ok(()); + }; + let status = match container.status { + Some(status) => status, + None => self.cli.status(container.vmid).await?, + }; + if status == cli::ContainerStatus::Running { + self.cli.stop(container.vmid).await?; + } + self.cli.destroy(container.vmid).await?; + Ok(()) + } + + async fn setup_script_process( + &self, + server: &crate::server::Server, + script: &crate::server::installation::InstallationScript, + ) -> Result<(Arc, StatusReceiver), anyhow::Error> { + let (resources, mut environment) = { + let configuration = server.configuration.read().await; + ( + Self::helper_resources(&configuration, &self.app_config)?, + configuration.environment(&self.app_config), + ) + }; + for (key, value) in &script.environment { + environment.push(format!( + "{key}={}", + match value { + serde_json::Value::String(value) => value.clone(), + other => other.to_string(), + } + )); + } + + let staging_path = self + .app_config + .tmp_data_path(server.uuid) + .join(format!("pve-script-{}", uuid::Uuid::new_v4())); + let log_path = staging_path.join("console.log"); + tokio::fs::create_dir_all(&staging_path).await?; + tokio::fs::write( + staging_path.join("script.sh"), + script.script.replace("\r\n", "\n"), + ) + .await?; + tokio::fs::File::create(&log_path).await?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o755)) + .await?; + } + + let mut created_vmid = None; + let min_vmid = self.app_config.load().runtime.pve_lxc.min_vmid; + for _ in 0..3 { + let _vmid_lock = self.cli.lock_vmid_allocation().await?; + let vmid = self.cli.next_vmid(min_vmid).await?; + match self + .create_helper_container( + server, + script, + Self::SCRIPT_HELPER_ROLE, + vmid, + resources.clone(), + environment.clone(), + &staging_path, + "/mnt/script", + "/mnt/script/script.sh", + &log_path, + ) + .await + { + Ok(()) => { + created_vmid = Some(vmid); + break; + } + Err(error) if Self::vmid_conflict(&error) => { + tracing::warn!(vmid, "retrying Proxmox script create after VMID collision"); + } + Err(error) => { + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(error); + } + } + } + let Some(vmid) = created_vmid else { + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(anyhow::anyhow!( + "failed to create Proxmox script helper after three VMID allocation collisions" + )); + }; + + let (status_tx, status_rx) = tokio::sync::mpsc::channel(1); + let handle = match PveHelperProcessHandle::new( + vmid, + self.cli.clone(), + server, + Arc::clone(&self.app_config), + log_path, + status_tx, + false, + false, + true, + Some(staging_path.clone()), + ) + .await + { + Ok(handle) => Arc::new(handle), + Err(error) => { + self.cli.destroy(vmid).await.ok(); + tokio::fs::remove_dir_all(&staging_path).await.ok(); + return Err(error.context("failed to create Proxmox script process handle")); + } + }; + + Ok((handle, status_rx)) + } + + async fn resolve_internal_target( + &self, + server: &crate::server::Server, + port: u16, + ) -> Result, anyhow::Error> { + Ok(self + .server_address(server.uuid) + .await? + .map(|address| std::net::SocketAddr::new(address, port))) + } + + async fn resolve_published_address(&self, server: &crate::server::Server) -> Option { + match self.server_address(server.uuid).await { + Ok(address) => address, + Err(error) => { + tracing::error!( + server = %server.uuid, + "failed to resolve Proxmox LXC address: {error:#}" + ); + None + } + } + } + + async fn container_refs( + &self, + servers: &[crate::server::Server], + ) -> HashMap { + let Some(node) = self.node.get() else { + tracing::error!("cannot inventory Proxmox containers before runtime boot"); + return HashMap::new(); + }; + let containers = match self.cli.list_containers().await { + Ok(containers) => containers, + Err(err) => { + tracing::error!("failed to list Proxmox LXC containers: {err:#}"); + return HashMap::new(); + } + }; + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let requested_servers: HashSet<_> = servers.iter().map(|server| server.uuid).collect(); + + let owned = containers + .into_iter() + .filter(|container| container.node == *node) + .filter_map(|container| { + let server_uuid = Self::server_from_tags(&tag_prefix, &container.tags)?; + requested_servers + .contains(&server_uuid) + .then_some((server_uuid, container)) + }) + .collect::>(); + + let runtime_statuses = match self.cli.node_runtime_statuses(node).await { + Ok(statuses) => statuses + .into_iter() + .map(|status| (status.vmid, status)) + .collect::>(), + Err(error) => { + tracing::error!("failed to list Proxmox LXC runtime status: {error:#}"); + return HashMap::new(); + } + }; + + let mut references = HashMap::new(); + for (server_uuid, container) in owned { + if container.status != Some(cli::ContainerStatus::Running) { + continue; + } + match runtime_statuses.get(&container.vmid) { + Some(status) if status.status == cli::ContainerStatus::Running => { + if let Some(pid) = status.pid.filter(|pid| *pid > 0) { + references.insert(server_uuid, format!("pid:{pid}")); + } else { + tracing::warn!( + server = %server_uuid, + vmid = container.vmid, + "running Proxmox LXC did not expose an init PID for private networking" + ); + } + } + Some(_) => {} + None => tracing::warn!( + server = %server_uuid, + vmid = container.vmid, + "Proxmox node inventory omitted an LXC needed for private networking" + ), + } + } + + references + } + + async fn used_ports( + &self, + ips: &[IpAddr], + ) -> Result>, anyhow::Error> { + if ips.is_empty() { + return Ok(HashMap::new()); + } + + let node = self + .node + .get() + .context("Proxmox VE LXC runtime has not been booted")?; + let requested = ips.iter().copied().collect::>(); + let tag_prefix = self.app_config.load().runtime.pve_lxc.tag_prefix.clone(); + let mut used = ips + .iter() + .copied() + .map(|ip| (ip, HashMap::>::new())) + .collect::>(); + + let mut containers = self.cli.list_containers().await?; + containers.sort_unstable_by_key(|container| container.vmid); + let runtime_statuses = self + .cli + .node_runtime_statuses(node) + .await? + .into_iter() + .map(|status| (status.vmid, status)) + .collect::>(); + + for container in containers.into_iter().filter(|container| { + container.node == *node && container.status == Some(cli::ContainerStatus::Running) + }) { + let interfaces = match self.cli.interfaces(&container.node, container.vmid).await { + Ok(interfaces) => interfaces, + Err(error) => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC during used-port discovery after interface lookup failed: {error:#}" + ); + continue; + } + }; + let matching_addresses = PveCli::container_addresses(&interfaces) + .into_iter() + .filter(|address| requested.contains(address)) + .collect::>(); + if matching_addresses.is_empty() { + continue; + } + + let runtime = match runtime_statuses.get(&container.vmid) { + Some(runtime) => runtime, + None => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC omitted from the node runtime inventory" + ); + continue; + } + }; + if runtime.status != cli::ContainerStatus::Running { + continue; + } + let Some(pid) = runtime.pid.filter(|pid| *pid > 0) else { + tracing::warn!( + vmid = container.vmid, + "skipping running Proxmox LXC without an init PID during used-port discovery" + ); + continue; + }; + let bound_ports = match Self::container_bound_ports(pid).await { + Ok(bound_ports) => bound_ports, + Err(error) => { + tracing::warn!( + vmid = container.vmid, + "skipping Proxmox LXC during used-port discovery after namespace inspection failed: {error:#}" + ); + continue; + } + }; + let server = Self::server_from_tags(&tag_prefix, &container.tags); + + for address in matching_addresses { + let Some(address_ports) = used.get_mut(&address) else { + continue; + }; + for bound in bound_ports + .iter() + .copied() + .filter(|bound| Self::bound_port_matches_address(*bound, address)) + { + address_ports + .entry(bound.port) + .and_modify(|owner| { + if *owner != server { + *owner = None; + } + }) + .or_insert(server); + } + } + } + + Ok(used + .into_iter() + .map(|(ip, ports)| { + let mut ports = ports + .into_iter() + .map(|(port, server)| UsedPort { port, server }) + .collect::>(); + ports.sort_unstable_by_key(|port| port.port); + (ip, ports) + }) + .collect()) + } +} + +#[cfg(test)] +mod tests; diff --git a/application/src/server/executor/pve_lxc/server_process.rs b/application/src/server/executor/pve_lxc/server_process.rs new file mode 100644 index 00000000..279df831 --- /dev/null +++ b/application/src/server/executor/pve_lxc/server_process.rs @@ -0,0 +1,1275 @@ +use super::*; + +pub(super) struct PveProcessHandle { + vmid: u32, + node: String, + cli: PveCli, + server: Weak, + app_config: Arc, + firewall: Arc, + log_path: PathBuf, + stdin_tx: tokio::sync::mpsc::Sender>, + stdout_ratelimited_rx: tokio::sync::broadcast::Receiver>, + stdout_rx: tokio::sync::broadcast::Receiver>, + started: Arc, + status_task: tokio::task::JoinHandle<()>, + stats_task: tokio::task::JoinHandle<()>, + console_task: tokio::task::JoinHandle<()>, +} + +impl PveProcessHandle { + const EXIT_STATUS_FILE: &'static str = ".wings-exit-status"; + // PVE's host-managed DHCP can lag CT init by several minutes during + // replacement starts. Keep the entrypoint alive for four real minutes. + pub(super) const DHCP_ROUTE_WAIT_INTERVAL_MILLIS: u32 = 250; + pub(super) const DHCP_ROUTE_WAIT_ATTEMPTS: u32 = 1920; + + pub(super) fn observed_status( + status: cli::ContainerStatus, + start_succeeded: bool, + seen_running: &mut bool, + exit_status: Option<(i32, bool)>, + ) -> Option { + match status { + cli::ContainerStatus::Running => { + *seen_running = true; + Some(super::ProcessStatus::Running) + } + cli::ContainerStatus::Stopped if !*seen_running && !start_succeeded => None, + cli::ContainerStatus::Stopped => { + let (exit_code, oom_killed) = exit_status.unwrap_or((-1, false)); + Some(super::ProcessStatus::Stopped { + exit_code, + oom_killed, + }) + } + } + } + + pub(super) fn parse_exit_status(contents: &str) -> Option<(i32, bool)> { + let mut exit_code = None; + let mut oom_killed = None; + for line in contents.lines() { + let (key, value) = line.split_once('=')?; + match key { + "exit_code" => exit_code = value.parse::().ok(), + "oom_killed" => { + oom_killed = match value { + "0" => Some(false), + "1" => Some(true), + _ => None, + } + } + _ => {} + } + } + Some((exit_code?, oom_killed?)) + } + + async fn read_exit_status(path: &std::path::Path) -> Option<(i32, bool)> { + let contents = tokio::fs::read_to_string(path).await.ok()?; + Self::parse_exit_status(&contents) + } + + fn exit_status_path(server: &crate::server::InnerServer) -> PathBuf { + std::path::Path::new(server.filesystem.base().as_str()).join(Self::EXIT_STATUS_FILE) + } + + fn managed_file_stage_name(target: &str) -> Result<&'static str, anyhow::Error> { + match target { + "/etc/machine-id" => Ok("machine-id"), + "/etc/hosts" => Ok("hosts"), + "/etc/passwd" => Ok("passwd"), + "/etc/group" => Ok("group"), + "/calagopus-entrypoint" => Ok("entrypoint"), + _ => Err(anyhow::anyhow!( + "unsupported managed Proxmox file mount target: {target}" + )), + } + } + + fn managed_file_stage_directory( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result { + let config = app_config.load(); + let root = config + .runtime + .pve_lxc + .managed_file_directory + .as_path(&config); + if !root.is_absolute() { + return Err(anyhow::anyhow!( + "runtime.pve_lxc.managed_file_directory must be an absolute path" + )); + } + Ok(root.join(server_uuid.to_compact_string())) + } + + pub(super) async fn cleanup_managed_file_staging( + app_config: &crate::config::Config, + server_uuid: uuid::Uuid, + ) -> Result<(), anyhow::Error> { + let path = Self::managed_file_stage_directory(app_config, server_uuid)?; + match tokio::fs::remove_dir_all(&path).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| { + format!( + "failed to remove managed Proxmox file staging directory {}", + path.display() + ) + }), + } + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn new( + vmid: u32, + node: String, + cli: PveCli, + stats_sampler: Arc, + server: &crate::server::Server, + app_config: Arc, + firewall: Arc, + status_tx: tokio::sync::mpsc::Sender, + fresh_log: bool, + suppress_initial_stopped: bool, + ) -> Result { + let log_path = Self::runtime_log_path(&app_config, server.uuid); + if let Some(parent) = log_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + if fresh_log { + tokio::fs::File::create(&log_path).await?; + } else { + tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .await?; + } + + let (stdin_tx, mut stdin_rx) = tokio::sync::mpsc::channel::>(150); + let websocket_log_count = app_config.load().system.websocket_log_count; + let (stdout_ratelimited_tx, stdout_ratelimited_rx) = + tokio::sync::broadcast::channel::>(websocket_log_count); + let (stdout_tx, stdout_rx) = tokio::sync::broadcast::channel::< + Arc, + >(websocket_log_count * 2); + + let started = Arc::new(AtomicBool::new(!suppress_initial_stopped)); + let state_started = Arc::clone(&started); + let state_cli = cli.clone(); + let state_server_uuid = server.uuid; + let state_exit_status_path = Self::exit_status_path(server); + let status_task = tokio::spawn(async move { + let mut seen_running = state_started.load(Ordering::Acquire); + let mut tick = tokio::time::interval(std::time::Duration::from_secs(2)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tick.tick().await; + let process_status = match state_cli.status(vmid).await { + Ok(status) => { + let exit_status = if status == cli::ContainerStatus::Stopped { + Self::read_exit_status(&state_exit_status_path).await + } else { + None + }; + match Self::observed_status( + status, + state_started.load(Ordering::Acquire), + &mut seen_running, + exit_status, + ) { + Some(status) => status, + None => continue, + } + } + Err(error) => { + tracing::warn!( + server = %state_server_uuid, + vmid, + "failed to read Proxmox LXC process state: {error:#}" + ); + continue; + } + }; + + if status_tx.send(process_status).await.is_err() { + break; + } + } + }); + + let stats_cli = cli.clone(); + let stats_node = node.clone(); + let stats_usage = server.resource_usage.clone(); + let stats_server = Arc::downgrade(&**server); + let stats_server_uuid = server.uuid; + let disk_bytes = server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(server.state.get_state()); + usage.disk_bytes = disk_bytes; + }); + let stats_task = tokio::spawn(async move { + let mut samples: Option = None; + let mut started_at: Option = None; + let mut previous_cpu_total = 0u64; + let mut previous_sample_at: Option = None; + let mut warned_unresolved_cgroup = false; + let mut resolve_tick = tokio::time::interval(std::time::Duration::from_secs(2)); + resolve_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + let Some(stats_server) = stats_server.upgrade() else { + break; + }; + let state = stats_server.state.get_state(); + if state == crate::server::state::ServerState::Offline { + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + resolve_tick.tick().await; + continue; + } + + if samples.is_none() { + resolve_tick.tick().await; + + let runtime_status = match stats_cli.runtime_status(&stats_node, vmid).await { + Ok(status) => status, + Err(error) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to resolve Proxmox LXC runtime stats process: {error:#}" + ); + continue; + } + }; + + if runtime_status.status != cli::ContainerStatus::Running { + let state = stats_server.state.get_state(); + let disk_bytes = stats_server.filesystem.limiter_usage().await; + stats_usage.send_modify(|usage| { + usage.wipe(state); + usage.disk_bytes = disk_bytes; + }); + continue; + } + + let Some(pid) = runtime_status.pid else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "running Proxmox LXC status did not include an init PID; resource stats are unavailable until it does" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + let Some(files) = super::cgroup::StatFiles::resolve(pid) else { + if !warned_unresolved_cgroup { + tracing::warn!( + server = %stats_server_uuid, + vmid, + pid, + "could not resolve the Proxmox LXC cgroup-v2 files; resource stats are unavailable until they can be resolved" + ); + warned_unresolved_cgroup = true; + } + continue; + }; + + samples = Some(stats_sampler.register(files)); + started_at = std::time::Instant::now().checked_sub( + std::time::Duration::from_secs(runtime_status.uptime_seconds), + ); + previous_cpu_total = 0; + previous_sample_at = None; + warned_unresolved_cgroup = false; + } + + let Some(receiver) = samples.as_mut() else { + continue; + }; + let sample = match tokio::time::timeout( + std::time::Duration::from_secs(3), + receiver.recv(), + ) + .await + { + Ok(Some(Ok(sample))) => sample, + Ok(Some(Err(error))) if error.kind() == std::io::ErrorKind::NotFound => { + samples = None; + started_at = None; + previous_cpu_total = 0; + previous_sample_at = None; + continue; + } + Ok(Some(Err(error))) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "failed to read Proxmox LXC cgroup stats: {error}" + ); + samples = None; + continue; + } + Ok(None) | Err(_) => { + tracing::warn!( + server = %stats_server_uuid, + vmid, + "Proxmox LXC cgroup stats sampler stopped delivering; resolving the container again" + ); + samples = None; + continue; + } + }; + + let cpu_absolute = if let Some(previous_at) = previous_sample_at { + let cpu_delta_ns = + sample.cpu_total_ns.saturating_sub(previous_cpu_total) as f64; + let wall_delta_ns = sample.at.duration_since(previous_at).as_nanos() as f64; + + if wall_delta_ns > 0.0 && cpu_delta_ns > 0.0 { + ((cpu_delta_ns / wall_delta_ns) * 100.0 * 1000.0).round() / 1000.0 + } else { + 0.0 + } + } else { + 0.0 + }; + previous_cpu_total = sample.cpu_total_ns; + previous_sample_at = Some(sample.at); + + let disk_bytes = stats_server.filesystem.limiter_usage().await; + let cpu_limit = stats_server.configuration.read().await.build.cpu_limit; + let cpu_limit_absolute = if cpu_limit > 0 { + u32::try_from(cpu_limit).unwrap_or(u32::MAX) + } else { + std::thread::available_parallelism().map_or(1, |threads| threads.get()) as u32 + * 100 + }; + let uptime = started_at + .map(|started_at| { + u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX) + }) + .unwrap_or(0); + let state = stats_server.state.get_state(); + + stats_usage.send_modify(|usage| { + usage.memory_bytes = sample.memory_bytes; + usage.memory_limit_bytes = sample.memory_limit_bytes; + usage.disk_bytes = disk_bytes; + usage.state = state; + usage.cpu_absolute = cpu_absolute; + usage.cpu_limit_absolute = cpu_limit_absolute; + usage.uptime = uptime; + + if let Some((rx_bytes, rx_packets, tx_bytes, tx_packets)) = sample.network { + usage.network.rx_bytes = rx_bytes; + usage.network.rx_packets = rx_packets; + usage.network.tx_bytes = tx_bytes; + usage.network.tx_packets = tx_packets; + } + }); + } + }); + + let console_cli = cli.clone(); + let console_app_config = Arc::clone(&app_config); + let console_log_max_bytes = app_config.load().runtime.pve_lxc.console_log_max_bytes; + let console_server = Arc::downgrade(&**server); + let console_log_path = log_path.clone(); + let console_task = tokio::spawn(async move { + let mut line_buffer = LineBuffer::new(); + let mut ratelimit_counter = 0; + let mut ratelimit_start = std::time::Instant::now(); + + loop { + match console_cli.status(vmid).await { + Ok(cli::ContainerStatus::Running) => {} + Ok(cli::ContainerStatus::Stopped) => { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + Err(error) => { + tracing::warn!( + vmid, + "failed to read Proxmox LXC state before console attach: {error:#}" + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + } + + let session = match console_cli.spawn_console(vmid) { + Ok(session) => session, + Err(error) => { + tracing::warn!(vmid, "failed to attach Proxmox LXC console: {error:#}"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + // `tokio::fs::File` serializes operations on a single handle. + // A blocking PTY read would therefore prevent stdin writes + // indefinitely if both directions shared one Tokio file. + let cli::ConsoleSession { + mut child, + mut reader, + mut writer, + } = session; + let mut log = match tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&console_log_path) + .await + { + Ok(log) => log, + Err(error) => { + tracing::error!(vmid, "failed to open Proxmox LXC runtime log: {error:#}"); + child.start_kill().ok(); + break; + } + }; + let mut log_size = log.metadata().await.map_or(0, |metadata| metadata.len()); + let mut buffer = vec![0u8; 8192]; + + loop { + tokio::select! { + read = reader.read(&mut buffer) => { + match read { + Ok(0) => break, + Ok(read) => { + let Some(data) = buffer.get(..read) else { + tracing::error!(vmid, read, "Proxmox console returned an invalid read length"); + break; + }; + line_buffer.extend(data); + while let Some(line) = line_buffer.next_line() { + if Self::is_pct_console_banner(line) { + continue; + } + if let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + true, + ).await { + tracing::error!(vmid, "failed to append Proxmox LXC runtime log: {error:#}"); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + let allow_ratelimit = { + ratelimit_counter += 1; + let config = console_app_config.load(); + if config.throttles.enabled + && config.throttles.line_reset_interval > 0 + && ratelimit_counter >= config.throttles.lines + { + if ratelimit_start.elapsed() + < std::time::Duration::from_millis( + config.throttles.line_reset_interval, + ) + { + if ratelimit_counter == config.throttles.lines + && let Some(server) = console_server.upgrade() + { + server.log_daemon_with_prelude( + "Server is outputting console data too quickly -- throttling...", + ); + } + false + } else { + ratelimit_counter = 0; + ratelimit_start = std::time::Instant::now(); + true + } + } else { + true + } + }; + + if allow_ratelimit { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + } + stdout_tx.send(line).ok(); + } + line_buffer.compact(); + } + Err(error) => { + tracing::warn!(vmid, "failed to read Proxmox LXC console: {error:#}"); + break; + } + } + } + data = stdin_rx.recv() => { + let Some(data) = data else { + child.start_kill().ok(); + return; + }; + if let Err(error) = writer.write_all(&data).await { + tracing::warn!(vmid, "failed to write Proxmox LXC console stdin: {error:#}"); + break; + } + } + result = child.wait() => { + if let Err(error) = result { + tracing::warn!(vmid, "failed waiting for Proxmox LXC console: {error:#}"); + } + break; + } + } + } + + child.start_kill().ok(); + if !matches!( + console_cli.status(vmid).await, + Ok(cli::ContainerStatus::Running) + ) { + if let Some(line) = line_buffer.flush() { + if !Self::is_pct_console_banner(line) + && let Err(error) = Self::write_bounded_log( + &mut log, + &mut log_size, + console_log_max_bytes, + line, + false, + ) + .await + { + tracing::error!( + vmid, + "failed to append final Proxmox LXC runtime log line: {error:#}" + ); + } + let line = Arc::new(compact_str::CompactString::from_utf8_lossy(line)); + if !Self::is_pct_console_banner(line.as_bytes()) { + stdout_ratelimited_tx.send(Arc::clone(&line)).ok(); + stdout_tx.send(line).ok(); + } + } + line_buffer = LineBuffer::new(); + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + }); + + Ok(Self { + vmid, + node, + cli, + server: Arc::downgrade(&**server), + app_config, + firewall, + log_path, + stdin_tx, + stdout_ratelimited_rx, + stdout_rx, + started, + status_task, + stats_task, + console_task, + }) + } + + fn runtime_log_path(app_config: &crate::config::Config, server_uuid: uuid::Uuid) -> PathBuf { + app_config + .resolve_as_path(|config| &config.system.log_directory) + .join("pve-lxc") + .join(format!("{server_uuid}.log")) + } + + pub(super) async fn write_bounded_log( + log: &mut tokio::fs::File, + size: &mut u64, + max_bytes: u64, + line: &[u8], + newline: bool, + ) -> Result<(), std::io::Error> { + let mut line = line; + let mut write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + if max_bytes > 0 && write_size > max_bytes { + let newline_size = usize::from(newline); + let keep = usize::try_from(max_bytes) + .unwrap_or(usize::MAX) + .saturating_sub(newline_size); + let start = line.len().saturating_sub(keep); + line = line.get(start..).unwrap_or_default(); + write_size = u64::try_from(line.len()) + .unwrap_or(u64::MAX) + .saturating_add(u64::from(newline)); + } + if max_bytes > 0 && size.saturating_add(write_size) > max_bytes { + log.set_len(0).await?; + log.seek(std::io::SeekFrom::Start(0)).await?; + *size = 0; + } + log.write_all(line).await?; + if newline { + log.write_all(b"\n").await?; + } + *size = size.saturating_add(write_size); + Ok(()) + } + + /// `pct console` forwards terminal key input and expects Enter as a carriage + /// return. Wings command producers use line feeds because Docker stdin is a + /// byte stream, so normalize line endings only at the PVE PTY boundary. + pub(super) fn normalize_console_input(data: Vec) -> Vec { + let mut normalized = Vec::with_capacity(data.len()); + let mut previous_was_carriage_return = false; + + for byte in data { + if byte == b'\n' { + if !previous_was_carriage_return { + normalized.push(b'\r'); + } + previous_was_carriage_return = false; + } else { + normalized.push(byte); + previous_was_carriage_return = byte == b'\r'; + } + } + + normalized + } + + pub(super) fn is_pct_console_banner(line: &[u8]) -> bool { + let line = String::from_utf8_lossy(line); + let line = line.trim_matches(|character: char| character.is_whitespace()); + line.starts_with("Connected to tty ") + || line.starts_with("Type to exit the console") + } + + fn get_server(&self) -> Result, anyhow::Error> { + self.server + .upgrade() + .ok_or_else(|| anyhow::anyhow!("server has been dropped")) + } + + pub(super) fn halt_signal(stop_type: &str, stop_value: Option<&str>) -> Option { + if stop_type != "signal" { + return None; + } + + Some( + match stop_value.map(str::to_uppercase).as_deref() { + Some("SIGABRT") => "SIGABRT", + Some("SIGINT") | Some("C") => "SIGINT", + Some("SIGTERM") => "SIGTERM", + Some("SIGQUIT") => "SIGQUIT", + _ => "SIGKILL", + } + .to_string(), + ) + } + + async fn managed_file_mounts( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + ) -> Result, anyhow::Error> { + #[cfg(not(unix))] + { + let _ = (server, app_config); + return Err(anyhow::anyhow!( + "managed Proxmox container files require a Unix host" + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let server_uuid = { + let configuration = server.configuration.read().await; + configuration.ensure_vmounts(app_config).await?; + configuration.uuid + }; + let (machine_id_enabled, tundra_enabled, passwd_enabled, passwd_directory) = { + let config = app_config.load(); + ( + config.system.machine_id.enabled, + config.tundra.enabled, + config.system.passwd.enabled, + config.system.passwd.directory.as_path(&config), + ) + }; + + let vmount_path = app_config.vmount_path(server_uuid); + let mut files = Vec::with_capacity(4); + if machine_id_enabled { + files.push((vmount_path.join("machine-id"), "/etc/machine-id")); + } + if tundra_enabled { + files.push((vmount_path.join("hosts"), "/etc/hosts")); + } + if passwd_enabled { + files.push((passwd_directory.join("passwd"), "/etc/passwd")); + files.push((passwd_directory.join("group"), "/etc/group")); + } + + let stage_directory = Self::managed_file_stage_directory(app_config, server_uuid)?; + let stage_root = stage_directory + .parent() + .context("managed Proxmox file staging directory did not have a parent")?; + tokio::fs::create_dir_all(stage_root) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging root {}", + stage_root.display() + ) + })?; + tokio::fs::set_permissions(stage_root, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging root permissions on {}", + stage_root.display() + ) + })?; + tokio::fs::create_dir_all(&stage_directory) + .await + .with_context(|| { + format!( + "failed to create managed Proxmox file staging directory {}", + stage_directory.display() + ) + })?; + tokio::fs::set_permissions(&stage_directory, std::fs::Permissions::from_mode(0o711)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file staging permissions on {}", + stage_directory.display() + ) + })?; + + let mut mounts = Vec::with_capacity(files.len() + 1); + for (source, target) in files { + let source = tokio::fs::canonicalize(&source).await.with_context(|| { + format!( + "failed to resolve managed Proxmox container file {}", + source.display() + ) + })?; + let metadata = tokio::fs::metadata(&source).await.with_context(|| { + format!( + "failed to inspect managed Proxmox container file {}", + source.display() + ) + })?; + if !metadata.is_file() { + return Err(anyhow::anyhow!( + "managed Proxmox container file {} is not a regular file", + source.display() + )); + } + + let staged_source = stage_directory.join(Self::managed_file_stage_name(target)?); + tokio::fs::copy(&source, &staged_source) + .await + .with_context(|| { + format!( + "failed to stage managed Proxmox container file {} at {}", + source.display(), + staged_source.display() + ) + })?; + tokio::fs::set_permissions(&staged_source, std::fs::Permissions::from_mode(0o644)) + .await + .with_context(|| { + format!( + "failed to set managed Proxmox file permissions on {}", + staged_source.display() + ) + })?; + let staged_source = + tokio::fs::canonicalize(&staged_source) + .await + .with_context(|| { + format!( + "failed to resolve staged managed Proxmox container file {}", + staged_source.display() + ) + })?; + let source_path = staged_source.into_os_string().into_string().map_err(|_| { + anyhow::anyhow!("managed Proxmox container file path is not valid UTF-8") + })?; + + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: target.to_string(), + }); + } + + let entrypoint_target = "/calagopus-entrypoint"; + let entrypoint_source = + stage_directory.join(Self::managed_file_stage_name(entrypoint_target)?); + let entrypoint_script = r#"#!/bin/sh +attempt=0 +while [ "$attempt" -lt __WINGS_DHCP_WAIT_ATTEMPTS__ ]; do + if grep -q '^eth0[[:space:]]*00000000[[:space:]]' /proc/net/route; then + break + fi + attempt=$((attempt + 1)) + sleep 0.25 +done +if [ "$attempt" -ge __WINGS_DHCP_WAIT_ATTEMPTS__ ]; then + echo 'timed out waiting for Proxmox DHCP' >&2 + exit 1 +fi + +read_oom_kills() { + value=0 + if [ -r /sys/fs/cgroup/memory.events ]; then + while read -r key count; do + if [ "$key" = oom_kill ]; then + value=$count + break + fi + done < /sys/fs/cgroup/memory.events + fi + printf '%s' "$value" +} + +child=0 +child_group=0 +forward_signal() { + if [ "$child" -gt 0 ]; then + if [ "$child_group" -eq 1 ]; then + kill "-$1" "-$child" 2>/dev/null || true + else + kill "-$1" "$child" 2>/dev/null || true + fi + fi +} +trap 'forward_signal HUP' HUP +trap 'forward_signal INT' INT +trap 'forward_signal QUIT' QUIT +trap 'forward_signal ABRT' ABRT +trap 'forward_signal TERM' TERM + +oom_before=$(read_oom_kills) +if command -v setsid >/dev/null 2>&1; then + setsid "$@" <&0 & + child_group=1 +else + "$@" <&0 & +fi +child=$! +while :; do + wait "$child" + status=$? + if ! kill -0 "$child" 2>/dev/null; then + break + fi +done +oom_after=$(read_oom_kills) +oom_killed=0 +if [ "$oom_after" -gt "$oom_before" ]; then + oom_killed=1 +fi + +marker=__WINGS_EXIT_STATUS_PATH__ +temporary="${marker}.tmp.$$" +( + umask 077 + printf 'exit_code=%s\noom_killed=%s\n' "$status" "$oom_killed" > "$temporary" +) +mv -f "$temporary" "$marker" +exit "$status" +"# + .replace( + "__WINGS_DHCP_WAIT_ATTEMPTS__", + &Self::DHCP_ROUTE_WAIT_ATTEMPTS.to_string(), + ) + .replace( + "__WINGS_EXIT_STATUS_PATH__", + &format!( + "{}/{}", + PveLxcExecutor::DATA_MOUNT_TARGET, + Self::EXIT_STATUS_FILE + ), + ); + tokio::fs::write(&entrypoint_source, entrypoint_script) + .await + .context("failed to stage Proxmox network-ready entrypoint")?; + tokio::fs::set_permissions(&entrypoint_source, std::fs::Permissions::from_mode(0o755)) + .await + .context("failed to make Proxmox network-ready entrypoint executable")?; + let entrypoint_source = tokio::fs::canonicalize(&entrypoint_source) + .await + .context("failed to resolve staged Proxmox network-ready entrypoint")?; + let source_path = entrypoint_source + .into_os_string() + .into_string() + .map_err(|_| { + anyhow::anyhow!("managed Proxmox entrypoint path is not valid UTF-8") + })?; + mounts.push(cli::ManagedFileMountSpec { + source_path, + target_path: entrypoint_target.to_string(), + }); + + Ok(mounts) + } + } + + pub(super) async fn runtime_config( + server: &crate::server::InnerServer, + app_config: &crate::config::Config, + cli: &PveCli, + vmid: u32, + ) -> Result { + let ( + environment, + panel_entrypoint, + cpuset_cpus, + io_weight, + memory_unlimited, + swap_unlimited, + ) = { + let configuration = server.configuration.read().await; + let resources = PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )?; + ( + configuration.environment(app_config), + configuration.entrypoint.clone(), + resources.cpuset_cpus.clone(), + PveCli::cgroup2_io_weight(configuration.build.io_weight)?, + resources.memory_unlimited, + resources.swap_unlimited, + ) + }; + let pids_limit = PveLxcExecutor::pids_limit(app_config); + let managed_file_mounts = Self::managed_file_mounts(server, app_config).await?; + let process = server.process_configuration.read().await; + let halt_signal = Self::halt_signal(&process.stop.r#type, process.stop.value.as_deref()); + let mut init_arguments = vec!["/calagopus-entrypoint".to_string()]; + if let Some(entrypoint) = panel_entrypoint { + if entrypoint.is_empty() { + return Err(anyhow::anyhow!("panel entrypoint override cannot be empty")); + } + init_arguments.extend(entrypoint); + } else { + let image_entrypoint = cli.oci_entrypoint(vmid).await?; + init_arguments.extend( + shell_words::split(&image_entrypoint) + .context("failed to parse OCI entrypoint for Proxmox LXC")?, + ); + } + let init_command = PveCli::encode_init_command(&init_arguments)?; + + Ok(cli::RuntimeConfigSpec { + vmid, + environment, + halt_signal, + init_command: Some(init_command), + console_logfile: None, + cpuset_cpus, + pids_limit, + io_weight, + memory_unlimited, + swap_unlimited, + managed_file_mounts: Some(managed_file_mounts), + }) + } + + pub(super) async fn sync_firewall(&self, wait_for_address: bool) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let has_rules = !server.configuration.read().await.firewall.is_empty(); + let edge_forwarding = PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some(); + let running = self.cli.status(self.vmid).await? == cli::ContainerStatus::Running; + + let container_ips = if running && (has_rules || edge_forwarding) { + let mut last_error = None; + let mut addresses = Vec::new(); + let attempts = if wait_for_address { 40 } else { 1 }; + + for attempt in 0..attempts { + match self.cli.interfaces(&self.node, self.vmid).await { + Ok(interfaces) => { + addresses = PveCli::container_addresses(&interfaces); + if !addresses.is_empty() { + break; + } + last_error = None; + } + Err(error) => last_error = Some(error), + } + + if attempt + 1 < attempts { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + + if addresses.is_empty() { + if let Some(error) = last_error { + return Err(error.context( + "failed to resolve Proxmox LXC address for firewall and edge forwarding", + )); + } + return Err(anyhow::anyhow!( + "running Proxmox LXC container {} has no usable bridged address; refusing to leave firewall or edge forwarding unapplied", + self.vmid + )); + } + + addresses + } else { + Vec::new() + }; + + let (mut spec, edge_forward) = { + let configuration = server.configuration.read().await; + let destination = container_ips.iter().find_map(|address| match address { + IpAddr::V4(address) => Some(*address), + IpAddr::V6(_) => None, + }); + let edge_forward = if edge_forwarding && running { + Some(PveLxcExecutor::edge_forward_payload( + &configuration, + destination + .context("running Proxmox LXC has no IPv4 address for edge forwarding")?, + )?) + } else { + None + }; + ( + PveLxcExecutor::firewall_spec(&configuration, container_ips, Some(self.vmid)), + edge_forward, + ) + }; + spec.files = Some(PveLxcExecutor::firewall_file_access(&server)); + self.firewall.sync(&spec).await?; + if edge_forwarding { + let (public_ip, payload) = edge_forward.flatten().unzip(); + PveLxcExecutor::sync_edge_forwarding(&self.app_config, server.uuid, public_ip, payload) + .await?; + } + Ok(()) + } + + pub(super) async fn stop_after_firewall_failure( + &self, + error: anyhow::Error, + context: &'static str, + ) -> anyhow::Error { + match self.cli.stop(self.vmid).await { + Ok(()) => error.context(format!("{context}; container was stopped")), + Err(stop_error) => error.context(format!( + "{context} and failed to stop the exposed container: {stop_error:#}" + )), + } + } +} + +impl Drop for PveProcessHandle { + fn drop(&mut self) { + self.status_task.abort(); + self.stats_task.abort(); + self.console_task.abort(); + } +} + +#[async_trait::async_trait] +impl ProcessHandle for PveProcessHandle { + async fn logs( + &self, + lines: Option, + ) -> Result, anyhow::Error> { + if let Some(lines) = lines { + let file = match tokio::fs::File::open(&self.log_path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Box::new(Cursor::new(Vec::new()))); + } + Err(error) => return Err(error.into()), + }; + return Ok(Box::new(crate::io::tail::async_tail(file, lines).await?)); + } + + Ok(Box::new(tokio::fs::File::open(&self.log_path).await?)) + } + + async fn send_stdin(&self, data: Vec) -> Result<(), anyhow::Error> { + self.stdin_tx + .send(Self::normalize_console_input(data)) + .await + .map_err(Into::into) + } + + async fn subscribe_stdout_lines_ratelimited( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_ratelimited_rx.resubscribe()) + } + + async fn subscribe_stdout_lines( + &self, + ) -> Result>, anyhow::Error> + { + Ok(self.stdout_rx.resubscribe()) + } + + async fn sync_configuration(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let resources = { + let configuration = server.configuration.read().await; + PveCli::panel_resources( + configuration.build.memory_limit, + configuration.build.overhead_memory, + configuration.build.swap, + configuration.build.cpu_limit, + configuration.build.threads.as_deref(), + )? + }; + self.cli.set_resources(self.vmid, &resources).await?; + self.cli + .apply_runtime_config( + &Self::runtime_config(&server, &self.app_config, &self.cli, self.vmid).await?, + ) + .await?; + if let Err(error) = self.sync_firewall(true).await { + let configuration = server.configuration.read().await; + let requires_network_policy = !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()); + drop(configuration); + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to synchronize Proxmox LXC network policy after configuration update", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state after configuration update: {error:#}" + ); + } + Ok(()) + } + + async fn start(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + match tokio::fs::remove_file(&exit_status_path).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to clear stale Proxmox LXC exit status {}", + exit_status_path.display() + ) + }); + } + } + self.cli.start(self.vmid).await?; + self.started.store(true, Ordering::Release); + + if let Err(error) = self.sync_firewall(true).await { + let requires_network_policy = match self.get_server() { + Ok(server) => { + let configuration = server.configuration.read().await; + !configuration.firewall.is_empty() + || (configuration.allocations.default.is_some() + && PveLxcExecutor::edge_forwarding_config(&self.app_config)?.is_some()) + } + Err(_) => true, + }; + + if requires_network_policy { + return Err(self + .stop_after_firewall_failure( + error, + "failed to apply Proxmox LXC network policy", + ) + .await); + } + + tracing::warn!( + vmid = self.vmid, + "failed to clear Proxmox LXC firewall state for a server without rules: {error:#}" + ); + } + + Ok(()) + } + + async fn stop(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let process = server.process_configuration.read().await; + let stop_type = process.stop.r#type.clone(); + let stop_value = process.stop.value.clone(); + drop(process); + + if stop_type == "command" { + let mut command = stop_value + .map(|value| value.as_bytes().to_vec()) + .unwrap_or_default(); + command.push(b'\n'); + self.stdin_tx.send(command).await.map_err(Into::into) + } else { + self.cli.request_shutdown(self.vmid).await + } + } + + async fn kill(&self) -> Result<(), anyhow::Error> { + let server = self.get_server()?; + let exit_status_path = Self::exit_status_path(&server); + if self.cli.status(self.vmid).await? == cli::ContainerStatus::Stopped { + return Ok(()); + } + match tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&exit_status_path) + .await + { + Ok(mut marker) => marker + .write_all(b"exit_code=137\noom_killed=0\n") + .await + .with_context(|| { + format!( + "failed to record forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + })?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to create forced Proxmox LXC exit status {}", + exit_status_path.display() + ) + }); + } + } + self.cli.stop(self.vmid).await + } +} diff --git a/application/src/server/executor/pve_lxc/tests.rs b/application/src/server/executor/pve_lxc/tests.rs new file mode 100644 index 00000000..c51b2d50 --- /dev/null +++ b/application/src/server/executor/pve_lxc/tests.rs @@ -0,0 +1,617 @@ +use super::*; + +#[test] +fn dhcp_route_wait_allows_the_configured_four_minutes() { + assert_eq!(PveProcessHandle::DHCP_ROUTE_WAIT_INTERVAL_MILLIS, 250); + assert_eq!(PveProcessHandle::DHCP_ROUTE_WAIT_ATTEMPTS, 1920); + assert_eq!( + u64::from(PveProcessHandle::DHCP_ROUTE_WAIT_INTERVAL_MILLIS) + * u64::from(PveProcessHandle::DHCP_ROUTE_WAIT_ATTEMPTS), + 480_000 + ); +} + +#[test] +fn successful_start_followed_by_first_stopped_probe_emits_stopped() { + let mut seen_running = false; + let status = PveProcessHandle::observed_status( + cli::ContainerStatus::Stopped, + true, + &mut seen_running, + Some((42, true)), + ); + assert!(matches!( + status, + Some(crate::server::executor::ProcessStatus::Stopped { + exit_code: 42, + oom_killed: true, + }) + )); +} + +#[test] +fn parses_managed_exit_status_strictly() { + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=137\noom_killed=1\n"), + Some((137, true)) + ); + assert_eq!( + PveProcessHandle::parse_exit_status("oom_killed=0\nexit_code=0\n"), + Some((0, false)) + ); + assert_eq!(PveProcessHandle::parse_exit_status("exit_code=0\n"), None); + assert_eq!( + PveProcessHandle::parse_exit_status("exit_code=nope\noom_killed=0\n"), + None + ); +} + +#[tokio::test] +async fn runtime_log_never_exceeds_its_retention_limit() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("console.log"); + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await + .unwrap(); + let mut size = 0; + + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"first", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"second", true) + .await + .unwrap(); + PveProcessHandle::write_bounded_log(&mut file, &mut size, 8, b"oversized-line", true) + .await + .unwrap(); + file.flush().await.unwrap(); + + let data = tokio::fs::read(&path).await.unwrap(); + assert_eq!(data, b"ed-line\n"); + assert_eq!(data.len(), 8); + assert_eq!(size, 8); +} + +#[test] +fn static_allocation_uses_explicit_isolated_vlan_network() { + assert_eq!( + PveLxcExecutor::network_for_allocation( + "vmbr0", + Some(30), + Some(24), + Some("10.0.30.1"), + Some("10.0.30.42"), + ) + .unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=10.0.30.42/24,gw=10.0.30.1,type=veth" + ); +} + +#[test] +fn static_allocation_never_derives_the_management_network() { + let error = + PveLxcExecutor::network_for_allocation("vmbr0", Some(30), None, None, Some("10.0.30.42")) + .unwrap_err(); + assert!(error.to_string().contains("network_prefix")); + + assert_eq!( + PveLxcExecutor::dhcp_network("vmbr0", Some(30)).unwrap(), + "name=eth0,bridge=vmbr0,firewall=1,tag=30,ip=dhcp,type=veth,host-managed=1" + ); +} + +#[test] +fn network_bridge_rejects_option_injection() { + assert!(PveLxcExecutor::dhcp_network("vmbr0 firewall=0", None).is_err()); + assert!(PveLxcExecutor::dhcp_network("vmbr0,firewall=0", None).is_err()); +} + +#[test] +fn accepts_pve_9_2_and_newer() { + assert_eq!( + PveVersion::parse("pve-manager/9.2.1/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); + assert_eq!( + PveVersion::parse("pve-manager/10.0.0/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(true) + ); +} + +#[test] +fn rejects_pve_older_than_9_2() { + assert_eq!( + PveVersion::parse("pve-manager/9.1.7/abcdef") + .ok() + .map(PveVersion::is_supported), + Some(false) + ); +} + +#[test] +fn pve_runtime_only_targets_the_local_node() { + assert_eq!( + PveLxcExecutor::runtime_node("", "pve-a").ok().as_deref(), + Some("pve-a") + ); + assert_eq!( + PveLxcExecutor::runtime_node("pve-a", "pve-a") + .ok() + .as_deref(), + Some("pve-a") + ); + assert!(PveLxcExecutor::runtime_node("pve-b", "pve-a").is_err()); +} + +#[test] +fn proc_net_parser_reports_listeners_and_udp_reservations() { + fn proc_ipv4(address: std::net::Ipv4Addr) -> String { + format!("{:08X}", u32::from_le_bytes(address.octets())) + } + + let any = proc_ipv4(std::net::Ipv4Addr::UNSPECIFIED); + let loopback = proc_ipv4(std::net::Ipv4Addr::LOCALHOST); + let tcp = format!( + " sl local_address rem_address st\n 0: {any}:63DD 00000000:0000 0A\n 1: {loopback}:C350 0100007F:9C40 01\n" + ); + let udp = format!( + " sl local_address rem_address st\n 0: {any}:4ABC 00000000:0000 07\n 1: {loopback}:D431 0100007F:0035 01\n 2: {any}:0000 00000000:0000 07\n" + ); + + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&tcp, ProcNetFamily::Ipv4, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 25565, + }] + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&udp, ProcNetFamily::Ipv4, false).unwrap(), + vec![ + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + BoundPort { + family: ProcNetFamily::Ipv4, + address: Some(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + port: 54321, + }, + ] + ); +} + +#[test] +fn proc_net_parser_decodes_ipv6_and_matches_ipv4_mapped_bindings() { + fn proc_ipv6(address: std::net::Ipv6Addr) -> String { + address + .octets() + .chunks_exact(4) + .map(|chunk| { + format!( + "{:08X}", + u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) + ) + }) + .collect::() + } + + let address: std::net::Ipv6Addr = "fd12:3456::20".parse().unwrap(); + let raw = proc_ipv6(address); + let table = format!( + " sl local_address rem_address st\n 0: {raw}:63DD 00000000000000000000000000000000:0000 0A\n" + ); + assert_eq!( + PveLxcExecutor::parse_proc_net_ports(&table, ProcNetFamily::Ipv6, true).unwrap(), + vec![BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6(address)), + port: 25565, + }] + ); + + let mapped = BoundPort { + family: ProcNetFamily::Ipv6, + address: Some(IpAddr::V6("::ffff:10.42.0.20".parse().unwrap())), + port: 25565, + }; + assert!(PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + mapped, + "10.42.0.21".parse().unwrap() + )); + assert!(PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv6, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); + assert!(!PveLxcExecutor::bound_port_matches_address( + BoundPort { + family: ProcNetFamily::Ipv4, + address: None, + port: 19132, + }, + "fd12:3456::20".parse().unwrap() + )); +} + +#[test] +fn ownership_tags_require_marker_and_server_uuid() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let server_tag = PveLxcExecutor::server_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), server_tag]; + + assert_eq!( + PveLxcExecutor::server_from_tags("calagopus", &tags), + Some(server_uuid) + ); + assert_eq!(PveLxcExecutor::server_from_tags("other", &tags), None); +} + +#[test] +fn helper_tags_are_distinct_from_servers_and_roles() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let installer_tag = PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ); + let tags = vec!["calagopus".to_string(), installer_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::helper_from_tags("calagopus", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), + Some(server_uuid) + ); + assert_eq!( + PveLxcExecutor::helper_from_tags("calagopus", PveLxcExecutor::SCRIPT_HELPER_ROLE, &tags,), + None + ); + assert_eq!( + PveLxcExecutor::helper_from_tags("other", PveLxcExecutor::INSTALLER_HELPER_ROLE, &tags,), + None + ); +} + +#[test] +fn replacement_tags_are_distinct_and_recoverable() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let replacement_tag = PveLxcExecutor::replacement_tag("calagopus", server_uuid); + let tags = vec!["calagopus".to_string(), replacement_tag]; + + assert_eq!(PveLxcExecutor::server_from_tags("calagopus", &tags), None); + assert_eq!( + PveLxcExecutor::replacement_from_tags("calagopus", &tags), + Some(server_uuid) + ); +} + +#[test] +fn image_drift_plan_recovers_staged_replacements() { + let desired = "calagopus-image-new"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old".to_string()], + }; + let staged = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&staged), desired), + ServerContainerPlan::ReplaceWithStaged { + old_vmid: 101, + staged_vmid: 102, + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&staged), desired), + ServerContainerPlan::RecoverStaged { staged_vmid: 102 } + ); +} + +#[test] +fn image_drift_plan_discards_stale_staging_without_replacing_good_container() { + let desired = "calagopus-image-current"; + let existing = cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec![desired.to_string()], + }; + let stale = cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: vec!["calagopus-image-old-attempt".to_string()], + }; + + assert_eq!( + PveLxcExecutor::plan_server_container(Some(&existing), Some(&stale), desired), + ServerContainerPlan::Reuse { + vmid: 101, + stale_replacement: Some(102), + } + ); + assert_eq!( + PveLxcExecutor::plan_server_container(None, Some(&stale), desired), + ServerContainerPlan::CreateFresh { + stale_replacement: Some(102), + } + ); +} + +#[test] +fn owned_container_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::server_tag("calagopus", server_uuid), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 101, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 102, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_server_container(&containers, "pve-a", "calagopus", server_uuid,) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_server_container(&containers, "pve-b", "calagopus", server_uuid,) + .ok() + .flatten(), + None + ); +} + +#[test] +fn owned_helper_lookup_rejects_duplicate_claims() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let tags = vec![ + "calagopus".to_string(), + PveLxcExecutor::helper_tag( + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ), + ]; + let containers = vec![ + cli::ClusterContainer { + vmid: 201, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Running), + tags: tags.clone(), + }, + cli::ClusterContainer { + vmid: 202, + node: "pve-a".to_string(), + status: Some(cli::ContainerStatus::Stopped), + tags, + }, + ]; + + assert!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::INSTALLER_HELPER_ROLE, + server_uuid, + ) + .is_err() + ); + assert_eq!( + PveLxcExecutor::owned_helper_container( + &containers, + "pve-a", + "calagopus", + PveLxcExecutor::SCRIPT_HELPER_ROLE, + server_uuid, + ) + .ok() + .flatten(), + None + ); +} + +#[test] +fn direct_bridge_firewall_spec_targets_container_addresses_without_published_bindings() { + let server_uuid = uuid::Uuid::from_u128(0x8f9ae020_60b9_4c5f_8ce2_8f0dfb8d7b31); + let mappings = HashMap::from([ + ( + compact_str::CompactString::from("10.42.0.20"), + vec![25566, 25565, 25565], + ), + (compact_str::CompactString::from("10.42.0.21"), vec![19132]), + ]); + let rules = vec![crate::server::firewall::FirewallRule { + action: crate::server::firewall::FirewallRuleAction::Deny, + protocols: HashSet::from([crate::server::firewall::FirewallRuleProtocol::Tcp]), + sources: Vec::new(), + ports: None, + source_file: None, + }]; + let spec = PveLxcExecutor::direct_bridge_firewall_spec( + server_uuid, + &mappings, + &rules, + vec![ + "fd12:3456::20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + "10.42.0.20".parse().unwrap(), + ], + None, + ); + + assert!(spec.bindings.is_empty()); + assert_eq!(spec.container_ports, vec![19132, 25565, 25566]); + assert_eq!( + spec.container_ips, + vec![ + "10.42.0.20".parse::().unwrap(), + "fd12:3456::20".parse::().unwrap(), + ] + ); + assert_eq!(spec.rules, rules); + + let concrete = crate::server::firewall::expand_rules(&spec); + assert_eq!(concrete.len(), 6); + assert!( + concrete + .iter() + .all(|rule| matches!(rule.dst, crate::server::firewall::RuleDst::Container { .. })) + ); +} + +#[test] +fn console_input_uses_terminal_enter_without_doubling_crlf() { + assert_eq!( + PveProcessHandle::normalize_console_input(b"list\n".to_vec()), + b"list\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(b"first\r\nsecond\n".to_vec()), + b"first\rsecond\r" + ); + assert_eq!( + PveProcessHandle::normalize_console_input(vec![0x03]), + vec![0x03] + ); + assert!(PveProcessHandle::is_pct_console_banner( + b"Connected to tty 0\r" + )); + assert!(PveProcessHandle::is_pct_console_banner( + b"Type to exit the console, to enter Ctrl+z itself\r" + )); + assert!(!PveProcessHandle::is_pct_console_banner( + b"[Server thread/INFO]: Connected to tty 0" + )); +} + +#[test] +fn halt_signal_matches_wings_stop_semantics() { + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGABRT")), + Some("SIGABRT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("c")), + Some("SIGINT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGTERM")), + Some("SIGTERM".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("SIGQUIT")), + Some("SIGQUIT".to_string()) + ); + assert_eq!( + PveProcessHandle::halt_signal("signal", Some("unexpected")), + Some("SIGKILL".to_string()) + ); + assert_eq!(PveProcessHandle::halt_signal("command", Some("stop")), None); +} + +#[test] +fn device_permissions_require_exact_pve_devn_semantics() { + let path = std::path::Path::new("/dev/kvm"); + + assert_eq!( + PveLxcExecutor::device_deny_write("r", path).ok(), + Some(true) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("rw", path).ok(), + Some(false) + ); + assert_eq!( + PveLxcExecutor::device_deny_write("wr", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("w", path).is_err()); + assert!(PveLxcExecutor::device_deny_write("m", path).is_err()); + assert_eq!( + PveLxcExecutor::device_deny_write("rwm", path).ok(), + Some(false) + ); + assert!(PveLxcExecutor::device_deny_write("rwx", path).is_err()); +} + +#[tokio::test] +async fn lxc_pid_limit_overrides_the_legacy_docker_setting() { + let config = crate::config::Config::mock(); + config + .mutate_in_place_for_testing() + .docker + .container_pid_limit = 256; + assert_eq!(PveLxcExecutor::pids_limit(&config), Some(256)); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .pids_limit = Some(1024); + assert_eq!(PveLxcExecutor::pids_limit(&config), Some(1024)); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .pids_limit = Some(0); + assert_eq!(PveLxcExecutor::pids_limit(&config), None); +} + +#[tokio::test] +async fn edge_ssh_user_is_configurable_and_validated() { + let config = crate::config::Config::mock(); + { + let inner = config.mutate_in_place_for_testing(); + inner.runtime.pve_lxc.edge_ssh_identity_path = "/root/edge-key".to_string(); + inner.runtime.pve_lxc.edge_known_hosts_path = "/root/known-hosts".to_string(); + inner.runtime.pve_lxc.edge_ssh_user = "edge-forward".to_string(); + } + + let resolved = PveLxcExecutor::edge_forwarding_config(&config) + .expect("valid edge configuration") + .expect("configured edge forwarding"); + assert_eq!(resolved.1, "edge-forward"); + + config + .mutate_in_place_for_testing() + .runtime + .pve_lxc + .edge_ssh_user = "root@host".to_string(); + assert!(PveLxcExecutor::edge_forwarding_config(&config).is_err()); +} diff --git a/application/src/server/firewall/iptables.rs b/application/src/server/firewall/iptables.rs index 42cc8863..1bf78f49 100644 --- a/application/src/server/firewall/iptables.rs +++ b/application/src/server/firewall/iptables.rs @@ -775,6 +775,7 @@ mod tests { fn rules() -> BTreeMap> { let spec = FirewallServerSpec { server: uuid::Uuid::from_str("abcdef12-3456-7890-abcd-ef1234567890").unwrap(), + target: None, bindings: vec![ FirewallBinding { ip: Some("192.168.1.5".parse().unwrap()), diff --git a/application/src/server/firewall/mod.rs b/application/src/server/firewall/mod.rs index 5a631175..d0f4f6a7 100644 --- a/application/src/server/firewall/mod.rs +++ b/application/src/server/firewall/mod.rs @@ -133,6 +133,9 @@ pub struct FirewallBinding { pub struct FirewallServerSpec { pub server: uuid::Uuid, + /// Runtime-specific destination for backends which attach policy to a + /// workload instead of the host forwarding path. + pub target: Option, pub bindings: Vec, pub container_ports: Vec, pub container_ips: Vec, @@ -140,6 +143,11 @@ pub struct FirewallServerSpec { pub files: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FirewallTarget { + ProxmoxLxc { vmid: u32, interface: String }, +} + impl FirewallServerSpec { #[inline] pub fn references_files(&self) -> bool { @@ -301,6 +309,99 @@ pub trait FirewallBackend: Send + Sync { async fn reconcile(&self, specs: &[FirewallServerSpec]) -> Result<(), anyhow::Error>; } +pub async fn create_host_local_with( + backend: FirewallBackendKind, + limits: sets::SourceFileLimits, +) -> Arc { + if backend == FirewallBackendKind::Disabled { + return Arc::new(noop::NoopFirewall::new(false)); + } + + if !cfg!(target_os = "linux") { + if backend != FirewallBackendKind::Auto { + tracing::warn!( + "docker.firewall.backend is set to {:?}, but host-local server firewalls are only supported on linux", + backend + ); + } + + return Arc::new(noop::NoopFirewall::new(true)); + } + + if std::path::Path::new("/.dockerenv").exists() || std::env::var("OCI_CONTAINER").is_ok() { + tracing::warn!( + "host-local server firewalls require Wings to run in the host network namespace; containerized Wings is not supported for this runtime" + ); + + return Arc::new(noop::NoopFirewall::new(true)); + } + + let runner = CommandRunner::Local; + match backend { + FirewallBackendKind::Nftables => { + warn_bridge_nf(&runner).await; + Arc::new(nftables::NftablesFirewall::new(Vec::new(), runner, limits)) + } + FirewallBackendKind::Iptables => { + warn_bridge_nf(&runner).await; + Arc::new(iptables::IptablesFirewall::new(Vec::new(), runner, limits).await) + } + FirewallBackendKind::Container => { + tracing::warn!( + "the container firewall backend is only available with the Docker runtime; host-local server firewall rules will not be applied" + ); + Arc::new(noop::NoopFirewall::new(true)) + } + FirewallBackendKind::Auto => { + if runner + .run( + "nft", + &["--check", "-f", "-"], + Some(b"add table inet wings\n"), + ) + .await + .is_ok() + { + tracing::info!("using nftables host-local server firewall backend"); + warn_bridge_nf(&runner).await; + + Arc::new(nftables::NftablesFirewall::new(Vec::new(), runner, limits)) + } else if runner + .run("iptables", &["-w", "-S", "FORWARD"], None) + .await + .is_ok() + { + tracing::info!("using iptables host-local server firewall backend"); + warn_bridge_nf(&runner).await; + + Arc::new(iptables::IptablesFirewall::new(Vec::new(), runner, limits).await) + } else { + tracing::warn!( + "neither nftables nor iptables are usable on this host, host-local server firewall rules will not be applied" + ); + + Arc::new(noop::NoopFirewall::new(true)) + } + } + FirewallBackendKind::Disabled => Arc::new(noop::NoopFirewall::new(false)), + } +} + +/// Removes Wings-owned host firewall state after another backend has committed +/// its replacement policy. Both host implementations are checked because the +/// previously selected `auto` backend may differ from the one usable now. +pub async fn clear_host_local(limits: sets::SourceFileLimits) { + let nftables = nftables::NftablesFirewall::new(Vec::new(), CommandRunner::Local, limits); + if let Err(error) = nftables.reconcile(&[]).await { + tracing::debug!("failed to clear stale Wings nftables policy: {error:#}"); + } + + let iptables = iptables::IptablesFirewall::new(Vec::new(), CommandRunner::Local, limits).await; + if let Err(error) = iptables.reconcile(&[]).await { + tracing::debug!("failed to clear stale Wings iptables policy: {error:#}"); + } +} + pub async fn create( config: &crate::config::Config, docker: &Arc, @@ -681,6 +782,7 @@ mod tests { fn spec(bindings: Vec, rules: Vec) -> FirewallServerSpec { FirewallServerSpec { server: uuid::Uuid::new_v4(), + target: None, bindings, container_ports: Vec::new(), container_ips: Vec::new(), diff --git a/application/src/server/firewall/nftables.rs b/application/src/server/firewall/nftables.rs index 082c02a0..44e9a86c 100644 --- a/application/src/server/firewall/nftables.rs +++ b/application/src/server/firewall/nftables.rs @@ -650,6 +650,7 @@ mod tests { fn rules() -> BTreeMap> { let spec = FirewallServerSpec { server: server(), + target: None, bindings: vec![ FirewallBinding { ip: Some("192.168.1.5".parse().unwrap()), @@ -730,6 +731,7 @@ mod tests { fn render_ruleset_declares_sets_and_matches_them_per_family() { let spec = FirewallServerSpec { server: server(), + target: None, bindings: vec![FirewallBinding { ip: None, port: 25565, diff --git a/application/src/server/firewall/sets.rs b/application/src/server/firewall/sets.rs index 512be652..b3ce5ef3 100644 --- a/application/src/server/firewall/sets.rs +++ b/application/src/server/firewall/sets.rs @@ -122,6 +122,45 @@ async fn fingerprint( })) } +pub async fn read_source_file( + filesystem: &crate::server::filesystem::cap::CapFilesystem, + path: &Path, + limits: SourceFileLimits, +) -> Result<(Vec, LoadStats), LoadError> { + let file = filesystem.async_open(path).await?; + let mut lines = tokio::io::BufReader::new(file).lines(); + let mut entries = Vec::new(); + let mut stats = LoadStats::default(); + let mut bytes: u64 = 0; + + while let Some(line) = lines.next_line().await? { + bytes += line.len() as u64 + 1; + if bytes > limits.max_bytes { + return Err(LoadError::TooLarge { + max_bytes: limits.max_bytes, + }); + } + + match parse_line(&line) { + None => {} + Some(Err(())) => stats.invalid += 1, + Some(Ok(entry)) => { + stats.entries += 1; + if stats.entries > limits.max_entries { + return Err(LoadError::TooManyEntries { + max_entries: limits.max_entries, + }); + } + entries.push(entry); + } + } + } + + entries.sort_unstable(); + entries.dedup(); + Ok((entries, stats)) +} + #[derive(Debug, Default, Clone, Copy)] pub struct LoadStats { pub entries: usize, diff --git a/application/src/server/mod.rs b/application/src/server/mod.rs index 2d0dd947..7cf6bcbc 100644 --- a/application/src/server/mod.rs +++ b/application/src/server/mod.rs @@ -511,6 +511,13 @@ impl Server { None => break, }; + #[cfg(unix)] + if let Some(tundra) = server.app_state.tundra.as_ref() + && tundra.hub.connected() + { + tundra.rebroadcast(); + } + if !server.filesystem.write_tracking_active() { server.filesystem.disk_checker_state_dirty.store(true, Ordering::Relaxed); } diff --git a/application/src/setup/mod.rs b/application/src/setup/mod.rs index b4958e41..e1bc5024 100644 --- a/application/src/setup/mod.rs +++ b/application/src/setup/mod.rs @@ -171,7 +171,7 @@ async fn probe( let container = std::env::var("OCI_CONTAINER").is_ok(); let data_directory = state.config.system.data_directory.as_path(&state.config); - let (cpu_count, memory_bytes, disk_bytes) = tokio::task::spawn_blocking(move || { + let (cpu_count, memory_bytes, local_disk_bytes) = tokio::task::spawn_blocking(move || { let mut system = sysinfo::System::new(); system.refresh_memory(); system.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing()); @@ -188,6 +188,10 @@ async fn probe( "failed to read system information", )?; + // Server data is bind-mounted from this filesystem for every backend, so + // its capacity is the only capacity the panel can safely allocate. + let disk_bytes = local_disk_bytes; + let ips = if container { BTreeSet::new() } else { diff --git a/application/src/stats.rs b/application/src/stats.rs index 6e1d6b5a..dc3933b6 100644 --- a/application/src/stats.rs +++ b/application/src/stats.rs @@ -1,5 +1,5 @@ use serde::Serialize; -use std::{path::Path, sync::Arc}; +use std::sync::Arc; use sysinfo::{Disks, Networks, System}; use utoipa::ToSchema; @@ -62,8 +62,12 @@ impl StatsManager { } } -impl Default for StatsManager { - fn default() -> Self { +impl StatsManager { + pub fn new(config: Arc) -> Self { + Self::new_with_config(Some(config)) + } + + fn new_with_config(config: Option>) -> Self { let stats = Arc::new(arc_swap::ArcSwap::new(Arc::new(SystemStats::default()))); std::thread::spawn({ @@ -77,6 +81,10 @@ impl Default for StatsManager { let mut sys = System::new_with_specifics(refresh_kind); let mut disks = Disks::new_with_refreshed_list(); let mut networks = Networks::new_with_refreshed_list(); + let data_directory = config.as_ref().map_or_else( + || std::path::PathBuf::from("/"), + |config| config.resolve_as_path(|inner| &inner.system.data_directory), + ); let cpu_model = sys .cpus() @@ -89,7 +97,7 @@ impl Default for StatsManager { sys.refresh_specifics(refresh_kind); networks.refresh(true); for disk in disks.list_mut() { - if disk.mount_point() == Path::new("/") { + if data_directory.starts_with(disk.mount_point()) { disk.refresh(); } } @@ -117,7 +125,11 @@ impl Default for StatsManager { let total_memory = sys.total_memory(); let used_memory = sys.used_memory(); - let disk = match disks.iter().find(|d| d.mount_point() == Path::new("/")) { + let disk = match disks + .iter() + .filter(|disk| data_directory.starts_with(disk.mount_point())) + .max_by_key(|disk| disk.mount_point().as_os_str().len()) + { Some(d) => d, None => match disks.first() { Some(d) => d, @@ -183,3 +195,9 @@ impl Default for StatsManager { Self { stats } } } + +impl Default for StatsManager { + fn default() -> Self { + Self::new_with_config(None) + } +} diff --git a/application/src/tundra/daemon.rs b/application/src/tundra/daemon.rs index 68417b33..6e868aa0 100644 --- a/application/src/tundra/daemon.rs +++ b/application/src/tundra/daemon.rs @@ -1,5 +1,6 @@ use super::TundraManager; use crate::{io::hash_reader::HashReader, routes::State}; +use anyhow::Context; use futures::StreamExt; use sha2::Digest; use std::{ @@ -15,6 +16,7 @@ const BINARY_NAME: &str = "tundra-node"; const SERVER_PLACEHOLDER: &str = "{server}"; const EXTRACT_CONTAINER_NAME: &str = "calagopus-wings-tundra-extract"; const SOURCE_ENTRYPOINT: &str = "/usr/bin/calagopus-tundra"; +const PIDFILE_NAME: &str = "tundra-node.pid"; const CREATE_ATTEMPTS: u32 = 5; const CREATE_BACKOFF: Duration = Duration::from_millis(200); @@ -441,7 +443,9 @@ impl Applied { } pub async fn stop(manager: &TundraManager) -> Result<(), anyhow::Error> { - let docker = manager.docker(); + let Some(docker) = manager.docker() else { + return stop_native(manager).await; + }; let inspect = match docker.inspect_container(CONTAINER_NAME, None).await { Ok(inspect) => inspect, Err(bollard::errors::Error::DockerResponseServerError { @@ -472,6 +476,124 @@ pub async fn stop(manager: &TundraManager) -> Result<(), anyhow::Error> { Ok(()) } +fn native_pid(manager: &TundraManager) -> Option { + let pid = std::fs::read_to_string(manager.data_dir.join("node").join(PIDFILE_NAME)) + .ok()? + .trim() + .parse::() + .ok()?; + let command = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?; + let expected_config = config_path(manager); + let command_matches = command + .split(|byte| *byte == 0) + .any(|argument| argument == expected_config.as_os_str().as_encoded_bytes()); + let executable = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?; + let expected_binary = binary_path(manager); + let executable_bytes = executable.as_os_str().as_encoded_bytes(); + let expected_bytes = expected_binary.as_os_str().as_encoded_bytes(); + let executable_matches = executable_bytes == expected_bytes + || executable_bytes.strip_suffix(b" (deleted)") == Some(expected_bytes); + (command_matches && executable_matches).then_some(pid) +} + +async fn stop_native(manager: &TundraManager) -> Result<(), anyhow::Error> { + if !manager.disabled() { + return Ok(()); + } + let Some(pid) = native_pid(manager) else { + return Ok(()); + }; + + let pid = rustix::process::Pid::from_raw(pid as i32) + .context("native tundra daemon PID was invalid")?; + rustix::process::kill_process(pid, rustix::process::Signal::TERM) + .context("failed to stop the native tundra daemon")?; + manager.hub.disconnect(); + Ok(()) +} + +async fn restart_native(manager: &TundraManager) -> Result<(), anyhow::Error> { + let output = tokio::process::Command::new(binary_path(manager)) + .arg("restart") + .arg("--config") + .arg(config_path(manager)) + .output() + .await + .context("failed to request a native tundra restart")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "native tundra restart failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(()) +} + +async fn ensure_native( + manager: &TundraManager, + wanted: &Applied, + tunnel_port: u16, +) -> Result<(), anyhow::Error> { + if let Some(pid) = native_pid(manager) { + let applied = manager.hub.request_metrics().await; + if applied + .as_ref() + .is_ok_and(|metrics| wanted.matches(metrics)) + { + return Ok(()); + } + + if manager.serving() && manager.restart_due() { + tracing::info!( + pid, + "requesting the native tundra daemon to apply its current binary and config" + ); + tokio::time::timeout(Duration::from_secs(30), restart_native(manager)).await??; + } + return Ok(()); + } + + let mut child = tokio::process::Command::new(binary_path(manager)) + .arg("--config") + .arg(config_path(manager)) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .context("failed to start the native tundra daemon")?; + let pid = child + .id() + .context("native tundra daemon did not expose a process ID")?; + let pid_path = manager.data_dir.join("node").join(PIDFILE_NAME); + if let Some(parent) = pid_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&pid_path, format!("{pid}\n")) + .await + .context("failed to write native tundra daemon PID file")?; + tokio::spawn(async move { + match child.wait().await { + Ok(status) if status.success() => { + tracing::info!(?pid, "native tundra daemon stopped"); + } + Ok(status) => { + tracing::error!(?pid, %status, "native tundra daemon exited"); + } + Err(error) => { + tracing::error!(?pid, "failed to wait for native tundra daemon: {error:#}"); + } + } + if tokio::fs::read_to_string(&pid_path) + .await + .is_ok_and(|contents| contents.trim() == pid.to_string()) + { + tokio::fs::remove_file(pid_path).await.ok(); + } + }); + tracing::info!(pid, tunnel_port, "started the native tundra daemon"); + Ok(()) +} + pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow::Error> { if !manager.serving() { return Ok(()); @@ -493,23 +615,34 @@ pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow let source = config.tundra.binary.as_path(&config); let source_image = config.tundra.source_image.clone(); let metrics_port = config.tundra.metrics_port; - let hosts_path = config - .system - .vmount_directory - .as_path(&config) - .join(SERVER_PLACEHOLDER) - .join("hosts"); + let hosts_path = match state.config.active_runtime_backend() { + crate::config::RuntimeBackend::PveLxc => config + .runtime + .pve_lxc + .managed_file_directory + .as_path(&config) + .join(SERVER_PLACEHOLDER) + .join("hosts"), + crate::config::RuntimeBackend::Auto | crate::config::RuntimeBackend::Docker => config + .system + .vmount_directory + .as_path(&config) + .join(SERVER_PLACEHOLDER) + .join("hosts"), + }; let image = config.tundra.image.clone(); let docker = manager.docker(); - let desired = create_body(state, manager, &image); drop(config); let refresh = !manager.images_refreshed.load(Ordering::Relaxed); if source.as_os_str().is_empty() { - ensure_image(&docker, state, &source_image, "source", refresh).await?; - extract_binary(&docker, manager, &source_image).await?; + let docker = docker + .as_ref() + .context("native tundra requires tundra.binary to point to a tundra-node executable")?; + ensure_image(docker, state, &source_image, "source", refresh).await?; + extract_binary(docker, manager, &source_image).await?; } else { let dest = binary_path(manager); let digest_path = image_digest_path(manager); @@ -545,6 +678,12 @@ pub async fn ensure(state: &State, manager: &TundraManager) -> Result<(), anyhow return Ok(()); } + let Some(docker) = docker else { + return ensure_native(manager, &wanted, own.tunnel_port).await; + }; + + let desired = create_body(state, manager, &image); + ensure_image(&docker, state, &image, "daemon", refresh).await?; manager.images_refreshed.store(true, Ordering::Relaxed); diff --git a/application/src/tundra/mod.rs b/application/src/tundra/mod.rs index 240e2bab..a4d98a58 100644 --- a/application/src/tundra/mod.rs +++ b/application/src/tundra/mod.rs @@ -71,7 +71,7 @@ pub struct TundraManager { pub data_dir: PathBuf, pub hub: hub::Hub, ca: ca::LocalCa, - docker: Arc, + docker: Option>, token: parking_lot::RwLock, control: parking_lot::Mutex, @@ -88,7 +88,7 @@ pub struct TundraManager { impl TundraManager { pub fn create( config: &crate::config::Config, - docker: Arc, + docker: Option>, ) -> Result, anyhow::Error> { let cfg = config.load(); let data_dir = cfg.tundra.data_directory.as_path(&cfg); @@ -134,8 +134,8 @@ impl TundraManager { } #[inline] - pub fn docker(&self) -> Arc { - Arc::clone(&self.docker) + pub fn docker(&self) -> Option> { + self.docker.as_ref().map(Arc::clone) } #[inline] @@ -203,10 +203,37 @@ impl TundraManager { self.control.lock().enriched.as_deref().cloned() } + fn supports_process_refs(metrics: &serde_json::Value) -> bool { + metrics + .get("capabilities") + .and_then(serde_json::Value::as_array) + .is_some_and(|capabilities| { + capabilities + .iter() + .any(|capability| capability.as_str() == Some("process_container_refs")) + }) + } + async fn rebuild(&self, state: &State) { let Some(cached) = self.cached() else { return; }; + + if state.config.active_runtime_backend() == crate::config::RuntimeBackend::PveLxc { + let supported = self + .hub + .request_metrics() + .await + .as_ref() + .is_ok_and(Self::supports_process_refs); + if !supported { + tracing::error!( + "refusing to publish Proxmox LXC references because the connected tundra-node does not advertise process_container_refs support" + ); + return; + } + } + let mut snapshot = Snapshot::clone(&cached); let cfg = state.config.load(); @@ -388,6 +415,19 @@ pub async fn run(state: State) { #[cfg(test)] mod tests { use super::*; + + #[test] + fn process_reference_capability_is_explicit() { + assert!(TundraManager::supports_process_refs(&serde_json::json!({ + "capabilities": ["process_container_refs"] + }))); + assert!(!TundraManager::supports_process_refs( + &serde_json::json!({}) + )); + assert!(!TundraManager::supports_process_refs(&serde_json::json!({ + "capabilities": ["some_future_capability"] + }))); + } use crate::remote::tundra::{Disabled, TunnelState}; fn manager(dir: &Path) -> TundraManager { @@ -395,7 +435,9 @@ mod tests { data_dir: dir.to_path_buf(), hub: hub::Hub::default(), ca: ca::LocalCa::load_or_create(dir).unwrap(), - docker: Arc::new(bollard::Docker::connect_with_local_defaults().unwrap()), + docker: Some(Arc::new( + bollard::Docker::connect_with_local_defaults().unwrap(), + )), token: parking_lot::RwLock::new(String::new()), control: parking_lot::Mutex::new(ControlState { cached: None, diff --git a/scripts/pve-lxc-smoke-test.sh b/scripts/pve-lxc-smoke-test.sh new file mode 100755 index 00000000..6455654e --- /dev/null +++ b/scripts/pve-lxc-smoke-test.sh @@ -0,0 +1,553 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="${IMAGE:-docker.io/nginxinc/nginx-unprivileged:alpine}" +TEMPLATE_STORAGE="${TEMPLATE_STORAGE:-local}" +ROOTFS_STORAGE="${ROOTFS_STORAGE:-local-lvm}" +BRIDGE="${BRIDGE:-vmbr0}" +ROOTFS_SIZE_GIB="${ROOTFS_SIZE_GIB:-2}" +MEMORY_MIB="${MEMORY_MIB:-256}" +SWAP_MIB="${SWAP_MIB:-256}" +HOST_DATA_UID="${HOST_DATA_UID:-1000}" +HOST_DATA_GID="${HOST_DATA_GID:-1000}" +KEEP_CT="${KEEP_CT:-0}" + +if [[ ${EUID} -ne 0 ]]; then + echo "error: run this smoke test as root on the Proxmox node" >&2 + exit 1 +fi + +for command in pveversion pvesh pct lxc-attach lxc-stop sha256sum perl tr awk stat; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "error: required command not found: $command" >&2 + exit 1 + fi +done + +node="$(pvesh get /cluster/status --output-format json | perl -MJSON::PP -0777 -e ' + my $rows = decode_json(); + for my $row (@$rows) { + next if ($row->{type} // q{}) ne q{node}; + next if !($row->{local} // 0); + print $row->{name} // q{}; + last; + } +')" +if [[ -z "$node" ]]; then + echo "error: Proxmox cluster status did not identify the local node" >&2 + exit 1 +fi +version="$(pveversion | sed -n 's#^pve-manager/\([0-9][0-9.]*\)/.*#\1#p')" +if [[ ! "$version" =~ ^([0-9]+)\.([0-9]+)(\.|$) ]]; then + echo "error: Proxmox VE 9.2 or newer is required; found ${version:-unknown}" >&2 + exit 1 +fi +major="${BASH_REMATCH[1]}" +minor="${BASH_REMATCH[2]}" +if [[ "$major" -lt 9 || ( "$major" -eq 9 && "$minor" -lt 2 ) ]]; then + echo "error: Proxmox VE 9.2 or newer is required; found ${version:-unknown}" >&2 + exit 1 +fi + +vmid="$(pvesh get /cluster/nextid)" +if [[ ! "$vmid" =~ ^[0-9]+$ || "$vmid" -le 0 ]]; then + echo "error: Proxmox returned an invalid next VMID: ${vmid:-empty}" >&2 + exit 1 +fi +# Mirror runtime.pve_lxc.min_vmid so the disposable CT does not squat a low +# ID reserved for infrastructure. Override with MIN_VMID=100 to test the floor. +min_vmid="${MIN_VMID:-200}" +if [[ ! "$min_vmid" =~ ^[0-9]+$ || "$min_vmid" -lt 100 ]]; then + echo "error: MIN_VMID must be a VMID at or above 100, got ${min_vmid:-empty}" >&2 + exit 1 +fi +if [[ "$vmid" -lt "$min_vmid" ]]; then + used_vmids="$(pvesh get /cluster/resources --type vm --output-format json | perl -ne 'while (/\"vmid\"\s*:\s*\"?(\d+)\"?/g) { print "$1\n" }' | sort -n -u)" + candidate="$min_vmid" + while printf '%s\n' "$used_vmids" | grep -qx "$candidate"; do + candidate=$((candidate + 1)) + done + vmid="$candidate" +fi +server_uuid="$(cat /proc/sys/kernel/random/uuid)" +run_tag="calagopus-smoke" +server_tag="calagopus-server-${server_uuid}" +normalized_image="$IMAGE" +last_component="${normalized_image##*/}" +if [[ "$last_component" != *:* ]]; then + normalized_image="${normalized_image}:latest" +fi +if [[ "$normalized_image" == *@* ]]; then + echo "error: digest image references are not supported by the current runtime" >&2 + exit 1 +fi + +image_digest="$(printf '%s' "$normalized_image" | sha256sum | awk '{print $1}')" +image_tag="calagopus-image-${image_digest:0:32}" +template_name="calagopus-oci-${image_digest}" +template_volid="${TEMPLATE_STORAGE}:vztmpl/${template_name}.tar" +data_dir="/var/lib/calagopus-pve-smoke/${server_uuid}" +managed_dir="/var/lib/calagopus-pve-smoke-managed/${server_uuid}" +managed_parent="${managed_dir%/*}" +managed_parent_created=0 +managed_parent_mode="" +created_ct=0 +config_file="$(mktemp /tmp/calagopus-smoke-config.XXXXXX)" + +cleanup() { + local status=0 + if [[ "$KEEP_CT" == "1" ]]; then + echo "KEEP_CT=1: leaving CT $vmid and $data_dir in place" + return + fi + + if pct config "$vmid" >"$config_file" 2>/dev/null; then + if grep -Eq "^tags: .*${run_tag}" "$config_file"; then + if pct status "$vmid" 2>/dev/null | grep -q running; then + pct stop "$vmid" >/dev/null 2>&1 || status=1 + fi + pct destroy "$vmid" --purge 1 >/dev/null 2>&1 || status=1 + else + echo "warning: refusing to clean CT $vmid because the smoke-test tag is missing" >&2 + status=1 + fi + fi + rm -f "$config_file" + rm -rf "$data_dir" + rm -rf "$managed_dir" + if [[ "$managed_parent_created" == "1" ]]; then + rmdir "$managed_parent" 2>/dev/null || true + elif [[ -n "$managed_parent_mode" && -d "$managed_parent" ]]; then + chmod "$managed_parent_mode" "$managed_parent" || status=1 + fi + if [[ "$status" -ne 0 ]]; then + echo "warning: smoke-test cleanup was incomplete" >&2 + fi +} +trap cleanup EXIT INT TERM + +json_field() { + local field="$1" + perl -MJSON::PP -0777 -e ' + my $field = shift @ARGV; + local $/; + my $json = decode_json(); + my $value = $json->{$field}; + print $value if defined $value; + ' "$field" +} + +ensure_template() { + if pvesh get "/nodes/${node}/storage/${TEMPLATE_STORAGE}/content" --content vztmpl --output-format json \ + | grep -Fq "\"volid\":\"${template_volid}\""; then + echo "template: reusing $template_volid" + return + fi + + echo "template: pulling $normalized_image" + local output upid task_json task_status exit_status + output="$(pvesh create "/nodes/${node}/storage/${TEMPLATE_STORAGE}/oci-registry-pull" \ + --reference "$normalized_image" \ + --filename "$template_name" \ + --output-format json)" + printf '%s\n' "$output" + + upid="$(printf '%s\n' "$output" \ + | sed -n -e 's/^"\(UPID:.*\)"$/\1/p' -e '/^UPID:/p' \ + | tail -n 1)" + if [[ -z "$upid" ]]; then + echo "error: OCI pull completed without a parseable UPID" >&2 + exit 1 + fi + + while true; do + task_json="$(pvesh get "/nodes/${node}/tasks/${upid}/status" --output-format json)" + task_status="$(printf '%s' "$task_json" | json_field status)" + if [[ "$task_status" == "stopped" ]]; then + exit_status="$(printf '%s' "$task_json" | json_field exitstatus)" + if [[ "$exit_status" != "OK" ]]; then + echo "error: OCI pull task failed: ${exit_status:-unknown}" >&2 + exit 1 + fi + break + fi + sleep 1 + done + + if ! pvesh get "/nodes/${node}/storage/${TEMPLATE_STORAGE}/content" --content vztmpl --output-format json \ + | grep -Fq "\"volid\":\"${template_volid}\""; then + echo "error: OCI pull task succeeded but $template_volid is absent" >&2 + exit 1 + fi +} + +create_ct() { + echo "ct: creating VMID $vmid" + pct create "$vmid" "$template_volid" \ + --hostname "calagopus-smoke-${vmid}" \ + --rootfs "${ROOTFS_STORAGE}:${ROOTFS_SIZE_GIB}" \ + --memory "$MEMORY_MIB" \ + --swap "$SWAP_MIB" \ + --net0 "name=eth0,bridge=${BRIDGE},ip=dhcp,type=veth,host-managed=1" \ + --unprivileged 1 \ + --cmode console \ + --onboot 0 \ + --cores 1 \ + --cpulimit 1 \ + --tags "calagopus;${run_tag};${server_tag};${image_tag}" + created_ct=1 +} + +configure_data_mount() { + local config uid gid + config="$(pct config "$vmid" | tr -d '\000')" + uid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.uid" {print $2; exit}')" + gid="$(printf '%s\n' "$config" | awk -F': *' '$1 == "lxc.init.gid" {print $2; exit}')" + if [[ ! "$uid" =~ ^[0-9]+$ || ! "$gid" =~ ^[0-9]+$ ]]; then + echo "error: OCI image did not resolve to numeric lxc.init.uid/gid values" >&2 + printf '%s\n' "$config" >&2 + exit 1 + fi + + echo "oci user: uid=$uid gid=$gid" + mkdir -p "$data_dir" + chown "$HOST_DATA_UID:$HOST_DATA_GID" "$data_dir" + chmod 0750 "$data_dir" + + pct set "$vmid" --mp0 \ + "${data_dir},mp=/home/container,backup=0,idmap=u:${uid}:${HOST_DATA_UID}:1;g:${gid}:${HOST_DATA_GID}:1" + + OCI_UID="$uid" + OCI_GID="$gid" +} + +wait_for_address() { + local attempt interfaces usable + for attempt in $(seq 1 30); do + interfaces="$(pvesh get "/nodes/${node}/lxc/${vmid}/interfaces" --output-format json 2>/dev/null || true)" + usable="$(printf '%s' "$interfaces" | perl -MJSON::PP -0777 -e ' + my $rows = eval { decode_json() } // []; + for my $row (@$rows) { + next if ($row->{name} // q{}) eq q{lo}; + for my $address (@{$row->{q{ip-addresses}} // []}) { + my $ip = $address->{q{ip-address}}; + next if !defined($ip); + next if $ip eq q{0.0.0.0} || $ip eq q{::} || $ip eq q{::1}; + next if $ip =~ /^127\./ || $ip =~ /^169\.254\./ || $ip =~ /^fe[89ab][0-9a-f]:/i; + print 1; + exit; + } + } + ')" + if [[ "$usable" == "1" ]]; then + printf '%s\n' "$interfaces" + return + fi + sleep 1 + done + echo "error: container did not report an interface address" >&2 + exit 1 +} + +verify_running_contracts() { + local runtime_json pid cgroup_file host_owner + runtime_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json)" + pid="$(printf '%s' "$runtime_json" | json_field pid)" + if [[ -z "$pid" || "$pid" == "0" || ! -d "/proc/$pid" ]]; then + echo "error: status/current did not expose a usable CT init PID" >&2 + printf '%s\n' "$runtime_json" >&2 + exit 1 + fi + echo "runtime status: $runtime_json" + echo "interfaces:" + wait_for_address + + lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'printf "calagopus-pve-smoke\n" > /home/container/smoke-marker && cat /home/container/smoke-marker' + + if [[ "$(cat "$data_dir/smoke-marker")" != "calagopus-pve-smoke" ]]; then + echo "error: host-backed /home/container marker mismatch" >&2 + exit 1 + fi + host_owner="$(stat -c '%u:%g' "$data_dir/smoke-marker")" + if [[ "$host_owner" != "${HOST_DATA_UID}:${HOST_DATA_GID}" ]]; then + echo "error: mount idmap produced host owner $host_owner, expected ${HOST_DATA_UID}:${HOST_DATA_GID}" >&2 + exit 1 + fi + echo "mount idmap: container ${OCI_UID}:${OCI_GID} -> host $host_owner" + + cgroup_file="/proc/${pid}/cgroup" + if [[ ! -r "$cgroup_file" ]]; then + echo "error: cannot read $cgroup_file" >&2 + exit 1 + fi + echo "cgroup: $(cat "$cgroup_file")" + + if [[ ! -r "/proc/${pid}/net/tcp" || ! -r "/proc/${pid}/net/tcp6" \ + || ! -r "/proc/${pid}/net/udp" || ! -r "/proc/${pid}/net/udp6" ]]; then + echo "error: CT network namespace proc files are unavailable through /proc/$pid/net" >&2 + exit 1 + fi + if ! awk 'NR > 1 && $4 == "0A" { found = 1 } END { exit !found }' \ + "/proc/${pid}/net/tcp" "/proc/${pid}/net/tcp6"; then + echo "error: no TCP listener was visible through the CT init PID network namespace" >&2 + exit 1 + fi + echo "proc net: tcp/tcp6/udp/udp6 are readable for init PID $pid" +} + +configure_runtime_contracts() { + local wrapper_path console_log managed_hosts + wrapper_path="$data_dir/runtime-wrapper.sh" + console_log="$data_dir/runtime-console.log" + managed_hosts="$managed_dir/hosts" + + if pct status "$vmid" | grep -q running; then + pct stop "$vmid" + fi + + cat >"$wrapper_path" <<'EOF' +#!/bin/sh +printf 'calagopus-init-wrapper %s\n' "${CALAGOPUS_SMOKE:-missing}" +exec nginx -g 'daemon off;' +EOF + chown "$HOST_DATA_UID:$HOST_DATA_GID" "$wrapper_path" + chmod 0755 "$wrapper_path" + + if [[ -d "$managed_parent" ]]; then + managed_parent_mode="$(stat -c '%a' "$managed_parent")" + else + mkdir -p "$managed_parent" + managed_parent_created=1 + fi + mkdir -p "$managed_dir" + chmod 0711 "$managed_parent" "$managed_dir" + cat >"$managed_hosts" <<'EOF' +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +# calagopus-managed-hosts +EOF + chmod 0644 "$managed_hosts" + rm -f "$console_log" + + perl -MPVE::LXC::Config - "$vmid" "$console_log" "$managed_hosts" <<'PERL' +use strict; +use warnings; + +my ($vmid, $console_log, $managed_hosts) = @ARGV; + +sub mount_target { + my ($entry) = @_; + return undef if !defined($entry); + my @fields = grep { length($_) } split(/\s+/, $entry); + return scalar(@fields) >= 2 ? $fields[1] : undef; +} + +PVE::LXC::Config->lock_config($vmid, sub { + my $conf = PVE::LXC::Config->load_config($vmid); + my @environment; + push @environment, split(/\0/, $conf->{env}, -1) + if defined($conf->{env}) && length($conf->{env}); + for my $entry (@{$conf->{lxc} // []}) { + next if ref($entry) ne 'ARRAY' || @$entry < 2; + push @environment, $entry->[1] + if $entry->[0] eq 'lxc.environment' || $entry->[0] eq 'lxc.environment.runtime'; + } + my %index; + for my $i (0 .. $#environment) { + my ($name) = split(/=/, $environment[$i], 2); + $index{$name} = $i if defined($name) && length($name); + } + my $smoke_environment = 'CALAGOPUS_SMOKE=runtime-config'; + if (exists($index{CALAGOPUS_SMOKE})) { + $environment[$index{CALAGOPUS_SMOKE}] = $smoke_environment; + } else { + push @environment, $smoke_environment; + } + + my @lxc = grep { + my $keep = ref($_) ne 'ARRAY' + || @$_ < 2 + || ($_->[0] ne 'lxc.environment' + && $_->[0] ne 'lxc.environment.runtime' + && $_->[0] ne 'lxc.signal.halt' + && $_->[0] ne 'lxc.init.cmd' + && $_->[0] ne 'lxc.console.logfile' + && $_->[0] ne 'lxc.cgroup2.memory.max' + && $_->[0] ne 'lxc.cgroup2.memory.swap.max'); + if ($keep && ref($_) eq 'ARRAY' && @$_ >= 2 && $_->[0] eq 'lxc.mount.entry') { + my $target = mount_target($_->[1]); + $keep = 0 if defined($target) && $target eq 'etc/hosts'; + } + $keep; + } @{$conf->{lxc} // []}; + + $conf->{env} = join("\0", @environment); + push @lxc, map { ['lxc.environment', $_] } @environment; + push @lxc, ['lxc.signal.halt', 'SIGTERM']; + push @lxc, ['lxc.init.cmd', '/bin/sh /home/container/runtime-wrapper.sh']; + push @lxc, ['lxc.console.logfile', $console_log]; + push @lxc, ['lxc.cgroup2.memory.max', 'max']; + push @lxc, ['lxc.cgroup2.memory.swap.max', 'max']; + push @lxc, [ + 'lxc.mount.entry', + "$managed_hosts etc/hosts none bind,ro,create=file 0 0", + ]; + $conf->{lxc} = \@lxc; + PVE::LXC::Config->write_config($vmid, $conf); +}); +PERL + + CONSOLE_LOG="$console_log" +} + +wait_for_stopped() { + local attempt + for attempt in $(seq 1 40); do + if pct status "$vmid" | grep -q stopped; then + return + fi + sleep 0.25 + done + echo "error: CT $vmid did not stop after lxc-stop request" >&2 + exit 1 +} + +wait_for_running_pid() { + local attempt status_json pid + for attempt in $(seq 1 40); do + status_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json 2>/dev/null || true)" + pid="$(printf '%s' "$status_json" | json_field pid 2>/dev/null || true)" + if [[ -n "$pid" ]] && pct status "$vmid" 2>/dev/null | grep -q running; then + return + fi + sleep 0.25 + done + echo "error: CT $vmid did not expose a stable running PID" >&2 + exit 1 +} + +start_ct() { + local attempt output + for attempt in $(seq 1 20); do + if output="$(pct start "$vmid" 2>&1)"; then + return + fi + if [[ "$output" != *"monitor socket"* && "$output" != *"got timeout"* \ + && "$output" != *"unable to get PID for CT"* ]]; then + printf '%s\n' "$output" >&2 + return 1 + fi + if pct status "$vmid" 2>/dev/null | grep -q running; then + return + fi + echo "start: retrying after transient monitor teardown (attempt $attempt)" >&2 + sleep 0.5 + done + printf '%s\n' "$output" >&2 + return 1 +} + +verify_runtime_contracts() { + local attempt runtime_json pid cgroup_path memory_max swap_max + configure_runtime_contracts + start_ct + + for attempt in $(seq 1 40); do + if [[ -f "$CONSOLE_LOG" ]] \ + && grep -Fq 'calagopus-init-wrapper runtime-config' "$CONSOLE_LOG"; then + break + fi + sleep 0.25 + done + if [[ ! -f "$CONSOLE_LOG" ]] \ + || ! grep -Fq 'calagopus-init-wrapper runtime-config' "$CONSOLE_LOG"; then + echo "error: lxc.init.cmd/environment output was not captured by lxc.console.logfile" >&2 + exit 1 + fi + echo "runtime config: init command, environment, and console logfile are active" + + if ! lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'grep -Fq "calagopus-managed-hosts" /etc/hosts'; then + echo "error: managed read-only /etc/hosts bind was not visible in the CT" >&2 + exit 1 + fi + if lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'printf "unexpected-write\n" >> /etc/hosts' >/dev/null 2>&1; then + echo "error: managed /etc/hosts bind unexpectedly allowed writes" >&2 + exit 1 + fi + echo "managed file bind: /etc/hosts is visible and read-only" + + runtime_json="$(pvesh get "/nodes/${node}/lxc/${vmid}/status/current" --output-format json)" + pid="$(printf '%s' "$runtime_json" | json_field pid)" + cgroup_path="$(awk -F: '$1 == "0" {print $3; exit}' "/proc/${pid}/cgroup")" + if [[ -z "$cgroup_path" ]]; then + echo "error: the live PVE smoke test requires a cgroup v2 host" >&2 + exit 1 + fi + memory_max="$(cat "/sys/fs/cgroup${cgroup_path}/memory.max")" + swap_max="$(cat "/sys/fs/cgroup${cgroup_path}/memory.swap.max")" + if [[ "$memory_max" != "max" || "$swap_max" != "max" ]]; then + echo "error: PVE did not preserve unlimited cgroup2 memory settings on start" >&2 + echo "memory.max=$memory_max memory.swap.max=$swap_max" >&2 + exit 1 + fi + if ! pct config "$vmid" | grep -Fq 'lxc.cgroup2.memory.max: max' \ + || ! pct config "$vmid" | grep -Fq 'lxc.cgroup2.memory.swap.max: max'; then + echo "error: PVE dropped unlimited cgroup2 memory settings from CT config" >&2 + exit 1 + fi + echo "cgroup2: PVE preserved memory.max=max and memory.swap.max=max after start" + + verify_running_contracts + + lxc-stop -n "$vmid" --nowait --nokill + wait_for_stopped + echo "shutdown: lxc-stop --nowait --nokill honored lxc.signal.halt" + + start_ct + wait_for_running_pid + pct stop "$vmid" + if ! pct status "$vmid" | grep -q stopped; then + echo "error: pct stop did not hard-stop CT $vmid" >&2 + exit 1 + fi + echo "shutdown: pct stop hard-stop path works" +} + +stop_and_destroy_ct() { + if pct status "$vmid" | grep -q running; then + if ! pct shutdown "$vmid" --timeout 10; then + pct stop "$vmid" + fi + fi + pct destroy "$vmid" --purge 1 + created_ct=0 +} + +echo "PVE $version node=$node vmid=$vmid image=$normalized_image" +ensure_template +create_ct +configure_data_mount +start_ct +verify_running_contracts +verify_runtime_contracts + +# Recreate the rootfs while keeping the host-backed data path. This exercises the +# persistence assumption used by staged image replacement. +echo "replacement: recreating CT rootfs while preserving $data_dir" +stop_and_destroy_ct +create_ct +configure_data_mount +start_ct +persisted_marker="$(lxc-attach -n "$vmid" --clear-env -u "$OCI_UID" -g "$OCI_GID" -- \ + /bin/sh -c 'cat /home/container/smoke-marker')" +if [[ "$persisted_marker" != "calagopus-pve-smoke" ]]; then + echo "error: persistent server data did not survive CT rootfs replacement" >&2 + exit 1 +fi +echo "replacement: persistent marker survived rootfs recreation" +verify_running_contracts + +echo "PASS: PVE LXC OCI lifecycle/runtime-config/idmap/procfs/replacement contracts"