From 7c82ce668c41d48d2e7adc9f56438647c56b7ce1 Mon Sep 17 00:00:00 2001 From: nashzhou Date: Thu, 15 Oct 2026 16:00:44 +0800 Subject: [PATCH] feat(sandbox): isolate each firecracker process in its own memcg Under concurrency, pausing or resuming one sandbox decompresses its memory image back into the page cache, and the reclaim and per-memcg writeback that follows churns the shared global lists: the flush_mapping teardown destroys the cache every co-located sandbox just refilled, so one sandbox's memory pressure is paid by all of them. Move each spawned Firecracker process into a cgroup v2 leaf under /sys/fs/cgroup/aenv so its anonymous memory and page cache are charged to a private memcg. The base is self-provisioned during normal server startup, so a root-privileged server (AENV_RUN_USER=root) gets the isolation with no extra step; a startup provisioning failure warns with the full error chain, so a read-only hierarchy or an unprivileged deployment is visible instead of silently skipping the feature. Production servers run as the non-root runtime account, where the privileged half belongs to `server --setup-host`: it creates the base, enables memory control down to it, and hands the base directory to the runtime group. Leaves are pooled and reused across processes instead of created and destroyed per spawn: a spawn moves the new pid into an idle leaf, a stop hands the leaf back, and startup reconciliation removes leaves whose processes are gone. Per-spawn leaf churn does not survive load - mkdir, rmdir, and cgroup.procs writes all serialize on the kernel's global cgroup_mutex, every rmdir queues memcg-offline work on a max_active=1 workqueue that runs under that same mutex, and at high pause/resume rates the resulting convoy stalled fc_spawn in seconds. A recycled leaf keeps the previous occupant's page-cache charges; those are dead data the kernel reclaims inside that memcg, which is the scoping the feature wants. A leaf is only returned to the pool once its process was reaped, so an emergency kill cannot hand a live process's leaf to the next spawn, and a leaf that still rejects the pid write is never recycled. Isolation is best-effort, like the adjacent oom_score_adj write: without a cgroup v2 hierarchy, or without the provisioned base, Firecracker runs ungrouped and the reason is logged once per server process with the setup-host hint instead of once per spawned process. --- docs/src/internals/sandbox-testing.md | 19 +- src/sandbox/firecracker/cgroup.rs | 262 ++++++++++++++++++++++++++ src/sandbox/firecracker/instance.rs | 12 ++ src/sandbox/firecracker/mod.rs | 1 + src/sandbox/mod.rs | 2 +- src/setup/cgroup.rs | 99 ++++++++++ src/setup/mod.rs | 17 +- 7 files changed, 402 insertions(+), 10 deletions(-) create mode 100644 src/sandbox/firecracker/cgroup.rs create mode 100644 src/setup/cgroup.rs diff --git a/docs/src/internals/sandbox-testing.md b/docs/src/internals/sandbox-testing.md index 5cab2b4..080c27f 100644 --- a/docs/src/internals/sandbox-testing.md +++ b/docs/src/internals/sandbox-testing.md @@ -517,13 +517,20 @@ resumed_sandbox.stop().await?; owns AgentENV state and receives ublk device access. Normal server startup performs validation only and never invokes `sudo`. 3. Host setup installs a udev rule for `/dev/ublk-control`, `/dev/ublkc*`, and `/dev/ublkb*`, so the runtime group can access the control and dynamic device nodes. The same rule sets the I/O scheduler of each `/dev/ublkb*` to `none`. -4. Update `config/default.toml` paths, or point `AENV_CONFIG_PATH` to a custom config file. -5. Ensure `/dev/kvm` is accessible by the runtime user and the configured +4. Host setup provisions `/sys/fs/cgroup/aenv` with the memory controller + enabled and hands the directory to the runtime group, so the server can + move each Firecracker process into its own memcg leaf. A root-privileged + server (for example `AENV_RUN_USER=root make start-server-release`) + provisions the base itself at startup; unprivileged accounts need + `--setup-host`. Without a provisioned base the server warns once and + Firecracker processes stay in the root memcg. +5. Update `config/default.toml` paths, or point `AENV_CONFIG_PATH` to a custom config file. +6. Ensure `/dev/kvm` is accessible by the runtime user and the configured virtualization mode matches the host modules. -6. If you run template tests, ensure the host can run `regctl` (server setup installs it automatically from the `[regclient]` manifest entry) and access the registry for template `fromImage` resolution. -7. Run `scripts/tests/e2e/run_e2e.sh` for API-level E2E coverage. The runner exports `E2E_TEMPLATE_USER_IMAGE`. Suite `05_template_lifecycle.sh` also creates a template build with `E2E_SHORT_USER_IMAGE` to verify short-name image resolution. -8. Run `make test-agent-integration` to run the `agentenv` integration test modules in `tests/integration/` as a non-root user with the required capabilities, plus the Docker/MinIO-backed OSS snapshot repository test (`crates/e2e-tests/tests/snapshot_oss_e2e_test.rs`). -9. Run `make test-ublk` to run the `uvm-ublk` and `overlaybd` storage tests, including the Docker/MinIO-backed OSS backend test (`storage/overlaybd/tests/oss_backend_minio.rs`). +7. If you run template tests, ensure the host can run `regctl` (server setup installs it automatically from the `[regclient]` manifest entry) and access the registry for template `fromImage` resolution. +8. Run `scripts/tests/e2e/run_e2e.sh` for API-level E2E coverage. The runner exports `E2E_TEMPLATE_USER_IMAGE`. Suite `05_template_lifecycle.sh` also creates a template build with `E2E_SHORT_USER_IMAGE` to verify short-name image resolution. +9. Run `make test-agent-integration` to run the `agentenv` integration test modules in `tests/integration/` as a non-root user with the required capabilities, plus the Docker/MinIO-backed OSS snapshot repository test (`crates/e2e-tests/tests/snapshot_oss_e2e_test.rs`). +10. Run `make test-ublk` to run the `uvm-ublk` and `overlaybd` storage tests, including the Docker/MinIO-backed OSS backend test (`storage/overlaybd/tests/oss_backend_minio.rs`). ## 4.1) Firecracker Client and Runtime Upgrade Checklist diff --git a/src/sandbox/firecracker/cgroup.rs b/src/sandbox/firecracker/cgroup.rs new file mode 100644 index 0000000..f256bf3 --- /dev/null +++ b/src/sandbox/firecracker/cgroup.rs @@ -0,0 +1,262 @@ +//! Per-instance cgroup v2 isolation for the Firecracker process, over a pool +//! of recycled leaves. +//! +//! Each spawned Firecracker process moves into a cgroup v2 leaf under +//! `/sys/fs/cgroup/aenv`, so its anonymous memory and page cache are charged +//! to a private memcg: reclaim and per-memcg writeback triggered by one +//! sandbox (a resume decompressing its memory image back into the page cache, +//! for example) stay scoped to that sandbox instead of churning the shared +//! global lists that every co-located sandbox depends on. +//! +//! Leaves are pooled and reused across processes: a spawn moves the new pid +//! into an idle leaf and a stop hands the leaf back, so the pause/resume path +//! never creates or removes a cgroup. Per-spawn leaf churn does not survive +//! load: mkdir, rmdir, and `cgroup.procs` writes all serialize on the +//! kernel's global `cgroup_mutex`, and every rmdir queues memcg-offline work +//! on a single-slot workqueue that runs under that same mutex, so at +//! pause/resume rates the spawn path stalls behind a convoy it built itself. +//! A recycled leaf keeps the previous occupant's page-cache charges; those +//! are dead data the kernel reclaims inside that memcg, which is exactly the +//! scoping this module exists for. Startup reconciles the base by removing +//! leaves whose processes are gone. +//! +//! The server runs as the non-root runtime account, so `server --setup-host` +//! provisions the base and hands it to the runtime group; this module only +//! creates leaves in it. + +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{LazyLock, Mutex}; + +use anyhow::Context; +use tracing::{debug, info, warn}; + +/// Mount point of the cgroup v2 unified hierarchy. +const CGROUP_V2_ROOT: &str = "/sys/fs/cgroup"; +/// Parent cgroup holding the leaf pool; provisioned by `server --setup-host`. +const BASE_DIR_NAME: &str = "aenv"; +/// Leaf prefix shared by current (`firecracker-slot-`) and legacy +/// (`firecracker-`) leaves; reconciliation only touches directories +/// carrying it. +const LEAF_PREFIX: &str = "firecracker-"; +/// The controller the isolation exists for; unless `server --setup-host` has +/// enabled it down to the base, leaves cannot charge anything. +pub(crate) const MEMORY_CONTROLLER: &str = "memory"; + +/// Set after the first isolation failure so a mis-provisioned host warns once +/// instead of once per spawned process. +static UNAVAILABLE_WARNED: AtomicBool = AtomicBool::new(false); +/// Slot index for the next freshly created leaf. Monotonic per process; a +/// name collision with a leaf an older run left occupied just bumps it. +static NEXT_SLOT: AtomicU64 = AtomicU64::new(0); +/// Leaves whose process was reaped, ready for the next spawn. +static IDLE_LEAVES: LazyLock>> = LazyLock::new(|| Mutex::new(Vec::new())); + +/// The cgroup v2 unified hierarchy, or `None` on hosts without one. +pub(crate) fn cgroup_v2_root() -> Option<&'static Path> { + let root = Path::new(CGROUP_V2_ROOT); + root.join("cgroup.controllers").is_file().then_some(root) +} + +/// The base cgroup under `root` that holds the leaf pool. +pub(crate) fn base_dir(root: &Path) -> PathBuf { + root.join(BASE_DIR_NAME) +} + +fn is_leaf_name(name: &str) -> bool { + name.starts_with(LEAF_PREFIX) +} + +/// Removes leaves left behind by a previous run whose processes are gone: the +/// kernel empties a leaf when its process dies but never removes the +/// directory, and the pool only exists in the old process's memory. Leaves +/// that still hold a live process are left alone. Called once at server +/// startup, before any spawn can race it. +pub(crate) fn remove_unoccupied_leaves() { + let Some(root) = cgroup_v2_root() else { + return; + }; + let base = base_dir(root); + let Ok(entries) = fs::read_dir(&base) else { + return; + }; + let mut removed = 0; + for entry in entries.flatten() { + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let Some(name) = entry.file_name().into_string().ok() else { + continue; + }; + if !is_leaf_name(&name) { + continue; + } + let occupied = fs::read_to_string(entry.path().join("cgroup.procs")) + .map(|procs| !procs.trim().is_empty()) + .unwrap_or(true); + if occupied { + continue; + } + if let Err(err) = fs::remove_dir(entry.path()) { + if err.kind() != ErrorKind::NotFound { + warn!( + path = %entry.path().display(), + err = %err, + "failed to remove an unoccupied firecracker cgroup leaf" + ); + } + } else { + removed += 1; + } + } + if removed > 0 { + info!( + removed, + base = %base.display(), + "removed firecracker cgroup leaves left over from a previous run" + ); + } +} + +/// The leaf cgroup of one Firecracker process. Dropping the value returns the +/// leaf to the pool for the next spawn. +pub(crate) struct FirecrackerCgroup { + leaf: PathBuf, +} + +impl FirecrackerCgroup { + /// Moves the freshly spawned process into a pooled leaf. + /// + /// Isolation stays best-effort like the adjacent `oom_score_adj` write: + /// without a provisioned base the process runs ungrouped and the reason + /// is logged once, not once per spawn. + pub(crate) fn attach(pid: u32) -> Option { + let Some(root) = cgroup_v2_root() else { + debug!( + pid, + "no cgroup v2 hierarchy; firecracker runs without a per-instance cgroup" + ); + return None; + }; + match Self::populate_leaf(&base_dir(root), pid) { + Ok(leaf) => Some(Self { leaf }), + Err(err) => { + let message = format!( + "failed to move firecracker into a pooled cgroup leaf: {err:#}; \ + run `server --setup-host` as root to provision it" + ); + if !UNAVAILABLE_WARNED.swap(true, Ordering::Relaxed) { + warn!(pid, "{message}"); + } else { + debug!(pid, "{message}"); + } + None + } + } + } + + /// Pops an idle leaf and moves the process in; a leaf removed under us + /// (a concurrent server reconciling the shared base, say) falls back to a + /// freshly created one. A leaf that rejected the write is never recycled. + fn populate_leaf(base: &Path, pid: u32) -> anyhow::Result { + if let Some(leaf) = Self::take_idle_leaf() { + // posix_spawn has returned, so exec has been initiated: the move + // can still race firecracker's earliest startup, but its large + // allocations (guest memory) only happen later, on the snapshot + // load path, and the whole thread group moves either way. + if Self::move_process_into(&leaf, pid).is_ok() { + return Ok(leaf); + } + let _ = fs::remove_dir(&leaf); + debug!( + pid, + leaf = %leaf.display(), + "pooled cgroup leaf vanished; creating a fresh one" + ); + } + let leaf = Self::create_leaf(base)?; + Self::move_process_into(&leaf, pid) + .with_context(|| format!("move the process into {}", leaf.display()))?; + Ok(leaf) + } + + fn take_idle_leaf() -> Option { + IDLE_LEAVES.lock().unwrap().pop() + } + + fn move_process_into(leaf: &Path, pid: u32) -> std::io::Result<()> { + fs::write(leaf.join("cgroup.procs"), pid.to_string()) + } + + fn create_leaf(base: &Path) -> anyhow::Result { + let controllers_path = base.join("cgroup.controllers"); + let controllers = fs::read_to_string(&controllers_path) + .with_context(|| format!("read {}", controllers_path.display()))?; + anyhow::ensure!( + controllers + .split_whitespace() + .any(|c| c == MEMORY_CONTROLLER), + "memory controller is not enabled on {}; provision with `server --setup-host`", + base.display() + ); + loop { + let slot = NEXT_SLOT.fetch_add(1, Ordering::Relaxed); + let leaf = base.join(format!("{LEAF_PREFIX}slot-{slot}")); + match fs::create_dir(&leaf) { + Ok(()) => return Ok(leaf), + // An older run's leaf that reconciliation could not remove + // (its orphan process is still alive) keeps the name. + Err(err) if err.kind() == ErrorKind::AlreadyExists => {} + Err(err) => { + return Err(err) + .with_context(|| format!("create cgroup leaf {}", leaf.display())) + } + } + } + } +} + +impl Drop for FirecrackerCgroup { + fn drop(&mut self) { + // Recycle only a leaf the kernel has already emptied: the emergency + // drop path kills the child without reaping it, and handing a live + // process's leaf to the next spawn would pool two occupants into one + // memcg. An occupied leaf stays on disk for the next startup + // reconciliation, once its process is gone. + match fs::read_to_string(self.leaf.join("cgroup.procs")) { + Ok(procs) if procs.trim().is_empty() => { + IDLE_LEAVES + .lock() + .unwrap() + .push(std::mem::take(&mut self.leaf)); + } + Ok(_) => warn!( + path = %self.leaf.display(), + "firecracker cgroup leaf still occupied at drop; left for startup reconciliation" + ), + // Removed under us; nothing to recycle. + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => warn!( + path = %self.leaf.display(), + err = %err, + "could not read firecracker cgroup leaf occupancy; left for startup reconciliation" + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leaf_name_matches_current_and_legacy_leaves() { + assert!(is_leaf_name("firecracker-slot-0")); + assert!(is_leaf_name("firecracker-slot-42")); + assert!(is_leaf_name("firecracker-1234")); + assert!(!is_leaf_name("firecracker")); + assert!(!is_leaf_name("init-1")); + } +} diff --git a/src/sandbox/firecracker/instance.rs b/src/sandbox/firecracker/instance.rs index d7fdbcd..4c94c08 100644 --- a/src/sandbox/firecracker/instance.rs +++ b/src/sandbox/firecracker/instance.rs @@ -22,6 +22,7 @@ use serde_json::value::RawValue; use tokio::time::{self, Duration, Instant}; use tracing::{debug, trace, warn}; +use super::cgroup::FirecrackerCgroup; use super::mmds::MmdsMetadata; use super::socket::UnixSocketClient; use crate::privileges::{ScopedChild, ScopedSpawnSpec}; @@ -81,6 +82,9 @@ pub(crate) struct FirecrackerInstance { socket_path: PathBuf, stderr_path: Option, process: Option, + /// Set when the process moved into its own cgroup; dropping it returns + /// the leaf to the pool for the next spawn. + cgroup: Option, } impl FirecrackerInstance { @@ -92,6 +96,7 @@ impl FirecrackerInstance { socket_path, stderr_path: None, process: None, + cgroup: None, } } @@ -194,6 +199,11 @@ impl FirecrackerInstance { warn!(pid, oom_path, err = %e, "failed to set oom_score_adj for firecracker"); } + // A private cgroup charges the process's memory (guest RAM, page + // cache) to this sandbox alone, keeping the reclaim and writeback + // churn of pausing/resuming one sandbox off every co-located sandbox. + self.cgroup = FirecrackerCgroup::attach(pid); + self.process = Some(child); Ok(()) } @@ -285,6 +295,7 @@ impl FirecrackerInstance { let _ = child.wait().await; } } + self.cgroup = None; if self.socket_path.exists() { let _ = fs::remove_file(&self.socket_path); @@ -622,6 +633,7 @@ impl Drop for FirecrackerInstance { warn!(error = %err, "failed to kill sandbox process on drop"); } } + self.cgroup = None; // Only remove the socket if we actually spawned/owned the process. if had_child { diff --git a/src/sandbox/firecracker/mod.rs b/src/sandbox/firecracker/mod.rs index b12ee4d..5f2535e 100644 --- a/src/sandbox/firecracker/mod.rs +++ b/src/sandbox/firecracker/mod.rs @@ -4,6 +4,7 @@ //! [`FirecrackerSandboxFactory`] which wires sandbox configuration from the //! global [`ConfigManager`][crate::cfg::ConfigManager]. +pub(crate) mod cgroup; mod config; mod connector; mod factory; diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 0802815..d815109 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -3,7 +3,7 @@ mod backend; pub(crate) mod custom_extension; mod envd; mod extra_drive; -mod firecracker; +pub(crate) mod firecracker; pub mod manifest; mod metrics; #[cfg(test)] diff --git a/src/setup/cgroup.rs b/src/setup/cgroup.rs new file mode 100644 index 0000000..d5af4c2 --- /dev/null +++ b/src/setup/cgroup.rs @@ -0,0 +1,99 @@ +//! Host provisioning for the per-Firecracker cgroup base. +//! +//! Creating the cgroup base and enabling the memory controller on it are +//! privileged operations. A root-privileged server does both by itself at +//! startup ([`ensure_runtime_base`]); a server running as the non-root +//! runtime account relies on `--setup-host` ([`provision`]) to do them and +//! hand the base directory to the runtime group, so the server only ever +//! creates and removes its own leaves in it. + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use nix::unistd::{chown, Gid}; +use tracing::{info, warn}; + +use crate::sandbox::firecracker::cgroup::{ + base_dir, cgroup_v2_root, remove_unoccupied_leaves, MEMORY_CONTROLLER, +}; + +/// Creates the per-Firecracker cgroup base with the memory controller enabled +/// and hands it to the runtime group. A no-op on hosts without cgroup v2. +pub(super) fn provision(runtime_gid: Gid) -> Result<()> { + let Some(root) = cgroup_v2_root() else { + info!("no cgroup v2 hierarchy; per-firecracker memcg isolation is unavailable"); + return Ok(()); + }; + let base = prepare_base(root)?; + // Leaves are created by the server itself and stay owned by it; group + // write on the base is all the account needs. + chown(&base, None, Some(runtime_gid)) + .with_context(|| format!("set runtime group ownership on {}", base.display()))?; + fs::set_permissions(&base, fs::Permissions::from_mode(0o775)) + .with_context(|| format!("set runtime group access on {}", base.display()))?; + Ok(()) +} + +/// Self-provisions the base during normal server startup, so privileged +/// servers get the isolation without running `--setup-host` first. +/// Best-effort: an unprivileged server that cannot create the base stays +/// quiet here, and the per-spawn attach warns once with the setup-host hint. +pub(super) fn ensure_runtime_base() { + let Some(root) = cgroup_v2_root() else { + return; + }; + match prepare_base(root) { + Ok(base) => { + // The leaf pool lives in the previous process's memory; reclaim + // the directories it left behind before spawns begin. + remove_unoccupied_leaves(); + info!( + path = %base.display(), + "firecracker cgroup base is ready" + ) + } + Err(err) => warn!( + "could not provision the firecracker cgroup base at startup ({err:#}); \ + per-sandbox memcg isolation stays off until the server starts \ + privileged or `server --setup-host` runs as root" + ), + } +} + +/// Creates the base and enables the memory controller down to it. +fn prepare_base(root: &Path) -> Result { + let base = base_dir(root); + fs::create_dir_all(&base) + .with_context(|| format!("create firecracker cgroup base {}", base.display()))?; + enable_memory_controller(root, &base)?; + Ok(base) +} + +/// Enables the memory controller in the root and base subtree controls so +/// leaves under the base charge memory. The root is exempt from the +/// no-internal-process rule and the base only ever holds leaves, so both +/// writes are legal. +fn enable_memory_controller(root: &Path, base: &Path) -> Result<()> { + for dir in [root, base] { + let controllers_path = dir.join("cgroup.controllers"); + let available = fs::read_to_string(&controllers_path) + .with_context(|| format!("read {}", controllers_path.display()))?; + anyhow::ensure!( + available.split_whitespace().any(|c| c == MEMORY_CONTROLLER), + "memory controller is not available in {}", + dir.display() + ); + let subtree = dir.join("cgroup.subtree_control"); + let enabled = fs::read_to_string(&subtree) + .with_context(|| format!("read {}", subtree.display()))? + .split_whitespace() + .any(|c| c == MEMORY_CONTROLLER); + if !enabled { + fs::write(&subtree, format!("+{MEMORY_CONTROLLER}")) + .with_context(|| format!("enable memory control in {}", subtree.display()))?; + } + } + Ok(()) +} diff --git a/src/setup/mod.rs b/src/setup/mod.rs index e000212..3a88086 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -1,3 +1,4 @@ +mod cgroup; mod deps; mod hotplug; mod kvm; @@ -56,6 +57,12 @@ pub fn ensure_host(config: &AppConfig, runtime_user: &str, runtime_group: &str) ); } ublk::provision(runtime_group)?; + if let Err(err) = cgroup::provision(runtime_gid) { + warn!( + error = %err, + "per-firecracker memcg isolation is unavailable; firecracker processes stay in the root memcg" + ); + } overlaybd::install_system_default_config(&config.deps_path, runtime_gid)?; network_capacity::install_persistent_config().context("install /etc/sysctl.d/99-aenv.conf")?; fs::write("/proc/sys/net/ipv4/ip_forward", "1\n").context("enable host IPv4 forwarding")?; @@ -135,18 +142,22 @@ pub async fn ensure_environment( info!("checking ublk module and permissions"); ublk::check()?; - // 4. Download dependencies and generate overlaybd runtime configs. + // 4. Per-firecracker memcg base: privileged servers self-provision here; + // unprivileged ones rely on `server --setup-host` having run. + cgroup::ensure_runtime_base(); + + // 5. Download dependencies and generate overlaybd runtime configs. ensure_dependencies(config).await?; deps::write_generated_overlaybd_global_configs(config, p2p_facade_address)?; - // 5. set nofile limit to a relative large value + // 6. set nofile limit to a relative large value let (soft_limit, hard_limit) = getrlimit(Resource::RLIMIT_NOFILE).context("get rlimit")?; if soft_limit < 65536 { setrlimit(Resource::RLIMIT_NOFILE, 65536.min(hard_limit), hard_limit) .context("set rlimit")?; } - // 6. Host networking is provisioned separately and only validated here. + // 7. Host networking is provisioned separately and only validated here. let ip_forward = fs::read_to_string("/proc/sys/net/ipv4/ip_forward") .context("read host IPv4 forwarding setting")?; if ip_forward.trim() != "1" { -- 2.43.0