From 4045349c113c388dc091a9eed6bc25c8e019fc9a Mon Sep 17 00:00:00 2001
From: nashzhou <nashzzhou@gmail.com>
Date: Thu, 15 Oct 2026 09:44:49 +0800
Subject: [PATCH] perf(sandbox): cap firecracker nofile via prlimit launcher

Firecracker pre-grows its kernel fd table to the soft RLIMIT_NOFILE at
startup (resize_fdtable dups stdin onto limit - 1). Under the server
unit's LimitNOFILE=1048576, every firecracker process start allocates a
1M-entry table: an 8 MiB kvmalloc->vmalloc plus 256 KiB of bitmaps,
which stalls in direct reclaim under the sweep's memory pressure and
dominated the kernel side of the pause/resume profile (~32% in
resize_fdtable).

glibc's posix_spawn has no rlimit attribute, and a pre_exec hook would
force a fork+exec of the server, so route the spawn through prlimit(1):
it applies the capped limit and execs firecracker in place, keeping the
spawned pid, process group, netns, and descriptor handoff unchanged.
2048 is the jailer default and far above a microVM build's descriptor
count. check_runtime now validates prlimit via the runtime-commands
manifest, and the image installs util-linux explicitly.
---
 config/deps_manifest.toml           |  3 +++
 deploy/docker/Dockerfile.agentenv   |  1 +
 src/sandbox/firecracker/instance.rs | 18 ++++++++++++++++++
 3 files changed, 22 insertions(+)

diff --git a/config/deps_manifest.toml b/config/deps_manifest.toml
index e016834..614bef1 100644
--- a/config/deps_manifest.toml
+++ b/config/deps_manifest.toml
@@ -56,6 +56,9 @@ curl = { default = "curl" }
 "iptables-save" = { default = "iptables" }
 jq = { default = "jq" }
 "mkfs.ext4" = { default = "e2fsprogs" }
+# Launcher for the Firecracker process; caps the child's RLIMIT_NOFILE so
+# Firecracker's startup fd-table pre-growth stays small (see FIRECRACKER_NOFILE_LIMIT).
+prlimit = { default = "util-linux" }
 "resize2fs" = { default = "e2fsprogs" }
 sudo = { default = "sudo" }
 umoci = { default = "umoci" }
diff --git a/deploy/docker/Dockerfile.agentenv b/deploy/docker/Dockerfile.agentenv
index 1944fce..fc3651a 100644
--- a/deploy/docker/Dockerfile.agentenv
+++ b/deploy/docker/Dockerfile.agentenv
@@ -70,6 +70,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
         jq \
         sudo \
         umoci \
+        util-linux \
         zstd \
     && if [ -n "${EXTRA_RUNTIME_PACKAGES}" ]; then \
         printf '%s\n' "${EXTRA_RUNTIME_PACKAGES}" \
diff --git a/src/sandbox/firecracker/instance.rs b/src/sandbox/firecracker/instance.rs
index d7fdbcd..3907b54 100644
--- a/src/sandbox/firecracker/instance.rs
+++ b/src/sandbox/firecracker/instance.rs
@@ -42,6 +42,12 @@ pub(crate) const SANDBOX_TAP_IFACE_NAME: &str = "tap0";
 /// it is told about, so replacing an inherited one here is safe.
 const NET_TAP_FD: RawFd = 3;
 
+/// `RLIMIT_NOFILE` (soft and hard) for the spawned Firecracker process:
+/// `resize_fdtable` pre-grows the kernel fd table to the soft limit, and 2048
+/// is the jailer default — far above what a microVM build opens, but small
+/// enough that the table costs pages instead of megabytes.
+const FIRECRACKER_NOFILE_LIMIT: u64 = 2048;
+
 /// Hands Firecracker the sandbox TAP queue as a pre-opened, pre-configured
 /// descriptor (an `fdp:` `host_dev_name` spec) instead of letting it open the
 /// interface by name inside the sandbox network namespace.
@@ -161,8 +167,20 @@ impl FirecrackerInstance {
         let stderr = open_log_stdio(stderr_path)?;
         self.stderr_path = stderr_path.map(Path::to_path_buf);
 
+        // Firecracker pre-grows its kernel fd table to the soft RLIMIT_NOFILE at
+        // startup (`resize_fdtable` dups stdin onto `limit - 1`), so with the
+        // server unit's `LimitNOFILE=1048576` every process start pays a
+        // multi-megabyte fd-table allocation that stalls in memory reclaim.
+        // glibc's posix_spawn has no rlimit attribute, and a pre_exec hook would
+        // force a fork+exec of the server; prlimit applies the capped limit and
+        // execs Firecracker in place, keeping the pid, process group, netns,
+        // and descriptor handoff unchanged.
         let spec = ScopedSpawnSpec {
             argv: vec![
+                CString::from(c"/usr/bin/prlimit"),
+                CString::new(format!("--nofile={FIRECRACKER_NOFILE_LIMIT}"))
+                    .expect("numeric literal is NUL-free"),
+                CString::from(c"--"),
                 path_to_cstring(&firecracker_binary)?,
                 CString::from(c"--api-sock"),
                 path_to_cstring(&self.socket_path)?,
-- 
2.43.0