markmental/tuxdock
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Locked down tuxreaperd code so it can gracefully stop Apache and any other descendants. Signal bitmask instead of a single pending signal, so rapid consecutive signals are not lost.

Browse source
This commit is contained in:
mrkmntal 2026-08-29 12:06:41 -04:00
commit 1d7eca9016
4 changed files with 389 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

12
compile.sh
View file

@ -6,6 +6,7 @@ run_tests=1
web_view=0
web_port=8095
force_libc_reaper=0
debug_reaper=0
memory_limit=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || true)
if [ -z "$memory_limit" ] || [ "$memory_limit" = max ]; then
@ -46,8 +47,9 @@ while [ "$#" -gt 0 ]; do
fi
;;
--force-libc-reaper) force_libc_reaper=1 ;;
--debug-reaper) debug_reaper=1 ;;
*)
printf '%s\n' "Usage: $0 [--no-test] [--web-test-view [port]] [--force-libc-reaper]" >&2
printf '%s\n' "Usage: $0 [--no-test] [--web-test-view [port]] [--force-libc-reaper] [--debug-reaper]" >&2
exit 2
;;
esac
@ -84,6 +86,10 @@ command -v gcc >/dev/null 2>&1 || {
host_arch=$(uname -m)
asm_archs='x86_64 amd64 aarch64 arm64'
page_size=$(getconf PAGE_SIZE 2>/dev/null || printf '%s\n' 4096)
reaper_cflags=""
if [ "$debug_reaper" -eq 1 ]; then
reaper_cflags="-DTUXREAPERD_DEBUG"
fi
build_tuxreaperd() {
case " $asm_archs " in
*" $host_arch "*)
@ -92,7 +98,7 @@ build_tuxreaperd() {
if gcc -nostdlib -static -fno-stack-protector -fno-asynchronous-unwind-tables \
-fno-builtin -O2 -s -Wl,--build-id=none -Wl,-z,noseparate-code \
-Wl,-z,max-page-size=${page_size} \
-Wall -Wextra tuxreaperdasm.c -o build/tuxreaperd 2>/tmp/tuxreaperd-asm-build.log; then
-Wall -Wextra $reaper_cflags tuxreaperdasm.c -o build/tuxreaperd 2>/tmp/tuxreaperd-asm-build.log; then
printf '%s\n' "[compile] Freestanding tuxreaperd built successfully"
return 0
fi
@ -105,7 +111,7 @@ build_tuxreaperd() {
esac
printf '%s\n' "[compile] Building standard tuxreaperd for $host_arch"
gcc -O2 -static -Wall -Wextra tuxreaperdgnu.c -o build/tuxreaperd
gcc -O2 -static -Wall -Wextra $reaper_cflags tuxreaperdgnu.c -o build/tuxreaperd
}
build_tuxreaperd

13
tests/test_tuxreaperd.sh
View file

@ -147,6 +147,19 @@ else
failures=$((failures + 1))
fi
printf '%s\n' "[TUI] Verify tuxreaperd waits for descendants after main child exits"
descendant_start=$(date +%s)
run_raw "docker run tuxreaperd descendant-wait" \
docker run --rm -v "$reaper_bin:/usr/local/bin/tuxreaperd:ro" \
"$image" /usr/local/bin/tuxreaperd sh -c 'sleep 3 & exit 0' || true
descendant_elapsed=$(( $(date +%s) - descendant_start ))
if [ "$descendant_elapsed" -ge 3 ] && [ "$descendant_elapsed" -le 10 ]; then
printf '%s\n' "[PASS] tuxreaperd waited for descendant to finish (elapsed ${descendant_elapsed}s)"
else
printf '%s\n' "[FAIL] Expected ~3s wait, got ${descendant_elapsed}s"
failures=$((failures + 1))
fi
trap - EXIT INT TERM
if [ "$failures" -gt 0 ]; then
printf '%s\n' "tuxreaperd tests failed: $failures failure(s)" >&2

268
tuxreaperdasm.c
View file

@ -1,3 +1,8 @@
struct timespec {
long tv_sec;
long tv_nsec;
};
struct k_sigaction {
void (*handler)(int);
unsigned long flags;
@ -183,7 +188,30 @@ static inline long sys_close(int fd) {
__asm__ volatile (
"syscall"
: "=a"(ret)
: "a"(3), "D"((long)fd)
: "a"(3), "D"(fd)
: "rcx", "r11", "memory"
);
return ret;
}
static inline long sys_clock_gettime(int clk_id, struct timespec *tp) {
long ret;
register long r10 __asm__("r10") = 0;
__asm__ volatile (
"syscall"
: "=a"(ret)
: "a"(228), "D"((long)clk_id), "S"(tp), "r"(r10)
: "rcx", "r11", "memory"
);
return ret;
}
static inline long sys_nanosleep(const struct timespec *req, struct timespec *rem) {
long ret;
__asm__ volatile (
"syscall"
: "=a"(ret)
: "a"(35), "D"(req), "S"(rem)
: "rcx", "r11", "memory"
);
return ret;
@ -433,6 +461,32 @@ static inline long sys_close(int fd) {
return x0;
}
static inline long sys_clock_gettime(int clk_id, struct timespec *tp) {
register long x0 __asm__("x0") = (long)clk_id;
register long x1 __asm__("x1") = (long)tp;
register long x8 __asm__("x8") = 113; /* __NR_clock_gettime */
__asm__ volatile (
"svc #0"
: "+r"(x0)
: "r"(x1), "r"(x8)
: "memory", "cc"
);
return x0;
}
static inline long sys_nanosleep(const struct timespec *req, struct timespec *rem) {
register long x0 __asm__("x0") = (long)req;
register long x1 __asm__("x1") = (long)rem;
register long x8 __asm__("x8") = 101; /* __NR_nanosleep */
__asm__ volatile (
"svc #0"
: "+r"(x0)
: "r"(x1), "r"(x8)
: "memory", "cc"
);
return x0;
}
__asm__(
".text\n"
".globl _start\n"
@ -524,10 +578,9 @@ static void my_execvp(const char *file, char *const argv[], char *const envp[])
}
}
/* Apache hijacks SIGWINCH (window resize) for graceful shutdown. Yes,
really. So when the outside world sends SIGTERM, we have to translate
it to SIGWINCH for anything whose /proc/<pid>/exe smells like apache2.
This is insane, but Apache is a good web server so I give it a pass. */
/* Apache hijacks SIGWINCH for graceful shutdown. When the outside world
sends SIGTERM, we translate it to SIGWINCH for Apache processes and send
the original signal to everyone else. */
static const char * const apache_exes[] = {
"/usr/sbin/apache2",
"/usr/sbin/httpd",
@ -537,6 +590,9 @@ static const char * const apache_exes[] = {
#define APACHE_IN_SIG 15 /* SIGTERM */
#define APACHE_OUT_SIG 28 /* SIGWINCH */
#define DESCENDANT_TIMEOUT_SECONDS 60
#define BROADCAST_SCAN_DELAY_US 100000
#define AT_FDCWD -100
#define O_RDONLY 0
#define O_DIRECTORY 00200000
@ -549,6 +605,42 @@ struct linux_dirent64 {
char d_name[];
};
#ifdef TUXREAPERD_DEBUG
static void debug_write(const char *s) {
sys_write(2, s, my_strlen(s));
}
static void debug_write_int(long n) {
char buf[32];
int i = 0;
if (n < 0) {
buf[i++] = '-';
n = -n;
}
int start = i;
do {
buf[i++] = '0' + (n % 10);
n /= 10;
} while (n > 0);
for (int j = start, k = i - 1; j < k; j++, k--) {
char tmp = buf[j];
buf[j] = buf[k];
buf[k] = tmp;
}
sys_write(2, buf, i);
}
static void debug_msg(const char *msg) {
debug_write("[tuxreaperd] ");
debug_write(msg);
debug_write("\n");
}
#else
static void debug_write(const char *s) { (void)s; }
static void debug_write_int(long n) { (void)n; }
static void debug_msg(const char *msg) { (void)msg; }
#endif
static int my_strcmp(const char *a, const char *b) {
while (*a && (*a == *b)) {
a++;
@ -572,11 +664,14 @@ static long parse_pid(const char *s) {
}
/* Iterate /proc, read each process's /proc/<pid>/exe symlink, and send
out_sig to matching process groups while sending in_sig to everyone
else. Skip PID 1 (the reaper itself). */
out_sig to matching PIDs while sending in_sig to everyone else.
Skip PID 1 (the reaper itself). */
static void proc_remap_signal(int in_sig, int out_sig, const char *const *target_exes) {
int procfd = (int)sys_openat(AT_FDCWD, "/proc", O_RDONLY | O_DIRECTORY, 0);
if (procfd < 0) return;
if (procfd < 0) {
debug_msg("opendir(/proc) failed");
return;
}
char dirbuf[4096];
@ -618,26 +713,88 @@ static void proc_remap_signal(int in_sig, int out_sig, const char *const *target
}
}
if (matched) {
sys_kill(pid, out_sig);
} else {
sys_kill(pid, in_sig);
}
int sent_sig = matched ? out_sig : in_sig;
#ifdef TUXREAPERD_DEBUG
debug_write("[tuxreaperd] scan pid=");
debug_write_int(pid);
debug_write(" exe=");
debug_write(linkbuf);
debug_write(" matched=");
debug_write_int(matched);
debug_write(" sig=");
debug_write_int(sent_sig);
debug_write("\n");
#endif
sys_kill(pid, sent_sig);
}
}
sys_close(procfd);
}
static void sleep_us(long us) {
struct timespec req;
req.tv_sec = us / 1000000;
req.tv_nsec = (us % 1000000) * 1000;
sys_nanosleep(&req, 0);
}
static void broadcast_signal(int in_sig, int out_sig, const char *const *target_exes) {
#ifdef TUXREAPERD_DEBUG
debug_write("[tuxreaperd] broadcasting in_sig=");
debug_write_int(in_sig);
debug_write(" out_sig=");
debug_write_int(out_sig);
debug_write("\n");
#endif
proc_remap_signal(in_sig, out_sig, target_exes);
sleep_us(BROADCAST_SCAN_DELAY_US);
proc_remap_signal(in_sig, out_sig, target_exes);
}
static int count_descendants(void) {
int procfd = (int)sys_openat(AT_FDCWD, "/proc", O_RDONLY | O_DIRECTORY, 0);
if (procfd < 0) return 0;
int count = 0;
char dirbuf[4096];
for (;;) {
long n = sys_getdents64(procfd, dirbuf, sizeof(dirbuf));
if (n <= 0) break;
for (long pos = 0; pos < n;) {
struct linux_dirent64 *de = (struct linux_dirent64 *)(dirbuf + pos);
pos += de->d_reclen;
const char *name = de->d_name;
if (name[0] == '.') continue;
long pid = parse_pid(name);
if (pid <= 1) continue;
count++;
}
}
sys_close(procfd);
return count;
}
static long long monotonic_ms(void) {
struct timespec ts;
if (sys_clock_gettime(1 /* CLOCK_MONOTONIC */, &ts) < 0) return 0;
return (long long)ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
}
static volatile long g_main_child = 0;
static volatile int g_child_exited = 0;
static volatile int g_main_status = 0;
static volatile int g_pending_signal = 0;
static volatile int g_pending_signals = 0;
static void proxy_sig_handler(int sig) {
/* Just record the signal; the main loop does the /proc scan so we
stay async-signal-safe here. */
g_pending_signal = sig;
if (sig >= 1 && sig <= 31) {
g_pending_signals |= (1 << (sig - 1));
}
}
static void sigchld_handler(int sig) {
@ -653,6 +810,30 @@ static void sigchld_handler(int sig) {
}
}
static void drain_zombies(void) {
int status;
while (sys_wait4(-1, &status, 1 /* WNOHANG */, 0) > 0);
}
static void handle_pending_signals(void) {
int pending = g_pending_signals;
g_pending_signals = 0;
for (int sig = 1; sig <= 31; sig++) {
if (!(pending & (1 << (sig - 1)))) continue;
#ifdef TUXREAPERD_DEBUG
debug_write("[tuxreaperd] handling pending signal ");
debug_write_int(sig);
debug_write("\n");
#endif
if (sig == APACHE_IN_SIG) {
broadcast_signal(APACHE_IN_SIG, APACHE_OUT_SIG, apache_exes);
} else {
broadcast_signal(sig, sig, apache_exes);
}
}
}
#define WIFEXITED(s) (((s) & 0x7f) == 0)
#define WEXITSTATUS(s) (((s) >> 8) & 0xff)
#define WIFSIGNALED(s) ((((s) & 0x7f) != 0) && (((s) & 0x7f) != 0x7f))
@ -702,8 +883,8 @@ int main(int argc, char **argv, char **envp) {
g_main_child = sys_fork();
if (g_main_child == 0) {
/* Become the leader of a new process group so the parent can
target the whole workload tree with kill(-g_main_child, sig). */
/* Become the leader of a new process group so signals can be
broadcast to the whole workload tree. */
sys_setpgid(0, 0);
sys_rt_sigprocmask(2 /* SIG_SETMASK */, &old_mask, 0, sizeof(old_mask));
my_execvp(argv[1], &argv[1], envp);
@ -712,28 +893,45 @@ int main(int argc, char **argv, char **envp) {
sys_exit_group(127);
}
while (!g_child_exited) {
sys_rt_sigsuspend(&old_mask, sizeof(old_mask));
#ifdef TUXREAPERD_DEBUG
debug_write("[tuxreaperd] started main_child=");
debug_write_int(g_main_child);
debug_write("\n");
#endif
/* Dynamic zombie drain: reap any terminated descendants so they
do not clog the process table while the workload is running. */
int status;
while (sys_wait4(-1, &status, 1 /* WNOHANG */, 0) > 0);
long long child_exit_time_ms = 0;
if (g_pending_signal != 0) {
int sig = g_pending_signal;
g_pending_signal = 0;
if (sig == APACHE_IN_SIG) {
proc_remap_signal(APACHE_IN_SIG, APACHE_OUT_SIG, apache_exes);
} else {
/* Broadcast unmapped signals to the workload process group. */
sys_kill(-g_main_child, sig);
while (1) {
if (g_child_exited) {
long long now = monotonic_ms();
if (child_exit_time_ms == 0) {
child_exit_time_ms = now;
debug_msg("main child exited, waiting up to 60s for descendants");
}
int descendants = count_descendants();
#ifdef TUXREAPERD_DEBUG
debug_write("[tuxreaperd] descendants=");
debug_write_int(descendants);
debug_write("\n");
#endif
if (descendants == 0) {
debug_msg("no descendants remaining, exiting cleanly");
break;
}
if (now - child_exit_time_ms >= DESCENDANT_TIMEOUT_SECONDS * 1000LL) {
debug_msg("descendant timeout reached, exiting");
break;
}
}
sys_rt_sigsuspend(&old_mask, sizeof(old_mask));
drain_zombies();
handle_pending_signals();
}
int status;
while (sys_wait4(-1, &status, 1 /* WNOHANG */, 0) > 0);
drain_zombies();
if (WIFEXITED(g_main_status)) {
return WEXITSTATUS(g_main_status);

168
tuxreaperdgnu.c
View file

@ -8,11 +8,27 @@
#include <sys/wait.h>
#include <dirent.h>
#include <errno.h>
#include <time.h>
#ifdef TUXREAPERD_DEBUG
#include <stdarg.h>
static void debug(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
va_end(ap);
fputc('\n', stderr);
}
#else
static void debug(const char *fmt, ...) {
(void)fmt;
}
#endif
/* Apache hijacks SIGWINCH (window resize) for graceful shutdown. Yes,
really. So when the outside world sends SIGTERM, we have to translate
it to SIGWINCH for anything whose /proc/<pid>/exe smells like apache2.
This is insane, but Apache is a good web server so I give it a pass. */
really. So when the outside world sends SIGTERM, we translate it to
SIGWINCH for anything whose /proc/<pid>/exe smells like apache2.
Other processes receive the original signal unchanged. */
static const char * const apache_exes[] = {
"/usr/sbin/apache2",
"/usr/sbin/httpd",
@ -22,17 +38,25 @@ static const char * const apache_exes[] = {
#define APACHE_IN_SIG SIGTERM
#define APACHE_OUT_SIG SIGWINCH
/* How long to keep the container alive after the main child exits, waiting
for remaining descendants (e.g., Apache workers) to finish gracefully. */
#define DESCENDANT_TIMEOUT_SECONDS 60
/* Delay between the two /proc scans when broadcasting a signal, to catch
processes that spawned just after the first scan. */
#define BROADCAST_SCAN_DELAY_US 100000
static volatile pid_t g_main_child = 0;
static volatile sig_atomic_t g_child_exited = 0;
static volatile int g_main_status = 0;
static volatile sig_atomic_t g_pending_signal = 0;
static volatile sig_atomic_t g_pending_signals = 0;
static void sigchld_handler(int sig) {
(void)sig;
int status;
pid_t pid;
// Reap all terminated orphans
// Reap all terminated descendants. Note when the main child exits.
while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
if (pid == g_main_child) {
g_child_exited = 1;
@ -42,14 +66,18 @@ static void sigchld_handler(int sig) {
}
static void proxy_signal_handler(int sig) {
// Just record the signal; the main loop does the /proc scan so we
// stay async-signal-safe here.
g_pending_signal = sig;
// Record the signal in a bitmask. Signals 1..31 fit comfortably.
if (sig >= 1 && sig <= 31) {
g_pending_signals |= (sig_atomic_t)(1U << (sig - 1));
}
}
static void proc_remap_signal(int in_sig, int out_sig, const char *const *target_exes) {
DIR *proc = opendir("/proc");
if (!proc) return;
if (!proc) {
debug("[tuxreaperd] opendir(/proc) failed: %s", strerror(errno));
return;
}
struct dirent *de;
@ -77,29 +105,80 @@ static void proc_remap_signal(int in_sig, int out_sig, const char *const *target
}
}
if (matched) {
kill(pid, out_sig);
} else {
kill(pid, in_sig);
int sent_sig = matched ? out_sig : in_sig;
debug("[tuxreaperd] scan pid=%ld exe=%s matched=%d sending sig=%d",
pid, linkbuf, matched, sent_sig);
if (kill((pid_t)pid, sent_sig) < 0) {
debug("[tuxreaperd] kill(%ld, %d) failed: %s", pid, sent_sig, strerror(errno));
}
}
closedir(proc);
}
static void broadcast_signal(int in_sig, int out_sig, const char *const *target_exes) {
debug("[tuxreaperd] broadcasting in_sig=%d out_sig=%d", in_sig, out_sig);
proc_remap_signal(in_sig, out_sig, target_exes);
usleep(BROADCAST_SCAN_DELAY_US);
proc_remap_signal(in_sig, out_sig, target_exes);
}
static int count_descendants(void) {
DIR *proc = opendir("/proc");
if (!proc) return 0;
int count = 0;
struct dirent *de;
while ((de = readdir(proc)) != NULL) {
if (de->d_name[0] == '.') continue;
char *endptr;
long pid = strtol(de->d_name, &endptr, 10);
if (*endptr != '\0' || pid <= 1) continue;
count++;
}
closedir(proc);
return count;
}
static void drain_zombies(void) {
while (waitpid(-1, NULL, WNOHANG) > 0);
}
static void handle_pending_signals(void) {
sig_atomic_t pending = g_pending_signals;
g_pending_signals = 0;
for (int sig = 1; sig <= 31; sig++) {
if (!(pending & (sig_atomic_t)(1U << (sig - 1)))) continue;
debug("[tuxreaperd] handling pending signal %d", sig);
if (sig == APACHE_IN_SIG) {
broadcast_signal(APACHE_IN_SIG, APACHE_OUT_SIG, apache_exes);
} else {
broadcast_signal(sig, sig, apache_exes);
}
}
}
static long long monotonic_ms(void) {
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0;
return (long long)ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
}
int main(int argc, char *argv[]) {
if (argc < 2) {
fprintf(stderr, "Usage: %s <command> [args...]\n", argv[0]);
return 1;
}
// Become a subreaper for any reparented processes down the tree
// Become a subreaper for any reparented processes down the tree.
if (prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) < 0) {
perror("prctl(PR_SET_CHILD_SUBREAPER)");
return 1;
}
// Set up signal forwarding
// Set up signal forwarding.
struct sigaction sa_forward;
sa_forward.sa_handler = proxy_signal_handler;
sigemptyset(&sa_forward.sa_mask);
@ -111,14 +190,14 @@ int main(int argc, char *argv[]) {
sigaction(SIGUSR1, &sa_forward, NULL);
sigaction(SIGUSR2, &sa_forward, NULL);
// Set up SIGCHLD reaper handler
// Set up SIGCHLD reaper handler.
struct sigaction sa_chld;
sa_chld.sa_handler = sigchld_handler;
sigemptyset(&sa_chld.sa_mask);
sa_chld.sa_flags = SA_RESTART | SA_NOCLDSTOP;
sigaction(SIGCHLD, &sa_chld, NULL);
// Block the signals we wait on so the check/suspend loop is race-free
// Block the signals we wait on so the check/suspend loop is race-free.
sigset_t block_mask, old_mask;
sigemptyset(&block_mask);
sigaddset(&block_mask, SIGCHLD);
@ -130,7 +209,7 @@ int main(int argc, char *argv[]) {
sigaddset(&block_mask, SIGUSR2);
sigprocmask(SIG_BLOCK, &block_mask, &old_mask);
// Spawn the primary workload
// Spawn the primary workload.
g_main_child = fork();
if (g_main_child < 0) {
perror("fork");
@ -138,8 +217,8 @@ int main(int argc, char *argv[]) {
}
if (g_main_child == 0) {
// Become the leader of a new process group so the parent can
// target the whole workload tree with kill(-g_main_child, sig).
// Become the leader of a new process group so signals can be
// broadcast to the whole workload tree.
setpgid(0, 0);
sigprocmask(SIG_SETMASK, &old_mask, NULL);
execvp(argv[1], &argv[1]);
@ -147,28 +226,45 @@ int main(int argc, char *argv[]) {
_exit(127);
}
// Block until the primary process exits
while (!g_child_exited) {
sigsuspend(&old_mask);
debug("[tuxreaperd] started main_child=%d", (int)g_main_child);
// Dynamic zombie drain: reap any terminated descendants so they
// do not clog the process table while the workload is running.
while (waitpid(-1, NULL, WNOHANG) > 0);
long long child_exit_time_ms = 0;
if (g_pending_signal != 0) {
int sig = g_pending_signal;
g_pending_signal = 0;
if (sig == APACHE_IN_SIG) {
proc_remap_signal(APACHE_IN_SIG, APACHE_OUT_SIG, apache_exes);
} else {
// Broadcast unmapped signals to the workload process group.
kill(-g_main_child, sig);
while (1) {
int descendants = 0;
if (g_child_exited) {
long long now = monotonic_ms();
if (child_exit_time_ms == 0) {
child_exit_time_ms = now;
debug("[tuxreaperd] main child exited, waiting up to %ds for descendants",
DESCENDANT_TIMEOUT_SECONDS);
}
descendants = count_descendants();
debug("[tuxreaperd] descendants=%d", descendants);
if (descendants == 0) {
debug("[tuxreaperd] no descendants remaining, exiting cleanly");
break;
}
if (now - child_exit_time_ms >= DESCENDANT_TIMEOUT_SECONDS * 1000LL) {
debug("[tuxreaperd] descendant timeout reached, exiting");
break;
}
}
// Wait for a signal. When one arrives, reap and handle it, then loop
// back to re-evaluate descendants and timeouts.
sigsuspend(&old_mask);
drain_zombies();
handle_pending_signals();
}
// Final sweep of any remaining lingering zombies
while (waitpid(-1, NULL, WNOHANG) > 0);
// Final sweep of any remaining lingering zombies.
drain_zombies();
if (WIFEXITED(g_main_status)) {
return WEXITSTATUS(g_main_status);