[PATCH] use vfork under Linux before execve

[PATCH] use vfork under Linux before execve

[Eric Wong]

Given the prevalance of gigantic VM footprints due to current glibc
malloc and our potentially large number of threads, vfork can speed
up fork used for spawning iostat and SIGUSR2 upgrades.

vfork only pauses the spawning thread, so it will not affect other
I/O threads used in cmogstored; only the non-performance-critical
master thread.

Swapping 'fork()' for 'vfork()' in the following C test program
should show a large speedup under Linux.

Changing FILL to increase or decrease memory usage will respectively
decrease or increase performance improvement gain from vfork over
fork..

-----------------------------8<-------------------------
/* gcc -o x x.c -Wall -O2 -lpthread && ./x */

	#include <sys/types.h>
	#include <sys/time.h>
	#include <unistd.h>
	#include <pthread.h>
	#include <poll.h>
	#include <stdio.h>
	#include <sys/wait.h>
	#include <stdlib.h>
	#include <string.h>
	#define FILL (1024 * 1024)

static void *thfunc(void *p)
{
	void *ptr = malloc(FILL);
	memset(ptr, 1, FILL);
	poll(0, 0, -1);
	return 0;
}

int main(void)
{
	long i;
	void *ptr = malloc(FILL);
	memset(ptr, 1, FILL);

	for (i = 0; i < 100; i++) {
		pthread_t th;
		pthread_create(&th, 0, thfunc, (void *)i);
	}

	poll(0, 0, 1000);
	for (i = 0; i < 100; i++) {
		/* swapping fork with vfork increases performance on Linux */
		pid_t pid = fork();
		if (pid < 0) {
			fprintf(stderr, "ERROR: forking %m\n");
			return 1;
		}
		if (pid == 0) {
			char *argv[] = { "/bin/true", 0 };
			char *env[] = { 0 };
			execve(argv[0], argv, env);
			return 1;
		} else {
			int s;
			waitpid(pid, &s, 0);
		}
	}

	return 0;
}
---
 cmogstored.h     | 10 ++++++++++
 configure.ac     |  1 +
 iostat_process.c |  6 +++---
 upgrade.c        |  8 +++++---
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/cmogstored.h b/cmogstored.h
index 5aa7c01..4bb0716 100644
--- a/cmogstored.h
+++ b/cmogstored.h
@@ -360,6 +360,16 @@ void mog_oom_if_null(const void *);
 
 #define warn(...) error(0, 0, __VA_ARGS__)
 
+/*
+ * vfork is poorly-specified, but at least on Linux it improves
+ * performance when used for spawning iostat processes
+ */
+#if defined(HAVE_VFORK) && defined(__linux__)
+#  define mog_fork_for_exec() vfork()
+#else
+#  define mog_fork_for_exec() fork()
+#endif
+
 /* maxconns.c */
 void mog_set_maxconns(unsigned long);
 
diff --git a/configure.ac b/configure.ac
index 8bd98e2..0d551c3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -51,6 +51,7 @@ AC_CHECK_FUNCS([ioctl])
 AC_CHECK_FUNCS([sendfile])
 AC_CHECK_FUNCS([open_memstream])
 AC_CHECK_FUNCS([posix_fadvise])
+AC_CHECK_FUNCS([vfork])
 
 dnl need LIBS=-lfreebsd-glue (but not CFLAGS=-I/usr/include/freebsd)
 AC_CHECK_FUNCS([bsd_sendfile])
diff --git a/iostat_process.c b/iostat_process.c
index 90a291f..4d7dfbf 100644
--- a/iostat_process.c
+++ b/iostat_process.c
@@ -65,7 +65,7 @@ static void dup2_or_die(int oldfd, int newfd, const char *errdesc)
 
 	if (rc < 0) {
 		syslog(LOG_CRIT, "dup2(%s) failed: %m", errdesc);
-		abort();
+		_exit(1);
 	}
 }
 
@@ -96,7 +96,7 @@ static pid_t iostat_fork_exec(int out_fd)
 
 	cmd = exec_cmd(cmd);
 
-	iostat_pid = fork();
+	iostat_pid = mog_fork_for_exec();
 	if (iostat_pid < 0) {
 		syslog(LOG_ERR, "fork() for iostat failed: %m");
 	} else if (iostat_pid > 0) {
@@ -111,7 +111,7 @@ static pid_t iostat_fork_exec(int out_fd)
 		mog_intr_enable();
 		execl("/bin/sh", "sh", "-c", cmd, (char *)NULL);
 		syslog(LOG_CRIT, "execl(%s) failed: %m", cmd);
-		abort();
+		_exit(1);
 	}
 	mog_free(cmd);
 	return iostat_pid;
diff --git a/upgrade.c b/upgrade.c
index 1311ebe..82a7b28 100644
--- a/upgrade.c
+++ b/upgrade.c
@@ -125,13 +125,14 @@ pid_t mog_upgrade_spawn(void)
 
 	assert(dst[bytes - 1] == ',' && "not comma-terminated no listeners?");
 	dst[bytes - 1] = '\0'; /* kill the last comma */
+	start.envp[0] = dst;
 
-	pid = fork();
+	pid = mog_fork_for_exec();
 	if (pid == 0) {
-		start.envp[0] = dst;
 		mog_svc_upgrade_prepare();
 		execve(execfile, start.argv, start.envp);
-		die_errno("execve %s", execfile);
+		syslog(LOG_ERR, "execve %s failed for upgrade: %m", execfile);
+		_exit(2);
 	} else if (pid > 0) {
 		mog_process_register(pid, MOG_PROC_UPGRADE);
 		syslog(LOG_INFO, "upgrade spawned PID:%d", pid);
@@ -143,6 +144,7 @@ out:
 	/* find_in_path does not malloc if output == input */
 	if (execfile != start.argv[0])
 		mog_free(execfile);
+	start.envp[0] = 0;
 	free(dst);
 
 	return pid;
-- 
EW