rework SystemExec - use vfork wrapper (and lots of related stuff)

3 files changed, 173 insertions, 0 deletions

diff --git a/libs/vfork/README b/libs/vfork/README
new file mode 100644
index 0000000000..fb591a6984
--- /dev/null
+++ b/libs/vfork/README
@@ -0,0 +1,44 @@
+vfork-exec-wrapper
+==================
+
+A tiny tool that redirects stdio file-descriptors and executes a program.
+
+Motivation
+----------
+
+Ardour can start external helper applications for various purposes
+(e.g. video-server, video-monitor, plugin-scanner, post-export scripts,...)
+and has the need to bidirectionally communicate with the external app.
+
+On POSIX platforms (OSX, GNU/Linux, BSD,..) launching an external app is a
+combination of fork() and execve(2). The problem with that is that fork(2)
+duplicates the complete page-table (incl. allocated locked memory, and 
+file-descriptors) which -  even if fork(2) is done from a non-realtime
+thread - may cause audio I/O glitches or worse out-of-memory errors if
+the mlock(2) limit is reached.
+
+vfork(2) on the other hand "is a special case of clone(2). It is used to
+create new processes without copying the page tables of the parent process.
+It may be useful in performance-sensitive applications where a child is
+created which then immediately issues an execve(2)." [vfork man page].
+
+The problem with vfork(2) is that file-descriptors are not cloned, which
+makes bi-directional communication impossible without additional work.
+This is exactly what this vfork-exec-wrapper does: It takes a list of
+file-descriptors, re-directs them to stdio and calls execve(2) again.
+
+This code was previously in pbd/system_exec.cc (done after fork(2),
+which become a NOOP with vfork(2)).
+
+Usage
+-----
+
+ardour-exec-wrapper <file-des> <mode> <nice> <command> [args]
+
+ardour-exec-wrapper takes three pairs of file-descriptors, stderr mode,
+nice-level followed by the command to execute and optional arguments.
+
+The first set FDs is used to communicate failure back to the parent process.
+They are closed if execve(2) succeeds. The following two FDs are stdin and
+stdout. The mode specifies handling of stderr: 0: keep stderr, 1: close and
+ignore, 2: merge stderr into stdout.
diff --git a/libs/vfork/exec_wrapper.c b/libs/vfork/exec_wrapper.c
new file mode 100644
index 0000000000..4beb7a3b87
--- /dev/null
+++ b/libs/vfork/exec_wrapper.c
@@ -0,0 +1,101 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <signal.h>
+
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+#ifndef STDERR_FILENO
+#define STDERR_FILENO 2
+#endif
+
+extern char **environ;
+static void close_fd (int *fd) { if ((*fd) >= 0) close (*fd); *fd = -1; }
+
+int main(int argc, char *argv[]) {
+	if (argc < 10) {
+		// TODO: if argv > 3, assume pok[] is given, notifify parent.
+		// usage() and a man-page (help2man) would not be bad, either :)
+		return -1;
+	}
+
+	int pok[2];
+	int pin[2];
+	int pout[2];
+
+	pok[0]  = atoi(argv[1]);
+	pok[1]  = atoi(argv[2]);
+	pin[0]  = atoi(argv[3]);
+	pin[1]  = atoi(argv[4]);
+	pout[0] = atoi(argv[5]);
+	pout[1] = atoi(argv[6]);
+
+	int stderr_mode = atoi(argv[7]);
+	int nicelevel = atoi(argv[8]);
+
+	/* vfork()ed child process - exec external process */
+	close_fd(&pok[0]);
+	fcntl(pok[1], F_SETFD, FD_CLOEXEC);
+
+	close_fd(&pin[1]);
+	if (pin[0] != STDIN_FILENO) {
+		dup2(pin[0], STDIN_FILENO);
+	}
+	close_fd(&pin[0]);
+	close_fd(&pout[0]);
+	if (pout[1] != STDOUT_FILENO) {
+		dup2(pout[1], STDOUT_FILENO);
+	}
+
+	if (stderr_mode == 2) {
+		/* merge STDERR into output */
+		if (pout[1] != STDERR_FILENO) {
+			dup2(pout[1], STDERR_FILENO);
+		}
+	} else if (stderr_mode == 1) {
+		/* ignore STDERR */
+		close(STDERR_FILENO);
+	} else {
+		/* keep STDERR */
+	}
+
+	if (pout[1] != STDOUT_FILENO && pout[1] != STDERR_FILENO) {
+		close_fd(&pout[1]);
+	}
+
+	if (nicelevel !=0) {
+		nice(nicelevel);
+	}
+
+	/* copy current environment */
+	char **envp = NULL;
+	int i=0;
+	envp = (char **) calloc(1, sizeof(char*));
+	for (i=0;environ[i];++i) {
+		envp[i] = strdup(environ[i]);
+		envp = (char **) realloc(envp, (i+2) * sizeof(char*));
+	}
+	envp[i] = 0;
+
+#ifdef HAVE_SIGSET
+	sigset(SIGPIPE, SIG_DFL);
+#else
+	signal(SIGPIPE, SIG_DFL);
+#endif
+
+	/* all systems go */
+	execve(argv[9], &argv[9], envp);
+
+	/* if we reach here something went wrong.. */
+	char buf = 0;
+	(void) write(pok[1], &buf, 1 );
+	close_fd(&pok[1]);
+	exit(-1);
+	return -1;
+}
diff --git a/libs/vfork/wscript b/libs/vfork/wscript
new file mode 100644
index 0000000000..1301f215c2
--- /dev/null
+++ b/libs/vfork/wscript
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+from waflib.extras import autowaf as autowaf
+from waflib import TaskGen
+import os
+import sys
+
+# Mandatory variables
+top = '.'
+out = 'build'
+
+def options(opt):
+    autowaf.set_options(opt)
+
+def configure(conf):
+    conf.load('compiler_c')
+    autowaf.configure(conf)
+
+def build(bld):
+    if bld.env['build_target'] == 'mingw':
+        return
+    obj = bld (features = 'c cprogram')
+    obj.source = 'exec_wrapper.c'
+    obj.target = 'ardour-exec-wrapper'
+    obj.install_path = os.path.join(bld.env['LIBDIR'], 'ardour3/vfork')
+    obj.defines = [
+            '_POSIX_SOURCE',
+            '_XOPEN_SOURCE=700',
+    ]