rework SystemExec - use vfork wrapper (and lots of related stuff)

9 files changed, 319 insertions, 27 deletions

diff --git a/libs/ardour/ardour/system_exec.h b/libs/ardour/ardour/system_exec.h
new file mode 100644
index 0000000000..40e429720e
--- /dev/null
+++ b/libs/ardour/ardour/system_exec.h
@@ -0,0 +1,50 @@
+/*
+    Copyright (C) 2010 Paul Davis
+    Copyright (C) 2010-2014 Robin Gareus <robin@gareus.org>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+#ifndef _ardour_system_exec_h_
+#define _ardour_system_exec_h_
+
+#include "ardour/libardour_visibility.h"
+#include "pbd/system_exec.h"
+
+namespace ARDOUR {
+
+class LIBARDOUR_API SystemExec
+	: public PBD::SystemExec
+{
+
+public:
+	SystemExec (std::string c, std::string a = "");
+	SystemExec (std::string c, char ** a);
+	~SystemExec ();
+
+	int start (int stderr_mode = 1) {
+		return PBD::SystemExec::start(stderr_mode, _vfork_exec_wrapper);
+	}
+
+private:
+	static char * _vfork_exec_wrapper;
+
+}; /* end class */
+
+}; /* end namespace */
+
+#endif /* _libpbd_system_exec_h_ */
+
+
diff --git a/libs/ardour/system_exec.cc b/libs/ardour/system_exec.cc
new file mode 100644
index 0000000000..c99863f2e5
--- /dev/null
+++ b/libs/ardour/system_exec.cc
@@ -0,0 +1,60 @@
+/*
+    Copyright (C) 2010 Paul Davis
+    Copyright (C) 2010-2014 Robin Gareus <robin@gareus.org>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include <glibmm/miscutils.h>
+#include "pbd/pathscanner.h"
+#include "pbd/file_utils.h"
+#include "pbd/error.h"
+
+#include "ardour/filesystem_paths.h"
+#include "ardour/system_exec.h"
+
+using namespace ARDOUR;
+
+char * SystemExec::_vfork_exec_wrapper = NULL;
+
+static char *vfork_exec_wrapper_path() {
+	std::string vfork_exec_wrapper;
+	if (!PBD::find_file_in_search_path (
+				PBD::Searchpath(Glib::build_filename(ARDOUR::ardour_dll_directory(), "vfork")),
+				"ardour-exec-wrapper", vfork_exec_wrapper)) {
+		PBD::warning << "vfork exec wrapper not found..'" << endmsg;
+		return NULL;
+	}
+	return strdup(vfork_exec_wrapper.c_str());
+}
+
+SystemExec::SystemExec (std::string c, char ** a)
+	: PBD::SystemExec(c, a)
+{
+	if (!_vfork_exec_wrapper) {
+		_vfork_exec_wrapper = vfork_exec_wrapper_path();
+	}
+}
+
+SystemExec::SystemExec (std::string c, std::string a)
+	: PBD::SystemExec(c, a)
+{
+	if (!_vfork_exec_wrapper) {
+		_vfork_exec_wrapper = vfork_exec_wrapper_path();
+	}
+}
+
+SystemExec::~SystemExec() { }
diff --git a/libs/ardour/vst_info_file.cc b/libs/ardour/vst_info_file.cc
index fe434909f3..a0426edad1 100644
--- a/libs/ardour/vst_info_file.cc
+++ b/libs/ardour/vst_info_file.cc
@@ -40,12 +40,18 @@
 #include <glib/gstdio.h>
 #include <glibmm.h>
 
-#include "pbd/error.h"
 
-#ifndef VST_SCANNER_APP
-#include "pbd/system_exec.h"
+#ifdef VST_SCANNER_APP
+#define errormsg cerr
+#define warningmsg cerr
+#define endmsg endl
+#else
 #include "ardour/plugin_manager.h" // scanner_bin_path
 #include "ardour/rc_configuration.h"
+#include "ardour/system_exec.h"
+#include "pbd/error.h"
+#define errormsg PBD::error
+#define warningmsg PBD::warning
 #endif
 
 #include "ardour/filesystem_paths.h"
@@ -324,7 +330,7 @@ vstfx_write_info_file (FILE* fp, vector<VSTInfo *> *infos)
 	} else if (infos->size() == 1) {
 		vstfx_write_info_block(fp, infos->front());
 	} else {
-		PBD::error << "Zero plugins in VST." << endmsg; // XXX here? rather make this impossible before if it ain't already.
+		errormsg << "Zero plugins in VST." << endmsg; // XXX here? rather make this impossible before if it ain't already.
 	}
 }
 
@@ -508,7 +514,7 @@ vstfx_get_info_from_file(const char* dllpath, vector<VSTInfo*> *infos)
 		rv = vstfx_load_info_file(infofile, infos);
 		fclose (infofile);
 		if (!rv) {
-			PBD::warning << "Cannot get VST information form " << dllpath << ": info file load failed." << endmsg;
+			warningmsg << "Cannot get VST information form " << dllpath << ": info file load failed." << endmsg;
 		}
 	}
 	return rv;
@@ -774,7 +780,7 @@ vstfx_instantiate_and_get_info_lx (
 	VSTHandle* h;
 	VSTState* vstfx;
 	if (!(h = vstfx_load(dllpath))) {
-		PBD::warning << "Cannot get LinuxVST information from " << dllpath << ": load failed." << endmsg;
+		warningmsg << "Cannot get LinuxVST information from " << dllpath << ": load failed." << endmsg;
 		return false;
 	}
 
@@ -782,7 +788,7 @@ vstfx_instantiate_and_get_info_lx (
 
 	if (!(vstfx = vstfx_instantiate(h, simple_master_callback, 0))) {
 		vstfx_unload(h);
-		PBD::warning << "Cannot get LinuxVST information from " << dllpath << ": instantiation failed." << endmsg;
+		warningmsg << "Cannot get LinuxVST information from " << dllpath << ": instantiation failed." << endmsg;
 		return false;
 	}
 
@@ -803,7 +809,7 @@ vstfx_instantiate_and_get_info_fst (
 	VSTHandle* h;
 	VSTState* vstfx;
 	if(!(h = fst_load(dllpath))) {
-		PBD::warning << "Cannot get Windows VST information from " << dllpath << ": load failed." << endmsg;
+		warningmsg << "Cannot get Windows VST information from " << dllpath << ": load failed." << endmsg;
 		return false;
 	}
 
@@ -812,7 +818,7 @@ vstfx_instantiate_and_get_info_fst (
 	if(!(vstfx = fst_instantiate(h, simple_master_callback, 0))) {
 		fst_unload(&h);
 		vstfx_current_loading_id = 0;
-		PBD::warning << "Cannot get Windows VST information from " << dllpath << ": instantiation failed." << endmsg;
+		warningmsg << "Cannot get Windows VST information from " << dllpath << ": instantiation failed." << endmsg;
 		return false;
 	}
 	vstfx_current_loading_id = 0;
@@ -834,14 +840,14 @@ static char * _errorlog_dll = 0;
 static void parse_scanner_output (std::string msg, size_t /*len*/)
 {
 	if (!_errorlog_fd && !_errorlog_dll) {
-		PBD::error << "VST scanner: " << msg;
+		errormsg << "VST scanner: " << msg;
 		return;
 	}
 
 	if (!_errorlog_fd) {
 		if (!(_errorlog_fd = fopen(vstfx_errorfile_path(_errorlog_dll, 0).c_str(), "w"))) {
 			if (!(_errorlog_fd = fopen(vstfx_errorfile_path(_errorlog_dll, 1).c_str(), "w"))) {
-				PBD::error << "Cannot create plugin error-log for plugin " << _errorlog_dll;
+				errormsg << "Cannot create plugin error-log for plugin " << _errorlog_dll;
 				free(_errorlog_dll);
 				_errorlog_dll = NULL;
 			}
@@ -851,7 +857,7 @@ static void parse_scanner_output (std::string msg, size_t /*len*/)
 	if (_errorlog_fd) {
 		fprintf (_errorlog_fd, "%s\n", msg.c_str());
 	} else {
-		PBD::error << "VST scanner: " << msg;
+		errormsg << "VST scanner: " << msg;
 	}
 }
 
@@ -907,11 +913,11 @@ vstfx_get_info (const char* dllpath, enum ARDOUR::PluginType type, enum VSTScanM
 		argp[2] = 0;
 
 		set_error_log(dllpath);
-		PBD::SystemExec scanner (scanner_bin_path, argp);
+		ARDOUR::SystemExec scanner (scanner_bin_path, argp);
 		PBD::ScopedConnectionList cons;
 		scanner.ReadStdout.connect_same_thread (cons, boost::bind (&parse_scanner_output, _1 ,_2));
 		if (scanner.start (2 /* send stderr&stdout via signal */)) {
-			PBD::error << "Cannot launch VST scanner app '" << scanner_bin_path << "': "<< strerror(errno) << endmsg;
+			errormsg << "Cannot launch VST scanner app '" << scanner_bin_path << "': "<< strerror(errno) << endmsg;
 			close_error_log();
 			return infos;
 		} else {
@@ -977,7 +983,7 @@ vstfx_get_info (const char* dllpath, enum ARDOUR::PluginType type, enum VSTScanM
 	/* crate cache/whitelist */
 	infofile = vstfx_infofile_for_write (dllpath);
 	if (!infofile) {
-		PBD::warning << "Cannot cache VST information for " << dllpath << ": cannot create new FST info file." << endmsg;
+		warningmsg << "Cannot cache VST information for " << dllpath << ": cannot create new FST info file." << endmsg;
 		return infos;
 	} else {
 		vstfx_write_info_file (infofile, infos);
@@ -1005,7 +1011,7 @@ get_personal_vst_blacklist_dir() {
 	/* if the directory doesn't exist, try to create it */
 	if (!Glib::file_test (dir, Glib::FILE_TEST_IS_DIR)) {
 		if (g_mkdir (dir.c_str (), 0700)) {
-			PBD::error << "Cannot create VST blacklist folder '" << dir << "'" << endmsg;
+			errormsg << "Cannot create VST blacklist folder '" << dir << "'" << endmsg;
 			//exit(1);
 		}
 	}
@@ -1018,7 +1024,7 @@ get_personal_vst_info_cache_dir() {
 	/* if the directory doesn't exist, try to create it */
 	if (!Glib::file_test (dir, Glib::FILE_TEST_IS_DIR)) {
 		if (g_mkdir (dir.c_str (), 0700)) {
-			PBD::error << "Cannot create VST info folder '" << dir << "'" << endmsg;
+			errormsg << "Cannot create VST info folder '" << dir << "'" << endmsg;
 			//exit(1);
 		}
 	}
diff --git a/libs/ardour/wscript b/libs/ardour/wscript
index bbe7f56aa1..260c334a30 100644
--- a/libs/ardour/wscript
+++ b/libs/ardour/wscript
@@ -197,6 +197,7 @@ libardour_sources = [
         'speakers.cc',
         'srcfilesource.cc',
         'strip_silence.cc',
+        'system_exec.cc',
         'revision.cc',
         'tape_file_matcher.cc',
         'template_utils.cc',
diff --git a/libs/pbd/pbd/system_exec.h b/libs/pbd/pbd/system_exec.h
index 0ca523085a..1232175fec 100644
--- a/libs/pbd/pbd/system_exec.h
+++ b/libs/pbd/pbd/system_exec.h
@@ -107,7 +107,7 @@ class LIBPBD_API SystemExec
 		 * @return If the process is already running or was launched successfully
 		 * the function returns zero (0). A negative number indicates an error.
 		 */
-		int start (int stderr_mode = 1);
+		int start (int stderr_mode, const char *_vfork_exec_wrapper);
 		/** kill running child-process
 		 *
 		 * if a child process exists trt to shut it down by closing its STDIN.
diff --git a/libs/pbd/system_exec.cc b/libs/pbd/system_exec.cc
index ac0d37f0d0..7814cce96b 100644
--- a/libs/pbd/system_exec.cc
+++ b/libs/pbd/system_exec.cc
@@ -43,6 +43,8 @@
 #endif
 
 
+#define USE_VFORK
+
 #include "pbd/system_exec.h"
 
 using namespace std;
@@ -51,7 +53,7 @@ using namespace PBD;
 static void * interposer_thread (void *arg);
 static void close_fd (int& fd) { if (fd >= 0) ::close (fd); fd = -1; }
 
-#ifndef PLATFORM_WINDOWS
+#if (!defined PLATFORM_WINDOWS && defined NO_VFORK)
 /*
  * This function was part of libasyncns.
  * LGPL v2.1
@@ -146,7 +148,7 @@ static int close_allv(const int except_fds[]) {
 
 	return 0;
 }
-#endif /* not on windows */
+#endif /* not on windows, nor vfork */
 
 
 SystemExec::SystemExec (std::string c, std::string a)
@@ -324,7 +326,7 @@ SystemExec::is_running ()
 }
 
 int
-SystemExec::start (int stderr_mode)
+SystemExec::start (int stderr_mode, const char * /*vfork_exec_wrapper*/)
 {
 	char* working_dir = 0;
 
@@ -596,7 +598,7 @@ SystemExec::is_running ()
 }
 
 int
-SystemExec::start (int stderr_mode)
+SystemExec::start (int stderr_mode, const char *vfork_exec_wrapper)
 {
 	if (is_running()) {
 		return 0; // mmh what to return here?
@@ -608,7 +610,7 @@ SystemExec::start (int stderr_mode)
 		return -1;
 	}
 
-#ifdef USE_VFORK
+#ifndef NO_VFORK
 	r = ::vfork();
 #else
 	r = ::fork();
@@ -631,11 +633,11 @@ SystemExec::start (int stderr_mode)
 				/* child process returned from execve */
 				pid=0;
 				close_fd(pok[0]);
+				close_fd(pok[1]);
 				close_fd(pin[1]);
 				close_fd(pin[0]);
 				close_fd(pout[1]);
 				close_fd(pout[0]);
-				pin[1] = -1;
 				return -3;
 			} else if ( n==-1 ) {
 				 if ( errno==EAGAIN || errno==EINTR )
@@ -659,7 +661,7 @@ SystemExec::start (int stderr_mode)
 		return 0; /* all systems go - return to main */
 	}
 
-#ifndef USE_VFORK
+#ifdef NO_VFORK
 	/* child process - exec external process */
 	close_fd(pok[0]);
 	::fcntl(pok[1], F_SETFD, FD_CLOEXEC);
@@ -713,12 +715,12 @@ SystemExec::start (int stderr_mode)
 #else
 
 	/* XXX this should be done before vfork()
-	 * calling malloc here only increases the time we vfork() blocks
+	 * calling malloc here only increases the time vfork() blocks
 	 */
 	int argn = 0;
 	for (int i=0;argp[i];++i) { argn++; }
 	char **argx = (char **) malloc((argn + 10) * sizeof(char *));
-	argx[0] = strdup("/home/rgareus/src/git/ardourCairoCanvas/tools/exec_wrapper"); // XXX TODO
+	argx[0] = strdup(vfork_exec_wrapper); // XXX
 
 #define FDARG(NUM, FDN) \
 	argx[NUM] = (char*) calloc(6, sizeof(char)); snprintf(argx[NUM], 6, "%d", FDN);
diff --git a/libs/vfork/README b/libs/vfork/README
new file mode 100644
index 0000000000..fb591a6984
--- /dev/null
+++ b/libs/vfork/README
@@ -0,0 +1,44 @@
+vfork-exec-wrapper
+==================
+
+A tiny tool that redirects stdio file-descriptors and executes a program.
+
+Motivation
+----------
+
+Ardour can start external helper applications for various purposes
+(e.g. video-server, video-monitor, plugin-scanner, post-export scripts,...)
+and has the need to bidirectionally communicate with the external app.
+
+On POSIX platforms (OSX, GNU/Linux, BSD,..) launching an external app is a
+combination of fork() and execve(2). The problem with that is that fork(2)
+duplicates the complete page-table (incl. allocated locked memory, and 
+file-descriptors) which -  even if fork(2) is done from a non-realtime
+thread - may cause audio I/O glitches or worse out-of-memory errors if
+the mlock(2) limit is reached.
+
+vfork(2) on the other hand "is a special case of clone(2). It is used to
+create new processes without copying the page tables of the parent process.
+It may be useful in performance-sensitive applications where a child is
+created which then immediately issues an execve(2)." [vfork man page].
+
+The problem with vfork(2) is that file-descriptors are not cloned, which
+makes bi-directional communication impossible without additional work.
+This is exactly what this vfork-exec-wrapper does: It takes a list of
+file-descriptors, re-directs them to stdio and calls execve(2) again.
+
+This code was previously in pbd/system_exec.cc (done after fork(2),
+which become a NOOP with vfork(2)).
+
+Usage
+-----
+
+ardour-exec-wrapper <file-des> <mode> <nice> <command> [args]
+
+ardour-exec-wrapper takes three pairs of file-descriptors, stderr mode,
+nice-level followed by the command to execute and optional arguments.
+
+The first set FDs is used to communicate failure back to the parent process.
+They are closed if execve(2) succeeds. The following two FDs are stdin and
+stdout. The mode specifies handling of stderr: 0: keep stderr, 1: close and
+ignore, 2: merge stderr into stdout.
diff --git a/libs/vfork/exec_wrapper.c b/libs/vfork/exec_wrapper.c
new file mode 100644
index 0000000000..4beb7a3b87
--- /dev/null
+++ b/libs/vfork/exec_wrapper.c
@@ -0,0 +1,101 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <signal.h>
+
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+#ifndef STDERR_FILENO
+#define STDERR_FILENO 2
+#endif
+
+extern char **environ;
+static void close_fd (int *fd) { if ((*fd) >= 0) close (*fd); *fd = -1; }
+
+int main(int argc, char *argv[]) {
+	if (argc < 10) {
+		// TODO: if argv > 3, assume pok[] is given, notifify parent.
+		// usage() and a man-page (help2man) would not be bad, either :)
+		return -1;
+	}
+
+	int pok[2];
+	int pin[2];
+	int pout[2];
+
+	pok[0]  = atoi(argv[1]);
+	pok[1]  = atoi(argv[2]);
+	pin[0]  = atoi(argv[3]);
+	pin[1]  = atoi(argv[4]);
+	pout[0] = atoi(argv[5]);
+	pout[1] = atoi(argv[6]);
+
+	int stderr_mode = atoi(argv[7]);
+	int nicelevel = atoi(argv[8]);
+
+	/* vfork()ed child process - exec external process */
+	close_fd(&pok[0]);
+	fcntl(pok[1], F_SETFD, FD_CLOEXEC);
+
+	close_fd(&pin[1]);
+	if (pin[0] != STDIN_FILENO) {
+		dup2(pin[0], STDIN_FILENO);
+	}
+	close_fd(&pin[0]);
+	close_fd(&pout[0]);
+	if (pout[1] != STDOUT_FILENO) {
+		dup2(pout[1], STDOUT_FILENO);
+	}
+
+	if (stderr_mode == 2) {
+		/* merge STDERR into output */
+		if (pout[1] != STDERR_FILENO) {
+			dup2(pout[1], STDERR_FILENO);
+		}
+	} else if (stderr_mode == 1) {
+		/* ignore STDERR */
+		close(STDERR_FILENO);
+	} else {
+		/* keep STDERR */
+	}
+
+	if (pout[1] != STDOUT_FILENO && pout[1] != STDERR_FILENO) {
+		close_fd(&pout[1]);
+	}
+
+	if (nicelevel !=0) {
+		nice(nicelevel);
+	}
+
+	/* copy current environment */
+	char **envp = NULL;
+	int i=0;
+	envp = (char **) calloc(1, sizeof(char*));
+	for (i=0;environ[i];++i) {
+		envp[i] = strdup(environ[i]);
+		envp = (char **) realloc(envp, (i+2) * sizeof(char*));
+	}
+	envp[i] = 0;
+
+#ifdef HAVE_SIGSET
+	sigset(SIGPIPE, SIG_DFL);
+#else
+	signal(SIGPIPE, SIG_DFL);
+#endif
+
+	/* all systems go */
+	execve(argv[9], &argv[9], envp);
+
+	/* if we reach here something went wrong.. */
+	char buf = 0;
+	(void) write(pok[1], &buf, 1 );
+	close_fd(&pok[1]);
+	exit(-1);
+	return -1;
+}
diff --git a/libs/vfork/wscript b/libs/vfork/wscript
new file mode 100644
index 0000000000..1301f215c2
--- /dev/null
+++ b/libs/vfork/wscript
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+from waflib.extras import autowaf as autowaf
+from waflib import TaskGen
+import os
+import sys
+
+# Mandatory variables
+top = '.'
+out = 'build'
+
+def options(opt):
+    autowaf.set_options(opt)
+
+def configure(conf):
+    conf.load('compiler_c')
+    autowaf.configure(conf)
+
+def build(bld):
+    if bld.env['build_target'] == 'mingw':
+        return
+    obj = bld (features = 'c cprogram')
+    obj.source = 'exec_wrapper.c'
+    obj.target = 'ardour-exec-wrapper'
+    obj.install_path = os.path.join(bld.env['LIBDIR'], 'ardour3/vfork')
+    obj.defines = [
+            '_POSIX_SOURCE',
+            '_XOPEN_SOURCE=700',
+    ]