1 files changed, 231 insertions, 105 deletions

diff --git a/libs/ardour/vst_info_file.cc b/libs/ardour/vst_info_file.cc
index 50d94850d6..7b9b0855bd 100644
--- a/libs/ardour/vst_info_file.cc
+++ b/libs/ardour/vst_info_file.cc
@@ -49,6 +49,23 @@
 
 using namespace std;
 
+// TODO: make static parts of this file re-usable for standalone app
+// TODO: namespace public API into ARDOUR, ::Session or ::PluginManager
+//       consolidate vstfx_get_info_lx() and vstfx_get_info_fst()
+
+/* prototypes */
+#ifdef WINDOWS_VST_SUPPORT
+#include <fst.h>
+static bool
+vstfx_instantiate_and_get_info_fst (const char* dllpath, vector<VSTInfo*> *infos, int uniqueID);
+#endif
+
+#ifdef LXVST_SUPPORT
+static bool vstfx_instantiate_and_get_info_lx (const char* dllpath, vector<VSTInfo*> *infos, int uniqueID);
+#endif
+
+static int vstfx_current_loading_id = 0;
+
 static char *
 read_string (FILE *fp)
 {
@@ -85,6 +102,22 @@ read_int (FILE* fp, int* n)
 }
 
 static void
+vstfx_free_info (VSTInfo *info)
+{
+	for (int i = 0; i < info->numParams; i++) {
+		free (info->ParamNames[i]);
+		free (info->ParamLabels[i]);
+	}
+
+	free (info->name);
+	free (info->creator);
+	free (info->Category);
+	free (info->ParamNames);
+	free (info->ParamLabels);
+	free (info);
+}
+
+static void
 vstfx_clear_info_list (vector<VSTInfo *> *infos)
 {
 	for (vector<VSTInfo *>::iterator i = infos->begin(); i != infos->end(); ++i) {
@@ -212,15 +245,16 @@ vstfx_write_info_file (FILE* fp, vector<VSTInfo *> *infos)
 	} else if (infos->size() == 1) {
 		vstfx_write_info_block(fp, infos->front());
 	} else {
-		PBD::warning << "Zero plugins in VST." << endmsg; // XXX here?
+		PBD::error << "Zero plugins in VST." << endmsg; // XXX here? rather make this impossible before if it ain't already.
 	}
 }
 
 static string
-vstfx_infofile_path (char* dllpath, int personal)
+vstfx_infofile_path (const char* dllpath, int personal)
 {
 	string dir;
 	if (personal) {
+		// TODO use XDG_CACHE_HOME
 		dir = Glib::build_filename (Glib::get_home_dir (), ".fst");
 
 		/* If the directory doesn't exist, try to create it */
@@ -240,7 +274,7 @@ vstfx_infofile_path (char* dllpath, int personal)
 }
 
 static char *
-vstfx_infofile_stat (char *dllpath, struct stat* statbuf, int personal)
+vstfx_infofile_stat (const char *dllpath, struct stat* statbuf, int personal)
 {
 	if (strstr (dllpath, ".so" ) == 0 && strstr(dllpath, ".dll") == 0) {
 		return 0;
@@ -272,7 +306,7 @@ vstfx_infofile_stat (char *dllpath, struct stat* statbuf, int personal)
 
 
 static FILE *
-vstfx_infofile_for_read (char* dllpath)
+vstfx_infofile_for_read (const char* dllpath)
 {
 	struct stat own_statbuf;
 	struct stat sys_statbuf;
@@ -300,7 +334,7 @@ vstfx_infofile_for_read (char* dllpath)
 }
 
 static FILE *
-vstfx_infofile_create (char* dllpath, int personal)
+vstfx_infofile_create (const char* dllpath, int personal)
 {
 	if (strstr (dllpath, ".so" ) == 0 && strstr(dllpath, ".dll") == 0) {
 		return 0;
@@ -311,7 +345,7 @@ vstfx_infofile_create (char* dllpath, int personal)
 }
 
 static FILE *
-vstfx_infofile_for_write (char* dllpath)
+vstfx_infofile_for_write (const char* dllpath)
 {
 	FILE* f;
 
@@ -340,8 +374,8 @@ int vstfx_can_midi (VSTState* vstfx)
 	return false;
 }
 
-static VSTInfo *
-vstfx_info_from_plugin (VSTState* vstfx)
+static VSTInfo*
+vstfx_parse_vst_state (VSTState* vstfx)
 {
 	assert (vstfx);
 
@@ -374,9 +408,25 @@ vstfx_info_from_plugin (VSTState* vstfx)
 		info->creator = strdup (creator);
 	}
 
+
+	switch (plugin->dispatcher (plugin, effGetPlugCategory, 0, 0, 0, 0))
+	{
+		case kPlugCategEffect:         info->Category = strdup ("Effect"); break;
+		case kPlugCategSynth:          info->Category = strdup ("Synth"); break;
+		case kPlugCategAnalysis:       info->Category = strdup ("Anaylsis"); break;
+		case kPlugCategMastering:      info->Category = strdup ("Mastering"); break;
+		case kPlugCategSpacializer:    info->Category = strdup ("Spacializer"); break;
+		case kPlugCategRoomFx:         info->Category = strdup ("RoomFx"); break;
+		case kPlugSurroundFx:          info->Category = strdup ("SurroundFx"); break;
+		case kPlugCategRestoration:    info->Category = strdup ("Restoration"); break;
+		case kPlugCategOfflineProcess: info->Category = strdup ("Offline"); break;
+		case kPlugCategShell:          info->Category = strdup ("Shell"); break;
+		case kPlugCategGenerator:      info->Category = strdup ("Generator"); break;
+		default:                       info->Category = strdup ("Unknown"); break;
+	}
+
 	info->UniqueID = plugin->uniqueID;
 
-	info->Category = strdup("None"); /* XXX */
 	info->numInputs = plugin->numInputs;
 	info->numOutputs = plugin->numOutputs;
 	info->numParams = plugin->numParams;
@@ -405,45 +455,120 @@ vstfx_info_from_plugin (VSTState* vstfx)
 	return info;
 }
 
+static void
+vstfx_info_from_plugin (const char *dllpath, VSTState* vstfx, vector<VSTInfo *> *infos, int type)
+{
+	assert(vstfx);
+	VSTInfo *info;
+
+	if ((info = vstfx_parse_vst_state(vstfx))) {
+		infos->push_back(info);
+#if 1
+		/* If this plugin is a Shell and we are not already inside a shell plugin
+		 * read the info for all of the plugins contained in this shell.
+		 */
+		if (!strncmp (info->Category, "Shell", 5)
+		    && vstfx->handle->plugincnt == 1) {
+			int id;
+			vector< pair<int, string> > ids;
+			AEffect *plugin = vstfx->plugin;
+			string path = vstfx->handle->path;
+
+			do {
+				char name[65] = "Unknown\0";
+				id = plugin->dispatcher (plugin, effShellGetNextPlugin, 0, 0, name, 0);
+				ids.push_back(std::make_pair(id, name));
+			} while ( id != 0 );
+
+			switch(type) {
+#ifdef WINDOWS_VST_SUPPORT
+				case 1: fst_close(vstfx); break;
+#endif
+#ifdef LXVST_SUPPORT
+				case 2: vstfx_close (vstfx); break;
+#endif
+				default: assert(0); break;
+			}
+
+			for (vector< pair<int, string> >::iterator x = ids.begin(); x != ids.end(); ++x) {
+				id = (*x).first;
+				if (id == 0) continue;
+				/* recurse vstfx_get_info() */
+
+				bool ok;
+				switch (type) { // TODO use lib ardour's type
+#ifdef WINDOWS_VST_SUPPORT
+					case 1:  ok = vstfx_instantiate_and_get_info_fst(dllpath, infos, id); break;
+#endif
+#ifdef LXVST_SUPPORT
+					case 2:  ok = vstfx_instantiate_and_get_info_lx(dllpath, infos, id); break;
+#endif
+					default: ok = false;
+				}
+				if (ok) {
+					// One shell (some?, all?) does not report the actual plugin name
+					// even after the shelled plugin has been instantiated.
+					// Replace the name of the shell with the real name.
+					info = infos->back();
+					free (info->name);
+
+					if ((*x).second.length() == 0) {
+						info->name = strdup("Unknown");
+					}
+					else {
+						info->name = strdup ((*x).second.c_str());
+					}
+				}
+			}
+		} else {
+			switch(type) {
+#ifdef WINDOWS_VST_SUPPORT
+				case 1: fst_close(vstfx); break;
+#endif
+#ifdef LXVST_SUPPORT
+				case 2: vstfx_close (vstfx); break;
+#endif
+				default: assert(0); break;
+			}
+		}
+#endif
+	}
+}
+
 /* A simple 'dummy' audiomaster callback which should be ok,
    we will only be instantiating the plugin in order to get its info
 */
 
 static intptr_t
-simple_master_callback (AEffect *, int32_t opcode, int32_t, intptr_t, void *, float)
+simple_master_callback (AEffect *, int32_t opcode, int32_t, intptr_t, void *ptr, float)
 {
-#if 0
-	static const char* can_do_strings[] = {
-		X_("supportShell"),
-		X_("shellCategory")
+	const char* vstfx_can_do_strings[] = {
+		"supportShell",
+		"shellCategory"
 	};
-	static const int can_do_string_count = sizeof (can_do_strings) / sizeof (char*);
-	static int current_loading_id = 0;
-#endif
+	const int vstfx_can_do_string_count = 2;
 
 	if (opcode == audioMasterVersion) {
 		return 2;
 	}
-#if 0
 	else if (opcode == audioMasterCanDo) {
-		for (int i = 0; i < can_do_string_count; i++) {
-			if (! strcmp(can_do_strings[i], (const char*)ptr)) {
+		for (int i = 0; i < vstfx_can_do_string_count; i++) {
+			if (! strcmp(vstfx_can_do_strings[i], (const char*)ptr)) {
 				return 1;
 			}
 		}
 		return 0;
 	}
 	else if (opcode == audioMasterCurrentId) {
-		return current_loading_id;
+		return vstfx_current_loading_id;
 	}
-#endif
 	else {
 		return 0;
 	}
 }
 
 static bool
-vstfx_get_info_from_file(char* dllpath, vector<VSTInfo*> *infos)
+vstfx_get_info_from_file(const char* dllpath, vector<VSTInfo*> *infos)
 {
 	FILE* infofile;
 	bool rv = false;
@@ -457,94 +582,70 @@ vstfx_get_info_from_file(char* dllpath, vector<VSTInfo*> *infos)
 	return rv;
 }
 
-void
-vstfx_free_info (VSTInfo *info)
-{
-	for (int i = 0; i < info->numParams; i++) {
-		free (info->ParamNames[i]);
-		free (info->ParamLabels[i]);
-	}
-
-	free (info->name);
-	free (info->creator);
-	free (info->Category);
-	free (info->ParamNames);
-	free (info->ParamLabels);
-	free (info);
-}
-
-void
-vstfx_free_info_list (vector<VSTInfo *> *infos)
-{
-	for (vector<VSTInfo *>::iterator i = infos->begin(); i != infos->end(); ++i) {
-		vstfx_free_info(*i);
-	}
-	delete infos;
-}
-
 #ifdef LXVST_SUPPORT
-/** Try to get plugin info - first by looking for a .fsi cache of the
-    data, and if that doesn't exist, load the plugin, get its data and
-    then cache it for future ref
-*/
-
-vector<VSTInfo *> *
-vstfx_get_info_lx (char* dllpath)
+static bool
+vstfx_instantiate_and_get_info_lx (
+		const char* dllpath, vector<VSTInfo*> *infos, int uniqueID)
 {
-	FILE* infofile;
 	VSTHandle* h;
 	VSTState* vstfx;
-	vector<VSTInfo*> *infos = new vector<VSTInfo*>;
-
-	// TODO check blacklist
-	// TODO pre-check file extension ?
-
-	if (vstfx_get_info_from_file(dllpath, infos)) {
-		PBD::info << "using cache for LinuxVST plugin '" << dllpath << "'" << endmsg;
-		return infos;
-	}
-
 	if (!(h = vstfx_load(dllpath))) {
 		PBD::warning << "Cannot get LinuxVST information from " << dllpath << ": load failed." << endmsg;
-		return infos;
+		return false;
 	}
 
+	vstfx_current_loading_id = uniqueID;
+
 	if (!(vstfx = vstfx_instantiate(h, simple_master_callback, 0))) {
 		vstfx_unload(h);
 		PBD::warning << "Cannot get LinuxVST information from " << dllpath << ": instantiation failed." << endmsg;
-		return infos;
-	}
-
-	infofile = vstfx_infofile_for_write (dllpath);
-
-	if (!infofile) {
-		vstfx_close(vstfx);
-		vstfx_unload(h);
-		PBD::warning << "Cannot get LinuxVST information from " << dllpath << ": cannot create new FST info file." << endmsg;
-		return 0;
+		return false;
 	}
 
-	VSTInfo* info = vstfx_info_from_plugin (vstfx);
-	infos->push_back(info); //XXX
+	vstfx_current_loading_id = 0;
 
-	vstfx_write_info_file (infofile, infos);
-	fclose (infofile);
+	vstfx_info_from_plugin(dllpath, vstfx, infos, 2);
 
 	vstfx_close (vstfx);
 	vstfx_unload (h);
-	return infos;
+	return true;
 }
 #endif
 
 #ifdef WINDOWS_VST_SUPPORT
-#include <fst.h>
-
-vector<VSTInfo *> *
-vstfx_get_info_fst (char* dllpath)
+static bool
+vstfx_instantiate_and_get_info_fst (
+		const char* dllpath, vector<VSTInfo*> *infos, int uniqueID)
 {
-	FILE* infofile;
 	VSTHandle* h;
 	VSTState* vstfx;
+	if(!(h = fst_load(dllpath))) {
+		PBD::warning << "Cannot get VST information from " << dllpath << ": load failed." << endmsg;
+		return false;
+	}
+
+	vstfx_current_loading_id = uniqueID;
+
+	if(!(vstfx = fst_instantiate(h, simple_master_callback, 0))) {
+		fst_unload(&h);
+		vstfx_current_loading_id = 0;
+		PBD::warning << "Cannot get VST information from " << dllpath << ": instantiation failed." << endmsg;
+		return false;
+	}
+	vstfx_current_loading_id = 0;
+
+	vstfx_info_from_plugin(dllpath, vstfx, infos, 1);
+
+	fst_close(vstfx);
+	//fst_unload(&h); // XXX -> fst_close()
+	return true;
+}
+#endif
+
+static vector<VSTInfo *> *
+vstfx_get_info (const char* dllpath, int type)
+{
+	FILE* infofile;
 	vector<VSTInfo*> *infos = new vector<VSTInfo*>;
 
 	// TODO check blacklist
@@ -555,34 +656,59 @@ vstfx_get_info_fst (char* dllpath)
 		return infos;
 	}
 
-	if(!(h = fst_load(dllpath))) {
-		PBD::warning << "Cannot get VST information from " << dllpath << ": load failed." << endmsg;
-		return infos;
+	bool ok;
+	// TODO blacklist (in case instantiat fails)
+	switch (type) { // TODO use lib ardour's type
+#ifdef WINDOWS_VST_SUPPORT
+		case 1:  ok = vstfx_instantiate_and_get_info_fst(dllpath, infos, 0); break;
+#endif
+#ifdef LXVST_SUPPORT
+		case 2:  ok = vstfx_instantiate_and_get_info_lx(dllpath, infos, 0); break;
+#endif
+		default: ok = false;
 	}
 
-	if(!(vstfx = fst_instantiate(h, simple_master_callback, 0))) {
-		fst_unload(&h);
-		PBD::warning << "Cannot get VST information from " << dllpath << ": instantiation failed." << endmsg;
+	if (!ok) {
+		PBD::warning << "Cannot get VST information for " << dllpath << "." << endmsg;
 		return infos;
 	}
 
-	infofile = vstfx_infofile_for_write (dllpath);
+	// TODO un-blacklist
 
+	infofile = vstfx_infofile_for_write (dllpath);
 	if (!infofile) {
-		fst_close(vstfx);
-		//fst_unload(&h); // XXX -> fst_close()
-		PBD::warning << "Cannot get VST information from " << dllpath << ": cannot create new FST info file." << endmsg;
-		return 0;
+		PBD::warning << "Cannot cache VST information for " << dllpath << ": cannot create new FST info file." << endmsg;
+		return infos;
+	} else {
+		vstfx_write_info_file (infofile, infos);
+		fclose (infofile);
 	}
+	return infos;
+}
 
-	VSTInfo* info = vstfx_info_from_plugin(vstfx);
-	infos->push_back(info); //XXX
+/* *** public API *** */
 
-	vstfx_write_info_file (infofile, infos);
-	fclose (infofile);
+void
+vstfx_free_info_list (vector<VSTInfo *> *infos)
+{
+	for (vector<VSTInfo *>::iterator i = infos->begin(); i != infos->end(); ++i) {
+		vstfx_free_info(*i);
+	}
+	delete infos;
+}
 
-	fst_close(vstfx);
-	//fst_unload(&h); // XXX -> fst_close()
-	return infos;
+#ifdef LXVST_SUPPORT
+vector<VSTInfo *> *
+vstfx_get_info_lx (char* dllpath)
+{
+	return vstfx_get_info(dllpath, 2);
+}
+#endif
+
+#ifdef WINDOWS_VST_SUPPORT
+vector<VSTInfo *> *
+vstfx_get_info_fst (char* dllpath)
+{
+	return vstfx_get_info(dllpath, 1);
 }
 #endif