Add support for built-in file/url unzip/untar

This introduces new build-dependency: libarchive
(http://www.libarchive.org/)

4 files changed, 433 insertions, 1 deletions

diff --git a/libs/pbd/file_archive.cc b/libs/pbd/file_archive.cc
new file mode 100644
index 0000000000..f8700ce60c
--- /dev/null
+++ b/libs/pbd/file_archive.cc
@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2016 Robin Gareus <robin@gareus.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <cstdio>
+
+#include "pbd/gstdio_compat.h"
+#include <glibmm.h>
+
+#include <archive.h>
+#include <archive_entry.h>
+#include <curl/curl.h>
+
+#include "pbd/failed_constructor.h"
+#include "pbd/file_archive.h"
+
+using namespace PBD;
+
+static size_t
+write_callback (void* buffer, size_t size, size_t nmemb, void* d)
+{
+	FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
+	size_t realsize = size * nmemb;
+
+	p->lock ();
+	p->data = (uint8_t*) realloc ((void*) p->data, p->size + realsize);
+	memcpy (&p->data[p->size], buffer, realsize);
+	p->size += realsize;
+	p->signal ();
+	p->unlock ();
+	return realsize;
+}
+
+static void*
+get_url (void* arg)
+{
+	FileArchive::Request* r = (FileArchive::Request*) arg;
+	CURL* curl;
+
+	curl = curl_easy_init ();
+	curl_easy_setopt (curl, CURLOPT_URL, r->url);
+
+	curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_callback);
+	curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void*) &r->mp);
+	curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
+
+	curl_easy_perform (curl);
+	curl_easy_cleanup (curl);
+
+	r->mp.lock ();
+	r->mp.done = 1;
+	r->mp.signal ();
+	r->mp.unlock ();
+
+	return NULL;
+}
+
+static ssize_t
+ar_read (struct archive* a, void* d, const void** buff)
+{
+	FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
+	size_t rv;
+
+	p->lock ();
+	while (p->size == 0) {
+		if (p->done) {
+			p->unlock ();
+			return 0;
+		}
+		p->wait ();
+	}
+
+	rv = p->size > 8192 ? 8192 : p->size;
+	memcpy (p->buf, p->data, rv);
+	if (p->size > rv) {
+		memmove (p->data, &p->data[rv], p->size - rv);
+	}
+	p->size -= rv;
+	*buff = p->buf;
+	p->unlock ();
+	return rv;
+}
+
+static int
+ar_copy_data (struct archive *ar, struct archive *aw)
+{
+	for (;;) {
+		const void *buff;
+		size_t size;
+		int64_t offset;
+		int r;
+		r = archive_read_data_block (ar, &buff, &size, &offset);
+		if (r == ARCHIVE_EOF) {
+			return (ARCHIVE_OK);
+		}
+		if (r != ARCHIVE_OK) {
+			return (r);
+		}
+		r = archive_write_data_block (aw, buff, size, offset);
+		if (r != ARCHIVE_OK) {
+			fprintf (stderr, "Extract/Write Archive: %s", archive_error_string(aw));
+			return (r);
+		}
+	}
+}
+
+static struct archive*
+setup_archive ()
+{
+	struct archive* a;
+	a = archive_read_new ();
+	archive_read_support_filter_all (a);
+	archive_read_support_format_all (a);
+	return a;
+}
+
+
+FileArchive::FileArchive (const std::string& url)
+	: _req (url)
+{
+	if (!_req.url) {
+		fprintf (stderr, "Invalid Archive URL/filename\n");
+		throw failed_constructor ();
+	}
+}
+
+int
+FileArchive::inflate (const std::string& destdir)
+{
+	int rv = -1;
+	std::string pwd (Glib::get_current_dir ());
+
+	if (g_chdir (destdir.c_str ())) {
+		fprintf (stderr, "Archive: cannot chdir to '%s'\n", destdir.c_str ());
+		return rv;
+	}
+
+	if (_req.is_remote ()) {
+		rv = extract_url ();
+	} else {
+		rv = extract_file ();
+	}
+
+	g_chdir (pwd.c_str());
+	return rv;
+}
+
+std::vector<std::string>
+FileArchive::contents ()
+{
+	if (_req.is_remote ()) {
+		return contents_url ();
+	} else {
+		return contents_file ();
+	}
+}
+
+std::vector<std::string>
+FileArchive::contents_file ()
+{
+	struct archive* a = setup_archive ();
+	if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
+		fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
+		return std::vector<std::string> ();
+	}
+	return get_contents (a);
+}
+
+std::vector<std::string>
+FileArchive::contents_url ()
+{
+	_req.mp.reset ();
+	pthread_create (&_tid, NULL, get_url, (void*)&_req);
+
+	struct archive* a = setup_archive ();
+	archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
+	std::vector<std::string> rv (get_contents (a));
+
+	pthread_join (_tid, NULL);
+	return rv;
+}
+
+int
+FileArchive::extract_file ()
+{
+	struct archive* a = setup_archive ();
+	if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
+		fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
+		return -1;
+	}
+	return do_extract (a);
+}
+
+int
+FileArchive::extract_url ()
+{
+	_req.mp.reset ();
+	pthread_create (&_tid, NULL, get_url, (void*)&_req);
+
+	struct archive* a = setup_archive ();
+	archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
+	int rv = do_extract (a);
+
+	pthread_join (_tid, NULL);
+	return rv;
+}
+
+std::vector<std::string>
+FileArchive::get_contents (struct archive* a)
+{
+	std::vector<std::string> rv;
+	struct archive_entry* entry;
+	for (;;) {
+		int r = archive_read_next_header (a, &entry);
+		if (r == ARCHIVE_EOF) {
+			break;
+		}
+		if (r != ARCHIVE_OK) {
+			fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
+			break;
+		}
+		rv.push_back (archive_entry_pathname (entry));
+	}
+
+	archive_read_close (a);
+	archive_read_free (a);
+	return rv;
+}
+
+int
+FileArchive::do_extract (struct archive* a)
+{
+	int flags = ARCHIVE_EXTRACT_TIME;
+
+	int rv = 0;
+	struct archive_entry* entry;
+	struct archive *ext;
+
+	ext = archive_write_disk_new();
+	archive_write_disk_set_options(ext, flags);
+
+	for (;;) {
+		int r = archive_read_next_header (a, &entry);
+		if (r == ARCHIVE_EOF) {
+			break;
+		}
+		if (r != ARCHIVE_OK) {
+			fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
+			break;
+		}
+
+#if 0 // hacky alternative to chdir
+		const std::string full_path = Glib::build_filename (destdir, archive_entry_pathname (entry));
+		archive_entry_set_pathname (entry, full_path.c_str());
+#endif
+
+		r = archive_write_header(ext, entry);
+		if (r != ARCHIVE_OK) {
+			fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
+		} else {
+			ar_copy_data (a, ext);
+			r = archive_write_finish_entry (ext);
+			if (r != ARCHIVE_OK) {
+				fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
+				rv = -1;
+				break;
+			}
+		}
+	}
+
+	archive_read_close (a);
+	archive_read_free (a);
+	archive_write_close(ext);
+	archive_write_free(ext);
+	return rv;
+}
diff --git a/libs/pbd/pbd/file_archive.h b/libs/pbd/pbd/file_archive.h
new file mode 100644
index 0000000000..ad1f85941f
--- /dev/null
+++ b/libs/pbd/pbd/file_archive.h
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2016 Robin Gareus <robin@gareus.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+#ifndef _pbd_archive_h_
+#define _pbd_archive_h_
+
+#include <pthread.h>
+
+#include "pbd/signals.h"
+
+#ifndef LIBPBD_API
+#include "pbd/libpbd_visibility.h"
+#endif
+
+
+namespace PBD {
+
+class LIBPBD_API FileArchive
+{
+	public:
+		FileArchive (const std::string& url);
+
+		int inflate (const std::string& destdir);
+		std::vector<std::string> contents ();
+
+		//PBD::Signal2<void, size_t, size_t> progress; // TODO
+
+		struct MemPipe {
+			public:
+				MemPipe ()
+					: data (NULL)
+					, size (0)
+					, done (false)
+				{
+					pthread_mutex_init (&_lock, NULL);
+					pthread_cond_init (&_ready, NULL);
+				}
+
+				~MemPipe ()
+				{
+					lock ();
+					free (data);
+					unlock ();
+
+					pthread_mutex_destroy (&_lock);
+					pthread_cond_destroy (&_ready);
+				}
+
+				void reset ()
+				{
+					lock ();
+					free (data);
+					data = 0;
+					size = 0;
+					done = false;
+					unlock ();
+				}
+
+				void lock ()   { pthread_mutex_lock (&_lock); }
+				void unlock () { pthread_mutex_unlock (&_lock); }
+				void signal () { pthread_cond_signal (&_ready); }
+				void wait ()   { pthread_cond_wait (&_ready, &_lock); }
+
+				uint8_t  buf[8192];
+				uint8_t* data;
+				size_t   size;
+				bool     done;
+
+			private:
+				pthread_mutex_t _lock;
+				pthread_cond_t  _ready;
+		};
+
+		struct Request {
+			public:
+				Request (const std::string& u)
+				{
+					if (u.size () > 0) {
+						url = strdup (u.c_str());
+					} else {
+						url = NULL;
+					}
+				}
+
+				~Request ()
+				{
+					free (url);
+				}
+
+				bool is_remote () const
+				{
+					if (!strncmp (url, "https://", 8) || !strncmp (url, "http://", 7) || !strncmp (url, "ftp://", 6)) {
+						return true;
+					}
+					return false;
+				}
+
+				char* url;
+				MemPipe mp;
+		};
+
+	private:
+
+		int process_file ();
+		int process_url ();
+
+		std::vector<std::string> contents_url ();
+		std::vector<std::string> contents_file ();
+
+		int extract_url ();
+		int extract_file ();
+
+		int do_extract (struct archive* a);
+		std::vector<std::string> get_contents (struct archive *a);
+
+		bool is_url ();
+
+		Request   _req;
+		pthread_t _tid;
+};
+
+} /* namespace */
+#endif // _reallocpool_h_
diff --git a/libs/pbd/wscript b/libs/pbd/wscript
index 6f872c987f..4620fa7bc1 100644
--- a/libs/pbd/wscript
+++ b/libs/pbd/wscript
@@ -47,6 +47,7 @@ libpbd_sources = [
     'epa.cc',
     'error.cc',
     'ffs.cc',
+    'file_archive.cc',
     'file_utils.cc',
     'fpu.cc',
     'id.cc',
@@ -134,7 +135,7 @@ def build(bld):
     obj.includes     = ['.']
     obj.name         = 'libpbd'
     obj.target       = 'pbd'
-    obj.uselib       = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM'
+    obj.uselib       = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM ARCHIVE CURL'
     if sys.platform == 'darwin':
         TaskGen.task_gen.mappings['.mm'] = TaskGen.task_gen.mappings['.cc']
         if 'cocoa_open_uri.mm' not in obj.source:
diff --git a/wscript b/wscript
index ae6d35b878..c76e3b6ccc 100644
--- a/wscript
+++ b/wscript
@@ -970,6 +970,7 @@ def configure(conf):
     autowaf.check_pkg(conf, 'sndfile', uselib_store='SNDFILE', atleast_version='1.0.18', mandatory=True)
     autowaf.check_pkg(conf, 'giomm-2.4', uselib_store='GIOMM', atleast_version='2.2', mandatory=True)
     autowaf.check_pkg(conf, 'libcurl', uselib_store='CURL', atleast_version='7.0.0', mandatory=True)
+    autowaf.check_pkg(conf, 'libarchive', uselib_store='ARCHIVE', atleast_version='3.0.0', mandatory=True)
     autowaf.check_pkg(conf, 'liblo', uselib_store='LO', atleast_version='0.26', mandatory=True)
     autowaf.check_pkg(conf, 'taglib', uselib_store='TAGLIB', atleast_version='1.6', mandatory=True)
     autowaf.check_pkg(conf, 'vamp-sdk', uselib_store='VAMPSDK', atleast_version='2.1', mandatory=True)