From 53ee3e2e722eeac80a16dc4fcf15dc6c34e1e099 Mon Sep 17 00:00:00 2001 From: Robin Gareus Date: Tue, 13 Sep 2016 16:14:08 +0200 Subject: Add support for built-in file/url unzip/untar This introduces new build-dependency: libarchive (http://www.libarchive.org/) --- libs/pbd/file_archive.cc | 292 ++++++++++++++++++++++++++++++++++++++++++++ libs/pbd/pbd/file_archive.h | 138 +++++++++++++++++++++ libs/pbd/wscript | 3 +- wscript | 1 + 4 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 libs/pbd/file_archive.cc create mode 100644 libs/pbd/pbd/file_archive.h diff --git a/libs/pbd/file_archive.cc b/libs/pbd/file_archive.cc new file mode 100644 index 0000000000..f8700ce60c --- /dev/null +++ b/libs/pbd/file_archive.cc @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2016 Robin Gareus + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include + +#include "pbd/gstdio_compat.h" +#include + +#include +#include +#include + +#include "pbd/failed_constructor.h" +#include "pbd/file_archive.h" + +using namespace PBD; + +static size_t +write_callback (void* buffer, size_t size, size_t nmemb, void* d) +{ + FileArchive::MemPipe* p = (FileArchive::MemPipe*)d; + size_t realsize = size * nmemb; + + p->lock (); + p->data = (uint8_t*) realloc ((void*) p->data, p->size + realsize); + memcpy (&p->data[p->size], buffer, realsize); + p->size += realsize; + p->signal (); + p->unlock (); + return realsize; +} + +static void* +get_url (void* arg) +{ + FileArchive::Request* r = (FileArchive::Request*) arg; + CURL* curl; + + curl = curl_easy_init (); + curl_easy_setopt (curl, CURLOPT_URL, r->url); + + curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void*) &r->mp); + curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L); + + curl_easy_perform (curl); + curl_easy_cleanup (curl); + + r->mp.lock (); + r->mp.done = 1; + r->mp.signal (); + r->mp.unlock (); + + return NULL; +} + +static ssize_t +ar_read (struct archive* a, void* d, const void** buff) +{ + FileArchive::MemPipe* p = (FileArchive::MemPipe*)d; + size_t rv; + + p->lock (); + while (p->size == 0) { + if (p->done) { + p->unlock (); + return 0; + } + p->wait (); + } + + rv = p->size > 8192 ? 8192 : p->size; + memcpy (p->buf, p->data, rv); + if (p->size > rv) { + memmove (p->data, &p->data[rv], p->size - rv); + } + p->size -= rv; + *buff = p->buf; + p->unlock (); + return rv; +} + +static int +ar_copy_data (struct archive *ar, struct archive *aw) +{ + for (;;) { + const void *buff; + size_t size; + int64_t offset; + int r; + r = archive_read_data_block (ar, &buff, &size, &offset); + if (r == ARCHIVE_EOF) { + return (ARCHIVE_OK); + } + if (r != ARCHIVE_OK) { + return (r); + } + r = archive_write_data_block (aw, buff, size, offset); + if (r != ARCHIVE_OK) { + fprintf (stderr, "Extract/Write Archive: %s", archive_error_string(aw)); + return (r); + } + } +} + +static struct archive* +setup_archive () +{ + struct archive* a; + a = archive_read_new (); + archive_read_support_filter_all (a); + archive_read_support_format_all (a); + return a; +} + + +FileArchive::FileArchive (const std::string& url) + : _req (url) +{ + if (!_req.url) { + fprintf (stderr, "Invalid Archive URL/filename\n"); + throw failed_constructor (); + } +} + +int +FileArchive::inflate (const std::string& destdir) +{ + int rv = -1; + std::string pwd (Glib::get_current_dir ()); + + if (g_chdir (destdir.c_str ())) { + fprintf (stderr, "Archive: cannot chdir to '%s'\n", destdir.c_str ()); + return rv; + } + + if (_req.is_remote ()) { + rv = extract_url (); + } else { + rv = extract_file (); + } + + g_chdir (pwd.c_str()); + return rv; +} + +std::vector +FileArchive::contents () +{ + if (_req.is_remote ()) { + return contents_url (); + } else { + return contents_file (); + } +} + +std::vector +FileArchive::contents_file () +{ + struct archive* a = setup_archive (); + if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) { + fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a)); + return std::vector (); + } + return get_contents (a); +} + +std::vector +FileArchive::contents_url () +{ + _req.mp.reset (); + pthread_create (&_tid, NULL, get_url, (void*)&_req); + + struct archive* a = setup_archive (); + archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL); + std::vector rv (get_contents (a)); + + pthread_join (_tid, NULL); + return rv; +} + +int +FileArchive::extract_file () +{ + struct archive* a = setup_archive (); + if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) { + fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a)); + return -1; + } + return do_extract (a); +} + +int +FileArchive::extract_url () +{ + _req.mp.reset (); + pthread_create (&_tid, NULL, get_url, (void*)&_req); + + struct archive* a = setup_archive (); + archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL); + int rv = do_extract (a); + + pthread_join (_tid, NULL); + return rv; +} + +std::vector +FileArchive::get_contents (struct archive* a) +{ + std::vector rv; + struct archive_entry* entry; + for (;;) { + int r = archive_read_next_header (a, &entry); + if (r == ARCHIVE_EOF) { + break; + } + if (r != ARCHIVE_OK) { + fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a)); + break; + } + rv.push_back (archive_entry_pathname (entry)); + } + + archive_read_close (a); + archive_read_free (a); + return rv; +} + +int +FileArchive::do_extract (struct archive* a) +{ + int flags = ARCHIVE_EXTRACT_TIME; + + int rv = 0; + struct archive_entry* entry; + struct archive *ext; + + ext = archive_write_disk_new(); + archive_write_disk_set_options(ext, flags); + + for (;;) { + int r = archive_read_next_header (a, &entry); + if (r == ARCHIVE_EOF) { + break; + } + if (r != ARCHIVE_OK) { + fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a)); + break; + } + +#if 0 // hacky alternative to chdir + const std::string full_path = Glib::build_filename (destdir, archive_entry_pathname (entry)); + archive_entry_set_pathname (entry, full_path.c_str()); +#endif + + r = archive_write_header(ext, entry); + if (r != ARCHIVE_OK) { + fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext)); + } else { + ar_copy_data (a, ext); + r = archive_write_finish_entry (ext); + if (r != ARCHIVE_OK) { + fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext)); + rv = -1; + break; + } + } + } + + archive_read_close (a); + archive_read_free (a); + archive_write_close(ext); + archive_write_free(ext); + return rv; +} diff --git a/libs/pbd/pbd/file_archive.h b/libs/pbd/pbd/file_archive.h new file mode 100644 index 0000000000..ad1f85941f --- /dev/null +++ b/libs/pbd/pbd/file_archive.h @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2016 Robin Gareus + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ +#ifndef _pbd_archive_h_ +#define _pbd_archive_h_ + +#include + +#include "pbd/signals.h" + +#ifndef LIBPBD_API +#include "pbd/libpbd_visibility.h" +#endif + + +namespace PBD { + +class LIBPBD_API FileArchive +{ + public: + FileArchive (const std::string& url); + + int inflate (const std::string& destdir); + std::vector contents (); + + //PBD::Signal2 progress; // TODO + + struct MemPipe { + public: + MemPipe () + : data (NULL) + , size (0) + , done (false) + { + pthread_mutex_init (&_lock, NULL); + pthread_cond_init (&_ready, NULL); + } + + ~MemPipe () + { + lock (); + free (data); + unlock (); + + pthread_mutex_destroy (&_lock); + pthread_cond_destroy (&_ready); + } + + void reset () + { + lock (); + free (data); + data = 0; + size = 0; + done = false; + unlock (); + } + + void lock () { pthread_mutex_lock (&_lock); } + void unlock () { pthread_mutex_unlock (&_lock); } + void signal () { pthread_cond_signal (&_ready); } + void wait () { pthread_cond_wait (&_ready, &_lock); } + + uint8_t buf[8192]; + uint8_t* data; + size_t size; + bool done; + + private: + pthread_mutex_t _lock; + pthread_cond_t _ready; + }; + + struct Request { + public: + Request (const std::string& u) + { + if (u.size () > 0) { + url = strdup (u.c_str()); + } else { + url = NULL; + } + } + + ~Request () + { + free (url); + } + + bool is_remote () const + { + if (!strncmp (url, "https://", 8) || !strncmp (url, "http://", 7) || !strncmp (url, "ftp://", 6)) { + return true; + } + return false; + } + + char* url; + MemPipe mp; + }; + + private: + + int process_file (); + int process_url (); + + std::vector contents_url (); + std::vector contents_file (); + + int extract_url (); + int extract_file (); + + int do_extract (struct archive* a); + std::vector get_contents (struct archive *a); + + bool is_url (); + + Request _req; + pthread_t _tid; +}; + +} /* namespace */ +#endif // _reallocpool_h_ diff --git a/libs/pbd/wscript b/libs/pbd/wscript index 6f872c987f..4620fa7bc1 100644 --- a/libs/pbd/wscript +++ b/libs/pbd/wscript @@ -47,6 +47,7 @@ libpbd_sources = [ 'epa.cc', 'error.cc', 'ffs.cc', + 'file_archive.cc', 'file_utils.cc', 'fpu.cc', 'id.cc', @@ -134,7 +135,7 @@ def build(bld): obj.includes = ['.'] obj.name = 'libpbd' obj.target = 'pbd' - obj.uselib = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM' + obj.uselib = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM ARCHIVE CURL' if sys.platform == 'darwin': TaskGen.task_gen.mappings['.mm'] = TaskGen.task_gen.mappings['.cc'] if 'cocoa_open_uri.mm' not in obj.source: diff --git a/wscript b/wscript index ae6d35b878..c76e3b6ccc 100644 --- a/wscript +++ b/wscript @@ -970,6 +970,7 @@ def configure(conf): autowaf.check_pkg(conf, 'sndfile', uselib_store='SNDFILE', atleast_version='1.0.18', mandatory=True) autowaf.check_pkg(conf, 'giomm-2.4', uselib_store='GIOMM', atleast_version='2.2', mandatory=True) autowaf.check_pkg(conf, 'libcurl', uselib_store='CURL', atleast_version='7.0.0', mandatory=True) + autowaf.check_pkg(conf, 'libarchive', uselib_store='ARCHIVE', atleast_version='3.0.0', mandatory=True) autowaf.check_pkg(conf, 'liblo', uselib_store='LO', atleast_version='0.26', mandatory=True) autowaf.check_pkg(conf, 'taglib', uselib_store='TAGLIB', atleast_version='1.6', mandatory=True) autowaf.check_pkg(conf, 'vamp-sdk', uselib_store='VAMPSDK', atleast_version='2.1', mandatory=True) -- cgit v1.2.3