introduce new all-in-RAM MIDI datastructure and use it for MIDI playback

13 files changed, 426 insertions, 256 deletions

diff --git a/libs/ardour/ardour/disk_io.h b/libs/ardour/ardour/disk_io.h
index 52cfe45d0e..37aa75b6d2 100644
--- a/libs/ardour/ardour/disk_io.h
+++ b/libs/ardour/ardour/disk_io.h
@@ -28,7 +28,9 @@
 #include "pbd/rcu.h"
 
 #include "ardour/interpolation.h"
+#include "ardour/midi_buffer.h"
 #include "ardour/processor.h"
+#include "ardour/rt_midibuffer.h"
 
 namespace PBD {
 	template<class T> class PlaybackBuffer;
@@ -191,6 +193,8 @@ protected:
 	gint                         _samples_written_to_ringbuffer;
 	gint                         _samples_read_from_ringbuffer;
 
+	RTMidiBuffer _mbuf;
+
 	static void get_location_times (const Location* location, samplepos_t* start, samplepos_t* end, samplepos_t* length);
 };
 
diff --git a/libs/ardour/ardour/midi_playlist.h b/libs/ardour/ardour/midi_playlist.h
index 82ec7b65e1..cd2ccddef7 100644
--- a/libs/ardour/ardour/midi_playlist.h
+++ b/libs/ardour/ardour/midi_playlist.h
@@ -88,6 +88,8 @@ public:
 	                 uint32_t                         chan_n = 0,
 	                 MidiChannelFilter*               filter = NULL);
 
+	void dump (Evoral::EventSink<samplepos_t>&, MidiChannelFilter*);
+
 	int set_state (const XMLNode&, int version);
 
 	bool destroy_region (boost::shared_ptr<Region>);
diff --git a/libs/ardour/ardour/midi_region.h b/libs/ardour/ardour/midi_region.h
index ad3b1c5044..2b99ede256 100644
--- a/libs/ardour/ardour/midi_region.h
+++ b/libs/ardour/ardour/midi_region.h
@@ -118,6 +118,11 @@ class LIBARDOUR_API MidiRegion : public Region
 
 	void clobber_sources (boost::shared_ptr<MidiSource> source);
 
+	int dump_to (Evoral::EventSink<samplepos_t>& dst,
+	              uint32_t                        chan_n,
+	              NoteMode                        mode,
+	              MidiChannelFilter*              filter) const;
+
   protected:
 
 	virtual bool can_trim_start_before_source_start () const {
diff --git a/libs/ardour/ardour/rt_midibuffer.h b/libs/ardour/ardour/rt_midibuffer.h
new file mode 100644
index 0000000000..4b7a669128
--- /dev/null
+++ b/libs/ardour/ardour/rt_midibuffer.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2007-2016 David Robillard <d@drobilla.net>
+ * Copyright (C) 2007-2018 Paul Davis <paul@linuxaudiosystems.com>
+ * Copyright (C) 2009-2010 Carl Hetherington <carl@carlh.net>
+ * Copyright (C) 2009 Hans Baier <hansfbaier@googlemail.com>
+ * Copyright (C) 2014-2016 Robin Gareus <robin@gareus.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef __ardour_rt_midi_buffer_h__
+#define __ardour_rt_midi_buffer_h__
+
+#include <map>
+
+#include "evoral/Event.hpp"
+#include "evoral/EventSink.hpp"
+#include "ardour/types.h"
+
+namespace ARDOUR {
+
+class MidiBuffer;
+
+/**  */
+class LIBARDOUR_API RTMidiBuffer : public Evoral::EventSink<samplepos_t>
+{
+  public:
+	typedef samplepos_t TimeType;
+
+	RTMidiBuffer (size_t capacity);
+	~RTMidiBuffer();
+
+	void clear() { _size = 0; }
+	void resize(size_t);
+	size_t size() const { return _size; }
+
+	uint32_t write (TimeType time, Evoral::EventType type, uint32_t size, const uint8_t* buf);
+	uint32_t read (MidiBuffer& dst, samplepos_t start, samplepos_t end, samplecnt_t offset = 0);
+
+  private:
+	size_t _size;
+	size_t _capacity;
+	uint8_t* _data; ///< event data
+	typedef std::multimap<TimeType,size_t> Map;
+	Map _map;
+};
+
+} // namespace ARDOUR
+
+#endif // __ardour_rt_midi_buffer_h__
diff --git a/libs/ardour/disk_io.cc b/libs/ardour/disk_io.cc
index e001e58aa5..076912b1d0 100644
--- a/libs/ardour/disk_io.cc
+++ b/libs/ardour/disk_io.cc
@@ -57,6 +57,7 @@ DiskIOProcessor::DiskIOProcessor (Session& s, string const & str, Flag f)
 	, _need_butler (false)
 	, channels (new ChannelList)
 	, _midi_buf (0)
+	, _mbuf (0)
 	, _samples_written_to_ringbuffer (0)
 	, _samples_read_from_ringbuffer (0)
 {
@@ -189,6 +190,7 @@ DiskIOProcessor::configure_io (ChanCount in, ChanCount out)
 	if (in.n_midi() > 0 && !_midi_buf) {
 		const size_t size = _session.butler()->midi_diskstream_buffer_size();
 		_midi_buf = new MidiRingBuffer<samplepos_t>(size);
+		_mbuf.resize (1048576);
 		changed = true;
 	}
 
diff --git a/libs/ardour/disk_reader.cc b/libs/ardour/disk_reader.cc
index 1eda5b7fe6..41767238a0 100644
--- a/libs/ardour/disk_reader.cc
+++ b/libs/ardour/disk_reader.cc
@@ -380,7 +380,7 @@ DiskReader::run (BufferSet& bufs, samplepos_t start_sample, samplepos_t end_samp
 	/* MIDI data handling */
 
   midi:
-	if (!declick_in_progress() && bufs.count().n_midi() && _midi_buf) {
+	if (!declick_in_progress() && bufs.count().n_midi()) {
 		MidiBuffer* dst;
 
 		if (_no_disk_output) {
@@ -427,55 +427,9 @@ DiskReader::run (BufferSet& bufs, samplepos_t start_sample, samplepos_t end_samp
 			}
 		}
 
-		if (_playlists[DataType::MIDI]) {
-			/* MIDI butler needed part */
-
-			uint32_t samples_read = g_atomic_int_get(const_cast<gint*>(&_samples_read_from_ringbuffer));
-			uint32_t samples_written = g_atomic_int_get(const_cast<gint*>(&_samples_written_to_ringbuffer));
-
-			/*
-			  cerr << name() << " MDS written: " << samples_written << " - read: " << samples_read <<
-			  " = " << samples_written - samples_read
-			  << " + " << disk_samples_to_consume << " < " << midi_readahead << " = " << need_butler << ")" << endl;
-			*/
-
-			/* samples_read will generally be less than samples_written, but
-			 * immediately after an overwrite, we can end up having read some data
-			 * before we've written any. we don't need to trip an assert() on this,
-			 * but we do need to check so that the decision on whether or not we
-			 * need the butler is done correctly.
-			 */
-
-			/* furthermore..
-			 *
-			 * Doing heavy GUI operations[1] can stall also the butler.
-			 * The RT-thread meanwhile will happily continue and
-			 * ‘samples_read’ (from buffer to output) will become larger
-			 * than ‘samples_written’ (from disk to buffer).
-			 *
-			 * The disk-stream is now behind..
-			 *
-			 * In those cases the butler needs to be summed to refill the buffer (done now)
-			 * AND we need to skip (samples_read - samples_written). ie remove old events
-			 * before playback_sample from the rinbuffer.
-			 *
-			 * [1] one way to do so is described at #6170.
-			 * For me just popping up the context-menu on a MIDI-track header
-			 * of a track with a large (think beethoven :) midi-region also did the
-			 * trick. The playhead stalls for 2 or 3 sec, until the context-menu shows.
-			 *
-			 * In both cases the root cause is that redrawing MIDI regions on the GUI is still very slow
-			 * and can stall
-			 */
-			if (samples_read <= samples_written) {
-				if ((samples_written - samples_read) + disk_samples_to_consume < midi_readahead) {
-					butler_required = true;
-				}
-			} else {
-				butler_required = true;
-			}
-
-		}
+		/* All of MIDI is in RAM, no need to call the butler unless we
+		 * have to overwrite buffers because of a playlist change.
+		 */
 
 		_need_butler = butler_required;
 	}
@@ -494,6 +448,8 @@ DiskReader::pending_overwrite () const {
 	return g_atomic_int_get (&_pending_overwrite) != 0;
 }
 
+PBD::Timing minsert;
+
 void
 DiskReader::set_pending_overwrite ()
 {
@@ -553,25 +509,21 @@ DiskReader::overwrite_existing_buffers ()
 
   midi:
 
-	if (_midi_buf && _playlists[DataType::MIDI]) {
+	if (_playlists[DataType::MIDI]) {
 
-		/* Clear the playback buffer contents.  This is safe as long as the butler
-		   thread is suspended, which it should be.
-		*/
-		_midi_buf->reset ();
-		_midi_buf->reset_tracker ();
-
-		g_atomic_int_set (&_samples_read_from_ringbuffer, 0);
-		g_atomic_int_set (&_samples_written_to_ringbuffer, 0);
+		minsert.reset();minsert.start();
+		_mbuf.clear(); midi_playlist()->dump (_mbuf, 0);
+		minsert.update(); cerr << "Reading " << name()  << " took " << minsert.elapsed() << " microseconds, final size = " << _mbuf.size() << endl;
 
+#if 0
 		/* Resolve all currently active notes in the playlist.  This is more
 		   aggressive than it needs to be: ideally we would only resolve what is
 		   absolutely necessary, but this seems difficult and/or impossible without
 		   having the old data or knowing what change caused the overwrite.
 		*/
 		midi_playlist()->resolve_note_trackers (*_midi_buf, overwrite_sample);
+#endif
 
-		midi_read (overwrite_sample, _chunk_samples, false);
 		file_sample[DataType::MIDI] = overwrite_sample; // overwrite_sample was adjusted by ::midi_read() to the new position
 	}
 
@@ -616,20 +568,6 @@ DiskReader::seek (samplepos_t sample, bool complete_refill)
 		(*chan)->rbuf->reset ();
 	}
 
-	if (g_atomic_int_get (&_samples_read_from_ringbuffer) == 0) {
-		/* we haven't read anything since the last seek,
-		   so flush all note trackers to prevent
-		   wierdness
-		*/
-		reset_tracker ();
-	}
-
-	if (_midi_buf) {
-		_midi_buf->reset();
-	}
-	g_atomic_int_set(&_samples_read_from_ringbuffer, 0);
-	g_atomic_int_set(&_samples_written_to_ringbuffer, 0);
-
 	playback_sample = sample;
 	file_sample[DataType::AUDIO] = sample;
 	file_sample[DataType::MIDI] = sample;
@@ -663,16 +601,9 @@ DiskReader::can_internal_playback_seek (sampleoffset_t distance)
 		}
 	}
 
-	if (distance < 0) {
-		return true; // XXX TODO un-seek MIDI
-	}
-
-	/* 2. MIDI */
-
-	uint32_t samples_read    = g_atomic_int_get(&_samples_read_from_ringbuffer);
-	uint32_t samples_written = g_atomic_int_get(&_samples_written_to_ringbuffer);
+	/* 2. MIDI can always seek any distance */
 
-	return ((samples_written - samples_read) < distance);
+	return true;
 }
 
 void
@@ -1120,9 +1051,11 @@ DiskReader::move_processor_automation (boost::weak_ptr<Processor> p, list< Evora
 void
 DiskReader::reset_tracker ()
 {
+#if 0
 	if (_midi_buf) {
 		_midi_buf->reset_tracker ();
 	}
+#endif
 
 	boost::shared_ptr<MidiPlaylist> mp (midi_playlist());
 
@@ -1134,9 +1067,11 @@ DiskReader::reset_tracker ()
 void
 DiskReader::resolve_tracker (Evoral::EventSink<samplepos_t>& buffer, samplepos_t time)
 {
+#if 0
 	if (_midi_buf) {
 		_midi_buf->resolve_tracker(buffer, time);
 	}
+#endif
 
 	boost::shared_ptr<MidiPlaylist> mp (midi_playlist());
 
@@ -1154,7 +1089,9 @@ DiskReader::get_midi_playback (MidiBuffer& dst, samplepos_t start_sample, sample
 	MidiBuffer* target;
 	samplepos_t nframes = ::llabs (end_sample - start_sample);
 
-	assert (_midi_buf);
+	if (_mbuf.size() == 0) {
+		return;
+	}
 
 	if ((ms & MonitoringInput) == 0) {
 		/* Route::process_output_buffers() clears the buffer as-needed */
@@ -1168,15 +1105,6 @@ DiskReader::get_midi_playback (MidiBuffer& dst, samplepos_t start_sample, sample
 
 		Location* loc = _loop_location;
 
-		DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose (
-			             "%1 MDS pre-read read %8 offset = %9 @ %4..%5 from %2 write to %3, LOOPED ? %6 .. %7\n", _name,
-			             _midi_buf->get_read_ptr(), _midi_buf->get_write_ptr(), start_sample, end_sample,
-			             (loc ? loc->start() : -1), (loc ? loc->end() : -1), nframes, Port::port_offset()));
-
-		//cerr << "======== PRE ========\n";
-		//_midi_buf->dump (cerr);
-		//cerr << "----------------\n";
-
 		size_t events_read = 0;
 
 		if (loc) {
@@ -1192,7 +1120,7 @@ DiskReader::get_midi_playback (MidiBuffer& dst, samplepos_t start_sample, sample
 				   beyond the loop end.
 				*/
 
-				_midi_buf->resolve_tracker (*target, 0);
+				// _midi_buf->resolve_tracker (*target, 0);
 			}
 
 			/* for split-cycles we need to offset the events */
@@ -1215,37 +1143,28 @@ DiskReader::get_midi_playback (MidiBuffer& dst, samplepos_t start_sample, sample
 				if (first) {
 					DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("loop read #1, from %1 for %2\n",
 					                                                      effective_start, first));
-					events_read = _midi_buf->read (*target, effective_start, first);
+					events_read = _mbuf.read (*target, effective_start, effective_start + first);
 				}
 
 				if (second) {
 					DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("loop read #2, from %1 for %2\n",
 					                                                      loc->start(), second));
-					events_read += _midi_buf->read (*target, loc->start(), second);
+					events_read += _mbuf.read (*target, loc->start(), loc->start() + second);
 				}
 
 			} else {
 				DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("loop read #3, adjusted start as %1 for %2\n",
-				                                                      effective_start, nframes));
-				events_read = _midi_buf->read (*target, effective_start, effective_start + nframes);
+				                                                effective_start, nframes));
+				events_read = _mbuf.read (*target, effective_start, effective_start + nframes);
 			}
 		} else {
-			const size_t n_skipped = _midi_buf->skip_to (start_sample);
-			if (n_skipped > 0) {
-				warning << string_compose(_("MidiDiskstream %1: skipped %2 events, possible underflow"), id(), n_skipped) << endmsg;
-			}
 			DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("playback buffer read, from %1 to %2 (%3)", start_sample, end_sample, nframes));
-			events_read = _midi_buf->read (*target, start_sample, end_sample, Port::port_offset ());
+			events_read = _mbuf.read (*target, start_sample, end_sample, Port::port_offset ());
 		}
 
-		DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose (
-			             "%1 MDS events read %2 range %3 .. %4 rspace %5 wspace %6 r@%7 w@%8\n",
-			             _name, events_read, playback_sample, playback_sample + nframes,
-			             _midi_buf->read_space(), _midi_buf->write_space(),
-			             _midi_buf->get_read_ptr(), _midi_buf->get_write_ptr()));
+		DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("%1 MDS events read %2 range %3 .. %4\n", _name, events_read, playback_sample, playback_sample + nframes));
 	}
 
-	g_atomic_int_add (&_samples_read_from_ringbuffer, nframes);
 
 	if (!_no_disk_output && (ms & MonitoringInput)) {
 		dst.merge_from (*target, nframes);
@@ -1265,158 +1184,20 @@ DiskReader::get_midi_playback (MidiBuffer& dst, samplepos_t start_sample, sample
 		cerr << "----------------\n";
 	}
 #endif
-#if 0
-	cerr << "======== MIDI Disk Buffer ========\n";
-	_midi_buf->dump (cerr);
-	cerr << "----------------\n";
-#endif
 }
 
 /** @a start is set to the new sample position (TIME) read up to */
 int
 DiskReader::midi_read (samplepos_t& start, samplecnt_t dur, bool reversed)
 {
-	samplecnt_t this_read   = 0;
-	samplepos_t loop_end    = 0;
-	samplepos_t loop_start  = 0;
-	samplecnt_t loop_length = 0;
-	Location*  loc         = _loop_location;
-	samplepos_t effective_start = start;
-	Evoral::Range<samplepos_t>*  loop_range (0);
-
-	assert(_midi_buf);
-
-	DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("MDS::midi_read @ %1 cnt %2\n", start, dur));
-
-	boost::shared_ptr<MidiTrack> mt = boost::dynamic_pointer_cast<MidiTrack>(_route);
-	MidiChannelFilter* filter = mt ? &mt->playback_filter() : 0;
-	sampleoffset_t loop_offset = 0;
-
-	if (!reversed && loc) {
-		get_location_times (loc, &loop_start, &loop_end, &loop_length);
-	}
-
-	while (dur) {
-
-		/* take any loop into account. we can't read past the end of the loop. */
-
-		if (loc && !reversed) {
-
-			if (!loop_range) {
-				loop_range = new Evoral::Range<samplepos_t> (loop_start, loop_end-1); // inclusive semantics require -1
-			}
-
-			/* if we are (seamlessly) looping, ensure that the first sample we read is at the correct
-			   position within the loop.
-			*/
-
-			effective_start = loop_range->squish (effective_start);
-
-			if ((loop_end - effective_start) <= dur) {
-				/* too close to end of loop to read "dur", so
-				   shorten it.
-				*/
-				this_read = loop_end - effective_start;
-			} else {
-				this_read = dur;
-			}
-
-		} else {
-			this_read = dur;
-		}
-
-		if (this_read == 0) {
-			break;
-		}
-
-		this_read = min (dur,this_read);
-
-		DEBUG_TRACE (DEBUG::MidiDiskIO, string_compose ("MDS ::read at %1 for %2 loffset %3\n", effective_start, this_read, loop_offset));
-
-		if (midi_playlist()->read (*_midi_buf, effective_start, this_read, loop_range, 0, filter) != this_read) {
-			error << string_compose(
-					_("MidiDiskstream %1: cannot read %2 from playlist at sample %3"),
-					id(), this_read, start) << endmsg;
-			return -1;
-		}
-
-		g_atomic_int_add (&_samples_written_to_ringbuffer, this_read);
-
-		if (reversed) {
-
-			// Swap note ons with note offs here.  etc?
-			// Fully reversing MIDI requires look-ahead (well, behind) to find previous
-			// CC values etc.  hard.
-
-		} else {
-
-			/* adjust passed-by-reference argument (note: this is
-			   monotonic and does not reflect looping.
-			*/
-			start += this_read;
-
-			/* similarly adjust effective_start, but this may be
-			   readjusted for seamless looping as we continue around
-			   the loop.
-			*/
-			effective_start += this_read;
-		}
-
-		dur -= this_read;
-	}
-
 	return 0;
 }
 
 int
 DiskReader::refill_midi ()
 {
-	if (!_playlists[DataType::MIDI] || !_midi_buf) {
-		return 0;
-	}
-
-	const size_t  write_space = _midi_buf->write_space();
-	const bool reversed    = _session.transport_speed() < 0.0f;
-
-	DEBUG_TRACE (DEBUG::DiskIO, string_compose ("MIDI refill, write space = %1 file sample = %2\n", write_space, file_sample[DataType::MIDI]));
-
-	/* no space to write */
-	if (write_space == 0) {
-		return 0;
-	}
-
-	if (reversed) {
-		return 0;
-	}
-
-	/* at end: nothing to do */
-
-	samplepos_t ffm = file_sample[DataType::MIDI];
-
-	if (ffm == max_samplepos) {
-		return 0;
-	}
-
-	int ret = 0;
-	const uint32_t samples_read = g_atomic_int_get (&_samples_read_from_ringbuffer);
-	const uint32_t samples_written = g_atomic_int_get (&_samples_written_to_ringbuffer);
-
-	if ((samples_read < samples_written) && (samples_written - samples_read) >= midi_readahead) {
-		return 0;
-	}
-
-	samplecnt_t to_read = midi_readahead - ((samplecnt_t)samples_written - (samplecnt_t)samples_read);
-
-	to_read = min (to_read, (samplecnt_t) (max_samplepos - ffm));
-	to_read = min (to_read, (samplecnt_t) write_space);
-
-	if (midi_read (ffm, to_read, reversed)) {
-		ret = -1;
-	}
-
-	file_sample[DataType::MIDI] = ffm;
-
-	return ret;
+	/* nothing to do ... it's all in RAM thanks to overwrite */
+	return 0;
 }
 
 void
diff --git a/libs/ardour/midi_buffer.cc b/libs/ardour/midi_buffer.cc
index f0f82a1e29..9196d84de5 100644
--- a/libs/ardour/midi_buffer.cc
+++ b/libs/ardour/midi_buffer.cc
@@ -53,7 +53,7 @@ MidiBuffer::~MidiBuffer()
 }
 
 void
-MidiBuffer::resize(size_t size)
+MidiBuffer::resize (size_t size)
 {
 	if (_data && size < _capacity) {
 
@@ -65,11 +65,15 @@ MidiBuffer::resize(size_t size)
 		return;
 	}
 
-	cache_aligned_free (_data);
+	uint8_t* old_data = _data;
 
 	cache_aligned_malloc ((void**) &_data, size);
 
-	_size = 0;
+	if (_size) {
+		memcpy (_data, old_data, _size);
+	}
+
+	cache_aligned_free (old_data);
 	_capacity = size;
 
 	assert(_data);
@@ -222,6 +226,8 @@ MidiBuffer::push_back(TimeType time, size_t size, const uint8_t* data)
 	return true;
 }
 
+extern PBD::Timing minsert;
+
 bool
 MidiBuffer::insert_event(const Evoral::Event<TimeType>& ev)
 {
@@ -233,7 +239,7 @@ MidiBuffer::insert_event(const Evoral::Event<TimeType>& ev)
 	const size_t bytes_to_merge = stamp_size + ev.size();
 
 	if (_size + bytes_to_merge >= _capacity) {
-		cerr << "MidiBuffer::push_back failed (buffer is full)" << endl;
+		cerr << string_compose ("MidiBuffer::push_back failed (buffer is full: size: %1 capacity %2 new bytes %3)", _size, _capacity, bytes_to_merge) << endl;
 		PBD::stacktrace (cerr, 20);
 		return false;
 	}
@@ -254,8 +260,10 @@ MidiBuffer::insert_event(const Evoral::Event<TimeType>& ev)
 		insert_offset = m.offset;
 		break;
 	}
+
 	if (insert_offset == -1) {
-		return push_back(ev);
+		bool r = push_back(ev);
+		return r;
 	}
 
 	// don't use memmove - it may use malloc(!)
@@ -280,6 +288,20 @@ MidiBuffer::write(TimeType time, Evoral::EventType type, uint32_t size, const ui
 	return size;
 }
 
+uint32_t
+MidiBuffer::append(TimeType time, Evoral::EventType type, uint32_t size, const uint8_t* buf)
+{
+	const size_t bytes_to_merge = sizeof(TimeType) + size;
+
+	if (_size + bytes_to_merge >= _capacity) {
+		resize (_capacity + 8192);
+	}
+
+	return push_back (time, size, buf);
+}
+
+
+
 /** Reserve space for a new event in the buffer.
  *
  * This call is for copying MIDI directly into the buffer, the data location
diff --git a/libs/ardour/midi_playlist.cc b/libs/ardour/midi_playlist.cc
index 1522f17184..2231791de6 100644
--- a/libs/ardour/midi_playlist.cc
+++ b/libs/ardour/midi_playlist.cc
@@ -202,7 +202,7 @@ MidiPlaylist::read (Evoral::EventSink<samplepos_t>& dst,
 
 		/* Read from region into target. */
 		DEBUG_TRACE (DEBUG::MidiPlaylistIO, string_compose ("read from %1 at %2 for %3 LR %4 .. %5\n",
-		                                                    mr->name(), start, dur, 
+		                                                    mr->name(), start, dur,
 		                                                    (loop_range ? loop_range->from : -1),
 		                                                    (loop_range ? loop_range->to : -1)));
 		mr->read_at (tgt, start, dur, loop_range, tracker->cursor, chan_n, _note_mode, &tracker->tracker, filter);
@@ -489,3 +489,62 @@ MidiPlaylist::contained_automation()
 
 	return ret;
 }
+
+void
+MidiPlaylist::dump (Evoral::EventSink<samplepos_t>& dst, MidiChannelFilter* filter)
+{
+	typedef pair<MidiStateTracker*,samplepos_t> TrackerInfo;
+
+	Playlist::RegionReadLock rl (this);
+
+	DEBUG_TRACE (DEBUG::MidiPlaylistIO, "---- MidiPlaylist::dump-----\n");
+
+	std::vector< boost::shared_ptr<Region> > regs;
+
+	for (RegionList::iterator i = regions.begin(); i != regions.end(); ++i) {
+
+		/* check for the case of solo_selection */
+
+		if (_session.solo_selection_active() && SoloSelectedActive() && !SoloSelectedListIncludes ((const Region*) &(**i))) {
+			continue;
+		}
+
+		regs.push_back (*i);
+	}
+
+	/* If we are reading from a single region, we can read directly into dst.  Otherwise,
+	   we read into a temporarily list, sort it, then write that to dst.
+	*/
+	Evoral::EventList<samplepos_t>  evlist;
+	Evoral::EventSink<samplepos_t>& tgt = (regs.size() == 1) ? dst : evlist;
+
+	DEBUG_TRACE (DEBUG::MidiPlaylistIO, string_compose ("\t%1 regions to read, direct: %2\n", regs.size(), (regs.size() == 1)));
+
+	for (vector<boost::shared_ptr<Region> >::iterator i = regs.begin(); i != regs.end(); ++i) {
+
+		boost::shared_ptr<MidiRegion> mr = boost::dynamic_pointer_cast<MidiRegion>(*i);
+
+		if (!mr) {
+			continue;
+		}
+
+		DEBUG_TRACE (DEBUG::MidiPlaylistIO, string_compose ("dump from %1 at %2\n", mr->name()));
+		mr->dump_to (tgt, 0, _note_mode, filter);
+	}
+
+	if (!evlist.empty()) {
+		/* We've read from multiple regions into evlist, sort the event list by time. */
+		EventsSortByTimeAndType<samplepos_t> cmp;
+		evlist.sort (cmp);
+
+		/* Copy ordered events from event list to dst. */
+		for (Evoral::EventList<samplepos_t>::iterator e = evlist.begin(); e != evlist.end(); ++e) {
+			Evoral::Event<samplepos_t>* ev (*e);
+			dst.write (ev->time(), ev->event_type(), ev->size(), ev->buffer());
+			delete ev;
+		}
+	}
+
+	DEBUG_TRACE (DEBUG::MidiPlaylistIO, "---- End MidiPlaylist::dump ----\n");
+
+}
diff --git a/libs/ardour/midi_region.cc b/libs/ardour/midi_region.cc
index e48c5cf2b1..177555c0ba 100644
--- a/libs/ardour/midi_region.cc
+++ b/libs/ardour/midi_region.cc
@@ -481,6 +481,79 @@ MidiRegion::_read_at (const SourceList&              /*srcs*/,
 	return to_read;
 }
 
+
+int
+MidiRegion::dump_to (Evoral::EventSink<samplepos_t>& dst,
+                     uint32_t                        chan_n,
+                     NoteMode                        mode,
+                     MidiChannelFilter*              filter) const
+{
+	sampleoffset_t internal_offset = 0;
+
+	/* precondition: caller has verified that we cover the desired section */
+
+	assert(chan_n == 0);
+
+	if (muted()) {
+		return 0; /* read nothing */
+	}
+
+
+	/* dump pulls from zero to infinity ... */
+
+	if (_position) {
+		/* we are starting the read from before the start of the region */
+		internal_offset = 0;
+	} else {
+		/* we are starting the read from after the start of the region */
+		internal_offset = -_position;
+	}
+
+	if (internal_offset >= _length) {
+		return 0; /* read nothing */
+	}
+
+	boost::shared_ptr<MidiSource> src = midi_source(chan_n);
+
+	Glib::Threads::Mutex::Lock lm(src->mutex());
+
+	src->set_note_mode(lm, mode);
+
+#if 0
+	cerr << "MR " << name () << " read @ " << position << " + " << to_read
+	     << " dur was " << dur
+	     << " len " << _length
+	     << " l-io " << (_length - internal_offset)
+	     << " _position = " << _position
+	     << " _start = " << _start
+	     << " intoffset = " << internal_offset
+	     << " quarter_note = " << quarter_note()
+	     << " start_beat = " << _start_beats
+	     << endl;
+#endif
+
+	MidiCursor cursor;
+
+	/* This call reads events from a source and writes them to `dst' timed in session samples */
+
+	src->midi_read (
+		lm, // source lock
+		dst, // destination buffer
+		_position - _start, // start position of the source in session samples
+		_start + internal_offset, // where to start reading in the source
+		max_samplecnt,
+		0,
+		cursor,
+		0,
+		filter,
+		_filtered_parameters,
+		quarter_note(),
+		_start_beats);
+
+	return 0;
+}
+
+
 XMLNode&
 MidiRegion::state ()
 {
diff --git a/libs/ardour/midi_source.cc b/libs/ardour/midi_source.cc
index a2e8839651..2f6c62f4a8 100644
--- a/libs/ardour/midi_source.cc
+++ b/libs/ardour/midi_source.cc
@@ -38,6 +38,7 @@
 #include "pbd/xml++.h"
 #include "pbd/pthread_utils.h"
 #include "pbd/basename.h"
+#include "pbd/timing.h"
 
 #include "evoral/Control.hpp"
 #include "evoral/EventSink.hpp"
@@ -241,6 +242,7 @@ MidiSource::midi_read (const Lock&                        lm,
 
 	cursor.last_read_end = start + cnt;
 
+	samplepos_t time_samples = 0;
 	// Copy events in [start, start + cnt) into dst
 	for (; i != _model->end(); ++i) {
 
@@ -277,7 +279,7 @@ MidiSource::midi_read (const Lock&                        lm,
 				   destroying events in the model during read. */
 				Evoral::Event<Temporal::Beats> ev(*i, true);
 				if (!filter->filter(ev.buffer(), ev.size())) {
-					dst.write(time_samples, ev.event_type(), ev.size(), ev.buffer());
+					dst.write (time_samples, ev.event_type(), ev.size(), ev.buffer());
 				} else {
 					DEBUG_TRACE (DEBUG::MidiSourceIO,
 					             string_compose ("%1: filter event @ %2 type %3 size %4\n",
diff --git a/libs/ardour/rt_midibuffer.cc b/libs/ardour/rt_midibuffer.cc
new file mode 100644
index 0000000000..a440c7bb6f
--- /dev/null
+++ b/libs/ardour/rt_midibuffer.cc
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2019 Paul Davis <paul@linuxaudiosystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <iostream>
+
+#include "pbd/malign.h"
+#include "pbd/compose.h"
+#include "pbd/error.h"
+#include "pbd/debug.h"
+#include "pbd/stacktrace.h"
+
+#include "ardour/debug.h"
+#include "ardour/midi_buffer.h"
+#include "ardour/rt_midibuffer.h"
+
+using namespace std;
+using namespace ARDOUR;
+using namespace PBD;
+
+RTMidiBuffer::RTMidiBuffer (size_t capacity)
+	: _size (0)
+	, _capacity (0)
+	, _data (0)
+{
+	if (capacity) {
+		resize (capacity);
+		clear ();
+	}
+}
+
+RTMidiBuffer::~RTMidiBuffer()
+{
+	cache_aligned_free (_data);
+}
+
+void
+RTMidiBuffer::resize (size_t size)
+{
+	if (_data && size < _capacity) {
+
+		if (_size < size) {
+			/* truncate */
+			_size = size;
+		}
+
+		return;
+	}
+
+	uint8_t* old_data = _data;
+
+	cache_aligned_malloc ((void**) &_data, size);
+
+	if (_size) {
+		memcpy (_data, old_data, _size);
+	}
+
+	cache_aligned_free (old_data);
+	_capacity = size;
+
+	assert(_data);
+}
+
+uint32_t
+RTMidiBuffer::write (TimeType time, Evoral::EventType /*type*/, uint32_t size, const uint8_t* buf)
+{
+	/* This buffer stores only MIDI, we don't care about the value of "type" */
+
+	const size_t bytes_to_merge = sizeof (size) + size;
+
+	if (_size + bytes_to_merge > _capacity) {
+		resize (_capacity + 8192); // XXX 8192 is completely arbitrary
+	}
+
+	_map.insert (make_pair (time, _size));
+
+	uint8_t* addr = &_data[_size];
+
+	*(reinterpret_cast<uint32_t*>(addr)) = size;
+	addr += sizeof (size);
+	memcpy (addr, buf, size);
+
+	_size += bytes_to_merge;
+
+	return size;
+}
+
+uint32_t
+RTMidiBuffer::read (MidiBuffer& dst, samplepos_t start, samplepos_t end, samplecnt_t offset)
+{
+	Map::iterator iter = _map.lower_bound (start);
+	uint32_t count = 0;
+#ifndef NDEBUG
+	TimeType unadjusted_time;
+#endif
+
+	DEBUG_TRACE (DEBUG::MidiRingBuffer, string_compose ("read from %1 .. %2\n", start, end));
+
+	while ((iter != _map.end()) && (iter->first < end)) {
+
+		/* the event consists of a size followed by bytes of MIDI
+		 * data. It begins at _data[iter->second], which was stored in
+		 * our map when we wrote the event into the data structure.
+		 */
+
+		uint8_t* addr = &_data[iter->second];
+		TimeType evtime = iter->first;
+
+#ifndef NDEBUG
+		unadjusted_time = evtime;
+#endif
+		uint32_t size = *(reinterpret_cast<Evoral::EventType*>(addr));
+		addr += sizeof (size);
+
+		/* Adjust event times to be relative to 'start', taking
+		 * 'offset' into account.
+		 */
+
+		evtime -= start;
+		evtime += offset;
+
+		uint8_t* write_loc = dst.reserve (evtime, size);
+
+		if (write_loc == 0) {
+			DEBUG_TRACE (DEBUG::MidiRingBuffer, string_compose ("MidiRingBuffer: overflow in destination MIDI buffer, stopped after %1 events, dst size = %2\n", count, dst.size()));
+			cerr << string_compose ("MidiRingBuffer: overflow in destination MIDI buffer, stopped after %1 events, dst size = %1\n", count, dst.size()) << endl;
+			break;
+		}
+
+		DEBUG_TRACE (DEBUG::MidiRingBuffer, string_compose ("read event sz %1 @ %2\n", size, unadjusted_time));
+
+		memcpy (write_loc, addr, size);
+
+		++iter;
+		++count;
+	}
+
+	DEBUG_TRACE (DEBUG::MidiRingBuffer, string_compose ("total events found for %1 .. %2 = %3\n", start, end, count));
+	return count;
+}
diff --git a/libs/ardour/smf_source.cc b/libs/ardour/smf_source.cc
index 3f098fa928..9b7acb9402 100644
--- a/libs/ardour/smf_source.cc
+++ b/libs/ardour/smf_source.cc
@@ -34,6 +34,7 @@
 #include "pbd/file_utils.h"
 #include "pbd/stl_delete.h"
 #include "pbd/strsplit.h"
+#include "pbd/timing.h"
 
 #include "pbd/gstdio_compat.h"
 #include <glibmm/miscutils.h>
@@ -213,6 +214,8 @@ SMFSource::close ()
 	/* nothing to do: file descriptor is never kept open */
 }
 
+extern PBD::Timing minsert;
+
 /** All stamps in audio samples */
 samplecnt_t
 SMFSource::read_unlocked (const Lock&                    lock,
diff --git a/libs/ardour/wscript b/libs/ardour/wscript
index 4ab373b684..b66c3cf8c0 100644
--- a/libs/ardour/wscript
+++ b/libs/ardour/wscript
@@ -240,6 +240,7 @@ libardour_sources = [
         'strip_silence.cc',
         'system_exec.cc',
         'revision.cc',
+        'rt_midibuffer.cc',
         'tape_file_matcher.cc',
         'template_utils.cc',
         'tempo.cc',