jacob/songbird
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Voice Rework -- Events, Track Queues (#806)

Browse Source 
This implements a proof-of-concept for an improved audio frontend. The largest change is the introduction of events and event handling: both by time elapsed and by track events, such as ending or looping. Following on from this, the library now includes a basic, event-driven track queue system (which people seem to ask for unusually often). A new sample, `examples/13_voice_events`, demonstrates both the `TrackQueue` system and some basic events via the `~queue` and `~play_fade` commands.

Locks are removed from around the control of `Audio` objects, which should allow the backend to be moved to a more granular futures-based backend solution in a cleaner way.
This commit is contained in:
Kyle Simpson
2020-10-29 20:25:20 +00:00
committed by Alex M. M
commit 7e4392ae68
76 changed files with 8756 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

303
src/input/cached/compressed.rs Normal file
View File

@@ -0,0 +1,303 @@
use super::{apply_length_hint, compressed_cost_per_sec, default_config};
use crate::{
constants::*,
input::{
error::{Error, Result},
CodecType,
Container,
Input,
Metadata,
Reader,
},
};
use audiopus::{
coder::Encoder as OpusEncoder,
Application,
Bitrate,
Channels,
Error as OpusError,
ErrorCode as OpusErrorCode,
SampleRate,
};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use std::{
convert::TryInto,
io::{Error as IoError, ErrorKind as IoErrorKind, Read, Result as IoResult},
mem,
sync::atomic::{AtomicUsize, Ordering},
};
use streamcatcher::{Config, NeedsBytes, Stateful, Transform, TransformPosition, TxCatcher};
use tracing::{debug, trace};
/// A wrapper around an existing [`Input`] which compresses
/// the input using the Opus codec before storing it in memory.
///
/// The main purpose of this wrapper is to enable seeking on
/// incompatible sources (i.e., ffmpeg output) and to ease resource
/// consumption for commonly reused/shared tracks. [`Restartable`]
/// and [`Memory`] offer the same functionality with different
/// tradeoffs.
///
/// This is intended for use with larger, repeatedly used audio
/// tracks shared between sources, and stores the sound data
/// retrieved as **compressed Opus audio**. There is an associated memory cost,
/// but this is far smaller than using a [`Memory`].
///
/// [`Input`]: ../struct.Input.html
/// [`Memory`]: struct.Memory.html
/// [`Restartable`]: ../struct.Restartable.html
#[derive(Clone, Debug)]
pub struct Compressed {
/// Inner shared bytestore.
pub raw: TxCatcher<Box<Input>, OpusCompressor>,
/// Metadata moved out of the captured source.
pub metadata: Metadata,
/// Stereo-ness of the captured source.
pub stereo: bool,
}
impl Compressed {
/// Wrap an existing [`Input`] with an in-memory store, compressed using Opus.
///
/// [`Input`]: ../struct.Input.html
/// [`Metadata.duration`]: ../struct.Metadata.html#structfield.duration
pub fn new(source: Input, bitrate: Bitrate) -> Result<Self> {
Self::with_config(source, bitrate, None)
}
/// Wrap an existing [`Input`] with an in-memory store, compressed using Opus.
///
/// `config.length_hint` may be used to control the size of the initial chunk, preventing
/// needless allocations and copies. If this is not present, the value specified in
/// `source`'s [`Metadata.duration`] will be used.
///
/// [`Input`]: ../struct.Input.html
/// [`Metadata.duration`]: ../struct.Metadata.html#structfield.duration
pub fn with_config(source: Input, bitrate: Bitrate, config: Option<Config>) -> Result<Self> {
let channels = if source.stereo {
Channels::Stereo
} else {
Channels::Mono
};
let mut encoder = OpusEncoder::new(SampleRate::Hz48000, channels, Application::Audio)?;
encoder.set_bitrate(bitrate)?;
Self::with_encoder(source, encoder, config)
}
/// Wrap an existing [`Input`] with an in-memory store, compressed using a user-defined
/// Opus encoder.
///
/// `length_hint` functions as in [`new`]. This function's behaviour is undefined if your encoder
/// has a different sample rate than 48kHz, and if the decoder has a different channel count from the source.
///
/// [`Input`]: ../struct.Input.html
/// [`new`]: #method.new
pub fn with_encoder(
mut source: Input,
encoder: OpusEncoder,
config: Option<Config>,
) -> Result<Self> {
let bitrate = encoder.bitrate()?;
let cost_per_sec = compressed_cost_per_sec(bitrate);
let stereo = source.stereo;
let metadata = source.metadata.take();
let mut config = config.unwrap_or_else(|| default_config(cost_per_sec));
// apply length hint.
if config.length_hint.is_none() {
if let Some(dur) = metadata.duration {
apply_length_hint(&mut config, dur, cost_per_sec);
}
}
let raw = config
.build_tx(Box::new(source), OpusCompressor::new(encoder, stereo))
.map_err(Error::Streamcatcher)?;
Ok(Self {
raw,
metadata,
stereo,
})
}
/// Acquire a new handle to this object, creating a new
/// view of the existing cached data from the beginning.
pub fn new_handle(&self) -> Self {
Self {
raw: self.raw.new_handle(),
metadata: self.metadata.clone(),
stereo: self.stereo,
}
}
}
impl From<Compressed> for Input {
fn from(src: Compressed) -> Self {
Input::new(
true,
Reader::Compressed(src.raw),
CodecType::Opus
.try_into()
.expect("Default decoder values are known to be valid."),
Container::Dca { first_frame: 0 },
Some(src.metadata),
)
}
}
/// Transform applied inside [`Compressed`], converting a floating-point PCM
/// input stream into a DCA-framed Opus stream.
///
/// Created and managed by [`Compressed`].
///
/// [`Compressed`]: struct.Compressed.html
#[derive(Debug)]
pub struct OpusCompressor {
encoder: OpusEncoder,
last_frame: Vec<u8>,
stereo_input: bool,
frame_pos: usize,
audio_bytes: AtomicUsize,
}
impl OpusCompressor {
fn new(encoder: OpusEncoder, stereo_input: bool) -> Self {
Self {
encoder,
last_frame: Vec::with_capacity(4000),
stereo_input,
frame_pos: 0,
audio_bytes: Default::default(),
}
}
}
impl<T> Transform<T> for OpusCompressor
where
T: Read,
{
fn transform_read(&mut self, src: &mut T, buf: &mut [u8]) -> IoResult<TransformPosition> {
let output_start = mem::size_of::<u16>();
let mut eof = false;
let mut raw_len = 0;
let mut out = None;
let mut sample_buf = [0f32; STEREO_FRAME_SIZE];
let samples_in_frame = if self.stereo_input {
STEREO_FRAME_SIZE
} else {
MONO_FRAME_SIZE
};
// Purge old frame and read new, if needed.
if self.frame_pos == self.last_frame.len() + output_start || self.last_frame.is_empty() {
self.last_frame.resize(self.last_frame.capacity(), 0);
// We can't use `read_f32_into` because we can't guarantee the buffer will be filled.
for el in sample_buf[..samples_in_frame].iter_mut() {
match src.read_f32::<LittleEndian>() {
Ok(sample) => {
*el = sample;
raw_len += 1;
},
Err(e) if e.kind() == IoErrorKind::UnexpectedEof => {
eof = true;
break;
},
Err(e) => {
out = Some(Err(e));
break;
},
}
}
if out.is_none() && raw_len > 0 {
loop {
// NOTE: we don't index by raw_len because the last frame can be too small
// to occupy a "whole packet". Zero-padding is the correct behaviour.
match self
.encoder
.encode_float(&sample_buf[..samples_in_frame], &mut self.last_frame[..])
{
Ok(pkt_len) => {
trace!("Next packet to write has {:?}", pkt_len);
self.frame_pos = 0;
self.last_frame.truncate(pkt_len);
break;
},
Err(OpusError::Opus(OpusErrorCode::BufferTooSmall)) => {
// If we need more capacity to encode this frame, then take it.
trace!("Resizing inner buffer (+256).");
self.last_frame.resize(self.last_frame.len() + 256, 0);
},
Err(e) => {
debug!("Read error {:?} {:?} {:?}.", e, out, raw_len);
out = Some(Err(IoError::new(IoErrorKind::Other, e)));
break;
},
}
}
}
}
if out.is_none() {
// Write from frame we have.
let start = if self.frame_pos < output_start {
(&mut buf[..output_start])
.write_i16::<LittleEndian>(self.last_frame.len() as i16)
.expect(
"Minimum bytes requirement for Opus (2) should mean that an i16 \
may always be written.",
);
self.frame_pos += output_start;
trace!("Wrote frame header: {}.", self.last_frame.len());
output_start
} else {
0
};
let out_pos = self.frame_pos - output_start;
let remaining = self.last_frame.len() - out_pos;
let write_len = remaining.min(buf.len() - start);
buf[start..start + write_len]
.copy_from_slice(&self.last_frame[out_pos..out_pos + write_len]);
self.frame_pos += write_len;
trace!("Appended {} to inner store", write_len);
out = Some(Ok(write_len + start));
}
// NOTE: use of raw_len here preserves true sample length even if
// stream is extended to 20ms boundary.
out.unwrap_or_else(|| Err(IoError::new(IoErrorKind::Other, "Unclear.")))
.map(|compressed_sz| {
self.audio_bytes
.fetch_add(raw_len * mem::size_of::<f32>(), Ordering::Release);
if eof {
TransformPosition::Finished
} else {
TransformPosition::Read(compressed_sz)
}
})
}
}
impl NeedsBytes for OpusCompressor {
fn min_bytes_required(&self) -> usize {
2
}
}
impl Stateful for OpusCompressor {
type State = usize;
fn state(&self) -> Self::State {
self.audio_bytes.load(Ordering::Acquire)
}
}

40
src/input/cached/hint.rs Normal file
View File

@@ -0,0 +1,40 @@
use std::time::Duration;
use streamcatcher::Config;
/// Expected amount of time that an input should last.
#[derive(Copy, Clone, Debug)]
pub enum LengthHint {
/// Estimate of a source's length in bytes.
Bytes(usize),
/// Estimate of a source's length in time.
///
/// This will be converted to a bytecount at setup.
Time(Duration),
}
impl From<usize> for LengthHint {
fn from(size: usize) -> Self {
LengthHint::Bytes(size)
}
}
impl From<Duration> for LengthHint {
fn from(size: Duration) -> Self {
LengthHint::Time(size)
}
}
/// Modify the given cache configuration to initially allocate
/// enough bytes to store a length of audio at the given bitrate.
pub fn apply_length_hint<H>(config: &mut Config, hint: H, cost_per_sec: usize)
where
H: Into<LengthHint>,
{
config.length_hint = Some(match hint.into() {
LengthHint::Bytes(a) => a,
LengthHint::Time(t) => {
let s = t.as_secs() + if t.subsec_millis() > 0 { 1 } else { 0 };
(s as usize) * cost_per_sec
},
});
}

116
src/input/cached/memory.rs Normal file
View File

@@ -0,0 +1,116 @@
use super::{apply_length_hint, default_config, raw_cost_per_sec};
use crate::input::{
error::{Error, Result},
CodecType,
Container,
Input,
Metadata,
Reader,
};
use std::convert::{TryFrom, TryInto};
use streamcatcher::{Catcher, Config};
/// A wrapper around an existing [`Input`] which caches
/// the decoded and converted audio data locally in memory.
///
/// The main purpose of this wrapper is to enable seeking on
/// incompatible sources (i.e., ffmpeg output) and to ease resource
/// consumption for commonly reused/shared tracks. [`Restartable`]
/// and [`Compressed`] offer the same functionality with different
/// tradeoffs.
///
/// This is intended for use with small, repeatedly used audio
/// tracks shared between sources, and stores the sound data
/// retrieved in **uncompressed floating point** form to minimise the
/// cost of audio processing. This is a significant *3 Mbps (375 kiB/s)*,
/// or 131 MiB of RAM for a 6 minute song.
///
/// [`Input`]: ../struct.Input.html
/// [`Compressed`]: struct.Compressed.html
/// [`Restartable`]: ../struct.Restartable.html
#[derive(Clone, Debug)]
pub struct Memory {
/// Inner shared bytestore.
pub raw: Catcher<Box<Reader>>,
/// Metadata moved out of the captured source.
pub metadata: Metadata,
/// Codec used to read the inner bytestore.
pub kind: CodecType,
/// Stereo-ness of the captured source.
pub stereo: bool,
/// Framing mechanism for the inner bytestore.
pub container: Container,
}
impl Memory {
/// Wrap an existing [`Input`] with an in-memory store with the same codec and framing.
///
/// [`Input`]: ../struct.Input.html
pub fn new(source: Input) -> Result<Self> {
Self::with_config(source, None)
}
/// Wrap an existing [`Input`] with an in-memory store with the same codec and framing.
///
/// `length_hint` may be used to control the size of the initial chunk, preventing
/// needless allocations and copies. If this is not present, the value specified in
/// `source`'s [`Metadata.duration`] will be used, assuming that the source is uncompressed.
///
/// [`Input`]: ../struct.Input.html
/// [`Metadata.duration`]: ../struct.Metadata.html#structfield.duration
pub fn with_config(mut source: Input, config: Option<Config>) -> Result<Self> {
let stereo = source.stereo;
let kind = (&source.kind).into();
let container = source.container;
let metadata = source.metadata.take();
let cost_per_sec = raw_cost_per_sec(stereo);
let mut config = config.unwrap_or_else(|| default_config(cost_per_sec));
// apply length hint.
if config.length_hint.is_none() {
if let Some(dur) = metadata.duration {
apply_length_hint(&mut config, dur, cost_per_sec);
}
}
let raw = config
.build(Box::new(source.reader))
.map_err(Error::Streamcatcher)?;
Ok(Self {
raw,
metadata,
kind,
stereo,
container,
})
}
/// Acquire a new handle to this object, creating a new
/// view of the existing cached data from the beginning.
pub fn new_handle(&self) -> Self {
Self {
raw: self.raw.new_handle(),
metadata: self.metadata.clone(),
kind: self.kind,
stereo: self.stereo,
container: self.container,
}
}
}
impl TryFrom<Memory> for Input {
type Error = Error;
fn try_from(src: Memory) -> Result<Self> {
Ok(Input::new(
src.stereo,
Reader::Memory(src.raw),
src.kind.try_into()?,
src.container,
Some(src.metadata),
))
}
}

44
src/input/cached/mod.rs Normal file
View File

@@ -0,0 +1,44 @@
//! In-memory, shared input sources for reuse between calls, fast seeking, and
//! direct Opus frame passthrough.
mod compressed;
mod hint;
mod memory;
#[cfg(test)]
mod tests;
pub use self::{compressed::*, hint::*, memory::*};
use crate::constants::*;
use crate::input::utils;
use audiopus::Bitrate;
use std::{mem, time::Duration};
use streamcatcher::{Config, GrowthStrategy};
/// Estimates the cost, in B/s, of audio data compressed at the given bitrate.
pub fn compressed_cost_per_sec(bitrate: Bitrate) -> usize {
let framing_cost_per_sec = AUDIO_FRAME_RATE * mem::size_of::<u16>();
let bitrate_raw = match bitrate {
Bitrate::BitsPerSecond(i) => i,
Bitrate::Auto => 64_000,
Bitrate::Max => 512_000,
} as usize;
(bitrate_raw / 8) + framing_cost_per_sec
}
/// Calculates the cost, in B/s, of raw floating-point audio data.
pub fn raw_cost_per_sec(stereo: bool) -> usize {
utils::timestamp_to_byte_count(Duration::from_secs(1), stereo)
}
/// Provides the default config used by a cached source.
///
/// This maps to the default configuration in [`streamcatcher`], using
/// a constant chunk size of 5s worth of audio at the given bitrate estimate.
///
/// [`streamcatcher`]: https://docs.rs/streamcatcher/0.1.0/streamcatcher/struct.Config.html
pub fn default_config(cost_per_sec: usize) -> Config {
Config::new().chunk_size(GrowthStrategy::Constant(5 * cost_per_sec))
}

79
src/input/cached/tests.rs Normal file
View File

@@ -0,0 +1,79 @@
use super::*;
use crate::{
constants::*,
input::{error::Error, ffmpeg, Codec, Container, Input, Reader},
test_utils::*,
};
use audiopus::{coder::Decoder, Bitrate, Channels, SampleRate};
use byteorder::{LittleEndian, ReadBytesExt};
use std::io::{Cursor, Read};
#[tokio::test]
async fn streamcatcher_preserves_file() {
let input = make_sine(50 * MONO_FRAME_SIZE, true);
let input_len = input.len();
let mut raw = default_config(raw_cost_per_sec(true))
.build(Cursor::new(input.clone()))
.map_err(Error::Streamcatcher)
.unwrap();
let mut out_buf = vec![];
let read = raw.read_to_end(&mut out_buf).unwrap();
assert_eq!(input_len, read);
assert_eq!(input, out_buf);
}
#[test]
fn compressed_scans_frames_decodes_mono() {
let data = one_s_compressed_sine(false);
run_through_dca(data.raw);
}
#[test]
fn compressed_scans_frames_decodes_stereo() {
let data = one_s_compressed_sine(true);
run_through_dca(data.raw);
}
#[test]
fn compressed_triggers_valid_passthrough() {
let mut input = Input::from(one_s_compressed_sine(true));
assert!(input.supports_passthrough());
let mut opus_buf = [0u8; 10_000];
let mut signal_buf = [0i16; 1920];
let opus_len = input.read_opus_frame(&mut opus_buf[..]).unwrap();
let mut decoder = Decoder::new(SampleRate::Hz48000, Channels::Stereo).unwrap();
decoder
.decode(Some(&opus_buf[..opus_len]), &mut signal_buf[..], false)
.unwrap();
}
fn one_s_compressed_sine(stereo: bool) -> Compressed {
let data = make_sine(50 * MONO_FRAME_SIZE, stereo);
let input = Input::new(stereo, data.into(), Codec::FloatPcm, Container::Raw, None);
Compressed::new(input, Bitrate::BitsPerSecond(128_000)).unwrap()
}
fn run_through_dca(mut src: impl Read) {
let mut decoder = Decoder::new(SampleRate::Hz48000, Channels::Stereo).unwrap();
let mut pkt_space = [0u8; 10_000];
let mut signals = [0i16; 1920];
while let Ok(frame_len) = src.read_i16::<LittleEndian>() {
let pkt_len = src.read(&mut pkt_space[..frame_len as usize]).unwrap();
decoder
.decode(Some(&pkt_space[..pkt_len]), &mut signals[..], false)
.unwrap();
}
}