Driver: Automate (re)connection logic (#81)

This PR adds several enhancements to Driver connection logic:
* Driver (re)connection attempts now have a default timeout of around 10s.
* The driver will now attempt to retry full connection attempts using a user-provided strategy: currently, this defaults to 5 attempts under an exponential backoff strategy.
* The driver will now fire `DriverDisconnect` events at the end of any session -- this unifies (re)connection failure events with session expiry as seen in #76, which should provide users with enough detail to know *which* voice channel to reconnect to. Users still need to be careful to read the session/channel IDs to ensure that they aren't overwriting another join.

This has been tested using `cargo make ready`, and by setting low timeouts to force failures in the voice receive example (with some additional error handlers).

Closes #68.
This commit is contained in:
Kyle Simpson
2021-06-23 17:11:14 +01:00
parent 8381f8c461
commit 210e3ae584
17 changed files with 672 additions and 90 deletions

49
src/driver/retry/mod.rs Normal file
View File

@@ -0,0 +1,49 @@
//! Configuration for connection retries.
mod strategy;
pub use self::strategy::*;
use std::time::Duration;
/// Configuration to be used for retrying driver connection attempts.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Retry {
/// Strategy used to determine how long to wait between retry attempts.
///
/// *Defaults to an [`ExponentialBackoff`] from 0.25s
/// to 10s, with a jitter of `0.1`.*
///
/// [`ExponentialBackoff`]: Strategy::Backoff
pub strategy: Strategy,
/// The maximum number of retries to attempt.
///
/// `None` will attempt an infinite number of retries,
/// while `Some(0)` will attempt to connect *once* (no retries).
///
/// *Defaults to `Some(5)`.*
pub retry_limit: Option<usize>,
}
impl Default for Retry {
fn default() -> Self {
Self {
strategy: Strategy::Backoff(Default::default()),
retry_limit: Some(5),
}
}
}
impl Retry {
pub(crate) fn retry_in(
&self,
last_wait: Option<Duration>,
attempts: usize,
) -> Option<Duration> {
if self.retry_limit.map(|a| attempts < a).unwrap_or(true) {
Some(self.strategy.retry_in(last_wait))
} else {
None
}
}
}

View File

@@ -0,0 +1,84 @@
use rand::random;
use std::time::Duration;
/// Logic used to determine how long to wait between retry attempts.
#[derive(Clone, Copy, Debug, PartialEq)]
#[non_exhaustive]
pub enum Strategy {
/// The driver will wait for the same amount of time between each retry.
Every(Duration),
/// Exponential backoff waiting strategy, where the duration between
/// attempts (approximately) doubles each time.
Backoff(ExponentialBackoff),
}
impl Strategy {
pub(crate) fn retry_in(&self, last_wait: Option<Duration>) -> Duration {
match self {
Self::Every(t) => *t,
Self::Backoff(exp) => exp.retry_in(last_wait),
}
}
}
/// Exponential backoff waiting strategy.
///
/// Each attempt waits for twice the last delay plus/minus a
/// random jitter, clamped to a min and max value.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ExponentialBackoff {
/// Minimum amount of time to wait between retries.
///
/// *Defaults to 0.25s.*
pub min: Duration,
/// Maximum amount of time to wait between retries.
///
/// This will be clamped to `>=` min.
///
/// *Defaults to 10s.*
pub max: Duration,
/// Amount of uniform random jitter to apply to generated wait times.
/// I.e., 0.1 will add +/-10% to generated intervals.
///
/// This is restricted to within +/-100%.
///
/// *Defaults to `0.1`.*
pub jitter: f32,
}
impl Default for ExponentialBackoff {
fn default() -> Self {
Self {
min: Duration::from_millis(250),
max: Duration::from_secs(10),
jitter: 0.1,
}
}
}
impl ExponentialBackoff {
pub(crate) fn retry_in(&self, last_wait: Option<Duration>) -> Duration {
let attempt = last_wait.map(|t| 2 * t).unwrap_or(self.min);
let perturb = (1.0 - (self.jitter * 2.0 * (random::<f32>() - 1.0)))
.max(0.0)
.min(2.0);
let mut target_time = attempt.mul_f32(perturb);
// Now clamp target time into given range.
let safe_max = if self.max < self.min {
self.min
} else {
self.max
};
if target_time > safe_max {
target_time = safe_max;
}
if target_time < self.min {
target_time = self.min;
}
target_time
}
}