From a88deb2a24556f1d4b2efbacd730ab1e49991a88 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 28 Sep 2026 16:58:03 +0200 Subject: [PATCH 01/37] feat: queue up to 60 s of PCM and pace only while connected - Add LocalAudioTrackConfig::pcm_queue_capacity (default 60 s) and with_pcm_queue_capacity. The queue grows as needed. An overflow still drops the oldest samples and returns PcmQueueOverflow. - Add LocalAudioTrack::start_pacing and pause_pacing. write_pcm only fills the queue and does not start the pacer. - Start pacing when the publisher PeerConnection is connected and pause it for all other states. Pause it on an ICE restart. Before this change, webrtc-rs dropped audio written before DTLS connected and during reconnects. - Start pacing in gpt_realtime_bot from its OpenAI PeerConnection state. --- examples/gpt_realtime_bot.rs | 11 ++ src/rtc/error.rs | 2 +- src/rtc/join/connection.rs | 9 ++ src/rtc/join/publish.rs | 22 ++- src/rtc/join/reconnect_runtime.rs | 5 +- src/rtc/join/tests.rs | 46 +++++- src/rtc/peer/connection.rs | 68 ++++++++ src/rtc/peer/mod.rs | 2 + src/rtc/peer/publisher.rs | 5 + src/rtc/tracks/local.rs | 257 ++++++++++++++++++++++++------ 10 files changed, 375 insertions(+), 52 deletions(-) diff --git a/examples/gpt_realtime_bot.rs b/examples/gpt_realtime_bot.rs index 2168a2b..3523d8b 100644 --- a/examples/gpt_realtime_bot.rs +++ b/examples/gpt_realtime_bot.rs @@ -328,6 +328,17 @@ async fn configure_openai( .await .context("add audio track to the OpenAI PeerConnection")?, ); + let paced_mic = mic.clone(); + pc.on_peer_connection_state_change(Box::new(move |state| { + let paced_mic = paced_mic.clone(); + Box::pin(async move { + if state == RTCPeerConnectionState::Connected { + paced_mic.start_pacing().await; + } else { + paced_mic.pause_pacing(); + } + }) + })); let camera = LocalVideoTrack::h264().context("H264 track for OpenAI")?; spawn_rtcp_drain( diff --git a/src/rtc/error.rs b/src/rtc/error.rs index fa53994..1b966e7 100644 --- a/src/rtc/error.rs +++ b/src/rtc/error.rs @@ -132,7 +132,7 @@ pub enum RtcError { track_type: models::TrackType, }, - /// A PCM write exceeded the track's low-latency queue. The newest samples + /// A PCM write exceeded the track's PCM queue capacity. The newest samples /// were retained and this many oldest samples were discarded. #[error( "pcm queue overflow: dropped {dropped_samples} oldest samples \ diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index eca997f..6bedfc8 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -127,6 +127,15 @@ pub(super) fn register_connection_state( Box::pin(async move { tracing::debug!(label, ?state, "stream.rtc.pc.state"); tracer.trace("connectionstatechange", json!(state.to_string())); + // Not awaited: waiting for the media lock here can block ICE + // state delivery. The sync reads the current publisher state, so + // a subscriber change applies the same state again. + if core.is_generation_current(generation) { + let sync_core = core.clone(); + std::mem::drop(core.spawn_generation_task(generation, async move { + sync_core.sync_audio_pacing().await; + })); + } if state == RTCPeerConnectionState::Connected { ever_connected.store(true, Ordering::SeqCst); let mut lifecycle = core.lifecycle.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/src/rtc/join/publish.rs b/src/rtc/join/publish.rs index f4221a8..81f1a47 100644 --- a/src/rtc/join/publish.rs +++ b/src/rtc/join/publish.rs @@ -123,7 +123,9 @@ impl RtcCore { .user_id .clone(); self.add_published_track(&user_id, &session_id, track.track_type() as i32, None); - track.start_media(); + if publisher.connection_state() == RTCPeerConnectionState::Connected { + track.start_audio_pacing().await; + } signal .update_mute_states(signal::UpdateMuteStatesRequest { session_id: session_id.clone(), @@ -138,6 +140,24 @@ impl RtcCore { Ok(()) } + /// Pace the published audio only while the current publisher is connected. + pub(super) async fn sync_audio_pacing(&self) { + let media = self.media.lock().await; + let connected = self + .publisher_handles() + .await + .is_some_and(|(publisher, ..)| { + publisher.connection_state() == RTCPeerConnectionState::Connected + }); + for track in media.active_tracks() { + if connected { + track.start_audio_pacing().await; + } else { + track.pause_audio_pacing(); + } + } + } + pub(super) async fn register_publisher_tasks(&self, tasks: Vec>) { if tasks.is_empty() { return; diff --git a/src/rtc/join/reconnect_runtime.rs b/src/rtc/join/reconnect_runtime.rs index 03cff94..74773aa 100644 --- a/src/rtc/join/reconnect_runtime.rs +++ b/src/rtc/join/reconnect_runtime.rs @@ -4,8 +4,8 @@ use super::*; impl RtcCore { /// Re-publish every local track on the fresh publisher PC after a - /// REJOIN/MIGRATE. The tracks are the same `Arc`-backed handles, so the pacer - /// keeps running and simply binds to the new PeerConnection. + /// REJOIN/MIGRATE. The tracks are the same `Arc`-backed handles, so they bind + /// to the new PeerConnection and pacing resumes when it connects. pub(super) async fn restore_published_tracks(&self) -> Result<()> { let mut media = self.media.lock().await; let capabilities = self @@ -103,7 +103,6 @@ impl RtcCore { } else { self.add_published_track(&user_id, session_id, track.track_type() as i32, None); } - track.start_media(); } Ok(()) } diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index b508b4d..b47bc64 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -3,7 +3,7 @@ use super::*; use crate::client::ClientConfig; use crate::rtc::{ - ClientPublishOptions, LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig, + ClientPublishOptions, LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig, PcmFrame, PreferredVideoCodec, publish_options::H264_FMTP, }; use std::io::{Read, Write}; @@ -576,6 +576,50 @@ async fn stale_coordinator_stop_keeps_the_current_coordinator() { .expect("fake coordinator task"); } +#[tokio::test] +async fn published_audio_is_paced_only_while_the_publisher_is_connected() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let publisher = connection.publisher.clone(); + *core.connection.lock().await = Some(connection); + let audio = LocalAudioTrack::opus().expect("opus track"); + let tone = (0..960_u32) + .map(|n| { + (12_000.0 * (std::f64::consts::TAU * 440.0 * f64::from(n) / 48_000.0).sin()) as i16 + }) + .collect::>() + .repeat(50); + audio + .write_pcm(PcmFrame::mono(tone, 48_000)) + .await + .expect("one second fits the queue"); + publisher + .add_track(audio.webrtc_track()) + .await + .expect("add track"); + core.media + .lock() + .await + .begin_publish(LocalTrack::Audio(audio.clone()), 0); + + let (receiver, mut remote) = peer::connect_audio_receiver(&publisher).await; + + tokio::time::timeout(Duration::from_secs(5), remote.recv()) + .await + .expect("paced audio reaches the receiver") + .expect("remote track"); + publisher.close().await.expect("close publisher"); + wait_for( + Duration::from_secs(2), + || !audio.is_pacing(), + "closed publisher pauses pacing", + ) + .await; + let _ = receiver.close().await; + core.leave("test cleanup").await.expect("leave"); +} + #[tokio::test] async fn detached_connection_ignores_publish_options_from_its_sfu() { let core = test_core(); diff --git a/src/rtc/peer/connection.rs b/src/rtc/peer/connection.rs index f4a8936..d3fda04 100644 --- a/src/rtc/peer/connection.rs +++ b/src/rtc/peer/connection.rs @@ -256,6 +256,74 @@ async fn build_generic_offer( Ok(offer.sdp) } +/// Connect a receive-only audio peer to `sender` in this process, and return it +/// with the channel of its inbound tracks. Returns when `sender` is connected. +#[cfg(test)] +pub(crate) async fn connect_audio_receiver( + sender: &RTCPeerConnection, +) -> ( + Arc, + tokio::sync::mpsc::Receiver>, +) { + use std::time::Duration; + use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; + use webrtc::rtp_transceiver::RTCRtpTransceiverInit; + + // DTLS needs a process-default provider; `Call::join` installs it too. + let _ = rustls::crypto::ring::default_provider().install_default(); + let receiver = new_peer_connection(&[]).await.expect("receiver"); + receiver + .add_transceiver_from_kind( + RTPCodecType::Audio, + Some(RTCRtpTransceiverInit { + direction: RTCRtpTransceiverDirection::Recvonly, + send_encodings: vec![], + }), + ) + .await + .expect("receive transceiver"); + let (track_tx, track_rx) = tokio::sync::mpsc::channel(1); + receiver.on_track(Box::new(move |track, _, _| { + let track_tx = track_tx.clone(); + Box::pin(async move { + let _ = track_tx.send(track).await; + }) + })); + + let offer = sender.create_offer(None).await.expect("offer"); + let mut gathered = sender.gathering_complete_promise().await; + sender + .set_local_description(offer) + .await + .expect("set offer"); + let _ = gathered.recv().await; + let offer = sender.local_description().await.expect("offer"); + receiver + .set_remote_description(offer) + .await + .expect("apply offer"); + let answer = receiver.create_answer(None).await.expect("answer"); + let mut gathered = receiver.gathering_complete_promise().await; + receiver + .set_local_description(answer) + .await + .expect("set answer"); + let _ = gathered.recv().await; + let answer = receiver.local_description().await.expect("answer"); + sender + .set_remote_description(answer) + .await + .expect("apply answer"); + tokio::time::timeout(Duration::from_secs(10), async { + while sender.connection_state() != RTCPeerConnectionState::Connected { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("sender connects"); + (receiver, track_rx) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/rtc/peer/mod.rs b/src/rtc/peer/mod.rs index 0c8307a..70b4f30 100644 --- a/src/rtc/peer/mod.rs +++ b/src/rtc/peer/mod.rs @@ -12,6 +12,8 @@ mod ice; pub mod publisher; mod subscriber; +#[cfg(test)] +pub(crate) use connection::connect_audio_receiver; pub use connection::{generic_sdp, new_peer_connection, to_rtc_ice_servers, trace_peer_events}; pub(super) use ice::{PendingIce, register_ice_trickle}; pub(super) use subscriber::negotiate_subscriber; diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index 587257b..34db721 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -101,6 +101,11 @@ pub(crate) async fn restart_ice( } } } + // Media is lost until ICE connects again; the publisher `Connected` state + // resumes pacing. + for track in tracks { + track.pause_audio_pacing(); + } negotiate_publish(publisher, signal, session_id, tracks, publish_options).await } diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 5ca186a..13f971c 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -5,9 +5,11 @@ //! `StartWrite`). Three write paths feed the same outbound track: //! //! - [`LocalAudioTrack::write_pcm`] — raw [`PcmFrame`]s. Resampled to 48 kHz -//! mono and paced into 20 ms Opus frames by a background task that emits -//! silence on starve (stream-py `AudioStreamTrack` pacing). This is the PCM -//! republish / TTS-bot path. +//! mono, queued, and paced into 20 ms Opus frames by a background task that +//! emits silence on starve (stream-py `AudioStreamTrack` pacing). Pacing runs +//! after [`LocalAudioTrack::start_pacing`]; a published track paces only +//! while the SFU publisher is connected. This is the PCM republish / TTS-bot +//! path. //! - [`LocalAudioTrack::write_sample`] / [`LocalVideoTrack::write_sample`] — //! already-encoded media (Opus/VP8/…) plus a frame duration; the SDK //! packetizes and writes. The caller controls pacing. @@ -70,7 +72,7 @@ const VIDEO_BITRATE_KBPS: u32 = 1_000; const MAX_LOCAL_VIDEO_EDGE: u32 = 3_840; const MAX_LOCAL_VIDEO_PIXELS: u64 = 3_840 * 2_160; const MAX_LOCAL_VIDEO_I420_BYTES: usize = 3_840 * 2_160 * 3 / 2; -const PCM_QUEUE_CAPACITY_SAMPLES: usize = FRAME_SAMPLES_20MS * 10; +const PCM_QUEUE_CAPACITY: Duration = Duration::from_secs(60); const MAX_OPUS_PACKET_BYTES: usize = 1_500; const AUDIO_BITRATE_BPS: u32 = 32_000; @@ -324,10 +326,13 @@ struct AudioInner { core: TrackCore, /// Resampled 48 kHz mono PCM awaiting the 20 ms pacer. pcm: StdMutex>, + pcm_capacity_samples: usize, resampler: StdMutex, encoder: StdMutex, pacer: StdMutex>>, pacer_started: AtomicBool, + /// Set by `start_pacing`, cleared by `pause_pacing`. + pacing_enabled: AtomicBool, pcm_pacing: AtomicBool, write_guard: tokio::sync::Mutex<()>, } @@ -346,6 +351,9 @@ pub struct LocalAudioTrackConfig { pub expected_packet_loss_pct: u8, /// Discontinuous transmission: stop emitting packets during silence. pub dtx: bool, + /// Maximum PCM that [`LocalAudioTrack::write_pcm`] queues for the pacer. A + /// write above it drops the oldest queued samples. + pub pcm_queue_capacity: Duration, } impl Default for LocalAudioTrackConfig { @@ -355,6 +363,7 @@ impl Default for LocalAudioTrackConfig { inband_fec: true, expected_packet_loss_pct: EXPECTED_PACKET_LOSS_PCT, dtx: true, + pcm_queue_capacity: PCM_QUEUE_CAPACITY, } } } @@ -389,6 +398,13 @@ impl LocalAudioTrackConfig { self.dtx = dtx; self } + + /// Set the maximum PCM that [`LocalAudioTrack::write_pcm`] queues. + #[must_use] + pub fn with_pcm_queue_capacity(mut self, pcm_queue_capacity: Duration) -> Self { + self.pcm_queue_capacity = pcm_queue_capacity; + self + } } /// An outbound Opus audio track. @@ -433,14 +449,18 @@ impl LocalAudioTrack { encoder.set_inband_fec(config.inband_fec)?; encoder.set_packet_loss_perc(i32::from(config.expected_packet_loss_pct))?; encoder.set_dtx(config.dtx)?; + let pcm_capacity_samples = + (config.pcm_queue_capacity.as_secs_f64() * f64::from(OPUS_SAMPLE_RATE)) as usize; Ok(Self { inner: Arc::new(AudioInner { core, - pcm: StdMutex::new(VecDeque::with_capacity(PCM_QUEUE_CAPACITY_SAMPLES)), + pcm: StdMutex::new(VecDeque::new()), + pcm_capacity_samples, resampler: StdMutex::new(StreamResampler::to_opus_mono()), encoder: StdMutex::new(encoder), pacer: StdMutex::new(None), pacer_started: AtomicBool::new(false), + pacing_enabled: AtomicBool::new(false), pcm_pacing: AtomicBool::new(true), write_guard: tokio::sync::Mutex::new(()), }), @@ -449,19 +469,20 @@ impl LocalAudioTrack { /// Queue a PCM frame for the paced 20 ms Opus encoder. /// - /// The frame is resampled to 48 kHz mono and buffered for at most 200 ms; a - /// background task emits one Opus packet every 20 ms, writing silence when - /// the buffer runs dry. To keep interactive audio current, this method does - /// not backpressure a producer: overflow drops the oldest queued samples and - /// retains the newest audio. [`flush`](Self::flush) still drops all unsent - /// samples immediately for barge-in. + /// The frame is resampled to 48 kHz mono and buffered up to + /// [`LocalAudioTrackConfig::pcm_queue_capacity`]. While pacing runs (see + /// [`start_pacing`](Self::start_pacing)), a background task emits one Opus + /// packet every 20 ms, writing silence when the buffer runs dry. This method + /// does not backpressure a producer: overflow drops the oldest queued + /// samples and retains the newest audio. [`flush`](Self::flush) still drops + /// all unsent samples immediately for barge-in. /// /// # Errors /// /// Returns [`RtcError::PcmQueueOverflow`] after retaining the newest audio - /// when this write exceeds the 200 ms queue. The caller may continue writing; - /// the typed error makes overload observable without allowing stale audio to - /// accumulate. + /// when this write exceeds the queue capacity. The caller may continue + /// writing; the typed error makes overload observable without allowing + /// stale audio to accumulate. pub async fn write_pcm(&self, frame: PcmFrame) -> Result<()> { if self.inner.core.stopped.load(Ordering::SeqCst) { return Err(RtcError::IllegalState( @@ -479,21 +500,20 @@ impl LocalAudioTrack { }; let dropped = { let mut buf = self.inner.pcm.lock().unwrap_or_else(|e| e.into_inner()); - let dropped = push_bounded_pcm(&mut buf, resampled); + let dropped = push_bounded_pcm(&mut buf, resampled, self.inner.pcm_capacity_samples); if dropped > 0 { tracing::debug!( dropped_samples = dropped, - capacity_samples = PCM_QUEUE_CAPACITY_SAMPLES, + capacity_samples = self.inner.pcm_capacity_samples, "stream.rtc.audio.pcm_queue_overflow" ); } dropped }; - self.ensure_pacer(); if dropped > 0 { Err(RtcError::PcmQueueOverflow { dropped_samples: dropped, - capacity_samples: PCM_QUEUE_CAPACITY_SAMPLES, + capacity_samples: self.inner.pcm_capacity_samples, }) } else { Ok(()) @@ -569,6 +589,30 @@ impl LocalAudioTrack { buf.clear(); } + /// Start taking queued PCM: one 20 ms frame every 20 ms, or silence when + /// the queue is empty. Before this, [`write_pcm`](Self::write_pcm) only + /// fills the queue. + /// + /// [`Call::publish_audio`](crate::Call::publish_audio) starts and pauses + /// pacing with the SFU publisher connection. Call this yourself only for a + /// track on your own PeerConnection (see [`webrtc_track`](Self::webrtc_track)), + /// after that PeerConnection connects. + pub async fn start_pacing(&self) { + self.inner.pacing_enabled.store(true, Ordering::SeqCst); + self.ensure_pacer(); + } + + /// Stop taking queued PCM. The queue keeps its audio for the next + /// [`start_pacing`](Self::start_pacing). + pub fn pause_pacing(&self) { + self.inner.pacing_enabled.store(false, Ordering::SeqCst); + } + + #[cfg(test)] + pub(crate) fn is_pacing(&self) -> bool { + self.inner.pacing_enabled.load(Ordering::SeqCst) + } + /// Stop the pacer and reject further writes. Called by `stop_publish`/`leave`. pub(crate) fn stop(&self) { self.inner.core.stop(); @@ -589,7 +633,8 @@ impl LocalAudioTrack { /// [`Call::publish_audio`](crate::Call::publish_audio) does this for the /// SFU; you only need it to send the same audio to a second peer, such as an /// AI provider's Realtime endpoint. Every write path (`write_pcm` and - /// friends) feeds all bound senders. + /// friends) feeds all bound senders. A track that is not published paces + /// `write_pcm` audio only after [`start_pacing`](Self::start_pacing). pub fn webrtc_track(&self) -> Arc { self.inner.core.track.clone() } @@ -603,7 +648,7 @@ impl LocalAudioTrack { } /// Spawn the 20 ms PCM/silence pacing task (idempotent). - pub(crate) fn ensure_pacer(&self) { + fn ensure_pacer(&self) { if self .inner .pacer_started @@ -630,7 +675,8 @@ async fn pace_audio(inner: Arc) { if inner.core.stopped.load(Ordering::SeqCst) { return; } - if !inner.pcm_pacing.load(Ordering::SeqCst) { + if !inner.pcm_pacing.load(Ordering::SeqCst) || !inner.pacing_enabled.load(Ordering::SeqCst) + { continue; } let _write = inner.write_guard.lock().await; @@ -671,11 +717,11 @@ async fn pace_audio(inner: Arc) { } } -fn push_bounded_pcm(queue: &mut VecDeque, samples: Vec) -> usize { +fn push_bounded_pcm(queue: &mut VecDeque, samples: Vec, capacity: usize) -> usize { let overflow = queue .len() .saturating_add(samples.len()) - .saturating_sub(PCM_QUEUE_CAPACITY_SAMPLES); + .saturating_sub(capacity); let from_queue = overflow.min(queue.len()); queue.drain(..from_queue); let from_samples = overflow - from_queue; @@ -1926,10 +1972,19 @@ impl LocalTrack { } } - pub(crate) fn start_media(&self) { + pub(crate) async fn start_audio_pacing(&self) { + match self { + LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { + track.start_pacing().await; + } + LocalTrack::Video { .. } => {} + } + } + + pub(crate) fn pause_audio_pacing(&self) { match self { LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { - track.ensure_pacer(); + track.pause_pacing(); } LocalTrack::Video { .. } => {} } @@ -1992,6 +2047,7 @@ impl LocalTrack { #[cfg(test)] mod tests { use super::*; + use crate::rtc::peer; use crate::rtc::proto::models::{Codec, VideoDimension}; /// One 20 ms frame of 440 Hz tone: FEC and DTX both key off whether the @@ -2122,19 +2178,9 @@ mod tests { } #[tokio::test] - async fn write_pcm_paces_without_binding() { + async fn started_pacing_sends_silence_with_truthful_level() { let track = LocalAudioTrack::opus().expect("opus track"); - let frame = PcmFrame::mono(vec![1000; FRAME_SAMPLES_20MS], OPUS_SAMPLE_RATE); - track.write_pcm(frame).await.expect("write_pcm"); - // Give the pacer a couple of ticks; it must not panic writing silence. - tokio::time::sleep(Duration::from_millis(50)).await; - track.stop(); - } - - #[tokio::test] - async fn publication_starts_paced_silence_with_truthful_level() { - let track = LocalAudioTrack::opus().expect("opus track"); - LocalTrack::Audio(track.clone()).start_media(); + track.start_pacing().await; tokio::time::sleep(Duration::from_millis(30)).await; assert!(track.inner.pacer_started.load(Ordering::SeqCst)); assert_eq!(track.inner.core.audio_level.load(Ordering::Relaxed), 127); @@ -2150,9 +2196,9 @@ mod tests { .pcm .lock() .unwrap_or_else(|error| error.into_inner()); - queue.extend(std::iter::repeat_n(1, PCM_QUEUE_CAPACITY_SAMPLES - 2)); - assert_eq!(push_bounded_pcm(&mut queue, vec![2, 3, 4, 5]), 2); - assert_eq!(queue.len(), PCM_QUEUE_CAPACITY_SAMPLES); + queue.extend(std::iter::repeat_n(1, 4)); + assert_eq!(push_bounded_pcm(&mut queue, vec![2, 3, 4, 5], 6), 2); + assert_eq!(queue.len(), 6); assert_eq!(queue.back(), Some(&5)); } track.flush(); @@ -2167,17 +2213,26 @@ mod tests { } #[tokio::test] - async fn write_pcm_reports_typed_overflow_after_retaining_newest_audio() { + async fn write_pcm_above_the_default_minute_keeps_the_newest_samples() { let track = LocalAudioTrack::opus().expect("opus track"); - let samples = vec![7; PCM_QUEUE_CAPACITY_SAMPLES + FRAME_SAMPLES_20MS]; + let minute = 60 * OPUS_SAMPLE_RATE as usize; + track + .write_pcm(PcmFrame::mono(vec![1; minute], OPUS_SAMPLE_RATE)) + .await + .expect("a minute fits the default queue"); + let result = track - .write_pcm(PcmFrame::mono(samples, OPUS_SAMPLE_RATE)) + .write_pcm(PcmFrame::mono( + vec![2; FRAME_SAMPLES_20MS], + OPUS_SAMPLE_RATE, + )) .await; + assert!(matches!( result, Err(RtcError::PcmQueueOverflow { dropped_samples: FRAME_SAMPLES_20MS, - capacity_samples: PCM_QUEUE_CAPACITY_SAMPLES, + capacity_samples: 2_880_000, }) )); let queue = track @@ -2185,12 +2240,121 @@ mod tests { .pcm .lock() .unwrap_or_else(|error| error.into_inner()); - assert_eq!(queue.len(), PCM_QUEUE_CAPACITY_SAMPLES); - assert!(queue.iter().all(|sample| *sample == 7)); + assert_eq!(queue.len(), minute); + assert_eq!(queue.front(), Some(&1)); + assert!( + queue + .iter() + .rev() + .take(FRAME_SAMPLES_20MS) + .all(|sample| *sample == 2) + ); drop(queue); track.stop(); } + #[tokio::test] + async fn pcm_queue_capacity_is_set_per_track() { + let track = LocalAudioTrack::opus_with_config( + LocalAudioTrackConfig::default().with_pcm_queue_capacity(Duration::from_millis(100)), + ) + .expect("opus track"); + + let result = track + .write_pcm(PcmFrame::mono( + vec![7; FRAME_SAMPLES_20MS * 6], + OPUS_SAMPLE_RATE, + )) + .await; + + assert!(matches!( + result, + Err(RtcError::PcmQueueOverflow { + dropped_samples: FRAME_SAMPLES_20MS, + capacity_samples: 4_800, + }) + )); + track.stop(); + } + + #[tokio::test] + async fn pcm_written_before_pacing_starts_stays_queued() { + let track = LocalAudioTrack::opus_with_config( + LocalAudioTrackConfig::default().with_pcm_queue_capacity(Duration::from_millis(100)), + ) + .expect("opus track"); + track + .write_pcm(PcmFrame::mono(vec![7; 4_800], OPUS_SAMPLE_RATE)) + .await + .expect("100 ms fits the queue"); + + // A running pacer takes 960 samples every 20 ms. + tokio::time::sleep(Duration::from_millis(100)).await; + let result = track + .write_pcm(PcmFrame::mono( + vec![7; FRAME_SAMPLES_20MS], + OPUS_SAMPLE_RATE, + )) + .await; + + assert!(matches!( + result, + Err(RtcError::PcmQueueOverflow { + dropped_samples: FRAME_SAMPLES_20MS, + .. + }) + )); + track.stop(); + } + + #[tokio::test] + async fn pcm_queued_before_the_connection_is_sent_from_its_first_sample() { + let track = LocalAudioTrack::opus().expect("opus track"); + let minute = tone_20ms().repeat(3_000); + let queued = minute.len(); + track + .write_pcm(PcmFrame::mono(minute, OPUS_SAMPLE_RATE)) + .await + .expect("a minute fits the default queue"); + + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(track.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote_rx) = peer::connect_audio_receiver(&sender).await; + + track.start_pacing().await; + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + let mut received = 0; + let paced_until = tokio::time::Instant::now() + Duration::from_secs(1); + while tokio::time::Instant::now() < paced_until { + remote.read_rtp().await.expect("paced packet"); + received += 1; + } + track.pause_pacing(); + while let Ok(Ok(_)) = + tokio::time::timeout(Duration::from_millis(200), remote.read_rtp()).await + { + received += 1; + } + + let taken = queued + - track + .inner + .pcm + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len(); + assert_eq!(taken, received * FRAME_SAMPLES_20MS); + track.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + } + #[test] fn audio_level_dbov_maps_rms_to_rfc6464_levels() { assert_eq!(audio_level_dbov(0.0), 127, "digital silence"); @@ -2222,6 +2386,7 @@ mod tests { ); let loud = PcmFrame::mono(vec![i16::MAX / 2; FRAME_SAMPLES_20MS * 4], OPUS_SAMPLE_RATE); track.write_pcm(loud).await.expect("write_pcm"); + track.start_pacing().await; tokio::time::sleep(Duration::from_millis(60)).await; let level = track.inner.core.audio_level.load(Ordering::Relaxed); track.stop(); From 4589d7967e11db7c41a16e6a855b4fdb92bb3ece Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 28 Sep 2026 17:58:52 +0200 Subject: [PATCH 02/37] fix: make RTC tracks, join, and leave safe to drop and cancel - Enter the captured runtime when a RemoteTrack is dropped, so a drop on a thread without a runtime does not panic and still unsubscribes. - Reset a dropped join() to Idle and finish a dropped leave(), so the next join() is accepted. - Request 1280x720 when a video or screen-share subscription has no dimension. - Decode Opus into a reused buffer, so a queued frame holds only its samples. - Hold the audio track weakly in the pacer, so the pacer ends when the track is dropped. - Send CallEvent::CallEnded once when the SFU or the coordinator (call.ended) ends the call, then leave the call. --- src/rtc/join/connection.rs | 4 +- src/rtc/join/lifecycle.rs | 89 ++++++++++- src/rtc/join/mod.rs | 7 +- src/rtc/join/participants.rs | 3 + src/rtc/join/subscriptions_runtime.rs | 22 +-- src/rtc/join/tests.rs | 221 ++++++++++++++++++++++++++ src/rtc/subscriptions.rs | 12 +- src/rtc/tracks/local.rs | 37 ++++- src/rtc/tracks/remote.rs | 25 ++- tests/rtc_media.rs | 58 +++++++ 10 files changed, 447 insertions(+), 31 deletions(-) diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 6bedfc8..3109c24 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -400,9 +400,7 @@ pub(super) async fn handle_event( E::ParticipantMigrationComplete(_) => { core.complete_migration(context.generation); } - E::CallEnded(_) => { - let _ = core.events_tx.send(CallEvent::CallEnded); - } + E::CallEnded(_) => core.end_call(context.generation), E::PublisherAnswer(_) | E::JoinResponse(_) => { tracing::debug!("stream.rtc.unexpected_handshake_event"); } diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index 0a485c6..7f7f678 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -17,6 +17,10 @@ impl RtcCore { ) -> Result<()> { ensure_crypto_provider(); let generation = self.begin_join()?; + let _attempt = JoinAttempt { + core: self, + generation, + }; // A fresh unified session id for this join lifecycle; reused across // reconnects so the dashboard correlates the participant end-to-end. { @@ -644,6 +648,10 @@ impl RtcCore { pub async fn leave(&self, reason: impl Into) -> Result<()> { let reason = reason.into(); let generation = self.cancel_generation(); + let left = LeftCall { + core: self, + generation, + }; let connection = self.connection.lock().await.take(); if let Some(connection) = connection { @@ -658,8 +666,42 @@ impl RtcCore { } connection.teardown().await; } + drop(left); + self.stop_coordinator_events(generation).await; + Ok(()) + } + + /// The SFU or the coordinator reported the end of the call: send + /// [`CallEvent::CallEnded`] once and leave the call. + pub(super) fn end_call(self: &Arc, generation: u64) { + if !self.is_generation_current(generation) { + return; + } + let already_ended = std::mem::replace( + &mut self + .call_state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .ended, + true, + ); + if already_ended { + return; + } + let _ = self.events_tx.send(CallEvent::CallEnded); + let this = self.clone(); + // Not a generation task: `leave` ends the generation. + std::mem::drop(self.spawn_runtime_task(async move { + if this.is_generation_current(generation) { + let _ = this.leave("call ended").await; + } + })); + } + + /// Clear the call state of `generation` and set `Left`. + fn finish_leave(&self, generation: u64) { { - // A join that started during the awaits above owns these fields. + // A join that started during the leave owns these fields. let lifecycle = self.lifecycle.lock().unwrap_or_else(|e| e.into_inner()); if lifecycle.generation == generation { self.participants @@ -685,8 +727,45 @@ impl RtcCore { } } self.set_state_if_current(generation, CallingState::Left); - self.stop_coordinator_events(generation).await; - Ok(()) + } +} + +/// Abandons a join whose future is dropped while it is still joining. The join +/// cannot await its cleanup there, so a new generation stops its tasks, and +/// `Idle` allows the next join. +struct JoinAttempt<'a> { + core: &'a RtcCore, + generation: u64, +} + +impl Drop for JoinAttempt<'_> { + fn drop(&mut self) { + { + let mut lifecycle = self + .core + .lifecycle + .lock() + .unwrap_or_else(|e| e.into_inner()); + if lifecycle.generation != self.generation || lifecycle.state != CallingState::Joining { + return; + } + lifecycle.generation = lifecycle.generation.wrapping_add(1); + lifecycle.set_state(CallingState::Idle, &self.core.events_tx); + } + self.core.lifecycle_changed.notify_waiters(); + } +} + +/// Finishes a `leave` on drop, so a `leave` future that is dropped early still +/// leaves the call. The coordinator tasks stop on the generation change. +struct LeftCall<'a> { + core: &'a RtcCore, + generation: u64, +} + +impl Drop for LeftCall<'_> { + fn drop(&mut self) { + self.core.finish_leave(self.generation); } } @@ -737,7 +816,11 @@ impl RtcCore { event_core .apply_permissions_updated(&event, &local_user_id) .await; + let ended = event.event_type == "call.ended"; let _ = sender.send(CallEvent::Coordinator(event)); + if ended { + event_core.end_call(generation); + } } Ok(Some(_)) => {} Ok(None) => { diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index a14d292..64163c8 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -65,7 +65,9 @@ use super::reconnect::{ use super::sfu::signal::SignalClient; use super::sfu::ws::{self, SfuReceiver, SfuSender}; use super::stats::{self, StatsReporter, StatsReporterParts}; -use super::subscriptions::{SubscriptionConfig, SubscriptionTarget, TrackKey}; +use super::subscriptions::{ + DEFAULT_VIDEO_DIMENSION, SubscriptionConfig, SubscriptionTarget, TrackKey, +}; use super::tracer::Tracer; use super::tracks::{LocalTrack, RemoteParticipant, RemoteTrack}; @@ -240,7 +242,8 @@ pub enum CallEvent { IceRestarted(PeerType), /// The SFU reported an error for this participant. Error(SfuJoinError), - /// The call ended. + /// The call ended: the SFU or the coordinator (`call.ended`) reported it. + /// Sent once, after which the SDK leaves the call. CallEnded, /// The connection state changed. CallingStateChanged(CallingState), diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 537e7d7..1a5ac44 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -23,6 +23,8 @@ pub(super) struct CallStateCache { pub(super) started_at: Option, pub(super) e2ee_enabled: bool, pub(super) current_grants: Option, + /// Set by the first report of the call end. + pub(super) ended: bool, } impl RtcCore { @@ -107,6 +109,7 @@ impl RtcCore { started_at: state.started_at, e2ee_enabled: state.e2ee_enabled, current_grants: None, + ended: false, }; { let mut participants = self.participants.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index 1be0f62..34e0e5d 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -116,9 +116,10 @@ impl RtcCore { user_id: entry.user_id.clone(), session_id: entry.session_id.clone(), track_type: target.track_type as i32, - dimension: target.dimension.and_then(|(width, height)| { - is_video_type(target.track_type) - .then_some(models::VideoDimension { width, height }) + dimension: is_video_type(target.track_type).then(|| { + let (width, height) = + target.dimension.unwrap_or(DEFAULT_VIDEO_DIMENSION); + models::VideoDimension { width, height } }), }); } @@ -136,13 +137,11 @@ impl RtcCore { { continue; } - let dimension = if is_video_type(track_type) { - config - .video_dimension - .map(|(width, height)| models::VideoDimension { width, height }) - } else { - None - }; + let dimension = is_video_type(track_type).then(|| { + let (width, height) = + config.video_dimension.unwrap_or(DEFAULT_VIDEO_DIMENSION); + models::VideoDimension { width, height } + }); tracks.push(signal::TrackSubscriptionDetails { user_id: entry.user_id.clone(), session_id: entry.session_id.clone(), @@ -231,8 +230,11 @@ impl RtcCore { } let key = TrackKey::new(participant.session_id.clone(), track_type); let weak = Arc::downgrade(&self); + // The caller can drop the track on a thread without a runtime. + let runtime = tokio::runtime::Handle::current(); let on_drop = Box::new(move || { if let Some(core) = weak.upgrade() { + let _runtime = runtime.enter(); let task_core = core.clone(); std::mem::drop(core.spawn_generation_task(generation, async move { task_core diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index b47bc64..ef0cd96 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -162,6 +162,13 @@ async fn fake_sfu( /// A local coordinator WebSocket that sends `connection.ok`. It returns the /// REST base URL and a task that ends when the client socket closes. async fn fake_coordinator() -> (String, tokio::task::JoinHandle<()>) { + fake_coordinator_sending(Vec::new()).await +} + +/// [`fake_coordinator`] that also sends `events` after `connection.ok`. +async fn fake_coordinator_sending( + events: Vec, +) -> (String, tokio::task::JoinHandle<()>) { use futures_util::{SinkExt, StreamExt}; use tokio_tungstenite::tungstenite::Message; @@ -187,6 +194,12 @@ async fn fake_coordinator() -> (String, tokio::task::JoinHandle<()>) { )) .await .expect("send connection.ok"); + for event in events { + socket + .send(Message::Text(event.to_string().into())) + .await + .expect("send coordinator event"); + } while let Some(Ok(_)) = socket.next().await {} }); (format!("ws://{address}"), server) @@ -236,6 +249,21 @@ async fn establish_fake( (connection, sfu) } +/// The event loop context of the current `connection`. +fn event_context(core: &Arc, connection: &Connection) -> EventLoopContext { + EventLoopContext { + core: core.clone(), + subscriber: connection.subscriber.clone(), + publisher: connection.publisher.clone(), + signal: connection.signal.clone(), + session_id: connection.session_id.clone(), + pending_ice: connection.pending_ice.clone(), + generation: connection.generation, + ws_healthy: connection.ws_healthy.clone(), + reconnect_enabled: connection.reconnect_enabled.clone(), + } +} + /// The event loop context of `connection` after a migration detached it. fn detached_context(core: &Arc, connection: &Connection) -> EventLoopContext { connection.reconnect_enabled.store(false, Ordering::SeqCst); @@ -368,6 +396,140 @@ async fn leave_tears_down_the_stored_connection() { assert_eq!(spawned, completed); } +#[tokio::test] +async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() { + let (coordinator_ws_url, _coordinator) = fake_coordinator_sending(vec![ + json!({ "type": "call.ended", "call_cid": "default:test-call" }), + ]) + .await; + let core = test_core_with_config(ClientConfig { + coordinator_ws_url, + ..ClientConfig::default() + }); + let generation = prepare_joined_core(&core, "alice"); + let (connection, sfu) = establish_fake(&core, generation).await; + let context = event_context(&core, &connection); + *core.connection.lock().await = Some(connection); + let mut events = core.subscribe(); + let token = core.current_user_token().expect("user token"); + + core.connect_coordinator_events(generation, &token, "alice") + .await + .expect("coordinator events"); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Ok(CallEvent::CallEnded) = events.recv().await { + return; + } + } + }) + .await + .expect("call ended event"); + connection::handle_event( + &context, + sfu_event::EventPayload::CallEnded(event::CallEnded::default()), + ) + .await + .expect("handle SFU call ended"); + + wait_for( + Duration::from_secs(2), + || core.state() == CallingState::Left, + "the ended call is left", + ) + .await; + let mut repeated = 0; + while let Ok(event) = events.try_recv() { + if matches!(event, CallEvent::CallEnded) { + repeated += 1; + } + } + assert_eq!(repeated, 0, "call ended is reported once"); + let requests = requests_until_close(sfu).await; + assert!(requests.iter().any(|request| matches!( + request.request_payload, + Some(event::sfu_request::RequestPayload::LeaveCallRequest(_)) + ))); +} + +#[tokio::test] +async fn call_ended_twice_before_the_leave_is_reported_once() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let context = event_context(&core, &connection); + *core.connection.lock().await = Some(connection); + let mut events = core.subscribe(); + + // Neither call yields, so the spawned leave cannot run between them. + for _ in 0..2 { + connection::handle_event( + &context, + sfu_event::EventPayload::CallEnded(event::CallEnded::default()), + ) + .await + .expect("handle SFU call ended"); + } + + wait_for( + Duration::from_secs(2), + || core.state() == CallingState::Left, + "the ended call is left", + ) + .await; + let mut reported = 0; + while let Ok(event) = events.try_recv() { + if matches!(event, CallEvent::CallEnded) { + reported += 1; + } + } + assert_eq!(reported, 1); +} + +#[tokio::test] +async fn dropped_join_allows_a_new_join() { + let (coordinator_ws_url, coordinator) = fake_coordinator().await; + // Accepts connections but never answers, so the coordinator join call waits. + let silent = TcpListener::bind("127.0.0.1:0").expect("bind silent server"); + let core = test_core_with_config(ClientConfig { + coordinator_ws_url, + base_url: format!("http://{}", silent.local_addr().expect("silent address")), + ..ClientConfig::default() + }); + let token = crate::token::create_user_token( + b"test-secret", + "alice", + &crate::token::TokenOptions::default(), + ) + .expect("test user token"); + let mut data = JoinCallData::new("alice"); + data.location = Some("test-location".to_owned()); + + let join = tokio::time::timeout(Duration::from_millis(500), core.join(token, data)).await; + assert!(join.is_err(), "join waits for the coordinator"); + + assert_eq!(core.state(), CallingState::Idle); + tokio::time::timeout(Duration::from_secs(2), coordinator) + .await + .expect("coordinator socket closed") + .expect("fake coordinator task"); + core.begin_join().expect("a new join can start"); +} + +#[tokio::test] +async fn dropped_leave_still_leaves_the_call() { + let core = test_core(); + prepare_joined_core(&core, "alice"); + let connection_slot = core.connection.lock().await; + + let leave = tokio::time::timeout(Duration::from_millis(50), core.leave("dropped leave")).await; + assert!(leave.is_err(), "leave waits for the connection lock"); + drop(connection_slot); + + assert_eq!(core.state(), CallingState::Left); + core.begin_join().expect("a new join can start"); +} + #[tokio::test] async fn leave_closes_a_connection_owned_by_a_cancelled_join() { let core = test_core(); @@ -620,6 +782,65 @@ async fn published_audio_is_paced_only_while_the_publisher_is_connected() { core.leave("test cleanup").await.expect("leave"); } +#[tokio::test] +async fn remote_track_dropped_without_a_runtime_unsubscribes() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let (epoch, reconnect_enabled) = (connection.epoch, connection.reconnect_enabled.clone()); + *core.connection.lock().await = Some(connection); + let (remote_tx, remote_rx) = std::sync::mpsc::channel(); + *core + .on_track_cb + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(Arc::new(move |remote| { + let _ = remote_tx.send(remote); + })); + let audio = LocalAudioTrack::opus().expect("opus track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(audio.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut inbound) = peer::connect_audio_receiver(&sender).await; + audio.start_pacing().await; + let inbound = tokio::time::timeout(Duration::from_secs(5), inbound.recv()) + .await + .expect("inbound track") + .expect("inbound track channel"); + core.clone() + .handle_incoming_track( + generation, + epoch, + reconnect_enabled, + inbound, + Arc::downgrade(&receiver), + ) + .await; + let remote = remote_rx.recv().expect("remote track"); + let key = TrackKey::new(remote.participant().session_id.clone(), remote.track_type()); + + thread::spawn(move || drop(remote)) + .join() + .expect("a drop without a runtime does not panic"); + + wait_for( + Duration::from_secs(2), + || { + core.manual_unsub + .lock() + .unwrap_or_else(|error| error.into_inner()) + .contains(&key) + }, + "unsubscribe after the drop", + ) + .await; + audio.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + core.leave("test cleanup").await.expect("leave"); +} + #[tokio::test] async fn detached_connection_ignores_publish_options_from_its_sfu() { let core = test_core(); diff --git a/src/rtc/subscriptions.rs b/src/rtc/subscriptions.rs index 44ad069..da35266 100644 --- a/src/rtc/subscriptions.rs +++ b/src/rtc/subscriptions.rs @@ -13,6 +13,10 @@ use super::proto::models::TrackType; +/// Video dimension requested when a subscription gives none. The SFU rejects a +/// video or screen-share subscription without a dimension. +pub(crate) const DEFAULT_VIDEO_DIMENSION: (u32, u32) = (1280, 720); + /// A precise subscription to one participant session and track kind. #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[non_exhaustive] @@ -21,12 +25,13 @@ pub struct SubscriptionTarget { pub session_id: String, /// The remote track kind to receive. pub track_type: TrackType, - /// Optional preferred video dimensions sent as an SFU adaptation hint. + /// Preferred video dimensions sent as an SFU adaptation hint. `None` + /// requests 1280×720 for video and screen-share. pub dimension: Option<(u32, u32)>, } impl SubscriptionTarget { - /// Subscribe to `track_type` from `session_id` using the SFU's default size. + /// Subscribe to `track_type` from `session_id`, at 1280×720 for video. pub fn new(session_id: impl Into, track_type: TrackType) -> Self { Self { session_id: session_id.into(), @@ -55,7 +60,8 @@ pub struct SubscriptionConfig { pub video: bool, /// Subscribe to remote screen-share (video + audio). pub screen_share: bool, - /// Preferred video dimension hint sent to the SFU (width, height). + /// Preferred video dimension hint sent to the SFU (width, height). `None` + /// requests 1280×720 for video and screen-share. pub video_dimension: Option<(u32, u32)>, } diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 13f971c..8aa7d52 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -29,7 +29,7 @@ use std::collections::VecDeque; use std::num::NonZeroU8; use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU16, AtomicU32, Ordering}; -use std::sync::{Arc, Mutex as StdMutex}; +use std::sync::{Arc, Mutex as StdMutex, Weak}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bytes::Bytes; @@ -657,21 +657,25 @@ impl LocalAudioTrack { { return; } - let inner = self.inner.clone(); - let handle = tokio::spawn(async move { pace_audio(inner).await }); + let track = Arc::downgrade(&self.inner); + let handle = tokio::spawn(async move { pace_audio(track).await }); *self.inner.pacer.lock().unwrap_or_else(|e| e.into_inner()) = Some(handle); } } /// The 20 ms pacing loop: pull one Opus frame worth of PCM (or silence) every -/// tick, encode it, and packetize it onto the outbound track. -async fn pace_audio(inner: Arc) { +/// tick, encode it, and packetize it onto the outbound track. It ends when the +/// track is stopped or dropped. +async fn pace_audio(track: Weak) { let mut interval = tokio::time::interval(Duration::from_millis(20)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut scratch = vec![0i16; FRAME_SAMPLES_20MS]; let mut encoded = vec![0u8; MAX_OPUS_PACKET_BYTES]; loop { interval.tick().await; + let Some(inner) = track.upgrade() else { + return; + }; if inner.core.stopped.load(Ordering::SeqCst) { return; } @@ -2307,6 +2311,29 @@ mod tests { track.stop(); } + #[tokio::test] + async fn dropping_a_started_track_ends_its_pacer() { + let alive_tasks = || { + tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks() + }; + let before = alive_tasks(); + let track = LocalAudioTrack::opus().expect("opus track"); + track.start_pacing().await; + assert_eq!(alive_tasks(), before + 1); + + drop(track); + + tokio::time::timeout(Duration::from_secs(1), async { + while alive_tasks() != before { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the pacer ends with its track"); + } + #[tokio::test] async fn pcm_queued_before_the_connection_is_sent_from_its_first_sample() { let track = LocalAudioTrack::opus().expect("opus track"); diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index e17a28b..84047df 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -219,6 +219,8 @@ struct AudioDecode { /// rebuilt frame as long as the output buffer. A lost packet states no /// length, so the stream's own frame size is the best value to use. frame_samples: usize, + /// Decode output, reused for every packet. Queued frames are exact copies. + scratch: Vec, } /// How this track's payload is turned into something the caller can use. @@ -515,6 +517,7 @@ impl AudioDecode { last_seq: None, ready: VecDeque::new(), frame_samples: FRAME_SAMPLES_20MS, + scratch: vec![0; MAX_OPUS_FRAME_SAMPLES], } } @@ -571,17 +574,18 @@ impl AudioDecode { } else { MAX_OPUS_FRAME_SAMPLES }; - let mut out = vec![0i16; capacity]; - match self.decoder.decode(payload, &mut out, fec) { + match self + .decoder + .decode(payload, &mut self.scratch[..capacity], fec) + { Ok(samples) => { - out.truncate(samples); - if out.is_empty() { + if samples == 0 { return; } if !rebuilt { self.frame_samples = samples; } - self.ready.push_back(out); + self.ready.push_back(self.scratch[..samples].to_vec()); } Err(error) => { tracing::debug!(error = %error, "stream.rtc.remote.opus_decode_failed"); @@ -826,6 +830,17 @@ mod tests { } } + #[test] + fn a_decoded_frame_holds_only_its_samples() { + let packets = tone_packets(1, true); + let mut state = audio_decode(); + + state.push_packet(0, &packets[0]); + + let frame = state.take_frame().expect("decoded frame"); + assert_eq!(frame.capacity(), frame.len()); + } + #[test] fn a_lost_packet_is_rebuilt_from_the_next_one() { let packets = tone_packets(3, true); diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index 022cb1c..b1dc2ac 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -914,6 +914,64 @@ async fn publish_blue_video_reaches_raw_rtp_and_i420_decoder() { outcome.expect("VP9 RTP/decode test timed out"); } +/// A video subscription without a dimension hint is accepted and delivers video. +#[tokio::test] +async fn video_subscription_without_a_dimension_receives_video() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(120), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let video_a = LocalVideoTrack::vp9().expect("vp9 track"); + call_a + .publish_video(video_a.clone()) + .await + .expect("A publish_video"); + let feeder = spawn_blue_video(video_a); + + let mut rx_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig { + video_dimension: None, + ..SubscriptionConfig::audio_video() + }) + .await + .expect("B update_subscriptions without a dimension"); + recv_track( + &mut rx_b, + &user_a, + TrackType::Video, + Duration::from_secs(60), + ) + .await + .expect("B did not receive A's video track"); + + feeder.abort(); + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("dimensionless video subscription test timed out"); +} + #[tokio::test] async fn vp9_svc_preserves_one_ssrc_and_adapts_all_spatial_layers() { let Some(client) = common::client_or_skip() else { From 8793fefc4c3c4e6a2233037bad0aa190c7c6fb87 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Tue, 29 Sep 2026 17:29:03 +0200 Subject: [PATCH 03/37] fix: match JS backoff between join retries --- src/rtc/reconnect.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/rtc/reconnect.rs b/src/rtc/reconnect.rs index cc11aa0..c9d73d2 100644 --- a/src/rtc/reconnect.rs +++ b/src/rtc/reconnect.rs @@ -143,7 +143,7 @@ pub fn evaluate_join_failure( return JoinAttemptOutcome::Exhausted; } JoinAttemptOutcome::Retry { - delay: retry_interval(attempt + 1), + delay: retry_interval(attempt), switch_sfu: is_join_error_code || edge_failures >= 2, } } @@ -374,6 +374,21 @@ mod tests { } } + #[test] + fn join_retry_delays_stay_within_backoff_bounds() { + for (attempt, bounds) in [(0, 250..=500), (1, 250..=2500)] { + for _ in 0..100 { + match evaluate_join_failure(false, false, 1, attempt, 3) { + JoinAttemptOutcome::Retry { delay, .. } => { + let d = delay.as_millis(); + assert!(bounds.contains(&d), "attempt {attempt} out of range: {d}"); + } + other => panic!("expected retry, got {other:?}"), + } + } + } + } + #[test] fn rate_limiter_allows_max_then_blocks_within_window() { let mut rl = SlidingWindowRateLimiter::new(10, Duration::from_secs(120)); From 2074c92003f48e302f380162b046dc7e52569941 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 13:51:21 +0200 Subject: [PATCH 04/37] docs: state that event receivers get only later events --- src/rtc/client.rs | 4 +++- src/video/call.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/rtc/client.rs b/src/rtc/client.rs index 1d73aab..16b0ac7 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -243,7 +243,9 @@ pub struct RtcCall { } impl RtcCall { - /// Subscribe to the typed SFU event stream. + /// Subscribe to the typed SFU event stream. A receiver gets only events sent + /// after it subscribes, and [`RtcClient::join`] returns this handle after the + /// join. Read [`Self::participants`] and [`Self::call_state`] for earlier state. pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { self.core.subscribe() } diff --git a/src/video/call.rs b/src/video/call.rs index 4560543..62b166d 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -859,7 +859,8 @@ impl Call { } /// Subscribe to the typed SFU event stream (participant joined/left, tracks, - /// errors). Subscribe before or after [`Call::join`]. + /// errors). A receiver gets only events sent after it subscribes. Subscribe + /// before [`Call::join`] to get the join events, or read [`Call::participants`]. pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { self.rtc.subscribe() } From 0cee3bfd7b4bb8cfe969577696eb57d67e1025c1 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 14:13:37 +0200 Subject: [PATCH 05/37] feat: carry SFU track and call-ended data on call events --- CHANGELOG.md | 8 ++++ src/rtc/join/connection.rs | 16 ++++++-- src/rtc/join/lifecycle.rs | 10 +++-- src/rtc/join/mod.rs | 21 +++++++--- src/rtc/join/tests.rs | 79 ++++++++++++++++++++++++++++++++++++-- tests/rtc_media.rs | 5 +-- 6 files changed, 120 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b433800..f6ea897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ for a staging or local environment must set both fields. Code that builds `..ClientConfig::default()`. `DEFAULT_COORDINATOR_WS_URL` moved from `rtc::coordinator::ws` to the crate root. +### Track and call-ended events carry the SFU data + +`CallEvent::TrackPublished` and `CallEvent::TrackUnpublished` give `track_type` +as a `TrackType`, not an `i32`, and add `participant`. `TrackUnpublished` also +adds `cause`. `CallEvent::CallEnded` is now `CallEnded { reason }`: the SFU +reason, or `None` when the coordinator reported the end first. Patterns that +match these variants must use the new fields or `..`. + ## New Features ### Video REST: advanced call statistics and reporting diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 3109c24..211bbba 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -302,7 +302,8 @@ pub(super) async fn handle_event( let _ = core.events_tx.send(CallEvent::TrackPublished { user_id: ev.user_id, session_id: ev.session_id, - track_type: ev.r#type, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + participant: ev.participant, }); } E::TrackUnpublished(ev) => { @@ -312,7 +313,10 @@ pub(super) async fn handle_event( let _ = core.events_tx.send(CallEvent::TrackUnpublished { user_id: ev.user_id, session_id: ev.session_id, - track_type: ev.r#type, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + cause: models::TrackUnpublishReason::try_from(ev.cause) + .unwrap_or(models::TrackUnpublishReason::Unspecified), + participant: ev.participant, }); } E::DominantSpeakerChanged(ev) => { @@ -400,7 +404,13 @@ pub(super) async fn handle_event( E::ParticipantMigrationComplete(_) => { core.complete_migration(context.generation); } - E::CallEnded(_) => core.end_call(context.generation), + E::CallEnded(event) => core.end_call( + context.generation, + Some( + models::CallEndedReason::try_from(event.reason) + .unwrap_or(models::CallEndedReason::Unspecified), + ), + ), E::PublisherAnswer(_) | E::JoinResponse(_) => { tracing::debug!("stream.rtc.unexpected_handshake_event"); } diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index 7f7f678..e090f79 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -673,7 +673,11 @@ impl RtcCore { /// The SFU or the coordinator reported the end of the call: send /// [`CallEvent::CallEnded`] once and leave the call. - pub(super) fn end_call(self: &Arc, generation: u64) { + pub(super) fn end_call( + self: &Arc, + generation: u64, + reason: Option, + ) { if !self.is_generation_current(generation) { return; } @@ -688,7 +692,7 @@ impl RtcCore { if already_ended { return; } - let _ = self.events_tx.send(CallEvent::CallEnded); + let _ = self.events_tx.send(CallEvent::CallEnded { reason }); let this = self.clone(); // Not a generation task: `leave` ends the generation. std::mem::drop(self.spawn_runtime_task(async move { @@ -819,7 +823,7 @@ impl RtcCore { let ended = event.event_type == "call.ended"; let _ = sender.send(CallEvent::Coordinator(event)); if ended { - event_core.end_call(generation); + event_core.end_call(generation, None); } } Ok(Some(_)) => {} diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 64163c8..0392eb0 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -194,8 +194,10 @@ pub enum CallEvent { user_id: String, /// The publisher's session id. session_id: String, - /// The `TrackType` value. - track_type: i32, + /// The published track type. + track_type: TrackType, + /// The publisher, sent by the SFU only in large calls. + participant: Option, }, /// A track was unpublished. TrackUnpublished { @@ -203,8 +205,12 @@ pub enum CallEvent { user_id: String, /// The publisher's session id. session_id: String, - /// The `TrackType` value. - track_type: i32, + /// The unpublished track type. + track_type: TrackType, + /// Why the track was unpublished. + cause: models::TrackUnpublishReason, + /// The publisher, sent by the SFU only in large calls. + participant: Option, }, /// The dominant speaker changed. DominantSpeakerChanged { @@ -238,13 +244,16 @@ pub enum CallEvent { PublishQualityChanged(event::ChangePublishQuality), /// The current participant's publishing grants changed. CallGrantsUpdated(event::CallGrantsUpdated), - /// An SFU-directed ICE restart completed for a peer connection. + /// The SFU requested an ICE restart. The SDK restarts only the publisher. IceRestarted(PeerType), /// The SFU reported an error for this participant. Error(SfuJoinError), /// The call ended: the SFU or the coordinator (`call.ended`) reported it. /// Sent once, after which the SDK leaves the call. - CallEnded, + CallEnded { + /// The SFU reason, or `None` when the coordinator reported the end first. + reason: Option, + }, /// The connection state changed. CallingStateChanged(CallingState), } diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index ef0cd96..ec500ee 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -418,7 +418,7 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() .expect("coordinator events"); tokio::time::timeout(Duration::from_secs(2), async { loop { - if let Ok(CallEvent::CallEnded) = events.recv().await { + if let Ok(CallEvent::CallEnded { reason: None }) = events.recv().await { return; } } @@ -440,7 +440,7 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() .await; let mut repeated = 0; while let Ok(event) = events.try_recv() { - if matches!(event, CallEvent::CallEnded) { + if matches!(event, CallEvent::CallEnded { .. }) { repeated += 1; } } @@ -465,7 +465,9 @@ async fn call_ended_twice_before_the_leave_is_reported_once() { for _ in 0..2 { connection::handle_event( &context, - sfu_event::EventPayload::CallEnded(event::CallEnded::default()), + sfu_event::EventPayload::CallEnded(event::CallEnded { + reason: models::CallEndedReason::Kicked as i32, + }), ) .await .expect("handle SFU call ended"); @@ -479,13 +481,82 @@ async fn call_ended_twice_before_the_leave_is_reported_once() { .await; let mut reported = 0; while let Ok(event) = events.try_recv() { - if matches!(event, CallEvent::CallEnded) { + if matches!( + event, + CallEvent::CallEnded { + reason: Some(models::CallEndedReason::Kicked) + } + ) { reported += 1; } } assert_eq!(reported, 1); } +#[tokio::test] +async fn track_events_report_the_track_type_cause_and_participant() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let context = event_context(&core, &connection); + *core.connection.lock().await = Some(connection); + let mut events = core.subscribe(); + let bob = models::Participant { + user_id: "bob".to_owned(), + session_id: "bob-session".to_owned(), + ..Default::default() + }; + + connection::handle_event( + &context, + sfu_event::EventPayload::TrackPublished(event::TrackPublished { + user_id: "bob".to_owned(), + session_id: "bob-session".to_owned(), + r#type: TrackType::Audio as i32, + participant: Some(bob.clone()), + }), + ) + .await + .expect("handle track published"); + + let Ok(CallEvent::TrackPublished { + track_type, + participant, + .. + }) = events.try_recv() + else { + panic!("expected a track published event"); + }; + assert_eq!(track_type, TrackType::Audio); + assert_eq!(participant.as_ref(), Some(&bob)); + + connection::handle_event( + &context, + sfu_event::EventPayload::TrackUnpublished(event::TrackUnpublished { + user_id: "bob".to_owned(), + session_id: "bob-session".to_owned(), + r#type: TrackType::Audio as i32, + cause: models::TrackUnpublishReason::Moderation as i32, + participant: Some(bob.clone()), + }), + ) + .await + .expect("handle track unpublished"); + + let Ok(CallEvent::TrackUnpublished { + track_type, + cause, + participant, + .. + }) = events.try_recv() + else { + panic!("expected a track unpublished event"); + }; + assert_eq!(track_type, TrackType::Audio); + assert_eq!(cause, models::TrackUnpublishReason::Moderation); + assert_eq!(participant, Some(bob)); +} + #[tokio::test] async fn dropped_join_allows_a_new_join() { let (coordinator_ws_url, coordinator) = fake_coordinator().await; diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index b1dc2ac..8460741 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -380,7 +380,6 @@ async fn await_track_event( timeout: Duration, ) -> bool { use tokio::sync::broadcast::error::RecvError; - let want = track_type as i32; let deadline = tokio::time::sleep(timeout); tokio::pin!(deadline); loop { @@ -388,9 +387,9 @@ async fn await_track_event( () = &mut deadline => return false, recv = events.recv() => match recv { Ok(CallEvent::TrackPublished { user_id, track_type: tt, .. }) - if published && user_id == user && tt == want => return true, + if published && user_id == user && tt == track_type => return true, Ok(CallEvent::TrackUnpublished { user_id, track_type: tt, .. }) - if !published && user_id == user && tt == want => return true, + if !published && user_id == user && tt == track_type => return true, Ok(_) | Err(RecvError::Lagged(_)) => {} Err(RecvError::Closed) => return false, } From e3bdfa0192bb913ec7be4dc20979e932da1f8534 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 14:42:30 +0200 Subject: [PATCH 06/37] fix: send ParticipantCountChanged only when the count changes --- src/rtc/join/connection.rs | 5 +++-- src/rtc/join/participants.rs | 13 +++++++++---- src/rtc/join/tests.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 211bbba..bdee685 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -333,8 +333,9 @@ pub(super) async fn handle_event( .send(CallEvent::AudioLevelChanged(ev.audio_levels)); } E::HealthCheckResponse(event) => { - if let Some(participant_count) = event.participant_count { - core.update_participant_count(participant_count); + if let Some(participant_count) = event.participant_count + && core.update_participant_count(participant_count) + { let _ = core .events_tx .send(CallEvent::ParticipantCountChanged(participant_count)); diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 1a5ac44..7941110 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -265,11 +265,16 @@ impl RtcCore { } } - pub(super) fn update_participant_count(&self, participant_count: models::ParticipantCount) { - self.call_state + /// Returns whether the stored count changed. + pub(super) fn update_participant_count( + &self, + participant_count: models::ParticipantCount, + ) -> bool { + let mut state = self + .call_state .lock() - .unwrap_or_else(|error| error.into_inner()) - .participant_count = participant_count; + .unwrap_or_else(|error| error.into_inner()); + std::mem::replace(&mut state.participant_count, participant_count) != participant_count } pub(super) fn update_pins(&self, pins: Vec) { diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index ec500ee..bdcfe05 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -557,6 +557,38 @@ async fn track_events_report_the_track_type_cause_and_participant() { assert_eq!(participant, Some(bob)); } +#[tokio::test] +async fn participant_count_event_is_sent_only_when_the_count_changes() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let context = event_context(&core, &connection); + *core.connection.lock().await = Some(connection); + let mut events = core.subscribe(); + + for total in [2, 2, 3] { + connection::handle_event( + &context, + sfu_event::EventPayload::HealthCheckResponse(event::HealthCheckResponse { + participant_count: Some(models::ParticipantCount { + total, + anonymous: 0, + }), + }), + ) + .await + .expect("handle health check response"); + } + + let mut totals = Vec::new(); + while let Ok(event) = events.try_recv() { + if let CallEvent::ParticipantCountChanged(count) = event { + totals.push(count.total); + } + } + assert_eq!(totals, vec![2, 3]); +} + #[tokio::test] async fn dropped_join_allows_a_new_join() { let (coordinator_ws_url, coordinator) = fake_coordinator().await; From a3229fee41902f64ad23078456f433e23ab19eec Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 15:24:56 +0200 Subject: [PATCH 07/37] fix: unsubscribe only when the latest remote track is dropped --- src/rtc/join/lifecycle.rs | 4 +++ src/rtc/join/mod.rs | 6 ++++ src/rtc/join/subscriptions_runtime.rs | 17 +++++++++- src/rtc/join/tests.rs | 45 ++++++++++++++++++++------- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index e090f79..c6730ec 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -720,6 +720,10 @@ impl RtcCore { .lock() .unwrap_or_else(|e| e.into_inner()) .clear(); + self.delivered_tracks + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); self.own_capabilities .lock() .unwrap_or_else(|e| e.into_inner()) diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 0392eb0..54b1132 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -526,6 +526,10 @@ pub struct RtcCore { /// Tracks the caller explicitly dropped (unsubscribed); never re-subscribed /// until the publisher republishes them. manual_unsub: StdMutex>, + /// The id of the latest [`RemoteTrack`] delivered for each track. Only a + /// drop of that track unsubscribes. + delivered_tracks: StdMutex>, + next_remote_track_id: AtomicU64, /// Exact per-session subscriptions, or `None` while using the coarse policy. manual_subscriptions: StdMutex>>, /// Known participants keyed by session id (correlation + subscription build). @@ -590,6 +594,8 @@ impl RtcCore { sub_config: StdMutex::new(SubscriptionConfig::default()), subs_active: AtomicBool::new(false), manual_unsub: StdMutex::new(HashSet::new()), + delivered_tracks: StdMutex::new(HashMap::new()), + next_remote_track_id: AtomicU64::new(0), manual_subscriptions: StdMutex::new(None), participants: StdMutex::new(HashMap::new()), call_state: StdMutex::new(CallStateCache::default()), diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index 34e0e5d..ff33221 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -229,6 +229,11 @@ impl RtcCore { return; } let key = TrackKey::new(participant.session_id.clone(), track_type); + let track_id = self.next_remote_track_id.fetch_add(1, Ordering::SeqCst); + self.delivered_tracks + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(key.clone(), track_id); let weak = Arc::downgrade(&self); // The caller can drop the track on a thread without a runtime. let runtime = tokio::runtime::Handle::current(); @@ -238,7 +243,7 @@ impl RtcCore { let task_core = core.clone(); std::mem::drop(core.spawn_generation_task(generation, async move { task_core - .on_remote_track_dropped(generation, connection_epoch, key) + .on_remote_track_dropped(generation, connection_epoch, key, track_id) .await; })); } @@ -253,6 +258,7 @@ impl RtcCore { generation: u64, connection_epoch: u64, key: TrackKey, + track_id: u64, ) { { let connection = self.connection.lock().await; @@ -261,6 +267,15 @@ impl RtcCore { }) { return; } + let mut delivered = self + .delivered_tracks + .lock() + .unwrap_or_else(|e| e.into_inner()); + if delivered.get(&key) != Some(&track_id) { + return; + } + delivered.remove(&key); + drop(delivered); self.manual_unsub .lock() .unwrap_or_else(|e| e.into_inner()) diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index bdcfe05..2c88a7b 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -886,7 +886,7 @@ async fn published_audio_is_paced_only_while_the_publisher_is_connected() { } #[tokio::test] -async fn remote_track_dropped_without_a_runtime_unsubscribes() { +async fn only_the_latest_remote_track_unsubscribes_when_dropped_without_a_runtime() { let core = test_core(); let generation = prepare_joined_core(&core, "alice"); let (connection, _sfu) = establish_fake(&core, generation).await; @@ -911,17 +911,39 @@ async fn remote_track_dropped_without_a_runtime_unsubscribes() { .await .expect("inbound track") .expect("inbound track channel"); - core.clone() - .handle_incoming_track( - generation, - epoch, - reconnect_enabled, - inbound, - Arc::downgrade(&receiver), - ) - .await; - let remote = remote_rx.recv().expect("remote track"); + for _ in 0..2 { + core.clone() + .handle_incoming_track( + generation, + epoch, + reconnect_enabled.clone(), + inbound.clone(), + Arc::downgrade(&receiver), + ) + .await; + } + let stale = remote_rx.recv().expect("stale remote track"); + let remote = remote_rx.recv().expect("latest remote track"); let key = TrackKey::new(remote.participant().session_id.clone(), remote.track_type()); + let baseline = alive_tasks(); + + thread::spawn(move || drop(stale)) + .join() + .expect("a drop without a runtime does not panic"); + wait_for( + Duration::from_secs(2), + || alive_tasks() == baseline, + "the stale drop is handled", + ) + .await; + assert!( + !core + .manual_unsub + .lock() + .unwrap_or_else(|error| error.into_inner()) + .contains(&key), + "a stale drop must not unsubscribe the latest track" + ); thread::spawn(move || drop(remote)) .join() @@ -1567,6 +1589,7 @@ async fn stale_remote_track_drop_does_not_change_new_generation_subscriptions() first, 0, TrackKey::new("remote-session".to_owned(), TrackType::Audio), + 0, ) .await; From 191e2faec83f8c97acaf8d0071d404d610204546 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 15:46:41 +0200 Subject: [PATCH 08/37] feat: prepare an RtcCall before the join and share it with Call --- CHANGELOG.md | 8 ++++++ src/rtc/client.rs | 63 ++++++++++++++++++++++++++++++++++++----------- src/video/call.rs | 21 ++++++++++------ tests/rtc_join.rs | 26 +++++++++++++++---- 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6ea897..fed9e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,14 @@ match these variants must use the new fields or `..`. ## New Features +### A token-only client can prepare a call before the join + +`RtcClient::call` returns an `RtcCall` that is not joined yet, and +`RtcCall::join` joins it. Register `on_track` and subscribe before the join to +get the join events and tracks. `Call::rtc` gives the same `RtcCall` type for a +client with an API secret; both handles share one session. `RtcCall` also adds +`update_publish_options` and `set_disconnection_timeout`. + ### Video REST: advanced call statistics and reporting Application-level stats on `VideoClient` (`get_active_calls_status`, diff --git a/src/rtc/client.rs b/src/rtc/client.rs index 16b0ac7..74903eb 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -17,6 +17,7 @@ use crate::token::{self, TokenOptions}; use super::error::{Result, RtcError}; use super::join::{CallEvent, CallStateSnapshot, CallingState, JoinCallData, RtcCore}; use super::proto::models::TrackType; +use super::publish_options::ClientPublishOptions; use super::subscriptions::{SubscriptionConfig, SubscriptionTarget}; use super::tracks::{LocalAudioTrack, LocalTrack, LocalVideoTrack, RemoteParticipant, RemoteTrack}; @@ -47,9 +48,9 @@ where pub(crate) enum UserTokenSource { Static(String), Provider(Arc), + /// Mints a call-scoped token for the joining user from the API secret. ServerMinted { client: Arc, - user_id: String, call_cid: String, expiration: Duration, }, @@ -61,13 +62,11 @@ impl std::fmt::Debug for UserTokenSource { Self::Static(_) => f.debug_tuple("Static").field(&"").finish(), Self::Provider(_) => f.debug_tuple("Provider").field(&"").finish(), Self::ServerMinted { - user_id, call_cid, expiration, .. } => f .debug_struct("ServerMinted") - .field("user_id", user_id) .field("call_cid", call_cid) .field("expiration", expiration) .finish_non_exhaustive(), @@ -82,12 +81,11 @@ impl UserTokenSource { Self::Provider(provider) => provider.load_token().await.map_err(RtcError::from)?, Self::ServerMinted { client, - user_id, call_cid, expiration, } => token::create_user_token( client.api_secret(), - user_id, + expected_user_id, &TokenOptions { expiration: Some(*expiration), call_cids: Some(vec![call_cid.clone()]), @@ -221,6 +219,18 @@ impl RtcClient { self } + /// A handle for `:` that is not joined yet. Register + /// callbacks and subscribe before [`RtcCall::join`] to get the join events + /// and tracks. + pub fn call(&self, call_type: impl Into, call_id: impl Into) -> RtcCall { + let core = RtcCore::new(self.client.clone(), call_type.into(), call_id.into()); + core.set_disconnection_timeout(self.disconnection_timeout); + RtcCall { + core, + token_source: self.token_source.clone(), + } + } + /// Join `:` and return a live [`RtcCall`] handle. pub async fn join( &self, @@ -228,24 +238,49 @@ impl RtcClient { call_id: impl Into, data: JoinCallData, ) -> Result { - let core = RtcCore::new(self.client.clone(), call_type.into(), call_id.into()); - core.set_disconnection_timeout(self.disconnection_timeout); - core.join_with_token_source(self.token_source.clone(), data) - .await?; - Ok(RtcCall { core }) + let call = self.call(call_type, call_id); + call.join(data).await?; + Ok(call) } } -/// A joined call handle from [`RtcClient::join`]. +/// A call handle from [`RtcClient::call`], [`RtcClient::join`], or +/// [`Call::rtc`](crate::Call::rtc). #[derive(Clone)] pub struct RtcCall { core: Arc, + token_source: UserTokenSource, } impl RtcCall { + pub(crate) fn new(core: Arc, token_source: UserTokenSource) -> Self { + Self { core, token_source } + } + + /// Join the call as an SFU participant with the user token of its client. + /// Illegal (typed error) if already `JOINING`/`JOINED`. + pub async fn join(&self, data: JoinCallData) -> Result<()> { + self.core + .join_with_token_source(self.token_source.clone(), data) + .await + } + + /// Set the maximum reconnect duration. Zero keeps reconnecting indefinitely. + pub fn set_disconnection_timeout(&self, timeout: Duration) { + self.core.set_disconnection_timeout(timeout); + } + + /// Update publishing preferences for the next join generation. + /// + /// Call this before [`Self::join`]. Updates after joining starts emit a + /// warning and cannot affect the active join generation. + pub fn update_publish_options(&self, options: ClientPublishOptions) { + self.core.update_publish_options(options); + } + /// Subscribe to the typed SFU event stream. A receiver gets only events sent - /// after it subscribes, and [`RtcClient::join`] returns this handle after the - /// join. Read [`Self::participants`] and [`Self::call_state`] for earlier state. + /// after it subscribes. Subscribe before [`Self::join`] to get the join + /// events, or read [`Self::participants`] and [`Self::call_state`]. pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { self.core.subscribe() } @@ -462,13 +497,13 @@ mod tests { ); let source = UserTokenSource::ServerMinted { client: client.clone(), - user_id: "user".to_owned(), call_cid: "default:call".to_owned(), expiration: Duration::from_secs(600), }; let token = source.load("user").await.expect("mint token"); let claims = token::decode_token(client.api_secret(), &token).expect("verify token"); + assert_eq!(claims.user_id, "user"); assert_eq!(claims.call_cids, Some(vec!["default:call".to_owned()])); assert_eq!( claims.exp.zip(claims.iat).map(|(exp, iat)| exp - iat), diff --git a/src/video/call.rs b/src/video/call.rs index 62b166d..d0b7dbe 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -843,13 +843,20 @@ impl Call { /// and completes the SFU handshake. Illegal (typed error) if already /// `JOINING`/`JOINED`. Observe participants via [`Call::subscribe`]. pub async fn join(&self, data: crate::rtc::JoinCallData) -> crate::rtc::RtcResult<()> { - let source = crate::rtc::client::UserTokenSource::ServerMinted { - client: self.client.clone(), - user_id: data.user_id.clone(), - call_cid: self.cid(), - expiration: INTERNAL_RTC_TOKEN_LIFETIME, - }; - self.rtc.join_with_token_source(source, data).await + self.rtc().join(data).await + } + + /// The participant session of this handle as an [`RtcCall`](crate::rtc::RtcCall). + /// Both share one session; its join mints the user token as [`Call::join`] does. + pub fn rtc(&self) -> crate::rtc::RtcCall { + crate::rtc::RtcCall::new( + self.rtc.clone(), + crate::rtc::client::UserTokenSource::ServerMinted { + client: self.client.clone(), + call_cid: self.cid(), + expiration: INTERNAL_RTC_TOKEN_LIFETIME, + }, + ) } /// Leave the call, closing the SFU connection and PeerConnections. Succeeds diff --git a/tests/rtc_join.rs b/tests/rtc_join.rs index ca839df..b912af8 100644 --- a/tests/rtc_join.rs +++ b/tests/rtc_join.rs @@ -13,7 +13,7 @@ use std::time::Duration; use getstream::TokenOptions; use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; -use getstream::rtc::{CallEvent, JoinCallData, RtcClient}; +use getstream::rtc::{CallEvent, CallingState, JoinCallData, RtcClient}; use tokio::sync::broadcast::Receiver; /// Wait (up to `timeout`) for a `ParticipantJoined` whose `user_id` matches @@ -96,7 +96,8 @@ async fn two_sessions_join_and_observe_each_other() { // The whole join/observe/leave dance must finish well within a minute. let outcome = tokio::time::timeout(Duration::from_secs(90), async { let call_a = client.video().call("default", &call_id); - let call_b = client.video().call("default", &call_id); + // Session B joins through the `RtcCall` view of a server-client call. + let call_b = client.video().call("default", &call_id).rtc(); // Subscribe BEFORE joining so no participant event is missed. let rx_a = call_a.subscribe(); @@ -191,9 +192,11 @@ async fn provider_backed_client_loads_token_and_joins() { } /// Local decoding is not sufficient proof of authenticity: Stream must reject -/// a validly shaped participant token whose HS256 signature was altered. +/// a validly shaped participant token whose HS256 signature was altered. A +/// valid pre-minted token joins a call handle made before the join, and a +/// receiver subscribed before the join gets the join states. #[tokio::test] -async fn participant_token_signature_is_enforced() { +async fn preminted_token_client_gets_join_events_and_signature_is_enforced() { let Some(client) = common::client_or_skip() else { return; }; @@ -232,9 +235,22 @@ async fn participant_token_signature_is_enforced() { let outcome: Result<(), String> = tokio::time::timeout(Duration::from_secs(120), async { let allowed = RtcClient::new(client.api_key(), token) .map_err(|error| format!("build RTC client: {error}"))? - .join("default", &call_id, JoinCallData::new(&user_id)) + .call("default", &call_id); + let mut events = allowed.subscribe(); + allowed + .join(JoinCallData::new(&user_id)) .await .map_err(|error| format!("valid token failed to join: {error}"))?; + let mut states = Vec::new(); + while let Ok(event) = events.try_recv() { + if let CallEvent::CallingStateChanged(state) = event { + states.push(state); + } + } + if states != [CallingState::Joining, CallingState::Joined] { + let _ = allowed.leave().await; + return Err(format!("join states seen before the join: {states:?}")); + } allowed .leave() .await From ff934f8453b5c2b3c8632eeee309c92cb305caa6 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 15:52:19 +0200 Subject: [PATCH 09/37] fix: report only participant changes after a join or reconnect --- src/rtc/join/participants.rs | 59 +++++++++++++++++++++++---------- src/rtc/join/tests.rs | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 7941110..1702570 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -25,6 +25,8 @@ pub(super) struct CallStateCache { pub(super) current_grants: Option, /// Set by the first report of the call end. pub(super) ended: bool, + /// The local session id of the last join response. + pub(super) local_session_id: String, } impl RtcCore { @@ -86,7 +88,8 @@ impl RtcCore { } /// Replace the participants from an authoritative SFU join response when - /// its lifecycle generation is still active. + /// its lifecycle generation is still active, and report the participants + /// that joined, changed, or left since the previous join response. pub(super) fn apply_join_call_state_if_current( &self, generation: u64, @@ -100,20 +103,25 @@ impl RtcCore { } let state = call_state.unwrap_or_default(); let joined = state.participants.clone(); - *self - .call_state - .lock() - .unwrap_or_else(|error| error.into_inner()) = CallStateCache { - participant_count: state.participant_count.unwrap_or_default(), - pins: state.pins, - started_at: state.started_at, - e2ee_enabled: state.e2ee_enabled, - current_grants: None, - ended: false, - }; - { + let previous_session_id = std::mem::replace( + &mut *self + .call_state + .lock() + .unwrap_or_else(|error| error.into_inner()), + CallStateCache { + participant_count: state.participant_count.unwrap_or_default(), + pins: state.pins, + started_at: state.started_at, + e2ee_enabled: state.e2ee_enabled, + current_grants: None, + ended: false, + local_session_id: session_id.to_owned(), + }, + ) + .local_session_id; + let mut previous = { let mut participants = self.participants.lock().unwrap_or_else(|e| e.into_inner()); - participants.clear(); + let previous = std::mem::take(&mut *participants); let me = participants.entry(session_id.to_owned()).or_default(); me.user_id = user_id.to_owned(); me.session_id = session_id.to_owned(); @@ -139,12 +147,29 @@ impl RtcCore { .published .extend(participant.published_tracks.iter().copied()); } - } + previous + }; + // The local sessions, before and after a REJOIN, produce no events. + let is_local = |id: &str| id == session_id || id == previous_session_id; for participant in joined { - if participant.session_id != session_id { + let known = previous.remove(&participant.session_id); + if is_local(&participant.session_id) { + continue; + } + let event = match known { + None => CallEvent::ParticipantJoined(participant), + Some(entry) if entry.participant != participant => { + CallEvent::ParticipantUpdated(participant) + } + Some(_) => continue, + }; + let _ = self.events_tx.send(event); + } + for (id, entry) in previous { + if !is_local(&id) { let _ = self .events_tx - .send(CallEvent::ParticipantJoined(participant)); + .send(CallEvent::ParticipantLeft(entry.participant)); } } true diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 2c88a7b..49fed53 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1681,6 +1681,70 @@ fn call_state_snapshot_combines_join_state_and_incremental_sfu_updates() { assert!(state.current_grants.expect("grants").can_publish_audio); } +#[test] +fn join_state_reports_only_the_participant_changes_since_the_last_join() { + let core = test_core(); + let generation = core.begin_join().expect("join generation"); + let mut events = core.subscribe(); + let participant = |user_id: &str, session_id: &str| models::Participant { + user_id: user_id.to_owned(), + session_id: session_id.to_owned(), + ..Default::default() + }; + let call_state = |participants| { + Some(models::CallState { + participants, + ..Default::default() + }) + }; + let bob = participant("bob", "session-b"); + + assert!(core.apply_join_call_state_if_current( + generation, + "local-1", + "agent", + call_state(vec![ + participant("agent", "local-1"), + participant("alice", "session-a"), + bob.clone(), + ]), + )); + // A REJOIN: the local session changes, alice left, bob changed, carol joined. + assert!(core.apply_join_call_state_if_current( + generation, + "local-2", + "agent", + call_state(vec![ + participant("agent", "local-2"), + models::Participant { + name: "Bob".to_owned(), + ..bob + }, + participant("carol", "session-c"), + ]), + )); + + let mut reported = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + CallEvent::ParticipantJoined(p) => reported.push(("joined", p.session_id)), + CallEvent::ParticipantUpdated(p) => reported.push(("updated", p.session_id)), + CallEvent::ParticipantLeft(p) => reported.push(("left", p.session_id)), + _ => {} + } + } + assert_eq!( + reported, + [ + ("joined", "session-a".to_owned()), + ("joined", "session-b".to_owned()), + ("updated", "session-b".to_owned()), + ("joined", "session-c".to_owned()), + ("left", "session-a".to_owned()), + ] + ); +} + #[test] fn mute_state_builder_deduplicates_track_types() { let first = LocalTrack::Audio(LocalAudioTrack::opus().expect("first audio track")); From 510fe42896be6dd8806650fe0df29241a5ec0f34 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 18:35:22 +0200 Subject: [PATCH 10/37] fix: reuse the stopped audio sender when audio is published again A second audio transceiver failed the SFU negotiation (ParticipantMediaTransportFailure), which forced a REJOIN and made publish_audio return PARTICIPANT_NOT_FOUND. The new track now takes over the sender that stop_publish kept and continues its RTP sequence numbers and timestamps: the SFU drops a stream whose timestamps go back. --- src/rtc/join/publication.rs | 25 ++++++++++ src/rtc/join/publish.rs | 34 ++++++++++++-- src/rtc/peer/publisher.rs | 91 ++++++++++++++++++++++++++++++++++++- src/rtc/tracks/local.rs | 31 +++++++++++++ tests/rtc_media.rs | 56 +++++++++++++++++++++-- 5 files changed, 229 insertions(+), 8 deletions(-) diff --git a/src/rtc/join/publication.rs b/src/rtc/join/publication.rs index c50e424..2b43bc5 100644 --- a/src/rtc/join/publication.rs +++ b/src/rtc/join/publication.rs @@ -27,6 +27,9 @@ pub(super) struct LocalPublication { pub(super) struct MediaState { pub(super) publications: Vec, pub(super) publish_quality: HashMap<(i32, i32), Vec>, + /// The publish option id and stopped track of each sender that + /// `stop_publish` kept in the publisher envelope. + retired: Vec<(i32, LocalTrack)>, } impl MediaState { @@ -92,6 +95,28 @@ impl MediaState { Some(publication.track) } + /// Remove the publication of a stopped track whose sender stays in the + /// publisher envelope, and keep that sender for a later publish. + pub(super) fn retire(&mut self, track_id: &str) -> Option { + let publish_option_id = self.publications[self.position(track_id)?].publish_option_id; + let track = self.remove(track_id)?; + self.retired.push((publish_option_id, track.clone())); + Some(track) + } + + /// Take the stopped track of the latest sender that [`Self::retire`] kept + /// for this kind of track. + pub(super) fn take_retired( + &mut self, + track_type: TrackType, + publish_option_id: i32, + ) -> Option { + let position = self.retired.iter().rposition(|(option_id, retired)| { + retired.track_type() == track_type && *option_id == publish_option_id + })?; + Some(self.retired.remove(position).1) + } + pub(super) fn refresh_publish_options( &mut self, publish_options: &[models::PublishOption], diff --git a/src/rtc/join/publish.rs b/src/rtc/join/publish.rs index 81f1a47..94cdd04 100644 --- a/src/rtc/join/publish.rs +++ b/src/rtc/join/publish.rs @@ -63,9 +63,31 @@ impl RtcCore { }; let mut publisher_rtcp_tasks = Vec::new(); if status.is_none() { - publisher_rtcp_tasks = - match publisher::add_transceiver_for_track(&publisher, &track, &publish_options) - .await + // Reuse the audio sender that `stop_publish` kept, as JS `replaceTrack` + // does: a second audio m-line fails the SFU negotiation. Video RTCP + // readers keep the old track, and simulcast cannot be replaced. + let retired = if matches!( + track.track_type(), + TrackType::Audio | TrackType::ScreenShareAudio + ) { + media.take_retired(track.track_type(), publish_option_id) + } else { + None + }; + let reused = match retired { + Some(retired) => { + publisher::replace_retired_track(&publisher, &retired, &track, &publish_options) + .await? + } + None => false, + }; + if !reused { + publisher_rtcp_tasks = match publisher::add_transceiver_for_track( + &publisher, + &track, + &publish_options, + ) + .await { Ok(tasks) => tasks, Err(error) => { @@ -77,7 +99,11 @@ impl RtcCore { return Err(error); } }; + } media.begin_publish(track.clone(), publish_option_id); + if reused { + media.set_status(&track_id, PublicationStatus::PendingPublishMute); + } if let Some(layers) = media .publish_quality .get(&(publish_option_id, track.track_type() as i32)) @@ -218,7 +244,7 @@ impl RtcCore { if muted { self.remove_published_track(&session_id, track_type as i32); } - if let Some(removed) = media.remove(&track_id) { + if let Some(removed) = media.retire(&track_id) { removed.stop(); } Ok(()) diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index 34db721..25c48a2 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -232,6 +232,39 @@ pub(crate) async fn add_transceiver_for_track( Ok(tasks) } +/// Put `track` on the sender that still carries the stopped track `retired`, +/// as JS `replaceTrack` does. Returns `false` if `publisher` has no such sender. +pub(crate) async fn replace_retired_track( + publisher: &Arc, + retired: &LocalTrack, + track: &LocalTrack, + publish_options: &[PublishOption], +) -> Result { + let retired_track_id = retired.track_id(); + for transceiver in publisher.get_transceivers().await { + let sender = transceiver.sender().await; + if !sender + .track() + .await + .is_some_and(|bound| bound.id() == retired_track_id) + { + continue; + } + let option = publish_option(track, publish_options)?; + track.configure_for_publish(option)?; + track.continue_rtp_from(retired); + let physical = track.webrtc_tracks().into_iter().next().ok_or_else(|| { + RtcError::Media("local publication has no physical encodings".to_owned()) + })?; + sender + .replace_track(Some(physical)) + .await + .map_err(RtcError::from)?; + return Ok(true); + } + Ok(false) +} + fn spawn_rtcp_reader( sender: Arc, rid: Option, @@ -327,7 +360,63 @@ mod tests { use crate::rtc::peer; use crate::rtc::proto::event::VideoLayerSetting; use crate::rtc::proto::models::{Codec, VideoDimension}; - use crate::rtc::tracks::{LocalVideoTrack, LocalVideoTrackConfig}; + use crate::rtc::tracks::{LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig}; + use std::time::Duration; + + #[tokio::test] + async fn a_track_on_a_retired_sender_continues_its_rtp_timeline() { + let opus = [PublishOption { + track_type: TrackType::Audio as i32, + codec: Some(Codec { + name: "opus".to_owned(), + ..Default::default() + }), + ..Default::default() + }]; + let retired = LocalTrack::Audio(LocalAudioTrack::opus().expect("first track")); + let publisher = peer::new_peer_connection(&[]).await.expect("publisher"); + let rtcp_tasks = add_transceiver_for_track(&publisher, &retired, &opus) + .await + .expect("first transceiver"); + let (receiver, mut remote_rx) = peer::connect_audio_receiver(&publisher).await; + retired.start_audio_pacing().await; + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + let (mut last, _) = remote.read_rtp().await.expect("first packet"); + retired.pause_audio_pacing(); + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(200), remote.read_rtp()).await + { + last = packet; + } + retired.stop(); + + let track = LocalTrack::Audio(LocalAudioTrack::opus().expect("second track")); + assert!( + replace_retired_track(&publisher, &retired, &track, &opus) + .await + .expect("replace the retired track") + ); + track.start_audio_pacing().await; + let (next, _) = remote.read_rtp().await.expect("second packet"); + + assert_eq!( + next.header.sequence_number, + last.header.sequence_number.wrapping_add(1) + ); + assert_eq!( + next.header.timestamp, + last.header.timestamp.wrapping_add(960) + ); + track.stop(); + for task in rtcp_tasks { + task.abort(); + } + let _ = publisher.close().await; + let _ = receiver.close().await; + } fn video_option(name: &str) -> PublishOption { PublishOption { diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 8aa7d52..30f1c80 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -613,6 +613,25 @@ impl LocalAudioTrack { self.inner.pacing_enabled.load(Ordering::SeqCst) } + /// Continue the RTP sequence numbers and timestamps of `previous`, whose + /// sender this track takes over. The SFU drops a stream whose timestamps + /// go back. + fn continue_rtp_from(&self, previous: &LocalAudioTrack) { + let packetizer = previous + .inner + .core + .packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + *self + .inner + .core + .packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) = packetizer; + } + /// Stop the pacer and reject further writes. Called by `stop_publish`/`leave`. pub(crate) fn stop(&self) { self.inner.core.stop(); @@ -2001,6 +2020,18 @@ impl LocalTrack { } } + /// Continue the RTP timeline of the audio track `previous`, whose sender + /// this audio track takes over. + pub(crate) fn continue_rtp_from(&self, previous: &LocalTrack) { + if let ( + LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track), + LocalTrack::Audio(previous) | LocalTrack::ScreenShareAudio(previous), + ) = (self, previous) + { + track.continue_rtp_from(previous); + } + } + pub(crate) fn set_muted(&self, muted: bool) { match self { LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index 8460741..42304ce 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -21,8 +21,9 @@ use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - CallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, LocalVideoTrack, - PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SubscriptionConfig, VideoFrame, + CallEvent, CallingState, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, + LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SubscriptionConfig, + VideoFrame, }; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; @@ -1351,8 +1352,11 @@ async fn loud_publisher_is_reported_speaking_and_dominant() { /// typed event stream: it must see A's audio `TrackPublished`, then — once A /// stops the track — A's audio `TrackUnpublished`, confirming the SFU accepted /// the request and broadcast the corrected publication state to peers. +/// +/// A then publishes a new audio track on the same session. The publish must +/// succeed without a reconnect, and B must hear A again. #[tokio::test] -async fn stop_publish_of_sole_audio_is_accepted_and_unannounced() { +async fn sole_audio_can_be_stopped_and_published_again() { let Some(client) = common::client_or_skip() else { return; }; @@ -1366,6 +1370,7 @@ async fn stop_publish_of_sole_audio_is_accepted_and_unannounced() { let call_a = client.video().call("default", &call_id); let call_b = client.video().call("default", &call_id); + let mut tracks_b = track_sink(&call_b); call_b .join(JoinCallData::new(&user_b)) .await @@ -1380,6 +1385,7 @@ async fn stop_publish_of_sole_audio_is_accepted_and_unannounced() { .join(JoinCallData::new(&user_a)) .await .expect("A join"); + let mut events_a = call_a.subscribe(); let audio_a = LocalAudioTrack::opus().expect("opus track"); call_a .publish_audio(audio_a.clone()) @@ -1399,6 +1405,14 @@ async fn stop_publish_of_sole_audio_is_accepted_and_unannounced() { audio_published, "B never received A's audio TrackPublished before the stop" ); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(45), + ) + .await + .expect("B never received A's audio track"); // Stop the sole publication. Before the fix this renegotiated the // publisher with an empty track set and the SFU returned "Invalid @@ -1423,6 +1437,42 @@ async fn stop_publish_of_sole_audio_is_accepted_and_unannounced() { (mute state not propagated to peers)" ); + // Publish a new audio track on the same session, as after an unmute. + let audio_again = LocalAudioTrack::opus().expect("opus track"); + call_a + .publish_audio(audio_again.clone()) + .await + .expect("A publish_audio again on the same session"); + let feeder_again = spawn_tone(audio_again); + let audio_republished = await_track_event( + &mut events_b, + &user_a, + TrackType::Audio, + true, + Duration::from_secs(45), + ) + .await; + assert!( + audio_republished, + "B never received A's audio TrackPublished after the second publish" + ); + let rms = drain_rms(&remote_a, FRAME_20MS * 100, Duration::from_secs(30)).await; + feeder_again.abort(); + assert!( + rms > NON_SILENT_RMS, + "B got no audio after the second publish (rms={rms:.4})" + ); + let mut reconnected = false; + while let Ok(event) = events_a.try_recv() { + reconnected |= matches!( + event, + CallEvent::CallingStateChanged( + CallingState::Reconnecting | CallingState::Migrating + ) + ); + } + assert!(!reconnected, "A reconnected after the second publish"); + call_a.leave().await.expect("A leave"); call_b.leave().await.expect("B leave"); }) From 2a1d7f0c5a7edef5d1bc56c5562053d6235a07e8 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Wed, 30 Sep 2026 19:35:08 +0200 Subject: [PATCH 11/37] feat: add stable CallEvent names --- CHANGELOG.md | 8 ++++ src/rtc/join/connection.rs | 35 ++++++++++++++ src/rtc/join/tests.rs | 94 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fed9e01..db5032a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,14 @@ get the join events and tracks. `Call::rtc` gives the same `RtcCall` type for a client with an API secret; both handles share one session. `RtcCall` also adds `update_publish_options` and `set_disconnection_timeout`. +### Stable call event names + +`CallEvent::name` returns a stable name for each event: the `SfuEvent` field +name of its SFU source (for example `participant_joined`), `call_ended` from +both sources, the coordinator `type` of a `Coordinator` event (for example +`call.created`), and `calling_state_changed` or `participant_count_changed` for +the events of the SDK itself. + ### Video REST: advanced call statistics and reporting Application-level stats on `VideoClient` (`get_active_calls_status`, diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index bdee685..90cefb0 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -2,6 +2,41 @@ use super::*; +impl CallEvent { + /// The stable name of this event. The names are public API and do not + /// change. + /// + /// An event from the SFU has the `SfuEvent` field name of its source, for + /// example `participant_joined` or `change_publish_quality`. + /// [`CallEvent::CallEnded`] is `call_ended` from both sources, and a + /// [`CallEvent::Coordinator`] event has its coordinator `type`, for example + /// `call.created`. The events of the SDK itself are `calling_state_changed` + /// and `participant_count_changed`. + pub fn name(&self) -> &str { + match self { + Self::ParticipantJoined(_) => "participant_joined", + Self::ParticipantLeft(_) => "participant_left", + Self::ParticipantUpdated(_) => "participant_updated", + Self::Coordinator(event) => &event.event_type, + Self::TrackPublished { .. } => "track_published", + Self::TrackUnpublished { .. } => "track_unpublished", + Self::DominantSpeakerChanged { .. } => "dominant_speaker_changed", + Self::AudioLevelChanged(_) => "audio_level_changed", + Self::ConnectionQualityChanged(_) => "connection_quality_changed", + Self::ParticipantCountChanged(_) => "participant_count_changed", + Self::PinsUpdated(_) => "pins_updated", + Self::InboundStateChanged(_) => "inbound_state_notification", + Self::PublishOptionsChanged { .. } => "change_publish_options", + Self::PublishQualityChanged(_) => "change_publish_quality", + Self::CallGrantsUpdated(_) => "call_grants_updated", + Self::IceRestarted(_) => "ice_restart", + Self::Error(_) => "error", + Self::CallEnded { .. } => "call_ended", + Self::CallingStateChanged(_) => "calling_state_changed", + } + } +} + /// Build the SFU signaling WebSocket URL from `ws_endpoint`, appending the /// informational query params JS attaches (`attempt`, `user_id`, `api_key`, /// `user_session_id`, `cid`). Ported from JS `StreamSfuClient.createWebSocket`. diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 49fed53..c72657b 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1681,6 +1681,100 @@ fn call_state_snapshot_combines_join_state_and_incremental_sfu_updates() { assert!(state.current_grants.expect("grants").can_publish_audio); } +#[test] +fn every_call_event_has_its_stable_name() { + let events = [ + ( + CallEvent::ParticipantJoined(models::Participant::default()), + "participant_joined", + ), + ( + CallEvent::ParticipantLeft(models::Participant::default()), + "participant_left", + ), + ( + CallEvent::ParticipantUpdated(models::Participant::default()), + "participant_updated", + ), + ( + CallEvent::Coordinator(CoordinatorEvent { + event_type: "call.created".to_owned(), + raw: json!({ "type": "call.created" }), + }), + "call.created", + ), + ( + CallEvent::TrackPublished { + user_id: String::new(), + session_id: String::new(), + track_type: TrackType::Audio, + participant: None, + }, + "track_published", + ), + ( + CallEvent::TrackUnpublished { + user_id: String::new(), + session_id: String::new(), + track_type: TrackType::Audio, + cause: models::TrackUnpublishReason::UserMuted, + participant: None, + }, + "track_unpublished", + ), + ( + CallEvent::DominantSpeakerChanged { + user_id: String::new(), + session_id: String::new(), + }, + "dominant_speaker_changed", + ), + ( + CallEvent::AudioLevelChanged(Vec::new()), + "audio_level_changed", + ), + ( + CallEvent::ConnectionQualityChanged(Vec::new()), + "connection_quality_changed", + ), + ( + CallEvent::ParticipantCountChanged(models::ParticipantCount::default()), + "participant_count_changed", + ), + (CallEvent::PinsUpdated(Vec::new()), "pins_updated"), + ( + CallEvent::InboundStateChanged(Vec::new()), + "inbound_state_notification", + ), + ( + CallEvent::PublishOptionsChanged { + publish_options: Vec::new(), + reason: String::new(), + }, + "change_publish_options", + ), + ( + CallEvent::PublishQualityChanged(event::ChangePublishQuality::default()), + "change_publish_quality", + ), + ( + CallEvent::CallGrantsUpdated(event::CallGrantsUpdated::default()), + "call_grants_updated", + ), + (CallEvent::IceRestarted(PeerType::Subscriber), "ice_restart"), + (CallEvent::Error(SfuJoinError::from_event(None, 0)), "error"), + (CallEvent::CallEnded { reason: None }, "call_ended"), + ( + CallEvent::CallingStateChanged(CallingState::Joined), + "calling_state_changed", + ), + ]; + + for (event, name) in &events { + assert_eq!(event.name(), *name, "{event:?}"); + } +} + #[test] fn join_state_reports_only_the_participant_changes_since_the_last_join() { let core = test_core(); From 91543e5e40b6299c3d6bbd62fb125184befb7fb1 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Thu, 1 Oct 2026 11:51:51 +0200 Subject: [PATCH 12/37] feat: group CallEvent variants by their source --- CHANGELOG.md | 31 +++++--- src/rtc/join/connection.rs | 136 +++++++++++++++++++++++------------ src/rtc/join/lifecycle.rs | 6 +- src/rtc/join/mod.rs | 31 ++++++-- src/rtc/join/participants.rs | 10 +-- src/rtc/join/tests.rs | 113 +++++++++++++++++------------ src/rtc/mod.rs | 5 +- tests/rtc_join.rs | 10 ++- tests/rtc_media.rs | 18 ++--- 9 files changed, 238 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db5032a..ffe6d8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,23 @@ for a staging or local environment must set both fields. Code that builds `..ClientConfig::default()`. `DEFAULT_COORDINATOR_WS_URL` moved from `rtc::coordinator::ws` to the crate root. +### Call events are grouped by their source + +`CallEvent` has three variants: `Sfu(SfuCallEvent)` for the events from the +SFU, `Coordinator(CoordinatorEvent)` as before, and `Client(ClientCallEvent)` +for the events that the SDK itself produces (`CallingStateChanged` and +`CallEnded`). The SFU and client variants did not change; only their path did. +A pattern such as `CallEvent::ParticipantJoined(p)` becomes +`CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))`. + ### Track and call-ended events carry the SFU data -`CallEvent::TrackPublished` and `CallEvent::TrackUnpublished` give `track_type` -as a `TrackType`, not an `i32`, and add `participant`. `TrackUnpublished` also -adds `cause`. `CallEvent::CallEnded` is now `CallEnded { reason }`: the SFU -reason, or `None` when the coordinator reported the end first. Patterns that -match these variants must use the new fields or `..`. +`SfuCallEvent::TrackPublished` and `SfuCallEvent::TrackUnpublished` give +`track_type` as a `TrackType`, not an `i32`, and add `participant`. +`TrackUnpublished` also adds `cause`. `ClientCallEvent::CallEnded` is now +`CallEnded { reason }`: the SFU reason, or `None` when the coordinator reported +the end first. Patterns that match these variants must use the new fields or +`..`. ## New Features @@ -32,11 +42,12 @@ client with an API secret; both handles share one session. `RtcCall` also adds ### Stable call event names -`CallEvent::name` returns a stable name for each event: the `SfuEvent` field -name of its SFU source (for example `participant_joined`), `call_ended` from -both sources, the coordinator `type` of a `Coordinator` event (for example -`call.created`), and `calling_state_changed` or `participant_count_changed` for -the events of the SDK itself. +`CallEvent::name` returns a stable name for each event. `SfuCallEvent::name` +gives the `SfuEvent` field name of the source event (for example +`participant_joined`), and `participant_count_changed`. +`ClientCallEvent::name` gives `call_ended` (from both sources) or +`calling_state_changed`. A `Coordinator` event has its coordinator `type` (for +example `call.created`). ### Video REST: advanced call statistics and reporting diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 90cefb0..d5688c1 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -4,20 +4,27 @@ use super::*; impl CallEvent { /// The stable name of this event. The names are public API and do not - /// change. - /// - /// An event from the SFU has the `SfuEvent` field name of its source, for - /// example `participant_joined` or `change_publish_quality`. - /// [`CallEvent::CallEnded`] is `call_ended` from both sources, and a + /// change: see [`SfuCallEvent::name`] and [`ClientCallEvent::name`]. A /// [`CallEvent::Coordinator`] event has its coordinator `type`, for example - /// `call.created`. The events of the SDK itself are `calling_state_changed` - /// and `participant_count_changed`. + /// `call.created`. pub fn name(&self) -> &str { + match self { + Self::Sfu(event) => event.name(), + Self::Coordinator(event) => &event.event_type, + Self::Client(event) => event.name(), + } + } +} + +impl SfuCallEvent { + /// The stable `SfuEvent` field name of the source event, for example + /// `participant_joined` or `change_publish_quality`. + /// [`SfuCallEvent::ParticipantCountChanged`] is `participant_count_changed`. + pub fn name(&self) -> &'static str { match self { Self::ParticipantJoined(_) => "participant_joined", Self::ParticipantLeft(_) => "participant_left", Self::ParticipantUpdated(_) => "participant_updated", - Self::Coordinator(event) => &event.event_type, Self::TrackPublished { .. } => "track_published", Self::TrackUnpublished { .. } => "track_unpublished", Self::DominantSpeakerChanged { .. } => "dominant_speaker_changed", @@ -31,6 +38,15 @@ impl CallEvent { Self::CallGrantsUpdated(_) => "call_grants_updated", Self::IceRestarted(_) => "ice_restart", Self::Error(_) => "error", + } + } +} + +impl ClientCallEvent { + /// The stable name of this event: `call_ended` (from both sources) or + /// `calling_state_changed`. + pub fn name(&self) -> &'static str { + match self { Self::CallEnded { .. } => "call_ended", Self::CallingStateChanged(_) => "calling_state_changed", } @@ -297,16 +313,20 @@ pub(super) async fn handle_event( } E::ConnectionQualityChanged(event) => { core.update_connection_quality(&event.connection_quality_updates); - let _ = core.events_tx.send(CallEvent::ConnectionQualityChanged( - event.connection_quality_updates, - )); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::ConnectionQualityChanged( + event.connection_quality_updates, + ))); } E::ParticipantJoined(ev) => { if let Some(p) = ev.participant { core.upsert_participant(&p); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core.events_tx.send(CallEvent::ParticipantJoined(p)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))); } } E::ParticipantLeft(ev) => { @@ -314,7 +334,9 @@ pub(super) async fn handle_event( core.remove_participant(&p.session_id); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core.events_tx.send(CallEvent::ParticipantLeft(p)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::ParticipantLeft(p))); } } E::ParticipantUpdated(ev) => { @@ -322,7 +344,9 @@ pub(super) async fn handle_event( core.upsert_participant(&p); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core.events_tx.send(CallEvent::ParticipantUpdated(p)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::ParticipantUpdated(p))); } } E::TrackPublished(ev) => { @@ -334,38 +358,46 @@ pub(super) async fn handle_event( ); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core.events_tx.send(CallEvent::TrackPublished { - user_id: ev.user_id, - session_id: ev.session_id, - track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), - participant: ev.participant, - }); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::TrackPublished { + user_id: ev.user_id, + session_id: ev.session_id, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + participant: ev.participant, + })); } E::TrackUnpublished(ev) => { core.remove_published_track(&ev.session_id, ev.r#type); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core.events_tx.send(CallEvent::TrackUnpublished { - user_id: ev.user_id, - session_id: ev.session_id, - track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), - cause: models::TrackUnpublishReason::try_from(ev.cause) - .unwrap_or(models::TrackUnpublishReason::Unspecified), - participant: ev.participant, - }); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { + user_id: ev.user_id, + session_id: ev.session_id, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + cause: models::TrackUnpublishReason::try_from(ev.cause) + .unwrap_or(models::TrackUnpublishReason::Unspecified), + participant: ev.participant, + })); } E::DominantSpeakerChanged(ev) => { core.update_dominant_speaker(&ev.session_id); - let _ = core.events_tx.send(CallEvent::DominantSpeakerChanged { - user_id: ev.user_id, - session_id: ev.session_id, - }); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::DominantSpeakerChanged { + user_id: ev.user_id, + session_id: ev.session_id, + })); } E::AudioLevelChanged(ev) => { core.update_audio_levels(&ev.audio_levels); let _ = core .events_tx - .send(CallEvent::AudioLevelChanged(ev.audio_levels)); + .send(CallEvent::Sfu(SfuCallEvent::AudioLevelChanged( + ev.audio_levels, + ))); } E::HealthCheckResponse(event) => { if let Some(participant_count) = event.participant_count @@ -373,30 +405,40 @@ pub(super) async fn handle_event( { let _ = core .events_tx - .send(CallEvent::ParticipantCountChanged(participant_count)); + .send(CallEvent::Sfu(SfuCallEvent::ParticipantCountChanged( + participant_count, + ))); } } E::PinsUpdated(event) => { core.update_pins(event.pins.clone()); - let _ = core.events_tx.send(CallEvent::PinsUpdated(event.pins)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::PinsUpdated(event.pins))); } E::InboundStateNotification(event) => { core.update_inbound_state(&event.inbound_video_states); let _ = core .events_tx - .send(CallEvent::InboundStateChanged(event.inbound_video_states)); + .send(CallEvent::Sfu(SfuCallEvent::InboundStateChanged( + event.inbound_video_states, + ))); } E::ChangePublishOptions(event) => { core.apply_publish_options(context.generation, event.publish_options.clone()) .await?; - let _ = core.events_tx.send(CallEvent::PublishOptionsChanged { - publish_options: event.publish_options, - reason: event.reason, - }); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::PublishOptionsChanged { + publish_options: event.publish_options, + reason: event.reason, + })); } E::ChangePublishQuality(event) => { core.apply_publish_quality(&event).await; - let _ = core.events_tx.send(CallEvent::PublishQualityChanged(event)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::PublishQualityChanged(event))); } E::CallGrantsUpdated(event) => { core.update_call_grants(event.current_grants); @@ -412,7 +454,9 @@ pub(super) async fn handle_event( } } } - let _ = core.events_tx.send(CallEvent::CallGrantsUpdated(event)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::CallGrantsUpdated(event))); } E::IceRestart(event) => { let peer_type = @@ -420,12 +464,16 @@ pub(super) async fn handle_event( if peer_type == PeerType::PublisherUnspecified { core.restart_publisher_ice().await?; } - let _ = core.events_tx.send(CallEvent::IceRestarted(peer_type)); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::IceRestarted(peer_type))); } E::Error(err) => { let join_err = SfuJoinError::from_event(err.error, err.reconnect_strategy); let strategy = ReconnectStrategy::from_proto(err.reconnect_strategy); - let _ = core.events_tx.send(CallEvent::Error(join_err.clone())); + let _ = core + .events_tx + .send(CallEvent::Sfu(SfuCallEvent::Error(join_err.clone()))); if let Some(strategy) = strategy { core.trigger_reconnect(context.generation, strategy, join_err.message.clone()); } diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index c6730ec..97ce7a5 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -672,7 +672,7 @@ impl RtcCore { } /// The SFU or the coordinator reported the end of the call: send - /// [`CallEvent::CallEnded`] once and leave the call. + /// [`ClientCallEvent::CallEnded`] once and leave the call. pub(super) fn end_call( self: &Arc, generation: u64, @@ -692,7 +692,9 @@ impl RtcCore { if already_ended { return; } - let _ = self.events_tx.send(CallEvent::CallEnded { reason }); + let _ = self + .events_tx + .send(CallEvent::Client(ClientCallEvent::CallEnded { reason })); let this = self.clone(); // Not a generation task: `leave` ends the generation. std::mem::drop(self.spawn_runtime_task(async move { diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 54b1132..315fe31 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -176,18 +176,33 @@ pub enum CallingState { Left, } -/// A typed SFU event delivered on the [`Call`](crate::Call) event stream. +/// An event delivered on the [`Call`](crate::Call) event stream, grouped by its +/// source. #[derive(Debug, Clone)] #[non_exhaustive] +// `Sfu` carries `Participant` data (about 304 bytes), and the other variants are +// small. A `Box` would allocate for every SFU event and prevent nested +// `CallEvent::Sfu(SfuCallEvent::..)` patterns. +#[allow(clippy::large_enum_variant)] pub enum CallEvent { + /// An event from the SFU. + Sfu(SfuCallEvent), + /// A call-scoped coordinator WebSocket event. + Coordinator(CoordinatorEvent), + /// An event that the SDK itself produces. + Client(ClientCallEvent), +} + +/// An event from the SFU. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum SfuCallEvent { /// A participant joined the call. ParticipantJoined(models::Participant), /// A participant left the call. ParticipantLeft(models::Participant), /// A participant's user data changed. ParticipantUpdated(models::Participant), - /// A call-scoped coordinator WebSocket event. - Coordinator(CoordinatorEvent), /// A track was published (audio/video/screenshare). TrackPublished { /// The publisher's user id. @@ -248,6 +263,12 @@ pub enum CallEvent { IceRestarted(PeerType), /// The SFU reported an error for this participant. Error(SfuJoinError), +} + +/// An event that the SDK itself produces. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum ClientCallEvent { /// The call ended: the SFU or the coordinator (`call.ended`) reported it. /// Sent once, after which the SDK leaves the call. CallEnded { @@ -294,7 +315,9 @@ impl Lifecycle { fn set_state(&mut self, next: CallingState, events: &broadcast::Sender) { if self.state != next { self.state = next; - let _ = events.send(CallEvent::CallingStateChanged(next)); + let _ = events.send(CallEvent::Client(ClientCallEvent::CallingStateChanged( + next, + ))); } } } diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 1702570..37e8d93 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -157,19 +157,21 @@ impl RtcCore { continue; } let event = match known { - None => CallEvent::ParticipantJoined(participant), + None => SfuCallEvent::ParticipantJoined(participant), Some(entry) if entry.participant != participant => { - CallEvent::ParticipantUpdated(participant) + SfuCallEvent::ParticipantUpdated(participant) } Some(_) => continue, }; - let _ = self.events_tx.send(event); + let _ = self.events_tx.send(CallEvent::Sfu(event)); } for (id, entry) in previous { if !is_local(&id) { let _ = self .events_tx - .send(CallEvent::ParticipantLeft(entry.participant)); + .send(CallEvent::Sfu(SfuCallEvent::ParticipantLeft( + entry.participant, + ))); } } true diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index c72657b..094e19c 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -418,7 +418,9 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() .expect("coordinator events"); tokio::time::timeout(Duration::from_secs(2), async { loop { - if let Ok(CallEvent::CallEnded { reason: None }) = events.recv().await { + if let Ok(CallEvent::Client(ClientCallEvent::CallEnded { reason: None })) = + events.recv().await + { return; } } @@ -440,7 +442,7 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() .await; let mut repeated = 0; while let Ok(event) = events.try_recv() { - if matches!(event, CallEvent::CallEnded { .. }) { + if matches!(event, CallEvent::Client(ClientCallEvent::CallEnded { .. })) { repeated += 1; } } @@ -483,9 +485,9 @@ async fn call_ended_twice_before_the_leave_is_reported_once() { while let Ok(event) = events.try_recv() { if matches!( event, - CallEvent::CallEnded { + CallEvent::Client(ClientCallEvent::CallEnded { reason: Some(models::CallEndedReason::Kicked) - } + }) ) { reported += 1; } @@ -519,11 +521,11 @@ async fn track_events_report_the_track_type_cause_and_participant() { .await .expect("handle track published"); - let Ok(CallEvent::TrackPublished { + let Ok(CallEvent::Sfu(SfuCallEvent::TrackPublished { track_type, participant, .. - }) = events.try_recv() + })) = events.try_recv() else { panic!("expected a track published event"); }; @@ -543,12 +545,12 @@ async fn track_events_report_the_track_type_cause_and_participant() { .await .expect("handle track unpublished"); - let Ok(CallEvent::TrackUnpublished { + let Ok(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { track_type, cause, participant, .. - }) = events.try_recv() + })) = events.try_recv() else { panic!("expected a track unpublished event"); }; @@ -582,7 +584,7 @@ async fn participant_count_event_is_sent_only_when_the_count_changes() { let mut totals = Vec::new(); while let Ok(event) = events.try_recv() { - if let CallEvent::ParticipantCountChanged(count) = event { + if let CallEvent::Sfu(SfuCallEvent::ParticipantCountChanged(count)) = event { totals.push(count.total); } } @@ -1141,7 +1143,7 @@ fn state_events_arrive_in_the_order_of_the_state_changes() { barrier.wait(); let mut last = None; while let Ok(event) = events.try_recv() { - if let CallEvent::CallingStateChanged(state) = event { + if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { last = Some(state); } } @@ -1163,7 +1165,9 @@ fn setting_the_same_state_again_sends_no_event() { assert!(matches!( events.try_recv(), - Ok(CallEvent::CallingStateChanged(CallingState::Reconnecting)) + Ok(CallEvent::Client(ClientCallEvent::CallingStateChanged( + CallingState::Reconnecting + ))) )); assert!(events.try_recv().is_err()); } @@ -1177,7 +1181,9 @@ fn join_start_sends_joining() { assert!(matches!( events.try_recv(), - Ok(CallEvent::CallingStateChanged(CallingState::Joining)) + Ok(CallEvent::Client(ClientCallEvent::CallingStateChanged( + CallingState::Joining + ))) )); } @@ -1199,7 +1205,7 @@ async fn state_during_leave_matches_the_last_state_event() { let mut last = None; while let Ok(event) = events.try_recv() { - if let CallEvent::CallingStateChanged(state) = event { + if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { last = Some(state); } } @@ -1683,28 +1689,21 @@ fn call_state_snapshot_combines_join_state_and_incremental_sfu_updates() { #[test] fn every_call_event_has_its_stable_name() { - let events = [ + let sfu = [ ( - CallEvent::ParticipantJoined(models::Participant::default()), + SfuCallEvent::ParticipantJoined(models::Participant::default()), "participant_joined", ), ( - CallEvent::ParticipantLeft(models::Participant::default()), + SfuCallEvent::ParticipantLeft(models::Participant::default()), "participant_left", ), ( - CallEvent::ParticipantUpdated(models::Participant::default()), + SfuCallEvent::ParticipantUpdated(models::Participant::default()), "participant_updated", ), ( - CallEvent::Coordinator(CoordinatorEvent { - event_type: "call.created".to_owned(), - raw: json!({ "type": "call.created" }), - }), - "call.created", - ), - ( - CallEvent::TrackPublished { + SfuCallEvent::TrackPublished { user_id: String::new(), session_id: String::new(), track_type: TrackType::Audio, @@ -1713,7 +1712,7 @@ fn every_call_event_has_its_stable_name() { "track_published", ), ( - CallEvent::TrackUnpublished { + SfuCallEvent::TrackUnpublished { user_id: String::new(), session_id: String::new(), track_type: TrackType::Audio, @@ -1723,55 +1722,73 @@ fn every_call_event_has_its_stable_name() { "track_unpublished", ), ( - CallEvent::DominantSpeakerChanged { + SfuCallEvent::DominantSpeakerChanged { user_id: String::new(), session_id: String::new(), }, "dominant_speaker_changed", ), ( - CallEvent::AudioLevelChanged(Vec::new()), + SfuCallEvent::AudioLevelChanged(Vec::new()), "audio_level_changed", ), ( - CallEvent::ConnectionQualityChanged(Vec::new()), + SfuCallEvent::ConnectionQualityChanged(Vec::new()), "connection_quality_changed", ), ( - CallEvent::ParticipantCountChanged(models::ParticipantCount::default()), + SfuCallEvent::ParticipantCountChanged(models::ParticipantCount::default()), "participant_count_changed", ), - (CallEvent::PinsUpdated(Vec::new()), "pins_updated"), + (SfuCallEvent::PinsUpdated(Vec::new()), "pins_updated"), ( - CallEvent::InboundStateChanged(Vec::new()), + SfuCallEvent::InboundStateChanged(Vec::new()), "inbound_state_notification", ), ( - CallEvent::PublishOptionsChanged { + SfuCallEvent::PublishOptionsChanged { publish_options: Vec::new(), reason: String::new(), }, "change_publish_options", ), ( - CallEvent::PublishQualityChanged(event::ChangePublishQuality::default()), + SfuCallEvent::PublishQualityChanged(event::ChangePublishQuality::default()), "change_publish_quality", ), ( - CallEvent::CallGrantsUpdated(event::CallGrantsUpdated::default()), + SfuCallEvent::CallGrantsUpdated(event::CallGrantsUpdated::default()), "call_grants_updated", ), - (CallEvent::IceRestarted(PeerType::Subscriber), "ice_restart"), - (CallEvent::Error(SfuJoinError::from_event(None, 0)), "error"), - (CallEvent::CallEnded { reason: None }, "call_ended"), ( - CallEvent::CallingStateChanged(CallingState::Joined), + SfuCallEvent::IceRestarted(PeerType::Subscriber), + "ice_restart", + ), + ( + SfuCallEvent::Error(SfuJoinError::from_event(None, 0)), + "error", + ), + ] + .map(|(event, name)| (CallEvent::Sfu(event), name)); + let client = [ + (ClientCallEvent::CallEnded { reason: None }, "call_ended"), + ( + ClientCallEvent::CallingStateChanged(CallingState::Joined), "calling_state_changed", ), - ]; + ] + .map(|(event, name)| (CallEvent::Client(event), name)); + let coordinator = ( + CallEvent::Coordinator(CoordinatorEvent { + event_type: "call.created".to_owned(), + raw: json!({ "type": "call.created" }), + }), + "call.created", + ); + let events = sfu.into_iter().chain(client).chain([coordinator]); - for (event, name) in &events { - assert_eq!(event.name(), *name, "{event:?}"); + for (event, name) in events { + assert_eq!(event.name(), name, "{event:?}"); } } @@ -1821,9 +1838,15 @@ fn join_state_reports_only_the_participant_changes_since_the_last_join() { let mut reported = Vec::new(); while let Ok(event) = events.try_recv() { match event { - CallEvent::ParticipantJoined(p) => reported.push(("joined", p.session_id)), - CallEvent::ParticipantUpdated(p) => reported.push(("updated", p.session_id)), - CallEvent::ParticipantLeft(p) => reported.push(("left", p.session_id)), + CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p)) => { + reported.push(("joined", p.session_id)); + } + CallEvent::Sfu(SfuCallEvent::ParticipantUpdated(p)) => { + reported.push(("updated", p.session_id)); + } + CallEvent::Sfu(SfuCallEvent::ParticipantLeft(p)) => { + reported.push(("left", p.session_id)); + } _ => {} } } diff --git a/src/rtc/mod.rs b/src/rtc/mod.rs index 929df56..4c8388a 100644 --- a/src/rtc/mod.rs +++ b/src/rtc/mod.rs @@ -50,7 +50,10 @@ pub use error::{ SfuTimeoutError, TwirpError, WsConnectionError, is_join_error_code, }; pub use identity::{CLIENT_TYPE, SDK_TYPE, client_details, client_header}; -pub use join::{CallEvent, CallStateSnapshot, CallingState, JoinCallData, RtcCore}; +pub use join::{ + CallEvent, CallStateSnapshot, CallingState, ClientCallEvent, JoinCallData, RtcCore, + SfuCallEvent, +}; pub use pcm::chunk::Pad; pub use pcm::convert::G711_SAMPLE_RATE; pub use pcm::{ diff --git a/tests/rtc_join.rs b/tests/rtc_join.rs index b912af8..9724624 100644 --- a/tests/rtc_join.rs +++ b/tests/rtc_join.rs @@ -13,7 +13,9 @@ use std::time::Duration; use getstream::TokenOptions; use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; -use getstream::rtc::{CallEvent, CallingState, JoinCallData, RtcClient}; +use getstream::rtc::{ + CallEvent, CallingState, ClientCallEvent, JoinCallData, RtcClient, SfuCallEvent, +}; use tokio::sync::broadcast::Receiver; /// Wait (up to `timeout`) for a `ParticipantJoined` whose `user_id` matches @@ -29,7 +31,9 @@ async fn observe_participant( tokio::select! { () = &mut deadline => return false, event = rx.recv() => match event { - Ok(CallEvent::ParticipantJoined(p)) if p.user_id == other => return true, + Ok(CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))) if p.user_id == other => { + return true; + } Ok(_) => continue, // Lagged: keep waiting; the join event may still arrive. Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, @@ -243,7 +247,7 @@ async fn preminted_token_client_gets_join_events_and_signature_is_enforced() { .map_err(|error| format!("valid token failed to join: {error}"))?; let mut states = Vec::new(); while let Ok(event) = events.try_recv() { - if let CallEvent::CallingStateChanged(state) = event { + if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { states.push(state); } } diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index 42304ce..399dd29 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -21,9 +21,9 @@ use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - CallEvent, CallingState, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, - LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SubscriptionConfig, - VideoFrame, + CallEvent, CallingState, ClientCallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, + LocalTrack, LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, + SfuCallEvent, SubscriptionConfig, VideoFrame, }; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; @@ -387,9 +387,9 @@ async fn await_track_event( tokio::select! { () = &mut deadline => return false, recv = events.recv() => match recv { - Ok(CallEvent::TrackPublished { user_id, track_type: tt, .. }) + Ok(CallEvent::Sfu(SfuCallEvent::TrackPublished { user_id, track_type: tt, .. })) if published && user_id == user && tt == track_type => return true, - Ok(CallEvent::TrackUnpublished { user_id, track_type: tt, .. }) + Ok(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { user_id, track_type: tt, .. })) if !published && user_id == user && tt == track_type => return true, Ok(_) | Err(RecvError::Lagged(_)) => {} Err(RecvError::Closed) => return false, @@ -1232,7 +1232,7 @@ async fn await_speaking( tokio::select! { () = &mut deadline => return (level_seen, dominant_seen), received = events.recv() => match received { - Ok(CallEvent::AudioLevelChanged(levels)) => { + Ok(CallEvent::Sfu(SfuCallEvent::AudioLevelChanged(levels))) => { if levels .iter() .any(|l| l.session_id == session && l.is_speaking) @@ -1240,7 +1240,7 @@ async fn await_speaking( level_seen = true; } } - Ok(CallEvent::DominantSpeakerChanged { session_id, .. }) => { + Ok(CallEvent::Sfu(SfuCallEvent::DominantSpeakerChanged { session_id, .. })) => { if session_id == session { dominant_seen = true; } @@ -1466,9 +1466,9 @@ async fn sole_audio_can_be_stopped_and_published_again() { while let Ok(event) = events_a.try_recv() { reconnected |= matches!( event, - CallEvent::CallingStateChanged( + CallEvent::Client(ClientCallEvent::CallingStateChanged( CallingState::Reconnecting | CallingState::Migrating - ) + )) ); } assert!(!reconnected, "A reconnected after the second publish"); From bfe6c611a3f4f6d0d7fdd4f10b11924d4f7208c9 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Thu, 1 Oct 2026 14:40:16 +0200 Subject: [PATCH 13/37] feat: split call events into SFU, coordinator, and client streams --- CHANGELOG.md | 40 +++++---- src/rtc/client.rs | 28 +++--- src/rtc/join/connection.rs | 141 ++++++++++++----------------- src/rtc/join/lifecycle.rs | 21 ++--- src/rtc/join/mod.rs | 106 +++++++++------------- src/rtc/join/participants.rs | 8 +- src/rtc/join/tests.rs | 167 ++++++++++++++++++----------------- src/rtc/mod.rs | 6 +- src/video/call.rs | 27 +++--- tests/rtc_join.rs | 18 ++-- tests/rtc_media.rs | 28 +++--- tests/video_rest.rs | 29 +++--- 12 files changed, 279 insertions(+), 340 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffe6d8c..886d88c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,23 +12,28 @@ for a staging or local environment must set both fields. Code that builds `..ClientConfig::default()`. `DEFAULT_COORDINATOR_WS_URL` moved from `rtc::coordinator::ws` to the crate root. -### Call events are grouped by their source +### Call events come in one stream for each source -`CallEvent` has three variants: `Sfu(SfuCallEvent)` for the events from the -SFU, `Coordinator(CoordinatorEvent)` as before, and `Client(ClientCallEvent)` -for the events that the SDK itself produces (`CallingStateChanged` and -`CallEnded`). The SFU and client variants did not change; only their path did. -A pattern such as `CallEvent::ParticipantJoined(p)` becomes -`CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))`. +`Call`, `RtcCall` and `RtcCore` replace `subscribe()`, `on()`, `off()` and the +`CallEvent` enum with three streams: -### Track and call-ended events carry the SFU data +- `sfu_events()` gives `SfuCallEvent`: the events from the SFU, including + `CallEnded { reason }` for the SFU `call_ended`. +- `coordinator_events()` gives `CoordinatorEvent`: the call-scoped coordinator + events, including `call.ended`. +- `client_events()` gives `ClientCallEvent`: `CallingStateChanged`. + +The SDK leaves the call on the SFU `call_ended` or the coordinator +`call.ended`; the other one may then not arrive. `CallingStateChanged(Left)` is +the reliable end of the call. Each stream has its own buffer, so a lagging +receiver loses events only from its own stream. + +### Track events carry the SFU data `SfuCallEvent::TrackPublished` and `SfuCallEvent::TrackUnpublished` give `track_type` as a `TrackType`, not an `i32`, and add `participant`. -`TrackUnpublished` also adds `cause`. `ClientCallEvent::CallEnded` is now -`CallEnded { reason }`: the SFU reason, or `None` when the coordinator reported -the end first. Patterns that match these variants must use the new fields or -`..`. +`TrackUnpublished` also adds `cause`. Patterns that match these variants must +use the new fields or `..`. ## New Features @@ -42,12 +47,11 @@ client with an API secret; both handles share one session. `RtcCall` also adds ### Stable call event names -`CallEvent::name` returns a stable name for each event. `SfuCallEvent::name` -gives the `SfuEvent` field name of the source event (for example -`participant_joined`), and `participant_count_changed`. -`ClientCallEvent::name` gives `call_ended` (from both sources) or -`calling_state_changed`. A `Coordinator` event has its coordinator `type` (for -example `call.created`). +`SfuCallEvent::name` gives the stable `SfuEvent` field name of the source event +(for example `participant_joined` or `call_ended`), and +`participant_count_changed`. `ClientCallEvent::name` gives +`calling_state_changed`. A `CoordinatorEvent` has its coordinator `event_type` +(for example `call.created`). ### Video REST: advanced call statistics and reporting diff --git a/src/rtc/client.rs b/src/rtc/client.rs index 74903eb..e238e26 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -14,8 +14,11 @@ use crate::client::{Client, ClientConfig, NetworkLimits}; use crate::error::Result as CrateResult; use crate::token::{self, TokenOptions}; +use super::coordinator::ws::CoordinatorEvent; use super::error::{Result, RtcError}; -use super::join::{CallEvent, CallStateSnapshot, CallingState, JoinCallData, RtcCore}; +use super::join::{ + CallStateSnapshot, CallingState, ClientCallEvent, JoinCallData, RtcCore, SfuCallEvent, +}; use super::proto::models::TrackType; use super::publish_options::ClientPublishOptions; use super::subscriptions::{SubscriptionConfig, SubscriptionTarget}; @@ -278,24 +281,23 @@ impl RtcCall { self.core.update_publish_options(options); } - /// Subscribe to the typed SFU event stream. A receiver gets only events sent + /// Subscribe to the events from the SFU. A receiver gets only events sent /// after it subscribes. Subscribe before [`Self::join`] to get the join /// events, or read [`Self::participants`] and [`Self::call_state`]. - pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { - self.core.subscribe() + pub fn sfu_events(&self) -> tokio::sync::broadcast::Receiver { + self.core.sfu_events() } - /// Register a callback for typed call events. - pub fn on(&self, callback: F) -> tokio::task::AbortHandle - where - F: Fn(CallEvent) + Send + 'static, - { - self.core.on(callback) + /// Subscribe to the call-scoped coordinator events. A receiver gets only + /// events sent after it subscribes. + pub fn coordinator_events(&self) -> tokio::sync::broadcast::Receiver { + self.core.coordinator_events() } - /// Remove a callback registered with [`Self::on`]. - pub fn off(&self, handler: &tokio::task::AbortHandle) { - self.core.off(handler); + /// Subscribe to the events that the SDK itself produces. See + /// [`RtcCore::client_events`]. + pub fn client_events(&self) -> tokio::sync::broadcast::Receiver { + self.core.client_events() } /// The current calling state. diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index d5688c1..680dcf4 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -2,24 +2,11 @@ use super::*; -impl CallEvent { - /// The stable name of this event. The names are public API and do not - /// change: see [`SfuCallEvent::name`] and [`ClientCallEvent::name`]. A - /// [`CallEvent::Coordinator`] event has its coordinator `type`, for example - /// `call.created`. - pub fn name(&self) -> &str { - match self { - Self::Sfu(event) => event.name(), - Self::Coordinator(event) => &event.event_type, - Self::Client(event) => event.name(), - } - } -} - impl SfuCallEvent { /// The stable `SfuEvent` field name of the source event, for example - /// `participant_joined` or `change_publish_quality`. - /// [`SfuCallEvent::ParticipantCountChanged`] is `participant_count_changed`. + /// `participant_joined` or `change_publish_quality`. The names are public + /// API and do not change. [`SfuCallEvent::ParticipantCountChanged`] is + /// `participant_count_changed`. pub fn name(&self) -> &'static str { match self { Self::ParticipantJoined(_) => "participant_joined", @@ -38,16 +25,16 @@ impl SfuCallEvent { Self::CallGrantsUpdated(_) => "call_grants_updated", Self::IceRestarted(_) => "ice_restart", Self::Error(_) => "error", + Self::CallEnded { .. } => "call_ended", } } } impl ClientCallEvent { - /// The stable name of this event: `call_ended` (from both sources) or - /// `calling_state_changed`. + /// The stable name of this event: `calling_state_changed`. The names are + /// public API and do not change. pub fn name(&self) -> &'static str { match self { - Self::CallEnded { .. } => "call_ended", Self::CallingStateChanged(_) => "calling_state_changed", } } @@ -211,7 +198,7 @@ pub(super) fn register_connection_state( } /// The SFU WebSocket event loop: negotiate subscriber offers, add remote ICE -/// candidates, and fan out typed [`CallEvent`]s. +/// candidates, and fan out typed [`SfuCallEvent`]s. pub(super) async fn event_loop(mut receiver: SfuReceiver, context: EventLoopContext) { loop { let event = match receiver.recv().await { @@ -314,19 +301,17 @@ pub(super) async fn handle_event( E::ConnectionQualityChanged(event) => { core.update_connection_quality(&event.connection_quality_updates); let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ConnectionQualityChanged( + .sfu_events_tx + .send(SfuCallEvent::ConnectionQualityChanged( event.connection_quality_updates, - ))); + )); } E::ParticipantJoined(ev) => { if let Some(p) = ev.participant { core.upsert_participant(&p); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))); + let _ = core.sfu_events_tx.send(SfuCallEvent::ParticipantJoined(p)); } } E::ParticipantLeft(ev) => { @@ -334,9 +319,7 @@ pub(super) async fn handle_event( core.remove_participant(&p.session_id); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ParticipantLeft(p))); + let _ = core.sfu_events_tx.send(SfuCallEvent::ParticipantLeft(p)); } } E::ParticipantUpdated(ev) => { @@ -344,9 +327,7 @@ pub(super) async fn handle_event( core.upsert_participant(&p); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ParticipantUpdated(p))); + let _ = core.sfu_events_tx.send(SfuCallEvent::ParticipantUpdated(p)); } } E::TrackPublished(ev) => { @@ -358,87 +339,77 @@ pub(super) async fn handle_event( ); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::TrackPublished { - user_id: ev.user_id, - session_id: ev.session_id, - track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), - participant: ev.participant, - })); + let _ = core.sfu_events_tx.send(SfuCallEvent::TrackPublished { + user_id: ev.user_id, + session_id: ev.session_id, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + participant: ev.participant, + }); } E::TrackUnpublished(ev) => { core.remove_published_track(&ev.session_id, ev.r#type); core.recompute_subscriptions_for_generation(context.generation) .await?; - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { - user_id: ev.user_id, - session_id: ev.session_id, - track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), - cause: models::TrackUnpublishReason::try_from(ev.cause) - .unwrap_or(models::TrackUnpublishReason::Unspecified), - participant: ev.participant, - })); + let _ = core.sfu_events_tx.send(SfuCallEvent::TrackUnpublished { + user_id: ev.user_id, + session_id: ev.session_id, + track_type: TrackType::try_from(ev.r#type).unwrap_or(TrackType::Unspecified), + cause: models::TrackUnpublishReason::try_from(ev.cause) + .unwrap_or(models::TrackUnpublishReason::Unspecified), + participant: ev.participant, + }); } E::DominantSpeakerChanged(ev) => { core.update_dominant_speaker(&ev.session_id); let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::DominantSpeakerChanged { + .sfu_events_tx + .send(SfuCallEvent::DominantSpeakerChanged { user_id: ev.user_id, session_id: ev.session_id, - })); + }); } E::AudioLevelChanged(ev) => { core.update_audio_levels(&ev.audio_levels); let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::AudioLevelChanged( - ev.audio_levels, - ))); + .sfu_events_tx + .send(SfuCallEvent::AudioLevelChanged(ev.audio_levels)); } E::HealthCheckResponse(event) => { if let Some(participant_count) = event.participant_count && core.update_participant_count(participant_count) { let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ParticipantCountChanged( - participant_count, - ))); + .sfu_events_tx + .send(SfuCallEvent::ParticipantCountChanged(participant_count)); } } E::PinsUpdated(event) => { core.update_pins(event.pins.clone()); let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::PinsUpdated(event.pins))); + .sfu_events_tx + .send(SfuCallEvent::PinsUpdated(event.pins)); } E::InboundStateNotification(event) => { core.update_inbound_state(&event.inbound_video_states); - let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::InboundStateChanged( - event.inbound_video_states, - ))); + let _ = core.sfu_events_tx.send(SfuCallEvent::InboundStateChanged( + event.inbound_video_states, + )); } E::ChangePublishOptions(event) => { core.apply_publish_options(context.generation, event.publish_options.clone()) .await?; let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::PublishOptionsChanged { + .sfu_events_tx + .send(SfuCallEvent::PublishOptionsChanged { publish_options: event.publish_options, reason: event.reason, - })); + }); } E::ChangePublishQuality(event) => { core.apply_publish_quality(&event).await; let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::PublishQualityChanged(event))); + .sfu_events_tx + .send(SfuCallEvent::PublishQualityChanged(event)); } E::CallGrantsUpdated(event) => { core.update_call_grants(event.current_grants); @@ -455,8 +426,8 @@ pub(super) async fn handle_event( } } let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::CallGrantsUpdated(event))); + .sfu_events_tx + .send(SfuCallEvent::CallGrantsUpdated(event)); } E::IceRestart(event) => { let peer_type = @@ -465,15 +436,15 @@ pub(super) async fn handle_event( core.restart_publisher_ice().await?; } let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::IceRestarted(peer_type))); + .sfu_events_tx + .send(SfuCallEvent::IceRestarted(peer_type)); } E::Error(err) => { let join_err = SfuJoinError::from_event(err.error, err.reconnect_strategy); let strategy = ReconnectStrategy::from_proto(err.reconnect_strategy); let _ = core - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::Error(join_err.clone()))); + .sfu_events_tx + .send(SfuCallEvent::Error(join_err.clone())); if let Some(strategy) = strategy { core.trigger_reconnect(context.generation, strategy, join_err.message.clone()); } @@ -488,13 +459,13 @@ pub(super) async fn handle_event( E::ParticipantMigrationComplete(_) => { core.complete_migration(context.generation); } - E::CallEnded(event) => core.end_call( - context.generation, - Some( - models::CallEndedReason::try_from(event.reason) + E::CallEnded(event) => { + let _ = core.sfu_events_tx.send(SfuCallEvent::CallEnded { + reason: models::CallEndedReason::try_from(event.reason) .unwrap_or(models::CallEndedReason::Unspecified), - ), - ), + }); + core.end_call(context.generation); + } E::PublisherAnswer(_) | E::JoinResponse(_) => { tracing::debug!("stream.rtc.unexpected_handshake_event"); } diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index 97ce7a5..ad46f16 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -671,13 +671,9 @@ impl RtcCore { Ok(()) } - /// The SFU or the coordinator reported the end of the call: send - /// [`ClientCallEvent::CallEnded`] once and leave the call. - pub(super) fn end_call( - self: &Arc, - generation: u64, - reason: Option, - ) { + /// The SFU or the coordinator reported the end of the call: leave the call + /// once. + pub(super) fn end_call(self: &Arc, generation: u64) { if !self.is_generation_current(generation) { return; } @@ -692,9 +688,6 @@ impl RtcCore { if already_ended { return; } - let _ = self - .events_tx - .send(CallEvent::Client(ClientCallEvent::CallEnded { reason })); let this = self.clone(); // Not a generation task: `leave` ends the generation. std::mem::drop(self.spawn_runtime_task(async move { @@ -760,7 +753,7 @@ impl Drop for JoinAttempt<'_> { return; } lifecycle.generation = lifecycle.generation.wrapping_add(1); - lifecycle.set_state(CallingState::Idle, &self.core.events_tx); + lifecycle.set_state(CallingState::Idle, &self.core.client_events_tx); } self.core.lifecycle_changed.notify_waiters(); } @@ -814,7 +807,7 @@ impl RtcCore { let cid = self.cid(); let local_user_id = user_id.to_owned(); - let sender = self.events_tx.clone(); + let sender = self.coordinator_events_tx.clone(); let event_core = self.clone(); let event_task = self.spawn_generation_task(generation, async move { loop { @@ -827,9 +820,9 @@ impl RtcCore { .apply_permissions_updated(&event, &local_user_id) .await; let ended = event.event_type == "call.ended"; - let _ = sender.send(CallEvent::Coordinator(event)); + let _ = sender.send(event); if ended { - event_core.end_call(generation, None); + event_core.end_call(generation); } } Ok(Some(_)) => {} diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 315fe31..9b00ada 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -8,7 +8,8 @@ //! backoff, unrecoverable abort, and SFU switching via `migrating_from`; //! - the SFU WebSocket handshake (`JoinRequest` → `JoinResponse`), subscriber //! answer negotiation, and ICE trickle; -//! - a typed [`CallEvent`] broadcast stream (participant joined/left, tracks, …); +//! - typed broadcast streams by source: [`SfuCallEvent`], coordinator events, +//! and [`ClientCallEvent`]; //! - the reconnect state machine (`RtcCore::run_reconnect`) driven by the pure //! decision logic in [`super::reconnect`], with dedup, the rejoin rate limiter, //! the ICE / negotiation limits, the disconnection timeout, and the @@ -176,24 +177,7 @@ pub enum CallingState { Left, } -/// An event delivered on the [`Call`](crate::Call) event stream, grouped by its -/// source. -#[derive(Debug, Clone)] -#[non_exhaustive] -// `Sfu` carries `Participant` data (about 304 bytes), and the other variants are -// small. A `Box` would allocate for every SFU event and prevent nested -// `CallEvent::Sfu(SfuCallEvent::..)` patterns. -#[allow(clippy::large_enum_variant)] -pub enum CallEvent { - /// An event from the SFU. - Sfu(SfuCallEvent), - /// A call-scoped coordinator WebSocket event. - Coordinator(CoordinatorEvent), - /// An event that the SDK itself produces. - Client(ClientCallEvent), -} - -/// An event from the SFU. +/// An event from the SFU, delivered by [`RtcCore::sfu_events`]. #[derive(Debug, Clone)] #[non_exhaustive] pub enum SfuCallEvent { @@ -263,18 +247,18 @@ pub enum SfuCallEvent { IceRestarted(PeerType), /// The SFU reported an error for this participant. Error(SfuJoinError), + /// The SFU ended the call. The SDK then leaves the call. + CallEnded { + /// Why the call ended. + reason: models::CallEndedReason, + }, } -/// An event that the SDK itself produces. +/// An event that the SDK itself produces, delivered by +/// [`RtcCore::client_events`]. #[derive(Debug, Clone)] #[non_exhaustive] pub enum ClientCallEvent { - /// The call ended: the SFU or the coordinator (`call.ended`) reported it. - /// Sent once, after which the SDK leaves the call. - CallEnded { - /// The SFU reason, or `None` when the coordinator reported the end first. - reason: Option, - }, /// The connection state changed. CallingStateChanged(CallingState), } @@ -312,12 +296,10 @@ struct Lifecycle { impl Lifecycle { /// Call with the lifecycle lock held, so events arrive in the order of the /// state changes. - fn set_state(&mut self, next: CallingState, events: &broadcast::Sender) { + fn set_state(&mut self, next: CallingState, events: &broadcast::Sender) { if self.state != next { self.state = next; - let _ = events.send(CallEvent::Client(ClientCallEvent::CallingStateChanged( - next, - ))); + let _ = events.send(ClientCallEvent::CallingStateChanged(next)); } } } @@ -523,7 +505,9 @@ pub struct RtcCore { api_key: String, call_type: String, call_id: String, - events_tx: broadcast::Sender, + sfu_events_tx: broadcast::Sender, + coordinator_events_tx: broadcast::Sender, + client_events_tx: broadcast::Sender, lifecycle: StdMutex, lifecycle_changed: Notify, connection: TokioMutex>, @@ -584,7 +568,6 @@ impl Drop for ReconnectClaim { impl RtcCore { /// Build a fresh (idle) core for a call handle. pub(crate) fn new(client: Arc, call_type: String, call_id: String) -> Arc { - let (events_tx, _rx) = broadcast::channel(256); Arc::new(Self { api_key: client.api_key().to_owned(), client, @@ -593,7 +576,9 @@ impl RtcCore { token_refresh: TokioMutex::new(()), call_type, call_id, - events_tx, + sfu_events_tx: broadcast::channel(256).0, + coordinator_events_tx: broadcast::channel(256).0, + client_events_tx: broadcast::channel(256).0, lifecycle: StdMutex::new(Lifecycle { state: CallingState::Idle, generation: 0, @@ -706,7 +691,7 @@ impl RtcCore { if guard.generation != generation { return false; } - guard.set_state(next, &self.events_tx); + guard.set_state(next, &self.client_events_tx); true } @@ -765,7 +750,7 @@ impl RtcCore { match guard.state { CallingState::Idle | CallingState::Left => { guard.generation = guard.generation.wrapping_add(1); - guard.set_state(CallingState::Joining, &self.events_tx); + guard.set_state(CallingState::Joining, &self.client_events_tx); guard.generation_publish_options = guard.publish_options; guard.failure_limits = FailureLimits::default(); guard.rate_limiter = SlidingWindowRateLimiter::rejoin_default(); @@ -914,9 +899,26 @@ impl RtcCore { Ok(()) } - /// Subscribe to the typed event stream. - pub fn subscribe(&self) -> broadcast::Receiver { - self.events_tx.subscribe() + /// Subscribe to the events from the SFU. A receiver gets only events sent + /// after it subscribes; subscribe before the join to get the join events. + pub fn sfu_events(&self) -> broadcast::Receiver { + self.sfu_events_tx.subscribe() + } + + /// Subscribe to the call-scoped coordinator events. A receiver gets only + /// events sent after it subscribes. + pub fn coordinator_events(&self) -> broadcast::Receiver { + self.coordinator_events_tx.subscribe() + } + + /// Subscribe to the events that the SDK itself produces. A receiver gets + /// only events sent after it subscribes. + /// + /// `CallingStateChanged(Left)` is the reliable end of the call: the SDK + /// leaves on the SFU `call_ended` or the coordinator `call.ended`, and the + /// other one may then not arrive. + pub fn client_events(&self) -> broadcast::Receiver { + self.client_events_tx.subscribe() } fn user_request_query(&self) -> Option> { @@ -953,34 +955,6 @@ impl RtcCore { (!token.is_empty()).then(|| (token.clone(), query)) } - /// Register a callback for typed call events. - /// - /// Rust callers receive the full [`CallEvent`] enum and can pattern-match - /// the variants they need. Pass the returned handle to [`Self::off`]. - pub fn on(&self, callback: F) -> tokio::task::AbortHandle - where - F: Fn(CallEvent) + Send + 'static, - { - let mut events = self.subscribe(); - tokio::spawn(async move { - loop { - match events.recv().await { - Ok(event) => callback(event), - Err(broadcast::error::RecvError::Lagged(skipped)) => { - tracing::warn!(skipped, "stream.rtc.event_handler_lagged"); - } - Err(broadcast::error::RecvError::Closed) => break, - } - } - }) - .abort_handle() - } - - /// Remove an event callback registered with [`Self::on`]. - pub fn off(&self, handler: &tokio::task::AbortHandle) { - handler.abort(); - } - /// The cached stats options from the last coordinator join. pub fn stats_options(&self) -> StatsOptions { self.stats_options diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 37e8d93..172d73f 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -163,15 +163,13 @@ impl RtcCore { } Some(_) => continue, }; - let _ = self.events_tx.send(CallEvent::Sfu(event)); + let _ = self.sfu_events_tx.send(event); } for (id, entry) in previous { if !is_local(&id) { let _ = self - .events_tx - .send(CallEvent::Sfu(SfuCallEvent::ParticipantLeft( - entry.participant, - ))); + .sfu_events_tx + .send(SfuCallEvent::ParticipantLeft(entry.participant)); } } true diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 094e19c..fb3ed7c 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -397,7 +397,7 @@ async fn leave_tears_down_the_stored_connection() { } #[tokio::test] -async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() { +async fn call_ended_from_the_coordinator_is_forwarded_and_leaves_the_call() { let (coordinator_ws_url, _coordinator) = fake_coordinator_sending(vec![ json!({ "type": "call.ended", "call_cid": "default:test-call" }), ]) @@ -408,9 +408,8 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() }); let generation = prepare_joined_core(&core, "alice"); let (connection, sfu) = establish_fake(&core, generation).await; - let context = event_context(&core, &connection); *core.connection.lock().await = Some(connection); - let mut events = core.subscribe(); + let mut events = core.coordinator_events(); let token = core.current_user_token().expect("user token"); core.connect_coordinator_events(generation, &token, "alice") @@ -418,21 +417,15 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() .expect("coordinator events"); tokio::time::timeout(Duration::from_secs(2), async { loop { - if let Ok(CallEvent::Client(ClientCallEvent::CallEnded { reason: None })) = - events.recv().await + if let Ok(event) = events.recv().await + && event.event_type == "call.ended" { return; } } }) .await - .expect("call ended event"); - connection::handle_event( - &context, - sfu_event::EventPayload::CallEnded(event::CallEnded::default()), - ) - .await - .expect("handle SFU call ended"); + .expect("call.ended event"); wait_for( Duration::from_secs(2), @@ -440,13 +433,6 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() "the ended call is left", ) .await; - let mut repeated = 0; - while let Ok(event) = events.try_recv() { - if matches!(event, CallEvent::Client(ClientCallEvent::CallEnded { .. })) { - repeated += 1; - } - } - assert_eq!(repeated, 0, "call ended is reported once"); let requests = requests_until_close(sfu).await; assert!(requests.iter().any(|request| matches!( request.request_payload, @@ -455,13 +441,13 @@ async fn call_ended_from_the_coordinator_is_reported_once_and_leaves_the_call() } #[tokio::test] -async fn call_ended_twice_before_the_leave_is_reported_once() { +async fn sfu_call_ended_is_forwarded_with_its_reason_and_leaves_once() { let core = test_core(); let generation = prepare_joined_core(&core, "alice"); - let (connection, _sfu) = establish_fake(&core, generation).await; + let (connection, sfu) = establish_fake(&core, generation).await; let context = event_context(&core, &connection); *core.connection.lock().await = Some(connection); - let mut events = core.subscribe(); + let mut events = core.sfu_events(); // Neither call yields, so the spawned leave cannot run between them. for _ in 0..2 { @@ -481,18 +467,60 @@ async fn call_ended_twice_before_the_leave_is_reported_once() { "the ended call is left", ) .await; - let mut reported = 0; + let mut forwarded = 0; while let Ok(event) = events.try_recv() { if matches!( event, - CallEvent::Client(ClientCallEvent::CallEnded { - reason: Some(models::CallEndedReason::Kicked) - }) + SfuCallEvent::CallEnded { + reason: models::CallEndedReason::Kicked + } ) { - reported += 1; + forwarded += 1; } } - assert_eq!(reported, 1); + assert_eq!(forwarded, 2, "each SFU call_ended is forwarded"); + let leaves = requests_until_close(sfu) + .await + .into_iter() + .filter(|request| { + matches!( + request.request_payload, + Some(event::sfu_request::RequestPayload::LeaveCallRequest(_)) + ) + }) + .count(); + assert_eq!(leaves, 1, "the call is left once"); +} + +#[tokio::test] +async fn each_event_goes_only_to_the_stream_of_its_source() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let context = event_context(&core, &connection); + *core.connection.lock().await = Some(connection); + let mut sfu = core.sfu_events(); + let mut client = core.client_events(); + let mut coordinator = core.coordinator_events(); + + connection::handle_event( + &context, + sfu_event::EventPayload::PinsUpdated(event::PinsChanged::default()), + ) + .await + .expect("handle pins"); + assert!(core.set_state_if_current(generation, CallingState::Reconnecting)); + + assert!(matches!(sfu.try_recv(), Ok(SfuCallEvent::PinsUpdated(_)))); + assert!(sfu.try_recv().is_err()); + assert!(matches!( + client.try_recv(), + Ok(ClientCallEvent::CallingStateChanged( + CallingState::Reconnecting + )) + )); + assert!(client.try_recv().is_err()); + assert!(coordinator.try_recv().is_err()); } #[tokio::test] @@ -502,7 +530,7 @@ async fn track_events_report_the_track_type_cause_and_participant() { let (connection, _sfu) = establish_fake(&core, generation).await; let context = event_context(&core, &connection); *core.connection.lock().await = Some(connection); - let mut events = core.subscribe(); + let mut events = core.sfu_events(); let bob = models::Participant { user_id: "bob".to_owned(), session_id: "bob-session".to_owned(), @@ -521,11 +549,11 @@ async fn track_events_report_the_track_type_cause_and_participant() { .await .expect("handle track published"); - let Ok(CallEvent::Sfu(SfuCallEvent::TrackPublished { + let Ok(SfuCallEvent::TrackPublished { track_type, participant, .. - })) = events.try_recv() + }) = events.try_recv() else { panic!("expected a track published event"); }; @@ -545,12 +573,12 @@ async fn track_events_report_the_track_type_cause_and_participant() { .await .expect("handle track unpublished"); - let Ok(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { + let Ok(SfuCallEvent::TrackUnpublished { track_type, cause, participant, .. - })) = events.try_recv() + }) = events.try_recv() else { panic!("expected a track unpublished event"); }; @@ -566,7 +594,7 @@ async fn participant_count_event_is_sent_only_when_the_count_changes() { let (connection, _sfu) = establish_fake(&core, generation).await; let context = event_context(&core, &connection); *core.connection.lock().await = Some(connection); - let mut events = core.subscribe(); + let mut events = core.sfu_events(); for total in [2, 2, 3] { connection::handle_event( @@ -584,7 +612,7 @@ async fn participant_count_event_is_sent_only_when_the_count_changes() { let mut totals = Vec::new(); while let Ok(event) = events.try_recv() { - if let CallEvent::Sfu(SfuCallEvent::ParticipantCountChanged(count)) = event { + if let SfuCallEvent::ParticipantCountChanged(count) = event { totals.push(count.total); } } @@ -1123,7 +1151,7 @@ async fn leave_cancels_reconnect_task_before_next_generation() { fn state_events_arrive_in_the_order_of_the_state_changes() { let core = test_core(); let generation = core.begin_join().expect("join generation"); - let mut events = core.subscribe(); + let mut events = core.client_events(); let rounds = 20_000; let barrier = Arc::new(std::sync::Barrier::new(3)); let workers = [CallingState::Joined, CallingState::Reconnecting].map(|state| { @@ -1142,10 +1170,8 @@ fn state_events_arrive_in_the_order_of_the_state_changes() { barrier.wait(); barrier.wait(); let mut last = None; - while let Ok(event) = events.try_recv() { - if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { - last = Some(state); - } + while let Ok(ClientCallEvent::CallingStateChanged(state)) = events.try_recv() { + last = Some(state); } assert_eq!(last, Some(core.state()), "round {round}"); } @@ -1158,16 +1184,16 @@ fn state_events_arrive_in_the_order_of_the_state_changes() { fn setting_the_same_state_again_sends_no_event() { let core = test_core(); let generation = core.begin_join().expect("join generation"); - let mut events = core.subscribe(); + let mut events = core.client_events(); assert!(core.set_state_if_current(generation, CallingState::Reconnecting)); assert!(core.set_state_if_current(generation, CallingState::Reconnecting)); assert!(matches!( events.try_recv(), - Ok(CallEvent::Client(ClientCallEvent::CallingStateChanged( + Ok(ClientCallEvent::CallingStateChanged( CallingState::Reconnecting - ))) + )) )); assert!(events.try_recv().is_err()); } @@ -1175,22 +1201,20 @@ fn setting_the_same_state_again_sends_no_event() { #[test] fn join_start_sends_joining() { let core = test_core(); - let mut events = core.subscribe(); + let mut events = core.client_events(); core.begin_join().expect("join generation"); assert!(matches!( events.try_recv(), - Ok(CallEvent::Client(ClientCallEvent::CallingStateChanged( - CallingState::Joining - ))) + Ok(ClientCallEvent::CallingStateChanged(CallingState::Joining)) )); } #[tokio::test] async fn state_during_leave_matches_the_last_state_event() { let core = test_core(); - let mut events = core.subscribe(); + let mut events = core.client_events(); core.begin_join().expect("join generation"); let connection_slot = core.connection.lock().await; let generation = core.generation(); @@ -1204,10 +1228,8 @@ async fn state_during_leave_matches_the_last_state_event() { .await; let mut last = None; - while let Ok(event) = events.try_recv() { - if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { - last = Some(state); - } + while let Ok(ClientCallEvent::CallingStateChanged(state)) = events.try_recv() { + last = Some(state); } assert_eq!(last, Some(core.state())); drop(connection_slot); @@ -1768,35 +1790,28 @@ fn every_call_event_has_its_stable_name() { SfuCallEvent::Error(SfuJoinError::from_event(None, 0)), "error", ), - ] - .map(|(event, name)| (CallEvent::Sfu(event), name)); - let client = [ - (ClientCallEvent::CallEnded { reason: None }, "call_ended"), ( - ClientCallEvent::CallingStateChanged(CallingState::Joined), - "calling_state_changed", + SfuCallEvent::CallEnded { + reason: models::CallEndedReason::Ended, + }, + "call_ended", ), - ] - .map(|(event, name)| (CallEvent::Client(event), name)); - let coordinator = ( - CallEvent::Coordinator(CoordinatorEvent { - event_type: "call.created".to_owned(), - raw: json!({ "type": "call.created" }), - }), - "call.created", - ); - let events = sfu.into_iter().chain(client).chain([coordinator]); + ]; - for (event, name) in events { + for (event, name) in sfu { assert_eq!(event.name(), name, "{event:?}"); } + assert_eq!( + ClientCallEvent::CallingStateChanged(CallingState::Joined).name(), + "calling_state_changed" + ); } #[test] fn join_state_reports_only_the_participant_changes_since_the_last_join() { let core = test_core(); let generation = core.begin_join().expect("join generation"); - let mut events = core.subscribe(); + let mut events = core.sfu_events(); let participant = |user_id: &str, session_id: &str| models::Participant { user_id: user_id.to_owned(), session_id: session_id.to_owned(), @@ -1838,15 +1853,9 @@ fn join_state_reports_only_the_participant_changes_since_the_last_join() { let mut reported = Vec::new(); while let Ok(event) = events.try_recv() { match event { - CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p)) => { - reported.push(("joined", p.session_id)); - } - CallEvent::Sfu(SfuCallEvent::ParticipantUpdated(p)) => { - reported.push(("updated", p.session_id)); - } - CallEvent::Sfu(SfuCallEvent::ParticipantLeft(p)) => { - reported.push(("left", p.session_id)); - } + SfuCallEvent::ParticipantJoined(p) => reported.push(("joined", p.session_id)), + SfuCallEvent::ParticipantUpdated(p) => reported.push(("updated", p.session_id)), + SfuCallEvent::ParticipantLeft(p) => reported.push(("left", p.session_id)), _ => {} } } diff --git a/src/rtc/mod.rs b/src/rtc/mod.rs index 4c8388a..d207670 100644 --- a/src/rtc/mod.rs +++ b/src/rtc/mod.rs @@ -8,7 +8,8 @@ //! The participant layer sits on top: the [`coordinator`] join REST, dual //! publisher/subscriber PeerConnections ([`peer`]), and the [`join`] state //! machine ([`join::RtcCore`]) with `max_join_retries`, Stream's reconnect -//! strategies, and typed [`join::CallEvent`]s. [`crate::Call::join`] and +//! strategies, and typed event streams by source ([`SfuCallEvent`], +//! [`CoordinatorEvent`], [`ClientCallEvent`]). [`crate::Call::join`] and //! [`crate::Call::leave`] are the high-level entry points; [`RtcClient`] is the //! lower-level user-token client. //! @@ -51,8 +52,7 @@ pub use error::{ }; pub use identity::{CLIENT_TYPE, SDK_TYPE, client_details, client_header}; pub use join::{ - CallEvent, CallStateSnapshot, CallingState, ClientCallEvent, JoinCallData, RtcCore, - SfuCallEvent, + CallStateSnapshot, CallingState, ClientCallEvent, JoinCallData, RtcCore, SfuCallEvent, }; pub use pcm::chunk::Pad; pub use pcm::convert::G711_SAMPLE_RATE; diff --git a/src/video/call.rs b/src/video/call.rs index d0b7dbe..9db5265 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -841,7 +841,7 @@ impl Call { /// Mints a finite, call-CID-scoped user token internally from the server secret, runs the /// coordinator join, establishes the publisher/subscriber PeerConnections, /// and completes the SFU handshake. Illegal (typed error) if already - /// `JOINING`/`JOINED`. Observe participants via [`Call::subscribe`]. + /// `JOINING`/`JOINED`. Observe participants via [`Call::sfu_events`]. pub async fn join(&self, data: crate::rtc::JoinCallData) -> crate::rtc::RtcResult<()> { self.rtc().join(data).await } @@ -865,24 +865,25 @@ impl Call { self.rtc.leave("user requested leave").await } - /// Subscribe to the typed SFU event stream (participant joined/left, tracks, + /// Subscribe to the events from the SFU (participant joined/left, tracks, /// errors). A receiver gets only events sent after it subscribes. Subscribe /// before [`Call::join`] to get the join events, or read [`Call::participants`]. - pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { - self.rtc.subscribe() + pub fn sfu_events(&self) -> tokio::sync::broadcast::Receiver { + self.rtc.sfu_events() } - /// Register a callback for typed call events. - pub fn on(&self, callback: F) -> tokio::task::AbortHandle - where - F: Fn(crate::rtc::CallEvent) + Send + 'static, - { - self.rtc.on(callback) + /// Subscribe to the call-scoped coordinator events. A receiver gets only + /// events sent after it subscribes. + pub fn coordinator_events( + &self, + ) -> tokio::sync::broadcast::Receiver { + self.rtc.coordinator_events() } - /// Remove a callback registered with [`Call::on`]. - pub fn off(&self, handler: &tokio::task::AbortHandle) { - self.rtc.off(handler); + /// Subscribe to the events that the SDK itself produces. See + /// [`RtcCore::client_events`](crate::rtc::RtcCore::client_events). + pub fn client_events(&self) -> tokio::sync::broadcast::Receiver { + self.rtc.client_events() } /// The current calling state (`Idle` / `Joining` / `Joined` / …). diff --git a/tests/rtc_join.rs b/tests/rtc_join.rs index 9724624..6a79f9b 100644 --- a/tests/rtc_join.rs +++ b/tests/rtc_join.rs @@ -13,15 +13,13 @@ use std::time::Duration; use getstream::TokenOptions; use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; -use getstream::rtc::{ - CallEvent, CallingState, ClientCallEvent, JoinCallData, RtcClient, SfuCallEvent, -}; +use getstream::rtc::{CallingState, ClientCallEvent, JoinCallData, RtcClient, SfuCallEvent}; use tokio::sync::broadcast::Receiver; /// Wait (up to `timeout`) for a `ParticipantJoined` whose `user_id` matches /// `other`. Returns `true` if observed. async fn observe_participant( - mut rx: Receiver, + mut rx: Receiver, other: String, timeout: Duration, ) -> bool { @@ -31,9 +29,7 @@ async fn observe_participant( tokio::select! { () = &mut deadline => return false, event = rx.recv() => match event { - Ok(CallEvent::Sfu(SfuCallEvent::ParticipantJoined(p))) if p.user_id == other => { - return true; - } + Ok(SfuCallEvent::ParticipantJoined(p)) if p.user_id == other => return true, Ok(_) => continue, // Lagged: keep waiting; the join event may still arrive. Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, @@ -104,8 +100,8 @@ async fn two_sessions_join_and_observe_each_other() { let call_b = client.video().call("default", &call_id).rtc(); // Subscribe BEFORE joining so no participant event is missed. - let rx_a = call_a.subscribe(); - let rx_b = call_b.subscribe(); + let rx_a = call_a.sfu_events(); + let rx_b = call_b.sfu_events(); // Session A joins first. call_a @@ -240,14 +236,14 @@ async fn preminted_token_client_gets_join_events_and_signature_is_enforced() { let allowed = RtcClient::new(client.api_key(), token) .map_err(|error| format!("build RTC client: {error}"))? .call("default", &call_id); - let mut events = allowed.subscribe(); + let mut events = allowed.client_events(); allowed .join(JoinCallData::new(&user_id)) .await .map_err(|error| format!("valid token failed to join: {error}"))?; let mut states = Vec::new(); while let Ok(event) = events.try_recv() { - if let CallEvent::Client(ClientCallEvent::CallingStateChanged(state)) = event { + if let ClientCallEvent::CallingStateChanged(state) = event { states.push(state); } } diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index 399dd29..e733f58 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -21,9 +21,9 @@ use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - CallEvent, CallingState, ClientCallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, - LocalTrack, LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, - SfuCallEvent, SubscriptionConfig, VideoFrame, + CallingState, ClientCallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, + LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SfuCallEvent, + SubscriptionConfig, VideoFrame, }; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; @@ -374,7 +374,7 @@ async fn drain_rms(remote: &RemoteTrack, target: usize, overall: Duration) -> f6 /// `TrackUnpublished` (`published == false`) event for `user`/`track_type`, /// draining unrelated events. Returns whether the event was observed. async fn await_track_event( - events: &mut tokio::sync::broadcast::Receiver, + events: &mut tokio::sync::broadcast::Receiver, user: &str, track_type: TrackType, published: bool, @@ -387,9 +387,9 @@ async fn await_track_event( tokio::select! { () = &mut deadline => return false, recv = events.recv() => match recv { - Ok(CallEvent::Sfu(SfuCallEvent::TrackPublished { user_id, track_type: tt, .. })) + Ok(SfuCallEvent::TrackPublished { user_id, track_type: tt, .. }) if published && user_id == user && tt == track_type => return true, - Ok(CallEvent::Sfu(SfuCallEvent::TrackUnpublished { user_id, track_type: tt, .. })) + Ok(SfuCallEvent::TrackUnpublished { user_id, track_type: tt, .. }) if !published && user_id == user && tt == track_type => return true, Ok(_) | Err(RecvError::Lagged(_)) => {} Err(RecvError::Closed) => return false, @@ -1218,7 +1218,7 @@ async fn publish_h264_video_b_decodes_i420_frame() { /// Await both an `AudioLevelChanged` naming `session` as speaking and a /// `DominantSpeakerChanged` naming it, within `timeout`. async fn await_speaking( - events: &mut tokio::sync::broadcast::Receiver, + events: &mut tokio::sync::broadcast::Receiver, session: &str, timeout: Duration, ) -> (bool, bool) { @@ -1232,7 +1232,7 @@ async fn await_speaking( tokio::select! { () = &mut deadline => return (level_seen, dominant_seen), received = events.recv() => match received { - Ok(CallEvent::Sfu(SfuCallEvent::AudioLevelChanged(levels))) => { + Ok(SfuCallEvent::AudioLevelChanged(levels)) => { if levels .iter() .any(|l| l.session_id == session && l.is_speaking) @@ -1240,7 +1240,7 @@ async fn await_speaking( level_seen = true; } } - Ok(CallEvent::Sfu(SfuCallEvent::DominantSpeakerChanged { session_id, .. })) => { + Ok(SfuCallEvent::DominantSpeakerChanged { session_id, .. }) => { if session_id == session { dominant_seen = true; } @@ -1281,7 +1281,7 @@ async fn loud_publisher_is_reported_speaking_and_dominant() { // Join and subscribe before any participant publishes audio. A speaker // selected before B joins is present in JoinResponse, and the SFU does // not replay the earlier DominantSpeakerChanged event to B. - let mut events_b = call_b.subscribe(); + let mut events_b = call_b.sfu_events(); call_b .join(JoinCallData::new(&user_b)) .await @@ -1379,13 +1379,13 @@ async fn sole_audio_can_be_stopped_and_published_again() { .update_subscriptions(SubscriptionConfig::audio_all()) .await .expect("B update_subscriptions"); - let mut events_b = call_b.subscribe(); + let mut events_b = call_b.sfu_events(); call_a .join(JoinCallData::new(&user_a)) .await .expect("A join"); - let mut events_a = call_a.subscribe(); + let mut events_a = call_a.client_events(); let audio_a = LocalAudioTrack::opus().expect("opus track"); call_a .publish_audio(audio_a.clone()) @@ -1466,9 +1466,9 @@ async fn sole_audio_can_be_stopped_and_published_again() { while let Ok(event) = events_a.try_recv() { reconnected |= matches!( event, - CallEvent::Client(ClientCallEvent::CallingStateChanged( + ClientCallEvent::CallingStateChanged( CallingState::Reconnecting | CallingState::Migrating - )) + ) ); } assert!(!reconnected, "A reconnected after the second publish"); diff --git a/tests/video_rest.rs b/tests/video_rest.rs index 27479b2..f6aa8fa 100644 --- a/tests/video_rest.rs +++ b/tests/video_rest.rs @@ -13,7 +13,7 @@ use getstream::models::{ StartClosedCaptionsRequest, StartFrameRecordingRequest, StopClosedCaptionsRequest, UpdateCallMembersRequest, UserRequest, }; -use getstream::rtc::{CallEvent, JoinCallData, LocalAudioTrack, LocalTrack, RtcError}; +use getstream::rtc::{CoordinatorEvent, JoinCallData, LocalAudioTrack, LocalTrack, RtcError}; use std::time::Duration; /// End-to-end call lifecycle: create → get → update members → query → end → delete. @@ -192,7 +192,7 @@ async fn audio_room_send_audio_permission_controls_publishing() { .expect("take permission test audio room live"); let participant = client.video().call("audio_room", &call_id); let outcome: Result<(), String> = tokio::time::timeout(Duration::from_secs(120), async { - let mut events = participant.subscribe(); + let mut events = participant.coordinator_events(); participant .join(JoinCallData::new(&publisher_id)) .await @@ -247,7 +247,7 @@ async fn audio_room_send_audio_permission_controls_publishing() { } async fn wait_for_audio_permission( - events: &mut tokio::sync::broadcast::Receiver, + events: &mut tokio::sync::broadcast::Receiver, expected: bool, ) -> Result<(), String> { let mut last_update = None; @@ -257,9 +257,6 @@ async fn wait_for_audio_permission( .recv() .await .map_err(|error| format!("permission event stream closed: {error}"))?; - let CallEvent::Coordinator(event) = event else { - continue; - }; if event.event_type != "call.permissions_updated" { continue; } @@ -332,10 +329,7 @@ async fn scoped_participant_service_lifecycle() { .await .map_err(|error| format!("enable incoming video failed: {error}"))?; - let (event_tx, mut event_rx) = tokio::sync::mpsc::unbounded_channel(); - let handler = call.on(move |event| { - let _ = event_tx.send(event); - }); + let mut events = call.coordinator_events(); let mut custom = getstream::models::CustomData::new(); custom.insert("source".to_owned(), serde_json::json!("rust-live-test")); server_call @@ -347,9 +341,8 @@ async fn scoped_participant_service_lifecycle() { .await .map_err(|error| format!("send_custom_event failed: {error}"))?; let custom_event = tokio::time::timeout(Duration::from_secs(10), async { - while let Some(event) = event_rx.recv().await { - if let getstream::rtc::CallEvent::Coordinator(event) = event - && event.event_type == "custom" + while let Ok(event) = events.recv().await { + if event.event_type == "custom" && event .raw .pointer("/custom/source") @@ -359,7 +352,7 @@ async fn scoped_participant_service_lifecycle() { return Ok(()); } } - Err("call event handler closed before receiving custom event".to_owned()) + Err("coordinator event stream ended before the custom event".to_owned()) }) .await .map_err(|_| "timed out waiting for custom coordinator event".to_owned()) @@ -382,9 +375,8 @@ async fn scoped_participant_service_lifecycle() { .await .map_err(|error| format!("send_closed_caption failed: {error}"))?; tokio::time::timeout(Duration::from_secs(10), async { - while let Some(event) = event_rx.recv().await { - if let getstream::rtc::CallEvent::Coordinator(event) = event - && event.event_type == "call.closed_caption" + while let Ok(event) = events.recv().await { + if event.event_type == "call.closed_caption" && event .raw .pointer("/closed_caption/text") @@ -394,14 +386,13 @@ async fn scoped_participant_service_lifecycle() { return Ok(()); } } - Err("call event handler closed before receiving closed caption".to_owned()) + Err("coordinator event stream ended before the closed caption".to_owned()) }) .await .map_err(|_| "timed out waiting for closed caption event".to_owned()) .and_then(|result| result) } .await; - call.off(&handler); caption_result?; call.send_reaction(SendVideoReactionRequest { From 26dfeb9fdd9002deaceaf0dcd2af4a0fb5c8791e Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Thu, 1 Oct 2026 16:00:01 +0200 Subject: [PATCH 14/37] feat: carry the RTP timestamp on decoded PCM frames --- CHANGELOG.md | 9 +++ src/rtc/pcm/frame.rs | 6 ++ src/rtc/tracks/remote.rs | 130 ++++++++++++++++++++++++++------------- 3 files changed, 101 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 886d88c..d209874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,15 @@ receiver loses events only from its own stream. `TrackUnpublished` also adds `cause`. Patterns that match these variants must use the new fields or `..`. +### Decoded audio frames carry their RTP timestamp + +`PcmFrame` adds `pts: Option`: the RTP timestamp of the first sample, in +units of 1/48000 s, wrapping like RTP. `RemoteTrack::next_pcm` sets it; a frame +rebuilt for a lost packet continues from the frame before it. Frames that the +application or a conversion builds have `None`, and `write_pcm` ignores the +field. Code that builds `PcmFrame` with a struct literal must set `pts` or use +`PcmFrame::new` / `PcmFrame::mono`. + ## New Features ### A token-only client can prepare a call before the join diff --git a/src/rtc/pcm/frame.rs b/src/rtc/pcm/frame.rs index 774281f..40df27a 100644 --- a/src/rtc/pcm/frame.rs +++ b/src/rtc/pcm/frame.rs @@ -20,6 +20,11 @@ pub struct PcmFrame { pub sample_rate: u32, /// Channel count (1 = mono, 2 = stereo). pub channels: u16, + /// The RTP timestamp of the first sample, in units of 1/48000 s (the Opus + /// RTP clock), wrapping like RTP. Set on frames from + /// [`RemoteTrack::next_pcm`](crate::rtc::RemoteTrack::next_pcm); `None` on + /// frames that the application or a conversion builds. + pub pts: Option, } impl PcmFrame { @@ -29,6 +34,7 @@ impl PcmFrame { samples, sample_rate, channels: channels.max(1), + pts: None, } } diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index 84047df..8e36ac0 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -214,7 +214,10 @@ struct VideoDecode { struct AudioDecode { decoder: opus::Decoder, last_seq: Option, - ready: VecDeque>, + ready: VecDeque, + /// The RTP timestamp right after the last queued frame. A frame rebuilt + /// for a lost packet starts here. + next_pts: u32, /// Length of the last frame decoded from a real packet. libopus makes a /// rebuilt frame as long as the output buffer. A lost packet states no /// length, so the stream's own frame size is the best value to use. @@ -362,7 +365,8 @@ impl RemoteTrack { } } - /// Decode and return the next audio frame as 48 kHz mono s16 PCM. + /// Decode and return the next audio frame as 48 kHz mono s16 PCM, with + /// [`PcmFrame::pts`] set to the RTP timestamp of its first sample. /// /// Skips empty/comfort-noise packets and returns `None` only when the track /// ends. Returns `None` immediately for non-audio tracks. Concurrent reads @@ -373,17 +377,21 @@ impl RemoteTrack { }; let _read_guard = self.read_gate.lock().await; loop { - if let Some(samples) = state.lock().unwrap_or_else(|e| e.into_inner()).take_frame() { - return Some(PcmFrame::mono(samples, OPUS_SAMPLE_RATE)); + if let Some(frame) = state.lock().unwrap_or_else(|e| e.into_inner()).take_frame() { + return Some(frame); } let pkt = self.read_rtp_inner().await?; if pkt.payload.is_empty() { continue; } let mut state = state.lock().unwrap_or_else(|e| e.into_inner()); - state.push_packet(pkt.header.sequence_number, &pkt.payload); - if let Some(samples) = state.take_frame() { - return Some(PcmFrame::mono(samples, OPUS_SAMPLE_RATE)); + state.push_packet( + pkt.header.sequence_number, + pkt.header.timestamp, + &pkt.payload, + ); + if let Some(frame) = state.take_frame() { + return Some(frame); } } } @@ -516,6 +524,7 @@ impl AudioDecode { decoder, last_seq: None, ready: VecDeque::new(), + next_pts: 0, frame_samples: FRAME_SAMPLES_20MS, scratch: vec![0; MAX_OPUS_FRAME_SAMPLES], } @@ -531,7 +540,7 @@ impl AudioDecode { /// Only the frame directly before `sequence_number` can be rebuilt from real /// audio: in-band FEC puts a low-quality copy of a frame into the *next* /// packet, so anything lost earlier had its copy in a lost packet too. - fn push_packet(&mut self, sequence_number: u16, payload: &[u8]) { + fn push_packet(&mut self, sequence_number: u16, rtp_timestamp: u32, payload: &[u8]) { let missing = match self.last_seq { None => 0, Some(last) => { @@ -553,18 +562,19 @@ impl AudioDecode { self.last_seq = Some(sequence_number); for _ in 1..missing { - self.decode_frame(&[], false); + self.decode_frame(&[], false, self.next_pts); } if missing > 0 { - self.decode_frame(payload, true); + self.decode_frame(payload, true, self.next_pts); } - self.decode_frame(payload, false); + self.decode_frame(payload, false, rtp_timestamp); } - /// Decode one frame and queue it. An empty `payload` makes libopus build a - /// replacement for a lost frame. `fec` takes the copy of the previous frame - /// out of `payload` instead of decoding `payload` itself. - fn decode_frame(&mut self, payload: &[u8], fec: bool) { + /// Decode one frame that starts at RTP timestamp `pts` and queue it. An + /// empty `payload` makes libopus build a replacement for a lost frame. + /// `fec` takes the copy of the previous frame out of `payload` instead of + /// decoding `payload` itself. + fn decode_frame(&mut self, payload: &[u8], fec: bool, pts: u32) { let rebuilt = fec || payload.is_empty(); // A real packet states its own length and the buffer is only an upper // bound. For a rebuilt frame the buffer length is the length libopus @@ -585,7 +595,10 @@ impl AudioDecode { if !rebuilt { self.frame_samples = samples; } - self.ready.push_back(self.scratch[..samples].to_vec()); + let mut frame = PcmFrame::mono(self.scratch[..samples].to_vec(), OPUS_SAMPLE_RATE); + frame.pts = Some(pts); + self.ready.push_back(frame); + self.next_pts = pts.wrapping_add(samples as u32); } Err(error) => { tracing::debug!(error = %error, "stream.rtc.remote.opus_decode_failed"); @@ -593,7 +606,7 @@ impl AudioDecode { } } - fn take_frame(&mut self) -> Option> { + fn take_frame(&mut self) -> Option { self.ready.pop_front() } } @@ -811,18 +824,25 @@ mod tests { .unwrap_or(0) } + /// The RTP timestamp of packet `sequence_number` in a 20 ms stream. + fn rtp(sequence_number: u16) -> u32 { + u32::from(sequence_number) * FRAME_SAMPLES_20MS as u32 + } + #[test] fn an_unbroken_sequence_yields_one_frame_per_packet() { let packets = tone_packets(4, true); let mut state = audio_decode(); for (index, packet) in packets.iter().enumerate() { - state.push_packet(index as u16, packet); + state.push_packet(index as u16, rtp(index as u16), packet); + let frame = state.take_frame(); assert_eq!( - state.take_frame().map(|frame| frame.len()), + frame.as_ref().map(|frame| frame.samples.len()), Some(FRAME_SAMPLES_20MS), "packet {index} should yield exactly one frame" ); + assert_eq!(frame.and_then(|frame| frame.pts), Some(rtp(index as u16))); assert!( state.take_frame().is_none(), "packet {index} queued extra frames" @@ -835,10 +855,10 @@ mod tests { let packets = tone_packets(1, true); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); let frame = state.take_frame().expect("decoded frame"); - assert_eq!(frame.capacity(), frame.len()); + assert_eq!(frame.samples.capacity(), frame.samples.len()); } #[test] @@ -846,20 +866,41 @@ mod tests { let packets = tone_packets(3, true); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); assert!(state.take_frame().is_some()); // Packet 1 never arrives; packet 2 carries a copy of frame 1. - state.push_packet(2, &packets[2]); + state.push_packet(2, rtp(2), &packets[2]); let rebuilt = state.take_frame().expect("rebuilt frame"); let current = state.take_frame().expect("current frame"); assert!(state.take_frame().is_none(), "only two frames are owed"); - assert_eq!(rebuilt.len(), FRAME_SAMPLES_20MS); - assert_eq!(current.len(), FRAME_SAMPLES_20MS); + assert_eq!(rebuilt.samples.len(), FRAME_SAMPLES_20MS); + assert_eq!(current.samples.len(), FRAME_SAMPLES_20MS); + assert_eq!((rebuilt.pts, current.pts), (Some(rtp(1)), Some(rtp(2)))); assert!( - peak(&rebuilt) > 1_000, + peak(&rebuilt.samples) > 1_000, "rebuilt frame is silent (peak {})", - peak(&rebuilt) + peak(&rebuilt.samples) + ); + } + + #[test] + fn timestamps_continue_over_a_lost_packet_and_the_rtp_wrap() { + let packets = tone_packets(3, true); + let mut state = audio_decode(); + let first = u32::MAX - (FRAME_SAMPLES_20MS as u32 - 1); + + state.push_packet(0, first, &packets[0]); + // Packet 1 is lost; its timestamp wraps to 0. + state.push_packet(2, first.wrapping_add(rtp(2)), &packets[2]); + + let mut timestamps = Vec::new(); + while let Some(frame) = state.take_frame() { + timestamps.push(frame.pts); + } + assert_eq!( + timestamps, + [Some(first), Some(0), Some(FRAME_SAMPLES_20MS as u32)] ); } @@ -868,14 +909,14 @@ mod tests { let packets = tone_packets(3, false); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); assert!(state.take_frame().is_some()); - state.push_packet(2, &packets[2]); + state.push_packet(2, rtp(2), &packets[2]); assert_eq!( ( - state.take_frame().map(|frame| frame.len()), - state.take_frame().map(|frame| frame.len()) + state.take_frame().map(|frame| frame.samples.len()), + state.take_frame().map(|frame| frame.samples.len()) ), (Some(FRAME_SAMPLES_20MS), Some(FRAME_SAMPLES_20MS)), "a lost packet still owes two frames without FEC" @@ -896,13 +937,13 @@ mod tests { let packets = tone_packets(4, true); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); // Packet 1 is overtaken by 2, so its frame is rebuilt here. - state.push_packet(2, &packets[2]); + state.push_packet(2, rtp(2), &packets[2]); let before_late = drain(&mut state); - state.push_packet(1, &packets[1]); + state.push_packet(1, rtp(1), &packets[1]); let late = drain(&mut state); - state.push_packet(3, &packets[3]); + state.push_packet(3, rtp(3), &packets[3]); let after_late = drain(&mut state); assert_eq!(late, 0, "a late packet must not repeat a frame"); @@ -918,9 +959,9 @@ mod tests { let packets = tone_packets(1, true); let mut state = audio_decode(); - state.push_packet(7, &packets[0]); + state.push_packet(7, rtp(7), &packets[0]); let first = drain(&mut state); - state.push_packet(7, &packets[0]); + state.push_packet(7, rtp(7), &packets[0]); let second = drain(&mut state); assert_eq!((first, second), (1, 0)); @@ -931,10 +972,10 @@ mod tests { let packets = tone_packets(5, true); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); assert!(state.take_frame().is_some()); // Packets 1, 2 and 3 are lost. - state.push_packet(4, &packets[4]); + state.push_packet(4, rtp(4), &packets[4]); let mut frames = 0; while state.take_frame().is_some() { @@ -948,9 +989,10 @@ mod tests { let packets = tone_packets(2, true); let mut state = audio_decode(); - state.push_packet(0, &packets[0]); + state.push_packet(0, rtp(0), &packets[0]); assert!(state.take_frame().is_some()); - state.push_packet(AUDIO_MAX_FILLED_PACKETS + 2, &packets[1]); + let far = AUDIO_MAX_FILLED_PACKETS + 2; + state.push_packet(far, rtp(far), &packets[1]); assert!(state.take_frame().is_some(), "the arriving packet decodes"); assert!( @@ -964,9 +1006,9 @@ mod tests { let packets = tone_packets(3, true); let mut state = audio_decode(); - state.push_packet(9, &packets[0]); + state.push_packet(9, rtp(9), &packets[0]); assert_eq!(drain(&mut state), 1); - state.push_packet(4, &packets[1]); + state.push_packet(4, rtp(4), &packets[1]); assert_eq!(drain(&mut state), 0); } @@ -975,7 +1017,7 @@ mod tests { fn a_corrupt_payload_queues_nothing() { let mut state = audio_decode(); - state.push_packet(0, &[0xff; 4]); + state.push_packet(0, rtp(0), &[0xff; 4]); assert!(state.take_frame().is_none()); } From a42330b97a86a2604ad5eb283521c749b77347b3 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Thu, 1 Oct 2026 22:49:24 +0200 Subject: [PATCH 15/37] feat: port the stream-py subscription config --- CHANGELOG.md | 20 ++ Cargo.lock | 1 + Cargo.toml | 1 + README.md | 9 +- examples/gpt_realtime_bot.rs | 10 +- src/rtc/join/mod.rs | 8 +- src/rtc/join/participants.rs | 9 +- src/rtc/join/subscriptions_runtime.rs | 60 +++-- src/rtc/join/tests.rs | 22 ++ src/rtc/mod.rs | 2 +- src/rtc/subscriptions.rs | 313 ++++++++++++++++++++------ tests/rtc_media.rs | 34 +-- 12 files changed, 359 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d209874..7d67b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,26 @@ application or a conversion builds have `None`, and `write_pcm` ignores the field. Code that builds `PcmFrame` with a struct literal must set `pts` or use `PcmFrame::new` / `PcmFrame::mono`. +### Subscription config has the stream-py shape + +`SubscriptionConfig` replaces `audio`, `video`, `screen_share` and +`video_dimension` with the fields of the stream-py `SubscriptionConfig`: + +- `default: TrackSubscriptionConfig` gives `track_types`, `video_dimension` + and `screenshare_dimension`. Screen-share video and screen-share audio are + now separate track types, and screen share has its own dimension. +- `role_filters` gives a rule by participant role. The first role of the + participant that has a rule selects it; other participants use `default`. +- `max_subscriptions` limits the number of tracks. The tracks of the + participants that the call learned about first are kept. + +`SubscriptionConfig::default()` now subscribes to nothing, and +`SubscriptionConfig::matches` is removed. The presets `audio_all`, +`audio_video`, `all` and `none` stay. The default video and screen-share +dimension is now 1920×1080 (it was 1280×720), also for a `SubscriptionTarget` +without a dimension. `Call::participants` gives the participants in the order +the call learned about them. + ## New Features ### A token-only client can prepare a call before the join diff --git a/Cargo.lock b/Cargo.lock index 57d9f64..e8690ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1149,6 +1149,7 @@ dependencies = [ "flate2", "futures-util", "hmac 0.13.0", + "indexmap", "openh264", "opus", "prost", diff --git a/Cargo.toml b/Cargo.toml index 66c9285..7a89f67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ base64 = "0.23.1" bytes = "1.12.1" futures-util = "0.3.34" hmac = "0.13.0" +indexmap = "2.14.2" prost = "0.14.4" prost-types = "0.14.4" reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "json", "charset", "http2", "stream"] } diff --git a/README.md b/README.md index 88eae92..7d9d77a 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ remote audio and video, transform it, and publish media back into the call. feedback, and client call-event reporting. - Join a call as a server-side SFU participant with retry, reconnect, and migration handling. -- Subscribe globally or by participant session to remote audio, video, and - screen-share tracks. +- Subscribe globally, by participant role, or by participant session to remote + audio, video, and screen-share tracks. - Observe typed participant, connection-quality, pin, grant, and inbound-pause state from the SFU. - Read Opus audio as PCM, decode VP8/VP9/H264 video as I420, or work with raw @@ -252,8 +252,9 @@ async fn run_media_bridge(call: &Call) -> Result<(), Box> For a complete bridge with cancellation, barge-in, audio and video processing, and deterministic cleanup, see [`gpt_realtime_bot`](https://github.com/GetStream/stream-video-rust/blob/main/examples/gpt_realtime_bot.rs). -For selective agents, use `Call::update_subscription_targets` with -`SubscriptionTarget` values instead of subscribing to every participant. A +For selective agents, set `role_filters` and `max_subscriptions` in +`SubscriptionConfig`, or use `Call::update_subscription_targets` with +`SubscriptionTarget` values for an exact list of participant sessions. A temporary `mute_track` / `unmute_track` preserves the same local track and sender; `stop_publish` remains terminal for that local track handle. The latest SFU view is available synchronously through `Call::call_state`. diff --git a/examples/gpt_realtime_bot.rs b/examples/gpt_realtime_bot.rs index 3523d8b..c6019eb 100644 --- a/examples/gpt_realtime_bot.rs +++ b/examples/gpt_realtime_bot.rs @@ -52,7 +52,7 @@ use getstream::models::{CallRequest, GetOrCreateCallRequest, MemberRequest, User use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ JoinCallData, LocalAudioTrack, LocalVideoTrack, RemoteTrack, RtcError, SubscriptionConfig, - VideoFrame, + TrackSubscriptionConfig, VideoFrame, }; use getstream::video::Call; use getstream::{Stream, TokenOptions}; @@ -744,8 +744,12 @@ pub async fn start_bot( if let Err(error) = call .update_subscriptions(SubscriptionConfig { - video_dimension: Some((640, 360)), - ..SubscriptionConfig::audio_video() + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Audio, TrackType::Video], + video_dimension: (640, 360), + ..Default::default() + }, + ..Default::default() }) .await .context("update_subscriptions") diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 9b00ada..0da3384 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -36,6 +36,7 @@ use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering} use std::sync::{Arc, Mutex as StdMutex, Weak}; use std::time::{Duration, Instant}; +use indexmap::IndexMap; use tokio::sync::{Mutex as TokioMutex, Notify, broadcast}; use tokio::task::JoinHandle; use url::Url; @@ -539,8 +540,9 @@ pub struct RtcCore { next_remote_track_id: AtomicU64, /// Exact per-session subscriptions, or `None` while using the coarse policy. manual_subscriptions: StdMutex>>, - /// Known participants keyed by session id (correlation + subscription build). - participants: StdMutex>, + /// Known participants keyed by session id (correlation + subscription build), + /// in the order the call learned about them. + participants: StdMutex>, /// Call-level state supplied by join and incremental SFU events. call_state: StdMutex, /// Serialized publisher negotiation and retryable local publication state. @@ -605,7 +607,7 @@ impl RtcCore { delivered_tracks: StdMutex::new(HashMap::new()), next_remote_track_id: AtomicU64::new(0), manual_subscriptions: StdMutex::new(None), - participants: StdMutex::new(HashMap::new()), + participants: StdMutex::new(IndexMap::new()), call_state: StdMutex::new(CallStateCache::default()), media: TokioMutex::new(MediaState::default()), active_subs: StdMutex::new(Vec::new()), diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 172d73f..3fef984 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -31,8 +31,9 @@ pub(super) struct CallStateCache { impl RtcCore { /// A snapshot of the participants currently known in the call (including this - /// session), built from the SFU participant state. Updated as - /// `ParticipantJoined` / `ParticipantLeft` events arrive. + /// session), built from the SFU participant state, in the order the call + /// learned about them. Updated as `ParticipantJoined` / `ParticipantLeft` + /// events arrive. pub fn participants(&self) -> Vec { let participants = self.participants.lock().unwrap_or_else(|e| e.into_inner()); participants @@ -152,7 +153,7 @@ impl RtcCore { // The local sessions, before and after a REJOIN, produce no events. let is_local = |id: &str| id == session_id || id == previous_session_id; for participant in joined { - let known = previous.remove(&participant.session_id); + let known = previous.shift_remove(&participant.session_id); if is_local(&participant.session_id) { continue; } @@ -196,7 +197,7 @@ impl RtcCore { self.participants .lock() .unwrap_or_else(|e| e.into_inner()) - .remove(session_id); + .shift_remove(session_id); } /// Record a newly-published track for a participant, learning the diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index ff33221..db52796 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -43,19 +43,26 @@ impl RtcCore { /// Enable or disable incoming video for every remote participant. pub async fn set_incoming_video_enabled(&self, enabled: bool) -> Result<()> { - let config = { - let mut config = self.sub_config.lock().unwrap_or_else(|e| e.into_inner()); - config.video = enabled; - config.video_dimension = None; - *config - }; + { + let mut guard = self.sub_config.lock().unwrap_or_else(|e| e.into_inner()); + let config = &mut *guard; + for rule in std::iter::once(&mut config.default).chain(config.role_filters.values_mut()) + { + rule.track_types + .retain(|track_type| *track_type != TrackType::Video); + if enabled { + rule.track_types.push(TrackType::Video); + } + rule.video_dimension = DEFAULT_VIDEO_DIMENSION; + } + } *self .manual_subscriptions .lock() .unwrap_or_else(|error| error.into_inner()) = None; self.subs_active.store(true, Ordering::SeqCst); self.recompute_subscriptions().await?; - tracing::debug!(enabled = config.video, "stream.rtc.incoming_video_updated"); + tracing::debug!(enabled, "stream.rtc.incoming_video_updated"); Ok(()) } @@ -85,7 +92,11 @@ impl RtcCore { } }; - let config = *self.sub_config.lock().unwrap_or_else(|e| e.into_inner()); + let config = self + .sub_config + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); let targets = self .manual_subscriptions .lock() @@ -124,32 +135,13 @@ impl RtcCore { }); } } else { - for entry in participants.values() { - if entry.session_id == session_id { - continue; - } - for &tt_i in &entry.published { - let Ok(track_type) = TrackType::try_from(tt_i) else { - continue; - }; - if !config.matches(track_type) - || manual.contains(&TrackKey::new(entry.session_id.clone(), track_type)) - { - continue; - } - let dimension = is_video_type(track_type).then(|| { - let (width, height) = - config.video_dimension.unwrap_or(DEFAULT_VIDEO_DIMENSION); - models::VideoDimension { width, height } - }); - tracks.push(signal::TrackSubscriptionDetails { - user_id: entry.user_id.clone(), - session_id: entry.session_id.clone(), - track_type: tt_i, - dimension, - }); - } - } + tracks = config.track_subscriptions( + participants + .values() + .map(|entry| &entry.participant) + .filter(|participant| participant.session_id != session_id), + &manual, + ); } } tracks.sort_by(|a, b| { diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index fb3ed7c..0f6f431 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1648,6 +1648,28 @@ fn participant_refresh_replaces_published_track_state() { assert!(entry.published.contains(&(TrackType::Audio as i32))); } +#[test] +fn participants_keep_the_order_in_which_the_call_learned_them() { + let core = test_core(); + let participant = |session_id: &str, user_id: &str| models::Participant { + user_id: user_id.to_owned(), + session_id: session_id.to_owned(), + ..Default::default() + }; + for session_id in ["h", "c", "f", "a", "g", "b", "e", "d"] { + core.upsert_participant(&participant(session_id, "user")); + } + core.remove_participant("f"); + core.upsert_participant(&participant("c", "changed")); + + let order: Vec<_> = core + .participants() + .into_iter() + .map(|participant| participant.session_id) + .collect(); + assert_eq!(order, ["h", "c", "a", "g", "b", "e", "d"]); +} + #[test] fn call_state_snapshot_combines_join_state_and_incremental_sfu_updates() { let core = test_core(); diff --git a/src/rtc/mod.rs b/src/rtc/mod.rs index d207670..c467c40 100644 --- a/src/rtc/mod.rs +++ b/src/rtc/mod.rs @@ -66,7 +66,7 @@ pub use reconnect::{ pub use sfu::signal::SignalClient; pub use sfu::ws::{SfuReceiver, SfuSender}; pub use stats::{DEFAULT_REPORTING_INTERVAL_MS, reporting_interval}; -pub use subscriptions::{SubscriptionConfig, SubscriptionTarget}; +pub use subscriptions::{SubscriptionConfig, SubscriptionTarget, TrackSubscriptionConfig}; pub use tracer::{TraceRecord, Tracer}; pub use tracks::{ Codec, LocalAudioTrack, LocalAudioTrackConfig, LocalTrack, LocalVideoTrack, diff --git a/src/rtc/subscriptions.rs b/src/rtc/subscriptions.rs index da35266..6beaba3 100644 --- a/src/rtc/subscriptions.rs +++ b/src/rtc/subscriptions.rs @@ -8,14 +8,17 @@ //! the concrete `TrackSubscriptionDetails` list and (re)sends it whenever the //! participants change. //! -//! The default policy subscribes to remote **audio** only (the backend-bot -//! default); video and screen-share are opt-in. +//! The policy has the shape of the stream-py `SubscriptionConfig`: a default +//! rule, rules by participant role, and a limit on the number of tracks. -use super::proto::models::TrackType; +use std::collections::{HashMap, HashSet}; + +use super::proto::models::{self, TrackType}; +use super::proto::signal; /// Video dimension requested when a subscription gives none. The SFU rejects a /// video or screen-share subscription without a dimension. -pub(crate) const DEFAULT_VIDEO_DIMENSION: (u32, u32) = (1280, 720); +pub(crate) const DEFAULT_VIDEO_DIMENSION: (u32, u32) = (1920, 1080); /// A precise subscription to one participant session and track kind. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -26,12 +29,12 @@ pub struct SubscriptionTarget { /// The remote track kind to receive. pub track_type: TrackType, /// Preferred video dimensions sent as an SFU adaptation hint. `None` - /// requests 1280×720 for video and screen-share. + /// requests 1920×1080 for video and screen-share. pub dimension: Option<(u32, u32)>, } impl SubscriptionTarget { - /// Subscribe to `track_type` from `session_id`, at 1280×720 for video. + /// Subscribe to `track_type` from `session_id`, at 1920×1080 for video. pub fn new(session_id: impl Into, track_type: TrackType) -> Self { Self { session_id: session_id.into(), @@ -48,79 +51,136 @@ impl SubscriptionTarget { } } -/// Which remote track kinds to subscribe to. -/// -/// Reactive: the call subscribes to every matching track published by every -/// other participant, and updates as participants publish/unpublish. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SubscriptionConfig { - /// Subscribe to remote audio. - pub audio: bool, - /// Subscribe to remote video. - pub video: bool, - /// Subscribe to remote screen-share (video + audio). - pub screen_share: bool, - /// Preferred video dimension hint sent to the SFU (width, height). `None` - /// requests 1280×720 for video and screen-share. - pub video_dimension: Option<(u32, u32)>, +/// The subscription rule for a group of participants. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrackSubscriptionConfig { + /// The remote track kinds to receive. + pub track_types: Vec, + /// Preferred camera video dimension (width, height), sent to the SFU as an + /// adaptation hint. + pub video_dimension: (u32, u32), + /// Preferred screen-share dimension (width, height), sent to the SFU as an + /// adaptation hint. + pub screenshare_dimension: (u32, u32), } -impl Default for SubscriptionConfig { - /// Audio-only — the backend-bot default (matches stream-py's usual path). +impl Default for TrackSubscriptionConfig { + /// No track kinds, 1920×1080 for video and screen-share. fn default() -> Self { Self { - audio: true, - video: false, - screen_share: false, - video_dimension: None, + track_types: Vec::new(), + video_dimension: DEFAULT_VIDEO_DIMENSION, + screenshare_dimension: DEFAULT_VIDEO_DIMENSION, } } } +/// Which remote tracks to subscribe to. +/// +/// Reactive: the call subscribes to the matching tracks of every other +/// participant, and updates as participants join, leave, change, and publish. +/// The default subscribes to nothing. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SubscriptionConfig { + /// The rule for a participant whose roles have no rule in `role_filters`. + pub default: TrackSubscriptionConfig, + /// Rules by participant role. The first role of the participant that has a + /// rule selects it. + pub role_filters: HashMap, + /// The maximum number of subscribed tracks. + pub max_subscriptions: Option, +} + impl SubscriptionConfig { - /// Subscribe to audio from all participants (the default). + /// Subscribe to audio from all participants. pub fn audio_all() -> Self { - Self::default() + Self { + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Audio], + ..Default::default() + }, + ..Default::default() + } } /// Subscribe to audio and video from all participants. pub fn audio_video() -> Self { Self { - audio: true, - video: true, - video_dimension: Some((1280, 720)), - ..Self::default() + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Audio, TrackType::Video], + ..Default::default() + }, + ..Default::default() } } - /// Subscribe to audio, video, and screen-share. + /// Subscribe to audio, video, screen-share, and screen-share audio. pub fn all() -> Self { Self { - audio: true, - video: true, - screen_share: true, - video_dimension: Some((1280, 720)), + default: TrackSubscriptionConfig { + track_types: vec![ + TrackType::Audio, + TrackType::Video, + TrackType::ScreenShare, + TrackType::ScreenShareAudio, + ], + ..Default::default() + }, + ..Default::default() } } /// Subscribe to nothing (unsubscribe from all). pub fn none() -> Self { - Self { - audio: false, - video: false, - screen_share: false, - video_dimension: None, - } + Self::default() } - /// Whether this policy subscribes to `track_type`. - pub fn matches(&self, track_type: TrackType) -> bool { - match track_type { - TrackType::Audio => self.audio, - TrackType::Video => self.video, - TrackType::ScreenShare | TrackType::ScreenShareAudio => self.screen_share, - TrackType::Unspecified => false, + /// The subscriptions to the tracks of `participants`, in their order, except + /// the tracks in `unsubscribed`. + pub(crate) fn track_subscriptions<'a>( + &self, + participants: impl IntoIterator, + unsubscribed: &HashSet, + ) -> Vec { + let mut tracks = Vec::new(); + for participant in participants { + let rule = self.rule_for(participant); + for &published in &participant.published_tracks { + let Ok(track_type) = TrackType::try_from(published) else { + continue; + }; + if !rule.track_types.contains(&track_type) + || unsubscribed + .contains(&TrackKey::new(participant.session_id.clone(), track_type)) + { + continue; + } + let dimension = match track_type { + TrackType::Video => Some(rule.video_dimension), + TrackType::ScreenShare => Some(rule.screenshare_dimension), + _ => None, + }; + tracks.push(signal::TrackSubscriptionDetails { + user_id: participant.user_id.clone(), + session_id: participant.session_id.clone(), + track_type: published, + dimension: dimension + .map(|(width, height)| models::VideoDimension { width, height }), + }); + } + } + if let Some(max) = self.max_subscriptions { + tracks.truncate(max); } + tracks + } + + fn rule_for(&self, participant: &models::Participant) -> &TrackSubscriptionConfig { + participant + .roles + .iter() + .find_map(|role| self.role_filters.get(role)) + .unwrap_or(&self.default) } } @@ -144,27 +204,150 @@ impl TrackKey { mod tests { use super::*; + fn participant( + session_id: &str, + roles: &[&str], + published: &[TrackType], + ) -> models::Participant { + models::Participant { + user_id: format!("user-{session_id}"), + session_id: session_id.to_owned(), + roles: roles.iter().map(|role| (*role).to_owned()).collect(), + published_tracks: published + .iter() + .map(|track_type| *track_type as i32) + .collect(), + ..Default::default() + } + } + + fn subscribed( + config: &SubscriptionConfig, + participants: &[models::Participant], + ) -> Vec<(String, TrackType)> { + config + .track_subscriptions(participants, &HashSet::new()) + .into_iter() + .map(|track| (track.session_id.clone(), track.track_type())) + .collect() + } + + fn rule(track_types: &[TrackType]) -> TrackSubscriptionConfig { + TrackSubscriptionConfig { + track_types: track_types.to_vec(), + ..Default::default() + } + } + #[test] - fn default_is_audio_only() { - let c = SubscriptionConfig::default(); - assert!(c.matches(TrackType::Audio)); - assert!(!c.matches(TrackType::Video)); - assert!(!c.matches(TrackType::ScreenShare)); + fn a_role_rule_replaces_the_default_rule() { + let config = SubscriptionConfig { + default: rule(&[TrackType::Audio]), + role_filters: HashMap::from([("host".to_owned(), rule(&[TrackType::Video]))]), + ..Default::default() + }; + let both = [TrackType::Audio, TrackType::Video]; + let participants = [ + participant("host", &["host"], &both), + participant("guest", &["user"], &both), + ]; + + assert_eq!( + subscribed(&config, &participants), + [ + ("host".to_owned(), TrackType::Video), + ("guest".to_owned(), TrackType::Audio), + ] + ); + } + + #[test] + fn the_first_role_of_the_participant_with_a_rule_wins() { + let config = SubscriptionConfig { + role_filters: HashMap::from([ + ("admin".to_owned(), rule(&[TrackType::Audio])), + ("host".to_owned(), rule(&[TrackType::Video])), + ]), + ..Default::default() + }; + let both = [TrackType::Audio, TrackType::Video]; + let participants = [ + participant("a", &["user", "host", "admin"], &both), + participant("b", &["admin", "host"], &both), + ]; + + assert_eq!( + subscribed(&config, &participants), + [ + ("a".to_owned(), TrackType::Video), + ("b".to_owned(), TrackType::Audio), + ] + ); } #[test] - fn audio_video_opts_in_video() { - let c = SubscriptionConfig::audio_video(); - assert!(c.matches(TrackType::Audio)); - assert!(c.matches(TrackType::Video)); - assert!(!c.matches(TrackType::ScreenShare)); + fn video_and_screen_share_get_their_own_dimensions() { + let config = SubscriptionConfig { + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Audio, TrackType::Video, TrackType::ScreenShare], + video_dimension: (640, 360), + screenshare_dimension: (2560, 1440), + }, + ..Default::default() + }; + let presenter = participant( + "presenter", + &[], + &[ + TrackType::Audio, + TrackType::Video, + TrackType::ScreenShare, + TrackType::ScreenShareAudio, + ], + ); + + let dimensions: Vec<_> = config + .track_subscriptions(&[presenter], &HashSet::new()) + .into_iter() + .map(|track| { + let dimension = track.dimension.map(|d| (d.width, d.height)); + (track.track_type(), dimension) + }) + .collect(); + + assert_eq!( + dimensions, + [ + (TrackType::Audio, None), + (TrackType::Video, Some((640, 360))), + (TrackType::ScreenShare, Some((2560, 1440))), + ] + ); } #[test] - fn none_matches_nothing() { - let c = SubscriptionConfig::none(); - assert!(!c.matches(TrackType::Audio)); - assert!(!c.matches(TrackType::Video)); + fn the_limit_keeps_the_first_tracks_in_participant_order() { + let config = SubscriptionConfig { + max_subscriptions: Some(2), + ..SubscriptionConfig::audio_all() + }; + let participants = ["c", "a", "b"].map(|id| participant(id, &[], &[TrackType::Audio])); + let first = |tracks: Vec| { + tracks + .into_iter() + .map(|track| track.session_id) + .collect::>() + }; + + assert_eq!( + first(config.track_subscriptions(&participants, &HashSet::new())), + ["c", "a"] + ); + let unsubscribed = HashSet::from([TrackKey::new("c", TrackType::Audio)]); + assert_eq!( + first(config.track_subscriptions(&participants, &unsubscribed)), + ["a", "b"] + ); } #[test] diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index e733f58..875950d 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -23,7 +23,7 @@ use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ CallingState, ClientCallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SfuCallEvent, - SubscriptionConfig, VideoFrame, + SubscriptionConfig, SubscriptionTarget, TrackSubscriptionConfig, VideoFrame, }; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; @@ -914,9 +914,9 @@ async fn publish_blue_video_reaches_raw_rtp_and_i420_decoder() { outcome.expect("VP9 RTP/decode test timed out"); } -/// A video subscription without a dimension hint is accepted and delivers video. +/// A video target without a dimension hint is accepted and delivers video. #[tokio::test] -async fn video_subscription_without_a_dimension_receives_video() { +async fn video_target_without_a_dimension_receives_video() { let Some(client) = common::client_or_skip() else { return; }; @@ -946,13 +946,11 @@ async fn video_subscription_without_a_dimension_receives_video() { .join(JoinCallData::new(&user_b)) .await .expect("B join"); + let session_a = call_a.session_id().await.expect("A session id"); call_b - .update_subscriptions(SubscriptionConfig { - video_dimension: None, - ..SubscriptionConfig::audio_video() - }) + .update_subscription_targets(vec![SubscriptionTarget::new(session_a, TrackType::Video)]) .await - .expect("B update_subscriptions without a dimension"); + .expect("B update_subscription_targets without a dimension"); recv_track( &mut rx_b, &user_a, @@ -1053,10 +1051,12 @@ async fn vp9_svc_preserves_one_ssrc_and_adapts_all_spatial_layers() { // transition accidentally. call_b .update_subscriptions(SubscriptionConfig { - audio: false, - video: true, - screen_share: false, - video_dimension: Some((320, 180)), + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Video], + video_dimension: (320, 180), + ..Default::default() + }, + ..Default::default() }) .await .map_err(|error| format!("VP9 SVC low-quality subscription failed: {error}"))?; @@ -1074,10 +1074,12 @@ async fn vp9_svc_preserves_one_ssrc_and_adapts_all_spatial_layers() { // picture with truthful SS dimensions after the encoder reconfiguration. call_b .update_subscriptions(SubscriptionConfig { - audio: false, - video: true, - screen_share: false, - video_dimension: Some((1280, 720)), + default: TrackSubscriptionConfig { + track_types: vec![TrackType::Video], + video_dimension: (1280, 720), + ..Default::default() + }, + ..Default::default() }) .await .map_err(|error| format!("VP9 SVC high-quality subscription failed: {error}"))?; From 972471ff27d31a76501f0b37467667ca9cba516e Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 00:39:35 +0200 Subject: [PATCH 16/37] feat: make the call event buffer size configurable --- CHANGELOG.md | 8 ++++++++ src/client.rs | 32 ++++++++++++++++++++++++++++++++ src/rtc/join/mod.rs | 7 ++++--- src/rtc/join/tests.rs | 22 ++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d67b66..9f4f913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,14 @@ client with an API secret; both handles share one session. `RtcCall` also adds `calling_state_changed`. A `CoordinatorEvent` has its coordinator `event_type` (for example `call.created`). +### Configurable call event buffer + +`ClientConfig::call_event_capacity` sets how many events each call event +stream keeps for a slow receiver. The default stays 256. A larger value makes a +lag less likely, but each call allocates all slots of its three streams. Code +that builds `ClientConfig` with a struct literal must set the new field or use +`..ClientConfig::default()`. + ### Video REST: advanced call statistics and reporting Application-level stats on `VideoClient` (`get_active_calls_status`, diff --git a/src/client.rs b/src/client.rs index 39ca7ac..9ceca59 100644 --- a/src/client.rs +++ b/src/client.rs @@ -32,6 +32,7 @@ const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(55); const DEFAULT_MAX_CONNS_PER_HOST: usize = 5; +const DEFAULT_CALL_EVENT_CAPACITY: usize = 256; /// Default maximum body accepted from coordinator HTTP endpoints (16 MiB). pub const DEFAULT_MAX_RESPONSE_BODY_BYTES: usize = 16 * 1024 * 1024; /// Default maximum inbound SFU/coordinator WebSocket frame and message size (4 MiB). @@ -120,6 +121,11 @@ pub struct ClientConfig { /// and known secret keys are redacted regardless. Other fields, including /// PII and secrets stored under custom keys, remain visible when enabled. pub log_bodies: bool, + /// Events that each call event stream (`sfu_events`, `coordinator_events`, + /// `client_events`) keeps for a slow receiver, rounded up to a power of two. + /// A receiver that falls further behind loses the oldest events. Each call + /// allocates all slots. Must be at least 1. Default 256. + pub call_event_capacity: usize, } impl Default for ClientConfig { @@ -133,6 +139,7 @@ impl Default for ClientConfig { max_conns_per_host: DEFAULT_MAX_CONNS_PER_HOST, retry: RetryConfig::default(), log_bodies: false, + call_event_capacity: DEFAULT_CALL_EVENT_CAPACITY, } } } @@ -148,6 +155,7 @@ pub(crate) struct Client { http: reqwest::Client, retry: RetryConfig, log_bodies: bool, + call_event_capacity: usize, max_response_body_bytes: usize, max_websocket_message_bytes: usize, stream_client_header: String, @@ -206,6 +214,14 @@ impl Client { config.coordinator_ws_url ))); } + // tokio broadcast channels panic outside this range. + if config.call_event_capacity == 0 || config.call_event_capacity > usize::MAX >> 1 { + return Err(Error::Config(format!( + "call event capacity {} must be between 1 and {}", + config.call_event_capacity, + usize::MAX >> 1 + ))); + } let http = reqwest::Client::builder() .pool_max_idle_per_host(config.max_conns_per_host) @@ -227,6 +243,7 @@ impl Client { http, retry: config.retry, log_bodies: config.log_bodies, + call_event_capacity: config.call_event_capacity, max_response_body_bytes: limits.max_response_body_bytes, max_websocket_message_bytes: limits.max_websocket_message_bytes, stream_client_header: format!("stream-rust-{}", env!("CARGO_PKG_VERSION")), @@ -245,6 +262,10 @@ impl Client { &self.coordinator_ws_url } + pub(crate) fn call_event_capacity(&self) -> usize { + self.call_event_capacity + } + /// The shared `reqwest` client (connection pool). Used by the RTC layer to /// reuse the pool for coordinator join + SFU Twirp calls. pub(crate) fn http(&self) -> &reqwest::Client { @@ -652,6 +673,17 @@ mod tests { assert!(matches!(error, Error::Config(_))); } + #[test] + fn call_event_capacity_must_be_at_least_one() { + let config = ClientConfig { + call_event_capacity: 0, + ..ClientConfig::default() + }; + let error = Client::new("key".to_owned(), "secret".to_owned(), config) + .expect_err("zero call event capacity"); + assert!(matches!(error, Error::Config(_))); + } + #[test] fn compatibility_limits_are_conservative_and_configurable() { let limits = NetworkLimits::default(); diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 0da3384..5cdd164 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -570,6 +570,7 @@ impl Drop for ReconnectClaim { impl RtcCore { /// Build a fresh (idle) core for a call handle. pub(crate) fn new(client: Arc, call_type: String, call_id: String) -> Arc { + let event_capacity = client.call_event_capacity(); Arc::new(Self { api_key: client.api_key().to_owned(), client, @@ -578,9 +579,9 @@ impl RtcCore { token_refresh: TokioMutex::new(()), call_type, call_id, - sfu_events_tx: broadcast::channel(256).0, - coordinator_events_tx: broadcast::channel(256).0, - client_events_tx: broadcast::channel(256).0, + sfu_events_tx: broadcast::channel(event_capacity).0, + coordinator_events_tx: broadcast::channel(event_capacity).0, + client_events_tx: broadcast::channel(event_capacity).0, lifecycle: StdMutex::new(Lifecycle { state: CallingState::Idle, generation: 0, diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 0f6f431..863fed0 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -523,6 +523,28 @@ async fn each_event_goes_only_to_the_stream_of_its_source() { assert!(coordinator.try_recv().is_err()); } +#[test] +fn a_call_event_stream_keeps_the_configured_number_of_events() { + let core = test_core_with_config(ClientConfig { + call_event_capacity: 2, + ..ClientConfig::default() + }); + let generation = core.begin_join().expect("test generation"); + let mut events = core.client_events(); + for state in [ + CallingState::Joined, + CallingState::Reconnecting, + CallingState::Joined, + ] { + assert!(core.set_state_if_current(generation, state)); + } + + assert!(matches!( + events.try_recv(), + Err(broadcast::error::TryRecvError::Lagged(1)) + )); +} + #[tokio::test] async fn track_events_report_the_track_type_cause_and_participant() { let core = test_core(); From d5024a6a96b08d253be68150c7d0b875f7f0c28f Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 12:02:59 +0200 Subject: [PATCH 17/37] fix: keep participant order and video dimension across rejoin and video toggle --- CHANGELOG.md | 3 +- src/rtc/join/participants.rs | 11 +++++ src/rtc/join/subscriptions_runtime.rs | 1 - src/rtc/join/tests.rs | 65 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4f913..7ff6aa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,8 @@ field. Code that builds `PcmFrame` with a struct literal must set `pts` or use `audio_video`, `all` and `none` stay. The default video and screen-share dimension is now 1920×1080 (it was 1280×720), also for a `SubscriptionTarget` without a dimension. `Call::participants` gives the participants in the order -the call learned about them. +the call learned about them. `set_incoming_video_enabled` keeps the configured +video dimension. ## New Features diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 3fef984..54f5045 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -123,6 +123,17 @@ impl RtcCore { let mut previous = { let mut participants = self.participants.lock().unwrap_or_else(|e| e.into_inner()); let previous = std::mem::take(&mut *participants); + // Known participants keep the order in which the call learned them. + let current: HashSet<&str> = joined + .iter() + .map(|participant| participant.session_id.as_str()) + .chain([session_id]) + .collect(); + for id in previous.keys() { + if current.contains(id.as_str()) { + participants.insert(id.clone(), ParticipantState::default()); + } + } let me = participants.entry(session_id.to_owned()).or_default(); me.user_id = user_id.to_owned(); me.session_id = session_id.to_owned(); diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index db52796..d3ee936 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -53,7 +53,6 @@ impl RtcCore { if enabled { rule.track_types.push(TrackType::Video); } - rule.video_dimension = DEFAULT_VIDEO_DIMENSION; } } *self diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 863fed0..4a6e17a 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1670,6 +1670,71 @@ fn participant_refresh_replaces_published_track_state() { assert!(entry.published.contains(&(TrackType::Audio as i32))); } +#[tokio::test] +async fn turning_incoming_video_off_and_on_keeps_the_video_dimension() { + let core = test_core(); + let config = SubscriptionConfig { + default: crate::rtc::TrackSubscriptionConfig { + track_types: vec![TrackType::Video], + video_dimension: (640, 360), + ..Default::default() + }, + ..Default::default() + }; + core.update_subscriptions(config.clone()) + .await + .expect("update subscriptions"); + core.set_incoming_video_enabled(false) + .await + .expect("video off"); + core.set_incoming_video_enabled(true) + .await + .expect("video on"); + + assert_eq!( + *core.sub_config.lock().unwrap_or_else(|e| e.into_inner()), + config + ); +} + +#[test] +fn a_join_response_keeps_the_order_of_known_participants() { + let core = test_core(); + let generation = core.begin_join().expect("test generation"); + let call_state = |session_ids: &[&str]| { + Some(models::CallState { + participants: session_ids + .iter() + .map(|session_id| models::Participant { + user_id: format!("user-{session_id}"), + session_id: (*session_id).to_owned(), + ..Default::default() + }) + .collect(), + ..Default::default() + }) + }; + assert!(core.apply_join_call_state_if_current( + generation, + "me", + "me", + call_state(&["c", "a", "b"]) + )); + assert!(core.apply_join_call_state_if_current( + generation, + "me", + "me", + call_state(&["d", "b", "a"]) + )); + + let order: Vec<_> = core + .participants() + .into_iter() + .map(|participant| participant.session_id) + .collect(); + assert_eq!(order, ["me", "a", "b", "d"]); +} + #[test] fn participants_keep_the_order_in_which_the_call_learned_them() { let core = test_core(); From dc7ae47958c025012d95bfef6627283160e0dc8a Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 12:23:27 +0200 Subject: [PATCH 18/37] fix: finish the leave teardown when the leave future is dropped --- src/rtc/join/lifecycle.rs | 41 ++++++++++++++++++++++++++------------ src/rtc/join/tests.rs | 42 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index ad46f16..2780189 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -644,8 +644,10 @@ impl RtcCore { impl RtcCore { /// Leave the call: send `leave`, close the PeerConnections and WebSocket, /// abort background tasks. Succeeds from any state, including `Joining` - /// (JS: force to a leaving state rather than waiting for `JOINED`). - pub async fn leave(&self, reason: impl Into) -> Result<()> { + /// (JS: force to a leaving state rather than waiting for `JOINED`). The + /// teardown runs in a runtime task, so it finishes when this future is + /// dropped. + pub async fn leave(self: &Arc, reason: impl Into) -> Result<()> { let reason = reason.into(); let generation = self.cancel_generation(); let left = LeftCall { @@ -653,18 +655,31 @@ impl RtcCore { generation, }; - let connection = self.connection.lock().await.take(); - if let Some(connection) = connection { - let session_id = connection.session_id.clone(); - // Record the leave reason so the final `SendStats` (drained by - // `teardown`) carries the end-of-call event (JS `call.leaveReason`). - connection.signal.trace("call.leaveReason", json!(reason)); - { - let mut sender = connection.sfu_sender.lock().await; - let _ = sender.send_leave(session_id, &reason).await; - let _ = sender.close().await; + let core = self.clone(); + let teardown = self.spawn_runtime_task(async move { + // A join that starts after a dropped leave owns a later connection. + let connection = core + .connection + .lock() + .await + .take_if(|connection| connection.generation < generation); + if let Some(connection) = connection { + let session_id = connection.session_id.clone(); + // Record the leave reason so the final `SendStats` (drained by + // `teardown`) carries the end-of-call event (JS `call.leaveReason`). + connection.signal.trace("call.leaveReason", json!(reason)); + { + let mut sender = connection.sfu_sender.lock().await; + let _ = sender.send_leave(session_id, &reason).await; + let _ = sender.close().await; + } + connection.teardown().await; } - connection.teardown().await; + }); + if let Err(error) = teardown.await + && error.is_panic() + { + std::panic::resume_unwind(error.into_panic()); } drop(left); self.stop_coordinator_events(generation).await; diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 4a6e17a..4de2bcb 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -674,7 +674,10 @@ async fn dropped_join_allows_a_new_join() { #[tokio::test] async fn dropped_leave_still_leaves_the_call() { let core = test_core(); - prepare_joined_core(&core, "alice"); + let generation = prepare_joined_core(&core, "alice"); + let (connection, sfu) = establish_fake(&core, generation).await; + let (subscriber, publisher) = (connection.subscriber.clone(), connection.publisher.clone()); + *core.connection.lock().await = Some(connection); let connection_slot = core.connection.lock().await; let leave = tokio::time::timeout(Duration::from_millis(50), core.leave("dropped leave")).await; @@ -682,9 +685,46 @@ async fn dropped_leave_still_leaves_the_call() { drop(connection_slot); assert_eq!(core.state(), CallingState::Left); + let requests = requests_until_close(sfu).await; + assert!(requests.iter().any(|request| matches!( + request.request_payload, + Some(event::sfu_request::RequestPayload::LeaveCallRequest(_)) + ))); + wait_for( + Duration::from_secs(2), + || { + subscriber.connection_state() == RTCPeerConnectionState::Closed + && publisher.connection_state() == RTCPeerConnectionState::Closed + }, + "closed peer connections", + ) + .await; core.begin_join().expect("a new join can start"); } +#[tokio::test] +async fn a_dropped_leave_keeps_the_connection_of_a_later_join() { + let core = test_core(); + prepare_joined_core(&core, "alice"); + let mut connection_slot = core.connection.lock().await; + let leave = tokio::time::timeout(Duration::from_millis(50), core.leave("dropped leave")).await; + assert!(leave.is_err(), "leave waits for the connection lock"); + + let generation = core.begin_join().expect("a new join can start"); + let (connection, _sfu) = establish_fake(&core, generation).await; + *connection_slot = Some(connection); + let (_, _, completed) = core.runtime_task_snapshot(); + drop(connection_slot); + wait_for( + Duration::from_secs(2), + || core.runtime_task_snapshot().2 > completed, + "the leave task", + ) + .await; + + assert!(core.connection.lock().await.is_some()); +} + #[tokio::test] async fn leave_closes_a_connection_owned_by_a_cancelled_join() { let core = test_core(); From 505915e9d2eb040578268c68c83546a86a7e2da7 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 13:17:55 +0200 Subject: [PATCH 19/37] fix: reject a PCM queue capacity below one 20 ms frame --- src/rtc/tracks/local.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 30f1c80..7cfd2d5 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -352,7 +352,8 @@ pub struct LocalAudioTrackConfig { /// Discontinuous transmission: stop emitting packets during silence. pub dtx: bool, /// Maximum PCM that [`LocalAudioTrack::write_pcm`] queues for the pacer. A - /// write above it drops the oldest queued samples. + /// write above it drops the oldest queued samples. The minimum is one 20 ms + /// frame. pub pcm_queue_capacity: Duration, } @@ -451,6 +452,12 @@ impl LocalAudioTrack { encoder.set_dtx(config.dtx)?; let pcm_capacity_samples = (config.pcm_queue_capacity.as_secs_f64() * f64::from(OPUS_SAMPLE_RATE)) as usize; + if pcm_capacity_samples < FRAME_SAMPLES_20MS { + return Err(RtcError::Media(format!( + "pcm queue capacity below one 20 ms frame: {:?}", + config.pcm_queue_capacity + ))); + } Ok(Self { inner: Arc::new(AudioInner { core, @@ -2312,6 +2319,23 @@ mod tests { track.stop(); } + #[test] + fn a_pcm_queue_shorter_than_one_frame_is_rejected() { + let with_capacity = |capacity| { + LocalAudioTrack::opus_with_config( + LocalAudioTrackConfig::default().with_pcm_queue_capacity(capacity), + ) + }; + + for capacity in [Duration::ZERO, Duration::from_millis(19)] { + assert!( + matches!(with_capacity(capacity), Err(RtcError::Media(_))), + "{capacity:?} holds less than one 20 ms frame" + ); + } + assert!(with_capacity(Duration::from_millis(20)).is_ok()); + } + #[tokio::test] async fn pcm_written_before_pacing_starts_stays_queued() { let track = LocalAudioTrack::opus_with_config( From f4acff708438e5457325a47fe5fc899a43e1a0a3 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 13:17:55 +0200 Subject: [PATCH 20/37] docs: state that set_incoming_video_enabled changes only video --- CHANGELOG.md | 5 ++++- src/rtc/client.rs | 5 +++++ src/rtc/join/subscriptions_runtime.rs | 5 +++++ src/video/call.rs | 8 +++++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff6aa7..b995b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,10 @@ field. Code that builds `PcmFrame` with a struct literal must set `pts` or use dimension is now 1920×1080 (it was 1280×720), also for a `SubscriptionTarget` without a dimension. `Call::participants` gives the participants in the order the call learned about them. `set_incoming_video_enabled` keeps the configured -video dimension. +video dimension and changes only the video track type of the current config. +Before an `update_subscriptions` call, `set_incoming_video_enabled(true)` now +subscribes to video only (it was audio and video), and `false` subscribes to +nothing (it was audio). ## New Features diff --git a/src/rtc/client.rs b/src/rtc/client.rs index e238e26..7764a5d 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -397,6 +397,11 @@ impl RtcCall { } /// Enable or disable incoming video from every remote participant. + /// + /// This adds or removes `TrackType::Video` in every rule of the current + /// [`SubscriptionConfig`] and keeps the other track types. The default + /// config has no track types, so before an `update_subscriptions` call, + /// `true` subscribes to video only. pub async fn set_incoming_video_enabled(&self, enabled: bool) -> Result<()> { self.core.set_incoming_video_enabled(enabled).await } diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index d3ee936..a564828 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -42,6 +42,11 @@ impl RtcCore { } /// Enable or disable incoming video for every remote participant. + /// + /// This adds or removes `TrackType::Video` in every rule of the current + /// [`SubscriptionConfig`] and keeps the other track types. The default + /// config has no track types, so before an `update_subscriptions` call, + /// `true` subscribes to video only. pub async fn set_incoming_video_enabled(&self, enabled: bool) -> Result<()> { { let mut guard = self.sub_config.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/src/video/call.rs b/src/video/call.rs index 9db5265..84fe2c9 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -1002,7 +1002,8 @@ impl Call { } /// Set the subscription policy and (re)send `UpdateSubscriptions`. The SFU - /// forwards no media until this is called; the default policy is audio-only. + /// forwards no media until this is called; `SubscriptionConfig::default()` + /// subscribes to nothing. pub async fn update_subscriptions( &self, config: crate::rtc::SubscriptionConfig, @@ -1019,6 +1020,11 @@ impl Call { } /// Enable or disable incoming video from every remote participant. + /// + /// This adds or removes `TrackType::Video` in every rule of the current + /// [`SubscriptionConfig`](crate::rtc::SubscriptionConfig) and keeps the + /// other track types. The default config has no track types, so before an + /// `update_subscriptions` call, `true` subscribes to video only. pub async fn set_incoming_video_enabled(&self, enabled: bool) -> crate::rtc::RtcResult<()> { self.rtc.set_incoming_video_enabled(enabled).await } From 61fa0d775872de442f0cac11dacba920fdee7cf1 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 13:27:24 +0200 Subject: [PATCH 21/37] fix: count the timestamps of lost audio frames back from the next packet --- CHANGELOG.md | 8 ++++---- src/rtc/tracks/remote.rs | 44 ++++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b995b2a..042fab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,10 +39,10 @@ use the new fields or `..`. `PcmFrame` adds `pts: Option`: the RTP timestamp of the first sample, in units of 1/48000 s, wrapping like RTP. `RemoteTrack::next_pcm` sets it; a frame -rebuilt for a lost packet continues from the frame before it. Frames that the -application or a conversion builds have `None`, and `write_pcm` ignores the -field. Code that builds `PcmFrame` with a struct literal must set `pts` or use -`PcmFrame::new` / `PcmFrame::mono`. +rebuilt for a lost packet counts back from the next packet that arrives. Frames +that the application or a conversion builds have `None`, and `write_pcm` ignores +the field. Code that builds `PcmFrame` with a struct literal must set `pts` or +use `PcmFrame::new` / `PcmFrame::mono`. ### Subscription config has the stream-py shape diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index 8e36ac0..77fc89d 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -215,9 +215,6 @@ struct AudioDecode { decoder: opus::Decoder, last_seq: Option, ready: VecDeque, - /// The RTP timestamp right after the last queued frame. A frame rebuilt - /// for a lost packet starts here. - next_pts: u32, /// Length of the last frame decoded from a real packet. libopus makes a /// rebuilt frame as long as the output buffer. A lost packet states no /// length, so the stream's own frame size is the best value to use. @@ -524,7 +521,6 @@ impl AudioDecode { decoder, last_seq: None, ready: VecDeque::new(), - next_pts: 0, frame_samples: FRAME_SAMPLES_20MS, scratch: vec![0; MAX_OPUS_FRAME_SAMPLES], } @@ -561,11 +557,18 @@ impl AudioDecode { }; self.last_seq = Some(sequence_number); - for _ in 1..missing { - self.decode_frame(&[], false, self.next_pts); + // Lost frames count back from the packet that arrived: DTX skips + // timestamps over silence but not sequence numbers. + let frame_samples = self.frame_samples as u32; + for back in (2..=missing).rev() { + self.decode_frame( + &[], + false, + rtp_timestamp.wrapping_sub(u32::from(back) * frame_samples), + ); } if missing > 0 { - self.decode_frame(payload, true, self.next_pts); + self.decode_frame(payload, true, rtp_timestamp.wrapping_sub(frame_samples)); } self.decode_frame(payload, false, rtp_timestamp); } @@ -598,7 +601,6 @@ impl AudioDecode { let mut frame = PcmFrame::mono(self.scratch[..samples].to_vec(), OPUS_SAMPLE_RATE); frame.pts = Some(pts); self.ready.push_back(frame); - self.next_pts = pts.wrapping_add(samples as u32); } Err(error) => { tracing::debug!(error = %error, "stream.rtc.remote.opus_decode_failed"); @@ -904,6 +906,32 @@ mod tests { ); } + #[test] + fn frames_lost_after_dtx_silence_take_their_timestamps_from_the_next_packet() { + let packets = tone_packets(4, true); + let mut state = audio_decode(); + // DTX skips 400 ms of timestamps and keeps the sequence numbers. + let after_silence = rtp(20); + + state.push_packet(0, rtp(0), &packets[0]); + assert!(state.take_frame().is_some()); + // Packets 1 and 2, the first ones after the silence, are lost. + state.push_packet(3, after_silence + rtp(2), &packets[3]); + + let mut timestamps = Vec::new(); + while let Some(frame) = state.take_frame() { + timestamps.push(frame.pts); + } + assert_eq!( + timestamps, + [ + Some(after_silence), + Some(after_silence + rtp(1)), + Some(after_silence + rtp(2)) + ] + ); + } + #[test] fn a_sender_without_fec_still_keeps_the_timeline() { let packets = tone_packets(3, false); From ab6a7e8a33d07cddafc3aa01f447ee0ed684ad27 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 13:32:16 +0200 Subject: [PATCH 22/37] fix: ignore speaking and quality fields when a rejoin compares participants --- src/rtc/join/participants.rs | 15 ++++++++++-- src/rtc/join/tests.rs | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/rtc/join/participants.rs b/src/rtc/join/participants.rs index 54f5045..0702c8b 100644 --- a/src/rtc/join/participants.rs +++ b/src/rtc/join/participants.rs @@ -170,10 +170,21 @@ impl RtcCore { } let event = match known { None => SfuCallEvent::ParticipantJoined(participant), - Some(entry) if entry.participant != participant => { + Some(ParticipantState { + participant: mut known, + .. + }) => { + // Their own SFU events report these fields, not + // `ParticipantUpdated`. + known.audio_level = participant.audio_level; + known.is_speaking = participant.is_speaking; + known.connection_quality = participant.connection_quality; + known.is_dominant_speaker = participant.is_dominant_speaker; + if known == participant { + continue; + } SfuCallEvent::ParticipantUpdated(participant) } - Some(_) => continue, }; let _ = self.sfu_events_tx.send(event); } diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 4de2bcb..b5f8e90 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -2020,6 +2020,51 @@ fn join_state_reports_only_the_participant_changes_since_the_last_join() { ); } +#[test] +fn speaking_and_quality_events_do_not_mark_a_participant_updated_after_a_rejoin() { + let core = test_core(); + let generation = core.begin_join().expect("join generation"); + let mut events = core.sfu_events(); + let bob = models::Participant { + user_id: "bob".to_owned(), + session_id: "session-b".to_owned(), + ..Default::default() + }; + let call_state = || { + Some(models::CallState { + participants: vec![bob.clone()], + ..Default::default() + }) + }; + + assert!(core.apply_join_call_state_if_current(generation, "local-1", "agent", call_state())); + core.update_audio_levels(&[event::AudioLevel { + user_id: "bob".to_owned(), + session_id: "session-b".to_owned(), + level: 0.37, + is_speaking: true, + }]); + core.update_connection_quality(&[event::ConnectionQualityInfo { + user_id: "bob".to_owned(), + session_id: "session-b".to_owned(), + connection_quality: models::ConnectionQuality::Excellent as i32, + }]); + core.update_dominant_speaker("session-b"); + // A REJOIN: bob has not changed. + assert!(core.apply_join_call_state_if_current(generation, "local-2", "agent", call_state())); + + let mut reported = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + SfuCallEvent::ParticipantJoined(p) => reported.push(("joined", p.session_id)), + SfuCallEvent::ParticipantUpdated(p) => reported.push(("updated", p.session_id)), + SfuCallEvent::ParticipantLeft(p) => reported.push(("left", p.session_id)), + _ => {} + } + } + assert_eq!(reported, [("joined", "session-b".to_owned())]); +} + #[test] fn mute_state_builder_deduplicates_track_types() { let first = LocalTrack::Audio(LocalAudioTrack::opus().expect("first audio track")); From c962a9f7681ea613aae697cb2c1c4d929c9f315b Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 13:43:26 +0200 Subject: [PATCH 23/37] fix: keep only audio senders for reuse after stop_publish --- src/rtc/join/publication.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rtc/join/publication.rs b/src/rtc/join/publication.rs index 2b43bc5..f49f2c0 100644 --- a/src/rtc/join/publication.rs +++ b/src/rtc/join/publication.rs @@ -27,7 +27,7 @@ pub(super) struct LocalPublication { pub(super) struct MediaState { pub(super) publications: Vec, pub(super) publish_quality: HashMap<(i32, i32), Vec>, - /// The publish option id and stopped track of each sender that + /// The publish option id and stopped track of each audio sender that /// `stop_publish` kept in the publisher envelope. retired: Vec<(i32, LocalTrack)>, } @@ -96,11 +96,17 @@ impl MediaState { } /// Remove the publication of a stopped track whose sender stays in the - /// publisher envelope, and keep that sender for a later publish. + /// publisher envelope. Keep an audio sender for a later publish; `publish` + /// never reuses a video sender. pub(super) fn retire(&mut self, track_id: &str) -> Option { let publish_option_id = self.publications[self.position(track_id)?].publish_option_id; let track = self.remove(track_id)?; - self.retired.push((publish_option_id, track.clone())); + if matches!( + track.track_type(), + TrackType::Audio | TrackType::ScreenShareAudio + ) { + self.retired.push((publish_option_id, track.clone())); + } Some(track) } From c6d3291d09a4cf34c6f33337bf0f4e1dce4d8ef4 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 14:10:03 +0200 Subject: [PATCH 24/37] fix: start each join with an empty call state --- src/rtc/join/lifecycle.rs | 55 ++++++++++++++++++++++----------------- src/rtc/join/mod.rs | 9 +++---- src/rtc/join/tests.rs | 44 +++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 29 deletions(-) diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index 2780189..437f4a1 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -718,34 +718,41 @@ impl RtcCore { // A join that started during the leave owns these fields. let lifecycle = self.lifecycle.lock().unwrap_or_else(|e| e.into_inner()); if lifecycle.generation == generation { - self.participants - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clear(); - *self - .call_state - .lock() - .unwrap_or_else(|error| error.into_inner()) = CallStateCache::default(); - self.active_subs - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clear(); - self.delivered_tracks - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clear(); - self.own_capabilities - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clear(); - *self - .reconnect_generation - .lock() - .unwrap_or_else(|e| e.into_inner()) = None; + self.clear_call_state(); } } self.set_state_if_current(generation, CallingState::Left); } + + /// Clear the state of one join: participants, call state, subscriptions, + /// capabilities, and the reconnect claim. The caller holds the lifecycle + /// lock. + pub(super) fn clear_call_state(&self) { + self.participants + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + *self + .call_state + .lock() + .unwrap_or_else(|error| error.into_inner()) = CallStateCache::default(); + self.active_subs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + self.delivered_tracks + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + self.own_capabilities + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clear(); + *self + .reconnect_generation + .lock() + .unwrap_or_else(|e| e.into_inner()) = None; + } } /// Abandons a join whose future is dropped while it is still joining. The join diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 5cdd164..800d206 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -746,13 +746,16 @@ impl RtcCore { } /// Guard against a double join (JS "call.join() shall be called only once"). - /// Transitions `Idle`/`Left` → `Joining` and starts a new generation. + /// Transitions `Idle`/`Left` → `Joining` and starts a new generation with + /// an empty call state. fn begin_join(&self) -> Result { let generation = { let mut guard = self.lifecycle.lock().unwrap_or_else(|e| e.into_inner()); match guard.state { CallingState::Idle | CallingState::Left => { guard.generation = guard.generation.wrapping_add(1); + // A dropped or failed join does not clear its call state. + self.clear_call_state(); guard.set_state(CallingState::Joining, &self.client_events_tx); guard.generation_publish_options = guard.publish_options; guard.failure_limits = FailureLimits::default(); @@ -767,10 +770,6 @@ impl RtcCore { } }; self.lifecycle_changed.notify_waiters(); - *self - .reconnect_generation - .lock() - .unwrap_or_else(|e| e.into_inner()) = None; self.reconnect_attempts.store(0, Ordering::SeqCst); Ok(generation) } diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index b5f8e90..5547385 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -2065,6 +2065,50 @@ fn speaking_and_quality_events_do_not_mark_a_participant_updated_after_a_rejoin( assert_eq!(reported, [("joined", "session-b".to_owned())]); } +#[test] +fn a_join_after_a_failed_join_reports_only_its_own_participants() { + let core = test_core(); + let call_state = |user_id: &str, session_id: &str| { + Some(models::CallState { + participants: vec![models::Participant { + user_id: user_id.to_owned(), + session_id: session_id.to_owned(), + ..Default::default() + }], + ..Default::default() + }) + }; + let failed = core.begin_join().expect("first join"); + assert!(core.apply_join_call_state_if_current( + failed, + "local-1", + "agent", + call_state("alice", "session-a"), + )); + // The join fails after its join response. + assert!(core.set_state_if_current(failed, CallingState::Idle)); + + let mut events = core.sfu_events(); + let generation = core.begin_join().expect("second join"); + assert!(core.apply_join_call_state_if_current( + generation, + "local-2", + "agent", + call_state("bob", "session-b"), + )); + + let mut reported = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + SfuCallEvent::ParticipantJoined(p) => reported.push(("joined", p.session_id)), + SfuCallEvent::ParticipantUpdated(p) => reported.push(("updated", p.session_id)), + SfuCallEvent::ParticipantLeft(p) => reported.push(("left", p.session_id)), + _ => {} + } + } + assert_eq!(reported, [("joined", "session-b".to_owned())]); +} + #[test] fn mute_state_builder_deduplicates_track_types() { let first = LocalTrack::Audio(LocalAudioTrack::opus().expect("first audio track")); From ff27c478d15eef3114d34aff2eea10b4d3ae1a89 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Fri, 2 Oct 2026 14:23:46 +0200 Subject: [PATCH 25/37] fix: stop the published tracks on leave --- CHANGELOG.md | 6 +++++ src/rtc/client.rs | 3 ++- src/rtc/join/lifecycle.rs | 9 ++++++- src/rtc/join/publication.rs | 8 ++++++ src/rtc/join/tests.rs | 50 +++++++++++++++++++++++++++++++++++++ src/video/call.rs | 3 ++- 6 files changed, 76 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 042fab9..18826bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,12 @@ Before an `update_subscriptions` call, `set_incoming_video_enabled(true)` now subscribes to video only (it was audio and video), and `false` subscribes to nothing (it was audio). +### Leave stops the published tracks + +`leave` stops every published local track, as JS does with `stopOnLeave`. A +write to a stopped track returns `RtcError::IllegalState`, so a later join must +publish new tracks. + ## New Features ### A token-only client can prepare a call before the join diff --git a/src/rtc/client.rs b/src/rtc/client.rs index 7764a5d..b2e16ee 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -406,7 +406,8 @@ impl RtcCall { self.core.set_incoming_video_enabled(enabled).await } - /// Leave the call, closing the SFU connection and PeerConnections. + /// Leave the call, closing the SFU connection and PeerConnections. The + /// published tracks stop; a later join needs new tracks. pub async fn leave(&self) -> Result<()> { self.core.leave("user requested leave").await } diff --git a/src/rtc/join/lifecycle.rs b/src/rtc/join/lifecycle.rs index 437f4a1..898e7a4 100644 --- a/src/rtc/join/lifecycle.rs +++ b/src/rtc/join/lifecycle.rs @@ -643,7 +643,8 @@ impl RtcCore { impl RtcCore { /// Leave the call: send `leave`, close the PeerConnections and WebSocket, - /// abort background tasks. Succeeds from any state, including `Joining` + /// abort background tasks, and stop the published tracks (JS + /// `stopOnLeave`). Succeeds from any state, including `Joining` /// (JS: force to a leaving state rather than waiting for `JOINED`). The /// teardown runs in a runtime task, so it finishes when this future is /// dropped. @@ -675,6 +676,12 @@ impl RtcCore { } connection.teardown().await; } + // A `publish` in progress holds the media lock until it ends. A + // join that starts after a dropped leave owns the later tracks. + let mut media = core.media.lock().await; + if core.is_generation_current(generation) { + media.stop_all(); + } }); if let Err(error) = teardown.await && error.is_panic() diff --git a/src/rtc/join/publication.rs b/src/rtc/join/publication.rs index f49f2c0..9608fbe 100644 --- a/src/rtc/join/publication.rs +++ b/src/rtc/join/publication.rs @@ -110,6 +110,14 @@ impl MediaState { Some(track) } + /// Stop the track of every publication and clear the state. The retired + /// tracks are already stopped. + pub(super) fn stop_all(&mut self) { + for publication in std::mem::take(self).publications { + publication.track.stop(); + } + } + /// Take the stopped track of the latest sender that [`Self::retire`] kept /// for this kind of track. pub(super) fn take_retired( diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 5547385..ecf616b 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -702,6 +702,56 @@ async fn dropped_leave_still_leaves_the_call() { core.begin_join().expect("a new join can start"); } +#[tokio::test] +async fn leave_stops_the_published_tracks() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + *core.connection.lock().await = Some(connection); + let audio = LocalAudioTrack::opus().expect("opus track"); + core.media + .lock() + .await + .begin_publish(LocalTrack::Audio(audio.clone()), 0); + + core.leave("test leave").await.expect("leave"); + + let write = audio.write_pcm(PcmFrame::mono(vec![0; 960], 48_000)).await; + assert!( + matches!(write, Err(RtcError::IllegalState(_))), + "a track published before leave is stopped: {write:?}" + ); +} + +#[tokio::test] +async fn a_dropped_leave_keeps_the_tracks_of_a_later_join() { + let core = test_core(); + prepare_joined_core(&core, "alice"); + let connection_slot = core.connection.lock().await; + let leave = tokio::time::timeout(Duration::from_millis(50), core.leave("dropped leave")).await; + assert!(leave.is_err(), "leave waits for the connection lock"); + + core.begin_join().expect("a new join can start"); + let audio = LocalAudioTrack::opus().expect("opus track"); + core.media + .lock() + .await + .begin_publish(LocalTrack::Audio(audio.clone()), 0); + let (_, _, completed) = core.runtime_task_snapshot(); + drop(connection_slot); + wait_for( + Duration::from_secs(2), + || core.runtime_task_snapshot().2 > completed, + "the leave task", + ) + .await; + + audio + .write_pcm(PcmFrame::mono(vec![0; 960], 48_000)) + .await + .expect("the later join keeps its track"); +} + #[tokio::test] async fn a_dropped_leave_keeps_the_connection_of_a_later_join() { let core = test_core(); diff --git a/src/video/call.rs b/src/video/call.rs index 84fe2c9..f6129be 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -860,7 +860,8 @@ impl Call { } /// Leave the call, closing the SFU connection and PeerConnections. Succeeds - /// from any state, including `JOINING`. + /// from any state, including `JOINING`. The published tracks stop; a later + /// join needs new tracks. pub async fn leave(&self) -> crate::rtc::RtcResult<()> { self.rtc.leave("user requested leave").await } From a79150104f5e9920eef87295d446bea62d1d4c14 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 12:40:28 +0200 Subject: [PATCH 26/37] fix: keep pacing published audio through publisher disconnects --- examples/gpt_realtime_bot.rs | 2 -- src/rtc/join/connection.rs | 11 ++++--- src/rtc/join/publish.rs | 15 ++++----- src/rtc/join/reconnect_runtime.rs | 2 +- src/rtc/join/tests.rs | 51 +++++++++++++++++-------------- src/rtc/peer/publisher.rs | 8 +---- src/rtc/tracks/local.rs | 33 +++++++------------- 7 files changed, 55 insertions(+), 67 deletions(-) diff --git a/examples/gpt_realtime_bot.rs b/examples/gpt_realtime_bot.rs index c6019eb..707e28c 100644 --- a/examples/gpt_realtime_bot.rs +++ b/examples/gpt_realtime_bot.rs @@ -334,8 +334,6 @@ async fn configure_openai( Box::pin(async move { if state == RTCPeerConnectionState::Connected { paced_mic.start_pacing().await; - } else { - paced_mic.pause_pacing(); } }) })); diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 680dcf4..5733af4 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -166,12 +166,13 @@ pub(super) fn register_connection_state( tracing::debug!(label, ?state, "stream.rtc.pc.state"); tracer.trace("connectionstatechange", json!(state.to_string())); // Not awaited: waiting for the media lock here can block ICE - // state delivery. The sync reads the current publisher state, so - // a subscriber change applies the same state again. - if core.is_generation_current(generation) { - let sync_core = core.clone(); + // state delivery. The task reads the current publisher state, so + // a subscriber change only repeats the check. + if state == RTCPeerConnectionState::Connected && core.is_generation_current(generation) + { + let pacing_core = core.clone(); std::mem::drop(core.spawn_generation_task(generation, async move { - sync_core.sync_audio_pacing().await; + pacing_core.start_audio_pacing_if_connected().await; })); } if state == RTCPeerConnectionState::Connected { diff --git a/src/rtc/join/publish.rs b/src/rtc/join/publish.rs index 94cdd04..476bbe2 100644 --- a/src/rtc/join/publish.rs +++ b/src/rtc/join/publish.rs @@ -166,8 +166,10 @@ impl RtcCore { Ok(()) } - /// Pace the published audio only while the current publisher is connected. - pub(super) async fn sync_audio_pacing(&self) { + /// Start pacing the published audio once the current publisher connects. + /// Pacing continues through later disconnects, so a reconnect adds no + /// delay; stream-py `AudioStreamTrack` also catches up after a gap. + pub(super) async fn start_audio_pacing_if_connected(&self) { let media = self.media.lock().await; let connected = self .publisher_handles() @@ -175,12 +177,11 @@ impl RtcCore { .is_some_and(|(publisher, ..)| { publisher.connection_state() == RTCPeerConnectionState::Connected }); + if !connected { + return; + } for track in media.active_tracks() { - if connected { - track.start_audio_pacing().await; - } else { - track.pause_audio_pacing(); - } + track.start_audio_pacing().await; } } diff --git a/src/rtc/join/reconnect_runtime.rs b/src/rtc/join/reconnect_runtime.rs index 74773aa..7dc7c61 100644 --- a/src/rtc/join/reconnect_runtime.rs +++ b/src/rtc/join/reconnect_runtime.rs @@ -5,7 +5,7 @@ use super::*; impl RtcCore { /// Re-publish every local track on the fresh publisher PC after a /// REJOIN/MIGRATE. The tracks are the same `Arc`-backed handles, so they bind - /// to the new PeerConnection and pacing resumes when it connects. + /// to the new PeerConnection and their pacing continues. pub(super) async fn restore_published_tracks(&self) -> Result<()> { let mut media = self.media.lock().await; let capabilities = self diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index ecf616b..4c70102 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -984,23 +984,13 @@ async fn stale_coordinator_stop_keeps_the_current_coordinator() { } #[tokio::test] -async fn published_audio_is_paced_only_while_the_publisher_is_connected() { +async fn published_audio_paces_from_the_first_connect_through_a_disconnect() { let core = test_core(); let generation = prepare_joined_core(&core, "alice"); let (connection, _sfu) = establish_fake(&core, generation).await; let publisher = connection.publisher.clone(); *core.connection.lock().await = Some(connection); let audio = LocalAudioTrack::opus().expect("opus track"); - let tone = (0..960_u32) - .map(|n| { - (12_000.0 * (std::f64::consts::TAU * 440.0 * f64::from(n) / 48_000.0).sin()) as i16 - }) - .collect::>() - .repeat(50); - audio - .write_pcm(PcmFrame::mono(tone, 48_000)) - .await - .expect("one second fits the queue"); publisher .add_track(audio.webrtc_track()) .await @@ -1009,21 +999,36 @@ async fn published_audio_is_paced_only_while_the_publisher_is_connected() { .lock() .await .begin_publish(LocalTrack::Audio(audio.clone()), 0); - - let (receiver, mut remote) = peer::connect_audio_receiver(&publisher).await; - - tokio::time::timeout(Duration::from_secs(5), remote.recv()) + // A second peer on the same track receives every paced packet. + let observer = peer::new_peer_connection(&[]).await.expect("observer"); + observer + .add_track(audio.webrtc_track()) + .await + .expect("add observed track"); + let (observer_receiver, mut observed) = peer::connect_audio_receiver(&observer).await; + let (receiver, _remote) = peer::connect_audio_receiver(&publisher).await; + let observed = tokio::time::timeout(Duration::from_secs(5), observed.recv()) .await - .expect("paced audio reaches the receiver") - .expect("remote track"); + .expect("pacing starts when the publisher connects") + .expect("observed track"); + publisher.close().await.expect("close publisher"); - wait_for( - Duration::from_secs(2), - || !audio.is_pacing(), - "closed publisher pauses pacing", - ) - .await; + let (at_close, _) = observed.read_rtp().await.expect("observed packet"); + let second_later = at_close.header.timestamp.wrapping_add(48_000); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let (packet, _) = observed.read_rtp().await.expect("observed packet"); + if packet.header.timestamp.wrapping_sub(second_later) < u32::MAX / 2 { + break; + } + } + }) + .await + .expect("pacing continues after the publisher closes"); + let _ = receiver.close().await; + let _ = observer_receiver.close().await; + let _ = observer.close().await; core.leave("test cleanup").await.expect("leave"); } diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index 25c48a2..e1435b6 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -101,11 +101,6 @@ pub(crate) async fn restart_ice( } } } - // Media is lost until ICE connects again; the publisher `Connected` state - // resumes pacing. - for track in tracks { - track.pause_audio_pacing(); - } negotiate_publish(publisher, signal, session_id, tracks, publish_options).await } @@ -385,13 +380,12 @@ mod tests { .expect("remote track") .expect("remote track channel"); let (mut last, _) = remote.read_rtp().await.expect("first packet"); - retired.pause_audio_pacing(); + retired.stop(); while let Ok(Ok((packet, _))) = tokio::time::timeout(Duration::from_millis(200), remote.read_rtp()).await { last = packet; } - retired.stop(); let track = LocalTrack::Audio(LocalAudioTrack::opus().expect("second track")); assert!( diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 7cfd2d5..a21afca 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -7,9 +7,10 @@ //! - [`LocalAudioTrack::write_pcm`] — raw [`PcmFrame`]s. Resampled to 48 kHz //! mono, queued, and paced into 20 ms Opus frames by a background task that //! emits silence on starve (stream-py `AudioStreamTrack` pacing). Pacing runs -//! after [`LocalAudioTrack::start_pacing`]; a published track paces only -//! while the SFU publisher is connected. This is the PCM republish / TTS-bot -//! path. +//! after [`LocalAudioTrack::start_pacing`]; a published track starts pacing +//! when the SFU publisher first connects and does not pause on a later +//! disconnect, so audio written during an outage is lost instead of delayed. +//! This is the PCM republish / TTS-bot path. //! - [`LocalAudioTrack::write_sample`] / [`LocalVideoTrack::write_sample`] — //! already-encoded media (Opus/VP8/…) plus a frame duration; the SDK //! packetizes and writes. The caller controls pacing. @@ -600,26 +601,23 @@ impl LocalAudioTrack { /// the queue is empty. Before this, [`write_pcm`](Self::write_pcm) only /// fills the queue. /// - /// [`Call::publish_audio`](crate::Call::publish_audio) starts and pauses - /// pacing with the SFU publisher connection. Call this yourself only for a - /// track on your own PeerConnection (see [`webrtc_track`](Self::webrtc_track)), - /// after that PeerConnection connects. + /// [`Call::publish_audio`](crate::Call::publish_audio) starts pacing when + /// the SFU publisher first connects, and pacing continues through later + /// disconnects. Call this yourself only for a track on your own + /// PeerConnection (see [`webrtc_track`](Self::webrtc_track)), after that + /// PeerConnection connects. pub async fn start_pacing(&self) { self.inner.pacing_enabled.store(true, Ordering::SeqCst); self.ensure_pacer(); } - /// Stop taking queued PCM. The queue keeps its audio for the next + /// Stop taking queued PCM. The queue keeps its audio, so audio written + /// while paused is sent late after the next /// [`start_pacing`](Self::start_pacing). pub fn pause_pacing(&self) { self.inner.pacing_enabled.store(false, Ordering::SeqCst); } - #[cfg(test)] - pub(crate) fn is_pacing(&self) -> bool { - self.inner.pacing_enabled.load(Ordering::SeqCst) - } - /// Continue the RTP sequence numbers and timestamps of `previous`, whose /// sender this track takes over. The SFU drops a stream whose timestamps /// go back. @@ -2011,15 +2009,6 @@ impl LocalTrack { } } - pub(crate) fn pause_audio_pacing(&self) { - match self { - LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { - track.pause_pacing(); - } - LocalTrack::Video { .. } => {} - } - } - pub(crate) fn stop(&self) { match self { LocalTrack::Audio(a) | LocalTrack::ScreenShareAudio(a) => a.stop(), From ff4d1dfec0356bdc2beae33ce9fdce1cbd1c64b6 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 12:40:34 +0200 Subject: [PATCH 27/37] feat: add LocalAudioTrackConfig::pace and hold new publications until the SFU connects --- CHANGELOG.md | 16 +++++ examples/gpt_realtime_bot.rs | 8 ++- src/rtc/join/publication.rs | 2 + src/rtc/join/publish.rs | 7 +- src/rtc/join/tests.rs | 55 +++++++++++++++ src/rtc/peer/publisher.rs | 10 +-- src/rtc/tracks/local.rs | 133 +++++++++++++++++++++++++++++------ 7 files changed, 201 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18826bd..82e9012 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,8 +74,24 @@ nothing (it was audio). write to a stopped track returns `RtcError::IllegalState`, so a later join must publish new tracks. +### The PCM queue holds up to 60 s + +`write_pcm` queues up to `LocalAudioTrackConfig::pcm_queue_capacity`, 60 s by +default (it was 200 ms). A producer that writes faster than real time gets +`PcmQueueOverflow` only when the queue is full. + ## New Features +### Pacing control for PCM tracks + +`LocalAudioTrackConfig::pace` (default `true`) starts pacing at the first +`write_pcm`, as before. With `with_pace(false)`, `write_pcm` only queues until +`LocalAudioTrack::start_pacing`; `pause_pacing` stops pacing and keeps the +queue. `Call::publish_audio` holds the queue of a new publication until the SFU +publisher first connects, so the audio written before the connect is not lost. +Pacing does not pause on a later disconnect: audio written during an outage is +lost, and a reconnect adds no delay. + ### A token-only client can prepare a call before the join `RtcClient::call` returns an `RtcCall` that is not joined yet, and diff --git a/examples/gpt_realtime_bot.rs b/examples/gpt_realtime_bot.rs index 707e28c..3ff4d7b 100644 --- a/examples/gpt_realtime_bot.rs +++ b/examples/gpt_realtime_bot.rs @@ -51,8 +51,8 @@ use anyhow::{Context, Result, anyhow, bail}; use getstream::models::{CallRequest, GetOrCreateCallRequest, MemberRequest, UserRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - JoinCallData, LocalAudioTrack, LocalVideoTrack, RemoteTrack, RtcError, SubscriptionConfig, - TrackSubscriptionConfig, VideoFrame, + JoinCallData, LocalAudioTrack, LocalAudioTrackConfig, LocalVideoTrack, RemoteTrack, RtcError, + SubscriptionConfig, TrackSubscriptionConfig, VideoFrame, }; use getstream::video::Call; use getstream::{Stream, TokenOptions}; @@ -321,7 +321,9 @@ async fn configure_openai( .await .context("create oai-events data channel")?; - let mic = LocalAudioTrack::opus().context("opus track for OpenAI")?; + // Paced from the OpenAI connect below, so audio written before it is kept. + let mic = LocalAudioTrack::opus_with_config(LocalAudioTrackConfig::default().with_pace(false)) + .context("opus track for OpenAI")?; spawn_rtcp_drain( &tasks, pc.add_track(mic.webrtc_track()) diff --git a/src/rtc/join/publication.rs b/src/rtc/join/publication.rs index 9608fbe..9df6bc6 100644 --- a/src/rtc/join/publication.rs +++ b/src/rtc/join/publication.rs @@ -63,6 +63,7 @@ impl MediaState { if let Some(position) = self.position(&track_id) { return self.publications[position].status; } + track.hold_audio_pacing(); self.publications.push(LocalPublication { track, status: PublicationStatus::PendingPublishNegotiation, @@ -80,6 +81,7 @@ impl MediaState { pub(super) fn remove(&mut self, track_id: &str) -> Option { let position = self.position(track_id)?; let publication = self.publications.remove(position); + publication.track.release_audio_pacing(); let key = ( publication.publish_option_id, publication.track.track_type() as i32, diff --git a/src/rtc/join/publish.rs b/src/rtc/join/publish.rs index 476bbe2..d3a2530 100644 --- a/src/rtc/join/publish.rs +++ b/src/rtc/join/publish.rs @@ -166,9 +166,10 @@ impl RtcCore { Ok(()) } - /// Start pacing the published audio once the current publisher connects. - /// Pacing continues through later disconnects, so a reconnect adds no - /// delay; stream-py `AudioStreamTrack` also catches up after a gap. + /// End the hold of each new audio publication once the current publisher + /// connects, and start its pacing. Pacing continues through later + /// disconnects, so a reconnect adds no delay; stream-py `AudioStreamTrack` + /// also catches up after a gap. pub(super) async fn start_audio_pacing_if_connected(&self) { let media = self.media.lock().await; let connected = self diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 4c70102..2e31cdd 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1006,6 +1006,16 @@ async fn published_audio_paces_from_the_first_connect_through_a_disconnect() { .await .expect("add observed track"); let (observer_receiver, mut observed) = peer::connect_audio_receiver(&observer).await; + audio + .write_pcm(PcmFrame::mono(vec![0; 960], 48_000)) + .await + .expect("write before the publisher connects"); + assert!( + tokio::time::timeout(Duration::from_millis(300), observed.recv()) + .await + .is_err(), + "a publication takes no audio before the publisher connects" + ); let (receiver, _remote) = peer::connect_audio_receiver(&publisher).await; let observed = tokio::time::timeout(Duration::from_secs(5), observed.recv()) .await @@ -1032,6 +1042,51 @@ async fn published_audio_paces_from_the_first_connect_through_a_disconnect() { core.leave("test cleanup").await.expect("leave"); } +#[tokio::test] +async fn a_failed_publish_does_not_hold_the_track() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (mut connection, _sfu) = establish_fake(&core, generation).await; + connection.publish_options = vec![models::PublishOption { + track_type: TrackType::Audio as i32, + codec: Some(models::Codec { + name: "opus".to_owned(), + ..Default::default() + }), + ..Default::default() + }]; + *core.connection.lock().await = Some(connection); + core.own_capabilities + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert("send-audio".to_owned()); + let audio = LocalAudioTrack::opus().expect("opus track"); + + // The fake SFU has no SetPublisher endpoint. + core.publish(LocalTrack::Audio(audio.clone())) + .await + .expect_err("publish without SetPublisher"); + + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(audio.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote) = peer::connect_audio_receiver(&sender).await; + audio + .write_pcm(PcmFrame::mono(vec![0; 960], 48_000)) + .await + .expect("write"); + tokio::time::timeout(Duration::from_secs(5), remote.recv()) + .await + .expect("the track paces on its own PeerConnection") + .expect("remote track"); + audio.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + core.leave("test cleanup").await.expect("leave"); +} + #[tokio::test] async fn only_the_latest_remote_track_unsubscribes_when_dropped_without_a_runtime() { let core = test_core(); diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index e1435b6..b496c29 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -368,13 +368,14 @@ mod tests { }), ..Default::default() }]; - let retired = LocalTrack::Audio(LocalAudioTrack::opus().expect("first track")); + let first = LocalAudioTrack::opus().expect("first track"); + let retired = LocalTrack::Audio(first.clone()); let publisher = peer::new_peer_connection(&[]).await.expect("publisher"); let rtcp_tasks = add_transceiver_for_track(&publisher, &retired, &opus) .await .expect("first transceiver"); let (receiver, mut remote_rx) = peer::connect_audio_receiver(&publisher).await; - retired.start_audio_pacing().await; + first.start_pacing().await; let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) .await .expect("remote track") @@ -387,13 +388,14 @@ mod tests { last = packet; } - let track = LocalTrack::Audio(LocalAudioTrack::opus().expect("second track")); + let second = LocalAudioTrack::opus().expect("second track"); + let track = LocalTrack::Audio(second.clone()); assert!( replace_retired_track(&publisher, &retired, &track, &opus) .await .expect("replace the retired track") ); - track.start_audio_pacing().await; + second.start_pacing().await; let (next, _) = remote.read_rtp().await.expect("second packet"); assert_eq!( diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index a21afca..b74b91c 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -6,9 +6,10 @@ //! //! - [`LocalAudioTrack::write_pcm`] — raw [`PcmFrame`]s. Resampled to 48 kHz //! mono, queued, and paced into 20 ms Opus frames by a background task that -//! emits silence on starve (stream-py `AudioStreamTrack` pacing). Pacing runs -//! after [`LocalAudioTrack::start_pacing`]; a published track starts pacing -//! when the SFU publisher first connects and does not pause on a later +//! emits silence on starve (stream-py `AudioStreamTrack` pacing). Pacing +//! starts at the first write, or at [`LocalAudioTrack::start_pacing`] when +//! [`LocalAudioTrackConfig::pace`] is off. A published track holds its queue +//! until the SFU publisher first connects and does not pause on a later //! disconnect, so audio written during an outage is lost instead of delayed. //! This is the PCM republish / TTS-bot path. //! - [`LocalAudioTrack::write_sample`] / [`LocalVideoTrack::write_sample`] — @@ -335,6 +336,11 @@ struct AudioInner { /// Set by `start_pacing`, cleared by `pause_pacing`. pacing_enabled: AtomicBool, pcm_pacing: AtomicBool, + /// [`LocalAudioTrackConfig::pace`]. + pace: bool, + /// Set while a new publication waits for the first SFU publisher connect. + /// The pacer takes no PCM while it is set. + held: AtomicBool, write_guard: tokio::sync::Mutex<()>, } @@ -356,6 +362,9 @@ pub struct LocalAudioTrackConfig { /// write above it drops the oldest queued samples. The minimum is one 20 ms /// frame. pub pcm_queue_capacity: Duration, + /// Start pacing at the first [`LocalAudioTrack::write_pcm`]. When `false`, + /// `write_pcm` only queues until [`LocalAudioTrack::start_pacing`]. + pub pace: bool, } impl Default for LocalAudioTrackConfig { @@ -366,6 +375,7 @@ impl Default for LocalAudioTrackConfig { expected_packet_loss_pct: EXPECTED_PACKET_LOSS_PCT, dtx: true, pcm_queue_capacity: PCM_QUEUE_CAPACITY, + pace: true, } } } @@ -407,6 +417,13 @@ impl LocalAudioTrackConfig { self.pcm_queue_capacity = pcm_queue_capacity; self } + + /// Enable or disable the pacing start at the first `write_pcm`. + #[must_use] + pub fn with_pace(mut self, pace: bool) -> Self { + self.pace = pace; + self + } } /// An outbound Opus audio track. @@ -470,6 +487,8 @@ impl LocalAudioTrack { pacer_started: AtomicBool::new(false), pacing_enabled: AtomicBool::new(false), pcm_pacing: AtomicBool::new(true), + pace: config.pace, + held: AtomicBool::new(false), write_guard: tokio::sync::Mutex::new(()), }), }) @@ -478,9 +497,11 @@ impl LocalAudioTrack { /// Queue a PCM frame for the paced 20 ms Opus encoder. /// /// The frame is resampled to 48 kHz mono and buffered up to - /// [`LocalAudioTrackConfig::pcm_queue_capacity`]. While pacing runs (see - /// [`start_pacing`](Self::start_pacing)), a background task emits one Opus - /// packet every 20 ms, writing silence when the buffer runs dry. This method + /// [`LocalAudioTrackConfig::pcm_queue_capacity`]. The first write starts + /// pacing when [`LocalAudioTrackConfig::pace`] is set (the default); + /// otherwise [`start_pacing`](Self::start_pacing) does. While pacing runs, a + /// background task emits one Opus packet every 20 ms, writing silence when + /// the buffer runs dry. This method /// does not backpressure a producer: overflow drops the oldest queued /// samples and retains the newest audio. [`flush`](Self::flush) still drops /// all unsent samples immediately for barge-in. @@ -518,6 +539,11 @@ impl LocalAudioTrack { } dropped }; + // Only a pacer that never started starts here, so a `pause_pacing` + // stays in force. + if self.inner.pace && !self.inner.pacer_started.load(Ordering::SeqCst) { + self.start_pacing().await; + } if dropped > 0 { Err(RtcError::PcmQueueOverflow { dropped_samples: dropped, @@ -598,14 +624,14 @@ impl LocalAudioTrack { } /// Start taking queued PCM: one 20 ms frame every 20 ms, or silence when - /// the queue is empty. Before this, [`write_pcm`](Self::write_pcm) only - /// fills the queue. + /// the queue is empty. With [`LocalAudioTrackConfig::pace`] set (the + /// default), the first [`write_pcm`](Self::write_pcm) calls this. /// - /// [`Call::publish_audio`](crate::Call::publish_audio) starts pacing when - /// the SFU publisher first connects, and pacing continues through later - /// disconnects. Call this yourself only for a track on your own - /// PeerConnection (see [`webrtc_track`](Self::webrtc_track)), after that - /// PeerConnection connects. + /// [`Call::publish_audio`](crate::Call::publish_audio) holds the queue + /// until the SFU publisher first connects, then starts pacing; pacing + /// continues through later disconnects. With `pace` off, call this for a + /// track on your own PeerConnection (see [`webrtc_track`](Self::webrtc_track)) + /// after that PeerConnection connects. pub async fn start_pacing(&self) { self.inner.pacing_enabled.store(true, Ordering::SeqCst); self.ensure_pacer(); @@ -618,6 +644,17 @@ impl LocalAudioTrack { self.inner.pacing_enabled.store(false, Ordering::SeqCst); } + /// Take no PCM until [`Self::release_pacing`]. A new publication holds its + /// track until the SFU publisher first connects. + pub(crate) fn hold_pacing(&self) { + self.inner.held.store(true, Ordering::SeqCst); + } + + /// End a [`Self::hold_pacing`]. Returns whether the track was held. + pub(crate) fn release_pacing(&self) -> bool { + self.inner.held.swap(false, Ordering::SeqCst) + } + /// Continue the RTP sequence numbers and timestamps of `previous`, whose /// sender this track takes over. The SFU drops a stream whose timestamps /// go back. @@ -657,8 +694,10 @@ impl LocalAudioTrack { /// [`Call::publish_audio`](crate::Call::publish_audio) does this for the /// SFU; you only need it to send the same audio to a second peer, such as an /// AI provider's Realtime endpoint. Every write path (`write_pcm` and - /// friends) feeds all bound senders. A track that is not published paces - /// `write_pcm` audio only after [`start_pacing`](Self::start_pacing). + /// friends) feeds all bound senders. With [`LocalAudioTrackConfig::pace`] + /// set, audio written before this PeerConnection connects is lost; turn it + /// off and call [`start_pacing`](Self::start_pacing) at the connect to keep + /// that audio. pub fn webrtc_track(&self) -> Arc { self.inner.core.track.clone() } @@ -703,7 +742,9 @@ async fn pace_audio(track: Weak) { if inner.core.stopped.load(Ordering::SeqCst) { return; } - if !inner.pcm_pacing.load(Ordering::SeqCst) || !inner.pacing_enabled.load(Ordering::SeqCst) + if !inner.pcm_pacing.load(Ordering::SeqCst) + || !inner.pacing_enabled.load(Ordering::SeqCst) + || inner.held.load(Ordering::SeqCst) { continue; } @@ -2000,10 +2041,32 @@ impl LocalTrack { } } + /// End the publication hold at the first SFU publisher connect and start + /// pacing. A track that is not held keeps its pacing state, so a + /// `pause_pacing` stays in force. pub(crate) async fn start_audio_pacing(&self) { match self { LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { - track.start_pacing().await; + if track.release_pacing() { + track.start_pacing().await; + } + } + LocalTrack::Video { .. } => {} + } + } + + pub(crate) fn hold_audio_pacing(&self) { + match self { + LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => track.hold_pacing(), + LocalTrack::Video { .. } => {} + } + } + + /// End the hold of a publication that did not complete. + pub(crate) fn release_audio_pacing(&self) { + match self { + LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track) => { + track.release_pacing(); } LocalTrack::Video { .. } => {} } @@ -2245,7 +2308,9 @@ mod tests { #[tokio::test] async fn write_pcm_above_the_default_minute_keeps_the_newest_samples() { - let track = LocalAudioTrack::opus().expect("opus track"); + let track = + LocalAudioTrack::opus_with_config(LocalAudioTrackConfig::default().with_pace(false)) + .expect("opus track"); let minute = 60 * OPUS_SAMPLE_RATE as usize; track .write_pcm(PcmFrame::mono(vec![1; minute], OPUS_SAMPLE_RATE)) @@ -2325,10 +2390,36 @@ mod tests { assert!(with_capacity(Duration::from_millis(20)).is_ok()); } + #[tokio::test] + async fn the_first_write_starts_pacing_by_default() { + let track = LocalAudioTrack::opus().expect("opus track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(track.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote_rx) = peer::connect_audio_receiver(&sender).await; + + track + .write_pcm(PcmFrame::mono(tone_20ms(), OPUS_SAMPLE_RATE)) + .await + .expect("write"); + + tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("paced audio reaches the receiver") + .expect("remote track channel"); + track.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + } + #[tokio::test] async fn pcm_written_before_pacing_starts_stays_queued() { let track = LocalAudioTrack::opus_with_config( - LocalAudioTrackConfig::default().with_pcm_queue_capacity(Duration::from_millis(100)), + LocalAudioTrackConfig::default() + .with_pcm_queue_capacity(Duration::from_millis(100)) + .with_pace(false), ) .expect("opus track"); track @@ -2380,7 +2471,9 @@ mod tests { #[tokio::test] async fn pcm_queued_before_the_connection_is_sent_from_its_first_sample() { - let track = LocalAudioTrack::opus().expect("opus track"); + let track = + LocalAudioTrack::opus_with_config(LocalAudioTrackConfig::default().with_pace(false)) + .expect("opus track"); let minute = tone_20ms().repeat(3_000); let queued = minute.len(); track From ec7ba82143d8893476b6efb11c2c88d74a1d6384 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 13:25:51 +0200 Subject: [PATCH 28/37] feat: reuse the stopped video sender for a new video track --- CHANGELOG.md | 10 ++ src/rtc/client.rs | 5 +- src/rtc/error.rs | 13 +++ src/rtc/join/publication.rs | 36 ++++--- src/rtc/join/publish.rs | 33 ++++--- src/rtc/join/tests.rs | 63 ++++++++++++ src/rtc/peer/connection.rs | 14 ++- src/rtc/peer/mod.rs | 2 +- src/rtc/peer/publisher.rs | 191 +++++++++++++++++++++++++++++++++--- src/rtc/tracks/local.rs | 87 ++++++++++++++-- src/video/call.rs | 6 +- tests/rtc_media.rs | 126 ++++++++++++++++++++++++ 12 files changed, 533 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e9012..91e3c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,16 @@ publisher first connects, so the audio written before the connect is not lost. Pacing does not pause on a later disconnect: audio written during an outage is lost, and a reconnect adds no delay. +### A stopped track can be replaced by a new track + +After `stop_publish`, a new audio, video, or screen-share track of the same kind +and publish option takes over the sender that the stop kept, as JS +`replaceTrack` does. Remote participants keep their remote track and get the +media of the new track. Before, a second video publish failed with +"participant not found". A simulcast track (`h264_simulcast`, `vp8_simulcast`) +cannot replace or be replaced: `publish` returns `RtcError::SimulcastReplace`. +Turn such a track off and on with `mute_track` and `unmute_track`. + ### A token-only client can prepare a call before the join `RtcClient::call` returns an `RtcCall` that is not joined yet, and diff --git a/src/rtc/client.rs b/src/rtc/client.rs index b2e16ee..7f75578 100644 --- a/src/rtc/client.rs +++ b/src/rtc/client.rs @@ -378,7 +378,10 @@ impl RtcCall { self.core.stop_noise_cancellation().await } - /// Stop publishing a local media track. + /// Stop publishing a local media track. A new track of the same kind that + /// is published later takes over its sender, except for a simulcast track + /// ([`RtcError::SimulcastReplace`]); use [`mute_track`](Self::mute_track) + /// for that. pub async fn stop_publish(&self, track: LocalTrack) -> Result<()> { self.core.stop_publish(track).await } diff --git a/src/rtc/error.rs b/src/rtc/error.rs index 1b966e7..29be991 100644 --- a/src/rtc/error.rs +++ b/src/rtc/error.rs @@ -132,6 +132,19 @@ pub enum RtcError { track_type: models::TrackType, }, + /// A stopped simulcast publication and a new track cannot share a sender: + /// webrtc-rs replaces the track only on a sender with one encoding. Turn a + /// simulcast track off and on with `mute_track` and `unmute_track` instead + /// of a stop. + #[error( + "a stopped {track_type:?} publication cannot change its track when either \ + track is simulcast; use mute_track and unmute_track" + )] + SimulcastReplace { + /// Publication kind requested by the caller. + track_type: models::TrackType, + }, + /// A PCM write exceeded the track's PCM queue capacity. The newest samples /// were retained and this many oldest samples were discarded. #[error( diff --git a/src/rtc/join/publication.rs b/src/rtc/join/publication.rs index 9df6bc6..b6870b0 100644 --- a/src/rtc/join/publication.rs +++ b/src/rtc/join/publication.rs @@ -27,7 +27,7 @@ pub(super) struct LocalPublication { pub(super) struct MediaState { pub(super) publications: Vec, pub(super) publish_quality: HashMap<(i32, i32), Vec>, - /// The publish option id and stopped track of each audio sender that + /// The publish option id and stopped track of each sender that /// `stop_publish` kept in the publisher envelope. retired: Vec<(i32, LocalTrack)>, } @@ -98,17 +98,11 @@ impl MediaState { } /// Remove the publication of a stopped track whose sender stays in the - /// publisher envelope. Keep an audio sender for a later publish; `publish` - /// never reuses a video sender. + /// publisher envelope, and keep that sender for a later publish. pub(super) fn retire(&mut self, track_id: &str) -> Option { let publish_option_id = self.publications[self.position(track_id)?].publish_option_id; let track = self.remove(track_id)?; - if matches!( - track.track_type(), - TrackType::Audio | TrackType::ScreenShareAudio - ) { - self.retired.push((publish_option_id, track.clone())); - } + self.retired.push((publish_option_id, track.clone())); Some(track) } @@ -120,19 +114,33 @@ impl MediaState { } } - /// Take the stopped track of the latest sender that [`Self::retire`] kept - /// for this kind of track. + /// The stopped track of the latest sender that [`Self::retire`] kept for + /// this kind of track. + pub(super) fn retired( + &self, + track_type: TrackType, + publish_option_id: i32, + ) -> Option<&LocalTrack> { + let position = self.retired_position(track_type, publish_option_id)?; + Some(&self.retired[position].1) + } + + /// Take the track that [`Self::retired`] returns. pub(super) fn take_retired( &mut self, track_type: TrackType, publish_option_id: i32, ) -> Option { - let position = self.retired.iter().rposition(|(option_id, retired)| { - retired.track_type() == track_type && *option_id == publish_option_id - })?; + let position = self.retired_position(track_type, publish_option_id)?; Some(self.retired.remove(position).1) } + fn retired_position(&self, track_type: TrackType, publish_option_id: i32) -> Option { + self.retired.iter().rposition(|(option_id, retired)| { + retired.track_type() == track_type && *option_id == publish_option_id + }) + } + pub(super) fn refresh_publish_options( &mut self, publish_options: &[models::PublishOption], diff --git a/src/rtc/join/publish.rs b/src/rtc/join/publish.rs index d3a2530..f837c17 100644 --- a/src/rtc/join/publish.rs +++ b/src/rtc/join/publish.rs @@ -63,25 +63,28 @@ impl RtcCore { }; let mut publisher_rtcp_tasks = Vec::new(); if status.is_none() { - // Reuse the audio sender that `stop_publish` kept, as JS `replaceTrack` - // does: a second audio m-line fails the SFU negotiation. Video RTCP - // readers keep the old track, and simulcast cannot be replaced. - let retired = if matches!( - track.track_type(), - TrackType::Audio | TrackType::ScreenShareAudio - ) { - media.take_retired(track.track_type(), publish_option_id) - } else { - None - }; - let reused = match retired { + // Reuse the sender that `stop_publish` kept, as JS `replaceTrack` + // does: a second m-line for the same publish option fails the SFU + // negotiation. + if media + .retired(track.track_type(), publish_option_id) + .is_some_and(|retired| retired.is_simulcast() || track.is_simulcast()) + { + return Err(RtcError::SimulcastReplace { + track_type: track.track_type(), + }); + } + let reader = match media.take_retired(track.track_type(), publish_option_id) { Some(retired) => { publisher::replace_retired_track(&publisher, &retired, &track, &publish_options) .await? } - None => false, + None => None, }; - if !reused { + let reused = reader.is_some(); + if let Some(reader) = reader { + self.register_publisher_tasks(vec![reader]).await; + } else { publisher_rtcp_tasks = match publisher::add_transceiver_for_track( &publisher, &track, @@ -208,6 +211,8 @@ impl RtcCore { /// (`Invalid SetPublisher request; ... new track must have the same envelope as /// previous`), so removing the sole sender and renegotiating an empty envelope /// fails on the wire; muting the track type is the wire-correct way to stop. + /// A later [`Self::publish`] of the same kind and publish option puts its + /// track on the kept sender. pub async fn stop_publish(self: &Arc, track: LocalTrack) -> Result<()> { let mut media = self.media.lock().await; let track_id = track.track_id(); diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 2e31cdd..6810d35 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1087,6 +1087,69 @@ async fn a_failed_publish_does_not_hold_the_track() { core.leave("test cleanup").await.expect("leave"); } +#[tokio::test] +async fn a_stopped_simulcast_publication_is_not_replaced() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (mut connection, _sfu) = establish_fake(&core, generation).await; + connection.publish_options = vec![models::PublishOption { + id: 1, + track_type: TrackType::Video as i32, + codec: Some(models::Codec { + name: "h264".to_owned(), + ..Default::default() + }), + ..Default::default() + }]; + let stopped = LocalTrack::Video { + track: LocalVideoTrack::h264_simulcast().expect("simulcast track"), + track_type: TrackType::Video, + }; + let reader_tasks = crate::rtc::peer::publisher::add_transceiver_for_track( + &connection.publisher, + &stopped, + &connection.publish_options, + ) + .await + .expect("simulcast transceiver"); + *core.connection.lock().await = Some(connection); + core.own_capabilities + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert("send-video".to_owned()); + { + let mut media = core.media.lock().await; + media.begin_publish(stopped.clone(), 1); + media.retire(&stopped.track_id()); + } + stopped.stop(); + + for track in [ + LocalVideoTrack::h264_simulcast().expect("new simulcast track"), + LocalVideoTrack::h264().expect("new single-layer track"), + ] { + let result = core + .publish(LocalTrack::Video { + track, + track_type: TrackType::Video, + }) + .await; + assert!( + matches!( + result, + Err(RtcError::SimulcastReplace { + track_type: TrackType::Video + }) + ), + "{result:?}" + ); + } + for task in reader_tasks { + task.abort(); + } + core.leave("test cleanup").await.expect("leave"); +} + #[tokio::test] async fn only_the_latest_remote_track_unsubscribes_when_dropped_without_a_runtime() { let core = test_core(); diff --git a/src/rtc/peer/connection.rs b/src/rtc/peer/connection.rs index d3fda04..e7e9279 100644 --- a/src/rtc/peer/connection.rs +++ b/src/rtc/peer/connection.rs @@ -264,6 +264,18 @@ pub(crate) async fn connect_audio_receiver( ) -> ( Arc, tokio::sync::mpsc::Receiver>, +) { + connect_receiver(sender, RTPCodecType::Audio).await +} + +/// [`connect_audio_receiver`] for a receive-only peer of `kind`. +#[cfg(test)] +pub(crate) async fn connect_receiver( + sender: &RTCPeerConnection, + kind: RTPCodecType, +) -> ( + Arc, + tokio::sync::mpsc::Receiver>, ) { use std::time::Duration; use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; @@ -274,7 +286,7 @@ pub(crate) async fn connect_audio_receiver( let receiver = new_peer_connection(&[]).await.expect("receiver"); receiver .add_transceiver_from_kind( - RTPCodecType::Audio, + kind, Some(RTCRtpTransceiverInit { direction: RTCRtpTransceiverDirection::Recvonly, send_encodings: vec![], diff --git a/src/rtc/peer/mod.rs b/src/rtc/peer/mod.rs index 70b4f30..5b7e079 100644 --- a/src/rtc/peer/mod.rs +++ b/src/rtc/peer/mod.rs @@ -13,7 +13,7 @@ pub mod publisher; mod subscriber; #[cfg(test)] -pub(crate) use connection::connect_audio_receiver; +pub(crate) use connection::{connect_audio_receiver, connect_receiver}; pub use connection::{generic_sdp, new_peer_connection, to_rtc_ice_servers, trace_peer_events}; pub(super) use ice::{PendingIce, register_ice_trickle}; pub(super) use subscriber::negotiate_subscriber; diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index b496c29..983b627 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -228,13 +228,14 @@ pub(crate) async fn add_transceiver_for_track( } /// Put `track` on the sender that still carries the stopped track `retired`, -/// as JS `replaceTrack` does. Returns `false` if `publisher` has no such sender. +/// as JS `replaceTrack` does. Returns the RTCP reader of `track`, or `None` if +/// `publisher` has no such sender. pub(crate) async fn replace_retired_track( publisher: &Arc, retired: &LocalTrack, track: &LocalTrack, publish_options: &[PublishOption], -) -> Result { +) -> Result>> { let retired_track_id = retired.track_id(); for transceiver in publisher.get_transceivers().await { let sender = transceiver.sender().await; @@ -251,13 +252,14 @@ pub(crate) async fn replace_retired_track( let physical = track.webrtc_tracks().into_iter().next().ok_or_else(|| { RtcError::Media("local publication has no physical encodings".to_owned()) })?; + let rid = physical.rid().map(str::to_owned); sender .replace_track(Some(physical)) .await .map_err(RtcError::from)?; - return Ok(true); + return Ok(Some(spawn_rtcp_reader(sender, rid, track.clone()))); } - Ok(false) + Ok(None) } fn spawn_rtcp_reader( @@ -267,12 +269,20 @@ fn spawn_rtcp_reader( ) -> JoinHandle<()> { tokio::spawn(async move { loop { - let result = match rid.as_deref() { - Some(rid) => { - let mut buffer = vec![0_u8; 1_500]; - sender.read_simulcast(&mut buffer, rid).await + let read = async { + match rid.as_deref() { + Some(rid) => { + let mut buffer = vec![0_u8; 1_500]; + sender.read_simulcast(&mut buffer, rid).await + } + None => sender.read_rtcp().await, } - None => sender.read_rtcp().await, + }; + let result = tokio::select! { + biased; + // A track that takes over this sender starts its own reader. + () = track.stopped() => break, + result = read => result, }; let Ok((packets, _attributes)) = result else { break; @@ -357,6 +367,7 @@ mod tests { use crate::rtc::proto::models::{Codec, VideoDimension}; use crate::rtc::tracks::{LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig}; use std::time::Duration; + use webrtc::rtp_transceiver::rtp_codec::RTPCodecType; #[tokio::test] async fn a_track_on_a_retired_sender_continues_its_rtp_timeline() { @@ -390,11 +401,10 @@ mod tests { let second = LocalAudioTrack::opus().expect("second track"); let track = LocalTrack::Audio(second.clone()); - assert!( - replace_retired_track(&publisher, &retired, &track, &opus) - .await - .expect("replace the retired track") - ); + let reader = replace_retired_track(&publisher, &retired, &track, &opus) + .await + .expect("replace the retired track") + .expect("the retired sender"); second.start_pacing().await; let (next, _) = remote.read_rtp().await.expect("second packet"); @@ -407,6 +417,159 @@ mod tests { last.header.timestamp.wrapping_add(960) ); track.stop(); + reader.abort(); + for task in rtcp_tasks { + task.abort(); + } + let _ = publisher.close().await; + let _ = receiver.close().await; + } + + #[tokio::test] + async fn a_video_track_on_a_retired_sender_continues_its_rtp_timeline() { + let vp9 = [video_option("VP9")]; + let first = LocalVideoTrack::vp9().expect("first track"); + let retired = LocalTrack::Video { + track: first.clone(), + track_type: TrackType::Video, + }; + let publisher = peer::new_peer_connection(&[]).await.expect("publisher"); + let rtcp_tasks = add_transceiver_for_track(&publisher, &retired, &vp9) + .await + .expect("first transceiver"); + let (receiver, mut remote_rx) = + peer::connect_receiver(&publisher, RTPCodecType::Video).await; + let frame = vec![128_u8; 320 * 240 * 3 / 2]; + let frame_time = Duration::from_millis(33); + first + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("first frame"); + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + for _ in 0..3 { + first + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("frame"); + } + retired.stop(); + let mut last = None; + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(200), remote.read_rtp()).await + { + last = Some(packet); + } + let last = last.expect("packets of the first track"); + + let second = LocalVideoTrack::vp9().expect("second track"); + let track = LocalTrack::Video { + track: second.clone(), + track_type: TrackType::Video, + }; + let reader = replace_retired_track(&publisher, &retired, &track, &vp9) + .await + .expect("replace the retired track") + .expect("the retired sender"); + second + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("second track frame"); + let (next, _) = tokio::time::timeout(Duration::from_secs(5), remote.read_rtp()) + .await + .expect("the receiver takes the packets of the second track") + .expect("second track packet"); + + assert_eq!( + next.header.sequence_number, + last.header.sequence_number.wrapping_add(1) + ); + assert_eq!( + next.header.timestamp, + last.header.timestamp.wrapping_add(2_970) + ); + assert!( + tokio::time::timeout(Duration::from_millis(200), remote_rx.recv()) + .await + .is_err(), + "the receiver keeps its remote track" + ); + track.stop(); + reader.abort(); + for task in rtcp_tasks { + task.abort(); + } + let _ = publisher.close().await; + let _ = receiver.close().await; + } + + #[tokio::test] + async fn a_keyframe_request_reaches_the_track_on_a_retired_sender() { + use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; + + let vp9 = [video_option("VP9")]; + let retired = LocalTrack::Video { + track: LocalVideoTrack::vp9().expect("first track"), + track_type: TrackType::Video, + }; + let publisher = peer::new_peer_connection(&[]).await.expect("publisher"); + let rtcp_tasks = add_transceiver_for_track(&publisher, &retired, &vp9) + .await + .expect("first transceiver"); + let (receiver, mut remote_rx) = + peer::connect_receiver(&publisher, RTPCodecType::Video).await; + retired.stop(); + let second = LocalVideoTrack::vp9().expect("second track"); + let track = LocalTrack::Video { + track: second.clone(), + track_type: TrackType::Video, + }; + let reader = replace_retired_track(&publisher, &retired, &track, &vp9) + .await + .expect("replace the retired track") + .expect("the retired sender"); + let frame = vec![128_u8; 320 * 240 * 3 / 2]; + let frame_time = Duration::from_millis(33); + second + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("first frame"); + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + // VP9 payload descriptor: B starts a frame, P marks an inter frame. + let starts_a_keyframe = |payload: &[u8]| payload[0] & 0x08 != 0 && payload[0] & 0x40 == 0; + + receiver + .write_rtcp(&[Box::new(PictureLossIndication { + sender_ssrc: 0, + media_ssrc: remote.ssrc(), + })]) + .await + .expect("send PLI"); + let mut keyframes = 0; + // Ten frames are far below the periodic keyframe interval. + for _ in 0..10 { + second + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("frame"); + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(100), remote.read_rtp()).await + { + keyframes += usize::from(starts_a_keyframe(&packet.payload)); + } + } + + assert!( + keyframes >= 2, + "the PLI forces a keyframe after the first one (keyframes: {keyframes})" + ); + track.stop(); + reader.abort(); for task in rtcp_tasks { task.abort(); } diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index b74b91c..478f243 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -35,7 +35,7 @@ use std::sync::{Arc, Mutex as StdMutex, Weak}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bytes::Bytes; -use tokio::sync::Semaphore; +use tokio::sync::{Notify, Semaphore}; use tokio::task::JoinHandle; use webrtc::api::media_engine::{MIME_TYPE_H264, MIME_TYPE_OPUS, MIME_TYPE_VP8, MIME_TYPE_VP9}; use webrtc::rtp::extension::HeaderExtension; @@ -118,6 +118,7 @@ struct TrackCore { fwd_seq: AtomicU16, fwd_init: AtomicBool, stopped: AtomicBool, + stop_notify: Notify, muted: AtomicBool, quality_paused: AtomicBool, track_id: String, @@ -171,6 +172,7 @@ impl TrackCore { fwd_seq: AtomicU16::new(0), fwd_init: AtomicBool::new(false), stopped: AtomicBool::new(false), + stop_notify: Notify::new(), muted: AtomicBool::new(false), quality_paused: AtomicBool::new(false), track_id, @@ -294,6 +296,18 @@ impl TrackCore { fn stop(&self) { self.stopped.store(true, Ordering::SeqCst); + self.stop_notify.notify_waiters(); + } + + /// Wait until [`Self::stop`]. + async fn stopped(&self) { + let notified = self.stop_notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.stopped.load(Ordering::SeqCst) { + return; + } + notified.await; } fn set_muted(&self, muted: bool) { @@ -1230,6 +1244,44 @@ impl LocalVideoTrack { } } + /// Continue the RTP sequence numbers and timestamps of the single-encoding + /// `previous`, whose sender this track takes over. The encoder state moves + /// here, so the sequence numbers and the VP9 picture ids continue. The SFU + /// drops a stream whose timestamps go back. + fn continue_rtp_from(&self, previous: &LocalVideoTrack) { + let (next_pts, rtp_ts) = { + let clock = previous + .inner + .clock + .lock() + .unwrap_or_else(|e| e.into_inner()); + (clock.next_pts, clock.rtp_ts) + }; + { + let mut clock = self.inner.clock.lock().unwrap_or_else(|e| e.into_inner()); + clock.next_pts = next_pts; + clock.rtp_ts = rtp_ts; + } + let (current, previous) = (&self.inner.encodings[0], &previous.inner.encodings[0]); + let state = previous + .encoder + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + *current.encoder.lock().unwrap_or_else(|e| e.into_inner()) = state; + let packetizer = previous + .core + .packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + *current + .core + .packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) = packetizer; + } + /// The underlying webrtc-rs track, for attaching this track to a /// PeerConnection you manage yourself (`pc.add_track(...)`). See /// [`LocalAudioTrack::webrtc_track`]. @@ -2079,15 +2131,36 @@ impl LocalTrack { } } + /// Whether the track sends several RID encodings. + pub(crate) fn is_simulcast(&self) -> bool { + matches!(self, LocalTrack::Video { track, .. } if track.inner.encodings.len() > 1) + } + + /// Wait until the track is stopped. + pub(crate) async fn stopped(&self) { + match self { + LocalTrack::Audio(a) | LocalTrack::ScreenShareAudio(a) => a.inner.core.stopped().await, + LocalTrack::Video { track, .. } => track.inner.encodings[0].core.stopped().await, + } + } + /// Continue the RTP timeline of the audio track `previous`, whose sender /// this audio track takes over. pub(crate) fn continue_rtp_from(&self, previous: &LocalTrack) { - if let ( - LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track), - LocalTrack::Audio(previous) | LocalTrack::ScreenShareAudio(previous), - ) = (self, previous) - { - track.continue_rtp_from(previous); + match (self, previous) { + ( + LocalTrack::Audio(track) | LocalTrack::ScreenShareAudio(track), + LocalTrack::Audio(previous) | LocalTrack::ScreenShareAudio(previous), + ) => track.continue_rtp_from(previous), + ( + LocalTrack::Video { track, .. }, + LocalTrack::Video { + track: previous, .. + }, + ) => { + track.continue_rtp_from(previous); + } + _ => {} } } diff --git a/src/video/call.rs b/src/video/call.rs index f6129be..6fcc9b4 100644 --- a/src/video/call.rs +++ b/src/video/call.rs @@ -997,7 +997,11 @@ impl Call { /// Stop publishing a previously published track. The publisher keeps its /// transceiver in the negotiated envelope and signals the stop to the SFU /// via `UpdateMuteStates` (no publisher renegotiation), matching - /// `stream-video-js`. + /// `stream-video-js`. A new track of the same kind that is published later + /// takes over that transceiver. A simulcast track cannot do this + /// ([`RtcError::SimulcastReplace`](crate::rtc::RtcError::SimulcastReplace)); + /// turn it off and on with [`mute_track`](Self::mute_track) and + /// [`unmute_track`](Self::unmute_track). pub async fn stop_publish(&self, track: crate::rtc::LocalTrack) -> crate::rtc::RtcResult<()> { self.rtc.stop_publish(track).await } diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index 875950d..e4f35f1 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -1483,3 +1483,129 @@ async fn sole_audio_can_be_stopped_and_published_again() { let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; outcome.expect("test 9 (stop sole publication) timed out"); } + +/// A stopped camera track is replaced by a new track +/// +/// A publishes VP9 video, stops it, and publishes a new VP9 track on the same +/// session. The new track takes over the sender that the stop kept, as JS +/// `replaceTrack` does, so B decodes the new track on the remote track it +/// already has, and A does not reconnect. +#[tokio::test] +async fn sole_video_can_be_stopped_and_published_again() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(150), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + let mut tracks_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig::audio_video()) + .await + .expect("B update_subscriptions"); + let mut events_b = call_b.sfu_events(); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let mut events_a = call_a.client_events(); + let video_a = LocalVideoTrack::vp9().expect("vp9 track"); + call_a + .publish_video(video_a.clone()) + .await + .expect("A publish_video"); + let feeder = spawn_blue_video(video_a.clone()); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Video, + Duration::from_secs(45), + ) + .await + .expect("B never received A's video track"); + tokio::time::timeout(Duration::from_secs(45), remote_a.next_video_frame()) + .await + .expect("B decoded no frame before the stop") + .expect("B's video track ended before the stop"); + + call_a + .stop_publish(LocalTrack::Video { + track: video_a, + track_type: TrackType::Video, + }) + .await + .expect("A stop_publish"); + feeder.abort(); + let video_unpublished = await_track_event( + &mut events_b, + &user_a, + TrackType::Video, + false, + Duration::from_secs(45), + ) + .await; + assert!( + video_unpublished, + "B never received A's video TrackUnpublished after stop_publish" + ); + // Frames of the first track that B still has queued. + while let Ok(Some(_)) = + tokio::time::timeout(Duration::from_secs(2), remote_a.next_video_frame()).await + { + } + + let video_again = LocalVideoTrack::vp9().expect("vp9 track"); + call_a + .publish_video(video_again.clone()) + .await + .expect("A publish_video again on the same session"); + let feeder_again = spawn_blue_video(video_again); + let video_republished = await_track_event( + &mut events_b, + &user_a, + TrackType::Video, + true, + Duration::from_secs(45), + ) + .await; + assert!( + video_republished, + "B never received A's video TrackPublished after the second publish" + ); + let frame = tokio::time::timeout(Duration::from_secs(45), remote_a.next_video_frame()) + .await + .expect("B decoded no frame of the new track on its remote track") + .expect("B's video track ended after the second publish"); + feeder_again.abort(); + assert_packed_blue_frame(&frame); + let mut reconnected = false; + while let Ok(event) = events_a.try_recv() { + reconnected |= matches!( + event, + ClientCallEvent::CallingStateChanged( + CallingState::Reconnecting | CallingState::Migrating + ) + ); + } + assert!(!reconnected, "A reconnected after the second publish"); + + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("stop and publish video again timed out"); +} From 63e5fa6ad412b65c25448f0fa986234a45743faf Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 13:34:03 +0200 Subject: [PATCH 29/37] perf: skip the Opus encode while an audio track is muted --- src/rtc/tracks/local.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 478f243..1454060 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -772,6 +772,10 @@ async fn pace_audio(track: Weak) { *slot = buf.pop_front().unwrap_or(0); } } + // A muted track sends nothing, so its audio is dropped without an encode. + if inner.core.is_output_paused() { + continue; + } // Measure what we are about to encode, including any silence fill, so a // starved pacer reports silence rather than the last spoken level. inner From 6bef4ae3e2deaa941583ed593d59322abc194ecd Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 14:09:52 +0200 Subject: [PATCH 30/37] fix: keep the audio RTP clock running while a paced track is muted --- src/rtc/tracks/local.rs | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 1454060..2a93860 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -773,7 +773,15 @@ async fn pace_audio(track: Weak) { } } // A muted track sends nothing, so its audio is dropped without an encode. + // The RTP clock keeps time, so the first packet after the mute shows the + // gap (RFC 3550 §5.1). if inner.core.is_output_paused() { + inner + .core + .packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .skip_samples(FRAME_SAMPLES_20MS as u32); continue; } // Measure what we are about to encode, including any silence fill, so a @@ -2491,6 +2499,47 @@ mod tests { let _ = receiver.close().await; } + #[tokio::test] + async fn the_rtp_clock_runs_while_a_paced_track_is_muted() { + let track = LocalAudioTrack::opus().expect("opus track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(track.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote_rx) = peer::connect_audio_receiver(&sender).await; + track.start_pacing().await; + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + let published = LocalTrack::Audio(track.clone()); + + published.set_muted(true); + let (mut before, _) = remote.read_rtp().await.expect("packet before the mute"); + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(100), remote.read_rtp()).await + { + before = packet; + } + tokio::time::sleep(Duration::from_millis(500)).await; + published.set_muted(false); + let (after, _) = tokio::time::timeout(Duration::from_secs(5), remote.read_rtp()) + .await + .expect("packet after the mute") + .expect("packet after the mute"); + + // The mute lasts at least 600 ms; a slow runtime can skip some ticks. + let skipped = after.header.timestamp.wrapping_sub(before.header.timestamp); + assert!( + skipped >= 10 * FRAME_SAMPLES_20MS as u32, + "the timestamp advanced {skipped} samples over the mute" + ); + track.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + } + #[tokio::test] async fn pcm_written_before_pacing_starts_stays_queued() { let track = LocalAudioTrack::opus_with_config( From 147e04a7c9d668b31f805bb78a94d08e386c8e3f Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 14:19:36 +0200 Subject: [PATCH 31/37] fix: keep the RTP clock running while encoded audio or video is muted --- src/rtc/tracks/local.rs | 149 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 135 insertions(+), 14 deletions(-) diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 2a93860..6605ee1 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -213,6 +213,11 @@ impl TrackCore { )); } if self.muted.load(Ordering::SeqCst) || self.quality_paused.load(Ordering::SeqCst) { + // The RTP clock keeps time while no packet goes out. + self.packetizer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .skip_samples(samples); return Ok(()); } let payload = Bytes::copy_from_slice(payload); @@ -872,6 +877,17 @@ struct VideoClock { rtp_ts: u32, } +impl VideoClock { + /// Return the presentation time and RTP timestamp of the next frame, and + /// move the clock past its duration. + fn advance(&mut self, dur_ms: i64, samples: u32) -> (i64, u32) { + let current = (self.next_pts, self.rtp_ts); + self.next_pts = self.next_pts.saturating_add(dur_ms); + self.rtp_ts = self.rtp_ts.wrapping_add(samples); + current + } +} + struct VideoEncoding { core: TrackCore, encoder: StdMutex>, @@ -1138,6 +1154,9 @@ impl LocalVideoTrack { "write to a stopped track".to_owned(), )); } + let dur_ms = i64::try_from(duration.as_millis().max(1)).unwrap_or(i64::MAX); + let samples = + (duration.as_secs_f64() * f64::from(self.inner.encodings[0].core.clock_rate)) as u32; if self .inner .encodings @@ -1147,6 +1166,12 @@ impl LocalVideoTrack { )) .all(|encoding| encoding.core.is_output_paused()) { + // The RTP clock keeps time while no frame goes out. + self.inner + .clock + .lock() + .unwrap_or_else(|e| e.into_inner()) + .advance(dur_ms, samples); return Ok(()); } if width == 0 || height == 0 || !width.is_multiple_of(2) || !height.is_multiple_of(2) { @@ -1205,10 +1230,6 @@ impl LocalVideoTrack { ))); } - let dur_ms = i64::try_from(duration.as_millis().max(1)).unwrap_or(i64::MAX); - let samples = - (duration.as_secs_f64() * f64::from(self.inner.encodings[0].core.clock_rate)) as u32; - let permit = self .inner .encode_gate @@ -1666,16 +1687,11 @@ fn encode_i420_layers( dur_ms: i64, samples: u32, ) -> Result)>> { - let (pts, timestamp) = { - let mut clock = inner - .clock - .lock() - .unwrap_or_else(|error| error.into_inner()); - let current = (clock.next_pts, clock.rtp_ts); - clock.next_pts = clock.next_pts.saturating_add(dur_ms); - clock.rtp_ts = clock.rtp_ts.wrapping_add(samples); - current - }; + let (pts, timestamp) = inner + .clock + .lock() + .unwrap_or_else(|error| error.into_inner()) + .advance(dur_ms, samples); let count = usize::from(inner.active_encoding_count.load(Ordering::SeqCst)); let mut output = Vec::with_capacity(count); for (index, encoding) in inner.encodings.iter().take(count).enumerate() { @@ -2540,6 +2556,111 @@ mod tests { let _ = receiver.close().await; } + #[tokio::test] + async fn the_rtp_clock_runs_while_encoded_audio_is_muted() { + let track = LocalAudioTrack::opus().expect("opus track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(track.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote_rx) = peer::connect_audio_receiver(&sender).await; + let silence = [0xf8u8, 0xff, 0xfe]; + let frame = Duration::from_millis(20); + let published = LocalTrack::Audio(track.clone()); + + for _ in 0..3 { + track.write_sample(&silence, frame).await.expect("write"); + } + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + let mut before = None; + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(100), remote.read_rtp()).await + { + before = Some(packet); + } + published.set_muted(true); + for _ in 0..10 { + track + .write_sample(&silence, frame) + .await + .expect("muted write"); + } + published.set_muted(false); + track.write_sample(&silence, frame).await.expect("write"); + let (after, _) = tokio::time::timeout(Duration::from_secs(5), remote.read_rtp()) + .await + .expect("packet after the mute") + .expect("packet after the mute"); + + let before = before.expect("packets before the mute"); + assert_eq!( + after.header.timestamp.wrapping_sub(before.header.timestamp), + 11 * FRAME_SAMPLES_20MS as u32 + ); + let _ = sender.close().await; + let _ = receiver.close().await; + } + + #[tokio::test] + async fn the_rtp_clock_runs_while_a_video_track_is_muted() { + use webrtc::rtp_transceiver::rtp_codec::RTPCodecType; + + let track = LocalVideoTrack::vp9().expect("vp9 track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(track.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut remote_rx) = peer::connect_receiver(&sender, RTPCodecType::Video).await; + let frame = vec![128_u8; 320 * 240 * 3 / 2]; + let frame_time = Duration::from_millis(33); + + for _ in 0..3 { + track + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("frame"); + } + let remote = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("remote track") + .expect("remote track channel"); + let mut before = None; + while let Ok(Ok((packet, _))) = + tokio::time::timeout(Duration::from_millis(100), remote.read_rtp()).await + { + before = Some(packet); + } + track.set_muted(true); + for _ in 0..10 { + track + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("muted frame"); + } + track.set_muted(false); + track + .write_i420(&frame, 320, 240, frame_time) + .await + .expect("frame"); + let (after, _) = tokio::time::timeout(Duration::from_secs(5), remote.read_rtp()) + .await + .expect("packet after the mute") + .expect("packet after the mute"); + + let before = before.expect("packets before the mute"); + assert_eq!( + after.header.timestamp.wrapping_sub(before.header.timestamp), + 11 * 2_970 + ); + let _ = sender.close().await; + let _ = receiver.close().await; + } + #[tokio::test] async fn pcm_written_before_pacing_starts_stays_queued() { let track = LocalAudioTrack::opus_with_config( From 77f8dd3c2d0621227fab685877f2fbd4ebcb325b Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 14:30:41 +0200 Subject: [PATCH 32/37] test: correct the default subscription in a live test comment --- tests/rtc_media.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index e4f35f1..f57f55e 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -751,7 +751,7 @@ async fn no_video_on_track_without_video_subscription() { .await .expect("A publish video"); - // B subscribes to audio only (the default). + // B subscribes to audio only. let mut rx_b = track_sink(&call_b); call_b .join(JoinCallData::new(&user_b)) From 3f5530c55f3e88ca02345b4818b95f6ed5885262 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 15:35:17 +0200 Subject: [PATCH 33/37] feat: deliver a dropped remote track again after its republish --- CHANGELOG.md | 8 + src/rtc/join/connection.rs | 4 +- src/rtc/join/mod.rs | 2 +- src/rtc/join/subscriptions_runtime.rs | 108 +++++++- src/rtc/join/tests.rs | 88 +++++- src/rtc/tracks/remote.rs | 19 +- tests/rtc_media.rs | 371 ++++++++++++++++++++++++++ 7 files changed, 575 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e3c39..a8bd00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,14 @@ media of the new track. Before, a second video publish failed with cannot replace or be replaced: `publish` returns `RtcError::SimulcastReplace`. Turn such a track off and on with `mute_track` and `unmute_track`. +### A dropped remote track comes back after a republish + +Dropping a `RemoteTrack` unsubscribes it. When the publisher publishes the +track again (for example after an unmute), `on_track` now delivers a new +`RemoteTrack` for it, also when the SFU sends the media on the receiver that +the dropped track used. Before, the track did not come back. A track that is +kept through a mute gets the media again and is not delivered a second time. + ### A token-only client can prepare a call before the join `RtcClient::call` returns an `RtcCall` that is not joined yet, and diff --git a/src/rtc/join/connection.rs b/src/rtc/join/connection.rs index 5733af4..942d853 100644 --- a/src/rtc/join/connection.rs +++ b/src/rtc/join/connection.rs @@ -140,8 +140,8 @@ pub(super) fn register_on_track( reconnect_enabled, track, weak_pc, - ) - .await; + None, + ); }) })); } diff --git a/src/rtc/join/mod.rs b/src/rtc/join/mod.rs index 800d206..4b3b805 100644 --- a/src/rtc/join/mod.rs +++ b/src/rtc/join/mod.rs @@ -71,7 +71,7 @@ use super::subscriptions::{ DEFAULT_VIDEO_DIMENSION, SubscriptionConfig, SubscriptionTarget, TrackKey, }; use super::tracer::Tracer; -use super::tracks::{LocalTrack, RemoteParticipant, RemoteTrack}; +use super::tracks::{LocalTrack, RemoteParticipant, RemoteTrack, RtpPacket}; use serde_json::json; diff --git a/src/rtc/join/subscriptions_runtime.rs b/src/rtc/join/subscriptions_runtime.rs index a564828..b8e2b22 100644 --- a/src/rtc/join/subscriptions_runtime.rs +++ b/src/rtc/join/subscriptions_runtime.rs @@ -182,14 +182,15 @@ impl RtcCore { /// Correlate an inbound track to a participant, build a [`RemoteTrack`], and /// deliver it to the `on_track` callback. Called by the subscriber PC, which /// passes itself as `subscriber` so the track can send RTCP keyframe - /// requests. - pub(super) async fn handle_incoming_track( + /// requests. `first_packet` is a packet already read from `track`. + pub(super) fn handle_incoming_track( self: Arc, generation: u64, connection_epoch: u64, reconnect_enabled: Arc, track: Arc, subscriber: Weak, + first_packet: Option, ) { if !self.is_generation_current(generation) || !reconnect_enabled.load(Ordering::SeqCst) { return; @@ -231,6 +232,7 @@ impl RtcCore { .unwrap_or_else(|e| e.into_inner()) .insert(key.clone(), track_id); let weak = Arc::downgrade(&self); + let (kept_track, kept_subscriber) = (track.clone(), subscriber.clone()); // The caller can drop the track on a thread without a runtime. let runtime = tokio::runtime::Handle::current(); let on_drop = Box::new(move || { @@ -238,37 +240,63 @@ impl RtcCore { let _runtime = runtime.enter(); let task_core = core.clone(); std::mem::drop(core.spawn_generation_task(generation, async move { - task_core - .on_remote_track_dropped(generation, connection_epoch, key, track_id) - .await; + if task_core + .clone() + .on_remote_track_dropped( + generation, + connection_epoch, + key.clone(), + track_id, + ) + .await + { + task_core + .deliver_again_when_published( + generation, + connection_epoch, + reconnect_enabled, + key, + kept_track, + kept_subscriber, + ) + .await; + } })); } }); - let remote = RemoteTrack::new(track, participant, track_type, subscriber, on_drop); + let remote = RemoteTrack::new( + track, + participant, + track_type, + subscriber, + on_drop, + first_packet, + ); cb(remote); } - /// The publisher dropped their inbound track handle → unsubscribe from it. + /// The consumer dropped their inbound track handle → unsubscribe from it. + /// Returns whether the dropped track was the latest one for its key. pub(super) async fn on_remote_track_dropped( self: Arc, generation: u64, connection_epoch: u64, key: TrackKey, track_id: u64, - ) { + ) -> bool { { let connection = self.connection.lock().await; if !connection.as_ref().is_some_and(|current| { current.generation == generation && current.epoch == connection_epoch }) { - return; + return false; } let mut delivered = self .delivered_tracks .lock() .unwrap_or_else(|e| e.into_inner()); if delivered.get(&key) != Some(&track_id) { - return; + return false; } delivered.remove(&key); drop(delivered); @@ -281,10 +309,68 @@ impl RtcCore { .is_connection_current(generation, connection_epoch) .await { - return; + return false; } if let Err(e) = self.recompute_subscriptions().await { tracing::debug!(error = %e, "stream.rtc.unsubscribe_on_drop_failed"); } + true + } + + /// Read and drop the packets of a dropped track. When a packet arrives + /// after a republish subscribed the track again, deliver the track again + /// with that packet first: webrtc-rs fires `on_track` only once for each + /// receiver. + async fn deliver_again_when_published( + self: Arc, + generation: u64, + connection_epoch: u64, + reconnect_enabled: Arc, + key: TrackKey, + track: Arc, + subscriber: Weak, + ) { + while let Ok((packet, _)) = track.read_rtp().await { + // A track that a new receiver delivered, or a new connection, ends + // the wait. + let replaced = self + .delivered_tracks + .lock() + .unwrap_or_else(|e| e.into_inner()) + .contains_key(&key); + if replaced + || !self + .is_connection_current(generation, connection_epoch) + .await + { + return; + } + // The drop set `manual_unsub`, and only a republish clears it. + let dropped = self + .manual_unsub + .lock() + .unwrap_or_else(|e| e.into_inner()) + .contains(&key); + let subscribed = !dropped + && self + .active_subs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .any(|details| { + details.session_id == key.session_id && details.track_type == key.track_type + }); + if subscribed { + self.handle_incoming_track( + generation, + connection_epoch, + reconnect_enabled, + track, + subscriber, + Some(packet), + ); + return; + } + } } } diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index 6810d35..e3af698 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -1177,15 +1177,14 @@ async fn only_the_latest_remote_track_unsubscribes_when_dropped_without_a_runtim .expect("inbound track") .expect("inbound track channel"); for _ in 0..2 { - core.clone() - .handle_incoming_track( - generation, - epoch, - reconnect_enabled.clone(), - inbound.clone(), - Arc::downgrade(&receiver), - ) - .await; + core.clone().handle_incoming_track( + generation, + epoch, + reconnect_enabled.clone(), + inbound.clone(), + Arc::downgrade(&receiver), + None, + ); } let stale = remote_rx.recv().expect("stale remote track"); let remote = remote_rx.recv().expect("latest remote track"); @@ -1231,6 +1230,77 @@ async fn only_the_latest_remote_track_unsubscribes_when_dropped_without_a_runtim core.leave("test cleanup").await.expect("leave"); } +#[tokio::test] +async fn a_dropped_track_is_delivered_again_only_after_it_is_published_again() { + let core = test_core(); + let generation = prepare_joined_core(&core, "alice"); + let (connection, _sfu) = establish_fake(&core, generation).await; + let (epoch, reconnect_enabled) = (connection.epoch, connection.reconnect_enabled.clone()); + *core.connection.lock().await = Some(connection); + let (remote_tx, mut remote_rx) = tokio::sync::mpsc::unbounded_channel(); + *core + .on_track_cb + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(Arc::new(move |remote| { + let _ = remote_tx.send(remote); + })); + let audio = LocalAudioTrack::opus().expect("opus track"); + let sender = peer::new_peer_connection(&[]).await.expect("sender"); + sender + .add_track(audio.webrtc_track()) + .await + .expect("add track"); + let (receiver, mut inbound) = peer::connect_audio_receiver(&sender).await; + audio.start_pacing().await; + let inbound = tokio::time::timeout(Duration::from_secs(5), inbound.recv()) + .await + .expect("inbound track") + .expect("inbound track channel"); + core.clone().handle_incoming_track( + generation, + epoch, + reconnect_enabled, + inbound, + Arc::downgrade(&receiver), + None, + ); + let remote = remote_rx.recv().await.expect("remote track"); + let session_id = remote.participant().session_id.clone(); + // The SFU was asked to send the track. + *core + .active_subs + .lock() + .unwrap_or_else(|error| error.into_inner()) = vec![signal::TrackSubscriptionDetails { + session_id: session_id.clone(), + track_type: TrackType::Audio as i32, + ..Default::default() + }]; + + // The publisher keeps sending after the drop. + drop(remote); + assert!( + tokio::time::timeout(Duration::from_millis(500), remote_rx.recv()) + .await + .is_err(), + "a dropped live track does not come back" + ); + + core.add_published_track("bob", &session_id, TrackType::Audio as i32, None); + let again = tokio::time::timeout(Duration::from_secs(5), remote_rx.recv()) + .await + .expect("the track is delivered again after the republish") + .expect("remote track"); + tokio::time::timeout(Duration::from_secs(5), again.next_pcm()) + .await + .expect("audio of the track delivered again") + .expect("pcm frame"); + + audio.stop(); + let _ = sender.close().await; + let _ = receiver.close().await; + core.leave("test cleanup").await.expect("leave"); +} + #[tokio::test] async fn detached_connection_ignores_publish_options_from_its_sfu() { let core = test_core(); diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index 77fc89d..72f13c1 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -238,7 +238,8 @@ enum Decode { /// An inbound media track from a remote participant. /// /// Not `Clone`: it owns the inbound stream and unsubscribes on drop. Wrap it in -/// an `Arc` if you need shared read handles. +/// an `Arc` if you need shared read handles. When the publisher publishes a +/// dropped track again, `on_track` delivers a new `RemoteTrack` for it. pub struct RemoteTrack { track: Arc, participant: RemoteParticipant, @@ -258,6 +259,8 @@ pub struct RemoteTrack { last_keyframe_request: StdMutex>, /// Invoked once on drop to unsubscribe from the SFU. on_drop: StdMutex>>, + /// A packet read before this track was built, returned by the first read. + first_packet: StdMutex>, } impl RemoteTrack { @@ -265,13 +268,15 @@ impl RemoteTrack { /// /// `subscriber` is the PeerConnection the track arrived on, used to send /// RTCP keyframe requests. `on_drop` is invoked exactly once when the track - /// is dropped so the call can retract the subscription. + /// is dropped so the call can retract the subscription. `first_packet` is + /// returned before the packets of `track`. pub(crate) fn new( track: Arc, participant: RemoteParticipant, track_type: TrackType, subscriber: Weak, on_drop: Box, + first_packet: Option, ) -> Self { let params = track.codec(); let codec = Codec { @@ -294,6 +299,7 @@ impl RemoteTrack { subscriber, last_keyframe_request: StdMutex::new(None), on_drop: StdMutex::new(Some(on_drop)), + first_packet: StdMutex::new(first_packet), } } @@ -317,6 +323,7 @@ impl RemoteTrack { track_type, Arc::downgrade(peer), Box::new(|| {}), + None, ) } @@ -353,6 +360,14 @@ impl RemoteTrack { } async fn read_rtp_inner(&self) -> Option { + let first = self + .first_packet + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + if first.is_some() { + return first; + } match self.track.read_rtp().await { Ok((pkt, _attr)) => Some(pkt), Err(e) => { diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index f57f55e..e85930b 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -1609,3 +1609,374 @@ async fn sole_video_can_be_stopped_and_published_again() { let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; outcome.expect("stop and publish video again timed out"); } + +/// An audio track that B drops while A is muted arrives again after the unmute +/// +/// The SFU sends the unmuted audio on the receiver that B already has, so +/// webrtc-rs fires no new `on_track`. The SDK delivers a new `RemoteTrack` for +/// that receiver when the audio arrives. +#[tokio::test] +async fn an_audio_track_dropped_while_muted_arrives_again_after_unmute() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(150), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + let mut tracks_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig::audio_all()) + .await + .expect("B update_subscriptions"); + let mut events_b = call_b.sfu_events(); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let audio_a = LocalAudioTrack::opus().expect("opus track"); + call_a + .publish_audio(audio_a.clone()) + .await + .expect("A publish_audio"); + let feeder = spawn_tone(audio_a); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(45), + ) + .await + .expect("B never received A's audio track"); + + call_a.mute_track(TrackType::Audio).await.expect("A mute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Audio, + false, + Duration::from_secs(45) + ) + .await, + "B never received A's audio TrackUnpublished after the mute" + ); + drop(remote_a); + call_a + .unmute_track(TrackType::Audio) + .await + .expect("A unmute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Audio, + true, + Duration::from_secs(45) + ) + .await, + "B never received A's audio TrackPublished after the unmute" + ); + let again = recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(30), + ) + .await + .expect("B did not get A's audio track again after the unmute"); + let rms = drain_rms(&again, FRAME_20MS * 100, Duration::from_secs(30)).await; + feeder.abort(); + assert!( + rms > NON_SILENT_RMS, + "B got no audio on the track delivered again (rms={rms:.4})" + ); + + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("audio track dropped while muted timed out"); +} + +/// A video track that B drops while A is muted arrives again after the unmute +#[tokio::test] +async fn a_video_track_dropped_while_muted_arrives_again_after_unmute() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(150), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + let mut tracks_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig::audio_video()) + .await + .expect("B update_subscriptions"); + let mut events_b = call_b.sfu_events(); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let video_a = LocalVideoTrack::vp9().expect("vp9 track"); + call_a + .publish_video(video_a.clone()) + .await + .expect("A publish_video"); + let feeder = spawn_blue_video(video_a); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Video, + Duration::from_secs(45), + ) + .await + .expect("B never received A's video track"); + + call_a.mute_track(TrackType::Video).await.expect("A mute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Video, + false, + Duration::from_secs(45) + ) + .await, + "B never received A's video TrackUnpublished after the mute" + ); + drop(remote_a); + call_a + .unmute_track(TrackType::Video) + .await + .expect("A unmute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Video, + true, + Duration::from_secs(45) + ) + .await, + "B never received A's video TrackPublished after the unmute" + ); + let again = recv_track( + &mut tracks_b, + &user_a, + TrackType::Video, + Duration::from_secs(30), + ) + .await + .expect("B did not get A's video track again after the unmute"); + let frame = tokio::time::timeout(Duration::from_secs(45), again.next_video_frame()) + .await + .expect("B decoded no frame on the track delivered again") + .expect("the video track delivered again ended"); + feeder.abort(); + assert_packed_blue_frame(&frame); + + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("video track dropped while muted timed out"); +} + +/// A track that B keeps through a mute gets the audio again after the unmute, +/// and B gets no second track for it +#[tokio::test] +async fn a_kept_track_resumes_after_unmute_without_a_second_track() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(150), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + let mut tracks_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig::audio_all()) + .await + .expect("B update_subscriptions"); + let mut events_b = call_b.sfu_events(); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let audio_a = LocalAudioTrack::opus().expect("opus track"); + call_a + .publish_audio(audio_a.clone()) + .await + .expect("A publish_audio"); + let feeder = spawn_tone(audio_a); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(45), + ) + .await + .expect("B never received A's audio track"); + + call_a.mute_track(TrackType::Audio).await.expect("A mute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Audio, + false, + Duration::from_secs(45) + ) + .await, + "B never received A's audio TrackUnpublished after the mute" + ); + call_a + .unmute_track(TrackType::Audio) + .await + .expect("A unmute"); + assert!( + await_track_event( + &mut events_b, + &user_a, + TrackType::Audio, + true, + Duration::from_secs(45) + ) + .await, + "B never received A's audio TrackPublished after the unmute" + ); + let rms = drain_rms(&remote_a, FRAME_20MS * 100, Duration::from_secs(30)).await; + assert!( + rms > NON_SILENT_RMS, + "B got no audio on its kept track after the unmute (rms={rms:.4})" + ); + assert!( + recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(3) + ) + .await + .is_none(), + "B got A's audio track a second time" + ); + feeder.abort(); + + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("kept track through a mute timed out"); +} + +/// A track that B drops while A keeps sending does not come back +#[tokio::test] +async fn a_dropped_live_track_does_not_come_back() { + let Some(client) = common::client_or_skip() else { + return; + }; + init_tracing(); + + let user_a = common::unique_id("a"); + let user_b = common::unique_id("b"); + let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; + + let outcome = tokio::time::timeout(Duration::from_secs(150), async { + let call_a = client.video().call("default", &call_id); + let call_b = client.video().call("default", &call_id); + + let mut tracks_b = track_sink(&call_b); + call_b + .join(JoinCallData::new(&user_b)) + .await + .expect("B join"); + call_b + .update_subscriptions(SubscriptionConfig::audio_all()) + .await + .expect("B update_subscriptions"); + + call_a + .join(JoinCallData::new(&user_a)) + .await + .expect("A join"); + let audio_a = LocalAudioTrack::opus().expect("opus track"); + call_a + .publish_audio(audio_a.clone()) + .await + .expect("A publish_audio"); + let feeder = spawn_tone(audio_a); + let remote_a = recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(45), + ) + .await + .expect("B never received A's audio track"); + let rms = drain_rms(&remote_a, FRAME_20MS * 25, Duration::from_secs(30)).await; + assert!(rms > NON_SILENT_RMS, "B got no audio before the drop"); + + drop(remote_a); + assert!( + recv_track( + &mut tracks_b, + &user_a, + TrackType::Audio, + Duration::from_secs(5) + ) + .await + .is_none(), + "B got A's audio track again after it dropped the track" + ); + feeder.abort(); + + call_a.leave().await.expect("A leave"); + call_b.leave().await.expect("B leave"); + }) + .await; + + let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; + outcome.expect("dropped live track timed out"); +} From b7bb737b5ac7e67ee9f724d40d4e2ef0d0e94064 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 15:35:25 +0200 Subject: [PATCH 34/37] docs: add the branch fixes to the changelog --- CHANGELOG.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8bd00e..893827d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,7 +78,9 @@ publish new tracks. `write_pcm` queues up to `LocalAudioTrackConfig::pcm_queue_capacity`, 60 s by default (it was 200 ms). A producer that writes faster than real time gets -`PcmQueueOverflow` only when the queue is full. +`PcmQueueOverflow` only when the queue is full. The capacity must hold at least +one 20 ms frame; `LocalAudioTrack::opus_with_config` returns `RtcError::Media` +for a smaller value. ## New Features @@ -144,6 +146,34 @@ on `Call` (`get_call_participant_session_metrics`, `query_call_session_participant_stats`, `get_call_session_participant_stats_timeline`). +## Fixes + +- After a join or a reconnect, `sfu_events()` gives `ParticipantJoined`, + `ParticipantUpdated` and `ParticipantLeft` only for the participants that + changed since the previous join response. Before, each join response sent + `ParticipantJoined` for every participant. A change of speaking state, audio + level, connection quality or dominant speaker does not count: its own event + reports it. +- Each join starts with an empty participant list and call state, also after + a join that failed or was dropped. A new join no longer reports + `ParticipantLeft` for the participants of the earlier attempt. +- `ParticipantCountChanged` comes only when the count changes, not with each + SFU health check. +- A dropped `RemoteTrack` unsubscribes its track only when it is the latest + track for that participant and track type. It can be dropped on a thread + without a Tokio runtime. +- A dropped `join` future sets the call back to `Idle`, so the next `join` is + accepted. A dropped `leave` future still sends the leave, closes the + connection and sets `Left`. +- The first join retry waits 250–500 ms and a later one up to 2.5 s, as in + stream-video-js. +- The pacer task of a `LocalAudioTrack` ends when the track is dropped. A + decoded `PcmFrame` holds only its own samples. +- A muted audio track no longer encodes its PCM. +- While a track is muted, its RTP clock keeps running, for audio and video. The + first packet after an unmute shows the length of the mute (RFC 3550). Before, + the timestamps continued from the last packet before the mute. + # v0.1.0-preview.2 docs.rs builds on current nightly. `doc_auto_cfg` was removed in 1.92 and From dbfd13cdfceabd75c0e5aa1ee76bdfc372d11f23 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 5 Oct 2026 20:27:17 +0200 Subject: [PATCH 35/37] feat!: remove H.264 and make the OpenAI example voice-only --- CHANGELOG.md | 17 +- Cargo.lock | 56 ----- Cargo.toml | 4 - README.md | 19 +- benches/media_baseline.rs | 89 +------ examples/gpt_realtime_bot.rs | 188 ++------------ src/rtc/codecs/h264.rs | 470 ----------------------------------- src/rtc/codecs/mod.rs | 2 - src/rtc/codecs/rtp_h264.rs | 360 --------------------------- src/rtc/join/tests.rs | 106 ++++---- src/rtc/peer/connection.rs | 16 +- src/rtc/peer/publisher.rs | 30 ++- src/rtc/publish_options.rs | 31 +-- src/rtc/tracks/local.rs | 204 +++------------ src/rtc/tracks/remote.rs | 71 +----- tests/gpt_realtime_bot.rs | 73 +----- tests/rtc_media.rs | 93 +------ 17 files changed, 167 insertions(+), 1662 deletions(-) delete mode 100644 src/rtc/codecs/h264.rs delete mode 100644 src/rtc/codecs/rtp_h264.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 893827d..d70857d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,17 @@ default (it was 200 ms). A producer that writes faster than real time gets one 20 ms frame; `LocalAudioTrack::opus_with_config` returns `RtcError::Media` for a smaller value. +### H.264 is removed + +The SDK no longer depends on OpenH264, so it neither encodes nor decodes H.264. +`LocalVideoTrack::h264`, `h264_with_config`, `h264_simulcast` and +`PreferredVideoCodec::H264` are removed, and `"h264"` no longer parses as a +`PreferredVideoCodec`. The publisher and the subscriber negotiate only VP8 and +VP9 video, so a publisher that sends only H.264 gives the agent no video track. +Use `vp9` or `vp9_svc` for camera video and `vp8` or `vp8_simulcast` for screen +share. The `gpt_realtime_bot` example is voice-only, because OpenAI Realtime +accepts only H.264 video. + ## New Features ### Pacing control for PCM tracks @@ -100,9 +111,9 @@ After `stop_publish`, a new audio, video, or screen-share track of the same kind and publish option takes over the sender that the stop kept, as JS `replaceTrack` does. Remote participants keep their remote track and get the media of the new track. Before, a second video publish failed with -"participant not found". A simulcast track (`h264_simulcast`, `vp8_simulcast`) -cannot replace or be replaced: `publish` returns `RtcError::SimulcastReplace`. -Turn such a track off and on with `mute_track` and `unmute_track`. +"participant not found". A simulcast track (`vp8_simulcast`) cannot replace or +be replaced: `publish` returns `RtcError::SimulcastReplace`. Turn such a track +off and on with `mute_track` and `unmute_track`. ### A dropped remote track comes back after a republish diff --git a/Cargo.lock b/Cargo.lock index e8690ce..bc8f83f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,12 +294,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "bytemuck" -version = "1.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" - [[package]] name = "byteorder" version = "1.5.0" @@ -1150,7 +1144,6 @@ dependencies = [ "futures-util", "hmac 0.13.0", "indexmap", - "openh264", "opus", "prost", "prost-build", @@ -1918,15 +1911,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "nasm-rs" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" -dependencies = [ - "log", -] - [[package]] name = "nix" version = "0.26.4" @@ -2020,27 +2004,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openh264" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef0655e143954965073374f5390411131590d0bc793208aabf7c6785430fa00" -dependencies = [ - "openh264-sys2", - "wide", -] - -[[package]] -name = "openh264-sys2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad97e73d98000c46623ec4719e4fd2d7f79076a75350af8ae3878abf682c071d" -dependencies = [ - "cc", - "nasm-rs", - "walkdir", -] - [[package]] name = "openssl-probe" version = "0.2.1" @@ -2904,15 +2867,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "safe_arch" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" -dependencies = [ - "bytemuck", -] - [[package]] name = "same-file" version = "1.0.6" @@ -4042,16 +3996,6 @@ dependencies = [ "rustix 0.38.44", ] -[[package]] -name = "wide" -version = "0.7.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" -dependencies = [ - "bytemuck", - "safe_arch", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 7a89f67..9a8e2ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,10 +80,6 @@ uuid = { version = "1.24.0", features = ["v4"] } env-libvpx-sys = { version = "5.1.3", features = ["generate"] } webrtc = "0.17.2" flate2 = "1.1.9" -# In-process H264 via the maintained ralfbiedert wrapper. Pin 0.8.1 because -# 0.9.x's wide dependency requires Rust 1.89 while this crate supports 1.88. -# OpenH264 is BSD-2-Clause; downstreams must assess H264 patent obligations. -openh264 = "=0.8.1" [dev-dependencies] anyhow = "1.0.86" diff --git a/README.md b/README.md index 7d9d77a..d351830 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ remote audio and video, transform it, and publish media back into the call. audio, video, and screen-share tracks. - Observe typed participant, connection-quality, pin, grant, and inbound-pause state from the SFU. -- Read Opus audio as PCM, decode VP8/VP9/H264 video as I420, or work with raw +- Read Opus audio as PCM, decode VP8/VP9 video as I420, or work with raw RTP packets. - Resample and rechannel PCM, convert it to 32-bit float, raw bytes, WAV, or G.711, and slice it into chunks and sliding windows. @@ -249,8 +249,8 @@ async fn run_media_bridge(call: &Call) -> Result<(), Box> } ``` -For a complete bridge with cancellation, barge-in, audio and video processing, -and deterministic cleanup, see [`gpt_realtime_bot`](https://github.com/GetStream/stream-video-rust/blob/main/examples/gpt_realtime_bot.rs). +For a complete voice bridge with cancellation, barge-in, and deterministic +cleanup, see [`gpt_realtime_bot`](https://github.com/GetStream/stream-video-rust/blob/main/examples/gpt_realtime_bot.rs). For selective agents, set `role_filters` and `max_subscriptions` in `SubscriptionConfig`, or use `Call::update_subscription_targets` with @@ -305,10 +305,9 @@ bitrate, and every codec has a `_with_config` constructor that takes it. Layered publishing is opt-in. `LocalVideoTrack::vp9_svc()` provides camera SVC with up to three spatial and temporal layers on one SSRC. -`LocalVideoTrack::h264_simulcast()` supports camera video and `LocalVideoTrack::vp8_simulcast()` supports screen share with a `q`/`h`/`f` RID -ladder on one m-line. Each of the three is a shortcut for the matching -`_with_config` call on a `server_managed` config. All three follow SFU quality +ladder on one m-line. Each of the two is a shortcut for the matching +`_with_config` call on a `server_managed` config. Both follow SFU quality updates. To publish fewer layers than the SFU offers, set `VideoLayering` directly and cap the counts. @@ -335,8 +334,7 @@ fn video_tracks() -> RtcResult<()> { }; let capped_svc = LocalVideoTrack::vp9_with_config(capped)?; - // H264 camera simulcast, and VP8 screen-share simulcast. - let h264_camera = LocalVideoTrack::h264_simulcast()?; + // VP8 screen-share simulcast. let screen_share = LocalVideoTrack::vp8_simulcast()?; Ok(()) } @@ -370,7 +368,7 @@ Join a call as a backend participant: cargo run --example join_call ``` -Run the Stream-to-OpenAI Realtime audio/video bridge: +Run the Stream-to-OpenAI Realtime voice bridge: ```bash cargo run --example gpt_realtime_bot @@ -406,9 +404,6 @@ tracing::info!(event_type = event.event_type(), "verified Stream webhook"); # } ``` -H264 can be subject to patent obligations in some jurisdictions. Applications -that distribute H264 functionality must assess their own requirements. - ## Contributing Contributions are welcome. See [CONTRIBUTING.md](https://github.com/GetStream/stream-video-rust/blob/main/CONTRIBUTING.md) for local diff --git a/benches/media_baseline.rs b/benches/media_baseline.rs index 180fab6..a477f7c 100644 --- a/benches/media_baseline.rs +++ b/benches/media_baseline.rs @@ -1,6 +1,6 @@ //! Reproducible local media baselines for production-hardening work. //! -//! These benchmarks intentionally exercise the same private VPx/H264 encoders, +//! These benchmarks intentionally exercise the same private VPx encoders, //! decoders, and RTP packetizers as the SDK without making benchmark hooks part //! of the public API. @@ -18,10 +18,6 @@ mod rtc; use rtc::codecs::rtp_vpx::VpxRtpPacketizer; use rtc::codecs::vpx::{VpxCodec, VpxDecoder, VpxEncoder}; -use rtc::codecs::{ - h264::{H264Decoder, H264Encoder}, - rtp_h264::H264RtpPacketizer, -}; const FRAME_DURATION_MS: i64 = 33; const RTP_PAYLOAD_MTU: usize = 1_200; @@ -63,18 +59,6 @@ fn encoded_keyframe(codec: VpxCodec, width: u32, height: u32) -> Vec { .data } -fn encoded_h264_keyframe(width: u32, height: u32) -> Vec { - let source = i420_fixture(width, height); - let mut encoder = H264Encoder::new(1_000_000).expect("create fixture H264 encoder"); - let mut encoded = Vec::new(); - let keyframe = encoder - .encode_into(&source, width, height, true, &mut encoded) - .expect("encode fixture H264 keyframe"); - assert!(keyframe, "fixture H264 frame must be a keyframe"); - assert!(!encoded.is_empty(), "fixture H264 frame must not be empty"); - encoded -} - fn bench_resampling(criterion: &mut Criterion) { let mut group = criterion.benchmark_group("resample"); for (name, input) in [ @@ -155,18 +139,6 @@ fn bench_rtp_packetization(criterion: &mut Criterion) { }, ); } - let frame = encoded_h264_keyframe(1_280, 720); - let mut packetizer = H264RtpPacketizer::default(); - group.throughput(Throughput::Bytes(frame.len() as u64)); - group.bench_function("h264_720p_keyframe", |bencher| { - bencher.iter(|| { - black_box( - packetizer - .packetize(black_box(&frame), RTP_PAYLOAD_MTU) - .expect("packetize H264 benchmark frame"), - ) - }); - }); group.finish(); } @@ -226,43 +198,6 @@ fn bench_vpx_decode(criterion: &mut Criterion) { group.finish(); } -fn bench_h264_encode(criterion: &mut Criterion) { - let mut group = criterion.benchmark_group("h264_encode"); - for (width, height) in [(640, 360), (1_280, 720)] { - let mut source = i420_fixture(width, height); - let mut encoder = H264Encoder::new(1_000_000).expect("create H264 encoder"); - let mut encoded = Vec::new(); - encoder - .encode_into(&source, width, height, true, &mut encoded) - .expect("warm H264 encoder"); - let mut frame_index = 0usize; - - group.throughput(Throughput::Bytes(source.len() as u64)); - group.bench_function( - format!("h264_{width}x{height}_realtime_sequence"), - |bencher| { - bencher.iter(|| { - let offset = frame_index % source.len(); - source[offset] = source[offset].wrapping_add(1); - let force_keyframe = frame_index.is_multiple_of(30); - let keyframe = encoder - .encode_into( - black_box(&source), - width, - height, - force_keyframe, - black_box(&mut encoded), - ) - .expect("encode H264 benchmark frame"); - frame_index = frame_index.wrapping_add(1); - black_box((keyframe, encoded.len())) - }); - }, - ); - } - group.finish(); -} - /// Bounded multi-track publisher load: per iteration, encode a fixed fan-out of /// concurrent audio and video tracks (3 Opus + 2 VP9 640x360) representing a /// realtime multi-track publisher/forwarder cycle. The point is a *bounded*, @@ -322,26 +257,6 @@ fn bench_multitrack_load(criterion: &mut Criterion) { group.finish(); } -fn bench_h264_decode(criterion: &mut Criterion) { - let mut group = criterion.benchmark_group("h264_decode"); - for (width, height) in [(640, 360), (1_280, 720)] { - let encoded = encoded_h264_keyframe(width, height); - let mut decoder = H264Decoder::new().expect("create H264 decoder"); - - group.throughput(Throughput::Bytes(encoded.len() as u64)); - group.bench_function(format!("h264_{width}x{height}_keyframe"), |bencher| { - bencher.iter(|| { - black_box( - decoder - .decode(black_box(&encoded), 90_000) - .expect("decode H264 benchmark frame"), - ) - }); - }); - } - group.finish(); -} - criterion_group! { name = media_baselines; config = Criterion::default() @@ -354,8 +269,6 @@ criterion_group! { bench_rtp_packetization, bench_vpx_encode, bench_vpx_decode, - bench_h264_encode, - bench_h264_decode, bench_multitrack_load } criterion_main!(media_baselines); diff --git a/examples/gpt_realtime_bot.rs b/examples/gpt_realtime_bot.rs index 3ff4d7b..a101831 100644 --- a/examples/gpt_realtime_bot.rs +++ b/examples/gpt_realtime_bot.rs @@ -5,12 +5,7 @@ //! ``` //! //! The bot joins a Stream call, connects to OpenAI Realtime, and bridges audio -//! in both directions. It also sends downscaled H264 video frames to OpenAI. -//! -//! The OpenAI video track uses H264. [`LocalVideoTrack`] performs the encode and -//! RTP packetization in-process through OpenH264. OpenH264 is BSD-2-Clause; -//! applications distributing H264 functionality must evaluate their own patent -//! obligations. +//! in both directions. //! //! # OpenAI Realtime handshake //! @@ -28,22 +23,20 @@ //! after ICE gathering completes. OpenAI advertises host candidates only, so //! that PeerConnection needs no ICE servers. //! -//! # OpenAI video is H264-only +//! # Voice only //! -//! Offering VP8, VP9, AV1, H265, and H264 returns an answer with nine H264 -//! payload types and nothing else (`packetization-mode` 0/1, profiles `42001f`, -//! `42e01f`, `640028`–`640033`). With a video transceiver in the offer and no -//! H264 encoder, applying that answer fails with `unable to start track, codec -//! is not supported by remote` and takes the *audio* leg down with it. OpenAI -//! cannot renegotiate video mid-session without losing session state, so this is -//! decided at connect time — hence the in-process H264 encode below. +//! OpenAI accepts only H264 on a WebRTC video track: an offer with VP8, VP9, +//! AV1, H265, and H264 gets an answer with H264 payload types and nothing else. +//! The SDK has no H264 encoder, and a video transceiver that cannot send the +//! answered codec fails `set_remote_description` and takes the audio leg down +//! with it. So the bot offers audio only. //! //! Env: `STREAM_API_KEY`, `STREAM_API_SECRET`, `OPENAI_API_KEY`, //! `OPENAI_REALTIME_MODEL` (fallback `OPENAI_MODEL`, default `gpt-realtime`), //! optional `EXAMPLE_BASE_URL` / `EXAMPLE_CALL_TYPE` / `EXAMPLE_CALL_ID`. use std::future::Future; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex as StdMutex}; use std::time::Duration; @@ -51,8 +44,7 @@ use anyhow::{Context, Result, anyhow, bail}; use getstream::models::{CallRequest, GetOrCreateCallRequest, MemberRequest, UserRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - JoinCallData, LocalAudioTrack, LocalAudioTrackConfig, LocalVideoTrack, RemoteTrack, RtcError, - SubscriptionConfig, TrackSubscriptionConfig, VideoFrame, + JoinCallData, LocalAudioTrack, LocalAudioTrackConfig, RemoteTrack, RtcError, SubscriptionConfig, }; use getstream::video::Call; use getstream::{Stream, TokenOptions}; @@ -78,8 +70,6 @@ const OPENAI_EVENT_CHANNEL: &str = "oai-events"; const OPENAI_CONNECT_TIMEOUT: Duration = Duration::from_secs(45); const TRACK_EVENT_CAPACITY: usize = 32; const BRIDGE_STOP_TIMEOUT: Duration = Duration::from_secs(5); -const VIDEO_SEND_INTERVAL: Duration = Duration::from_secs(1); -const VIDEO_SEND_MAX_EDGE: u32 = 512; /// OpenAI Realtime connection settings resolved from the environment. #[derive(Clone)] pub struct OpenAiConfig { @@ -116,28 +106,11 @@ impl BotHandle { self.progress.audio_seen.load(Ordering::Relaxed) } - /// Number of inbound video frames decoded by the bot. - pub fn video_frames_decoded(&self) -> u64 { - self.progress.frames_decoded.load(Ordering::Relaxed) - } - - /// Number of video frames encoded for OpenAI. - pub fn video_frames_encoded(&self) -> u64 { - self.progress.frames_encoded.load(Ordering::Relaxed) - } - /// The OpenAI PeerConnection's connection state (ICE + DTLS). pub fn openai_connection_state(&self) -> RTCPeerConnectionState { self.openai.pc.connection_state() } - /// The codec OpenAI negotiated on the video m-line, or `None` if its answer - /// carried no video at all. - pub async fn openai_video_codec(&self) -> Option { - let remote = self.openai.pc.remote_description().await?; - negotiated_video_codec(&remote.sdp) - } - /// Cancel and reap bridge tasks, close OpenAI, and leave the Stream call. pub async fn shutdown(self) -> Result<()> { let bridge_result = self.bridge.shutdown().await; @@ -151,7 +124,6 @@ struct OpenAiSession { pc: Arc, _events: Arc, mic: LocalAudioTrack, - camera: LocalVideoTrack, tasks: TaskGroup, } @@ -340,14 +312,6 @@ async fn configure_openai( }) })); - let camera = LocalVideoTrack::h264().context("H264 track for OpenAI")?; - spawn_rtcp_drain( - &tasks, - pc.add_track(camera.webrtc_track()) - .await - .context("add video track to the OpenAI PeerConnection")?, - ); - let weak_pc = Arc::downgrade(&pc); let to_stream = stream_audio.clone(); let audio_tasks = tasks.clone(); @@ -395,7 +359,7 @@ async fn configure_openai( }) })); - let instructions = "You are a friendly voice assistant on a live video call. \ + let instructions = "You are a friendly voice assistant on a live call. \ Greet the caller warmly in one short sentence, then answer questions concisely."; let offer = pc.create_offer(None).await.context("create offer")?; @@ -411,12 +375,6 @@ async fn configure_openai( .sdp; let answer = exchange_sdp(cfg, &offer_sdp, &session_config(cfg, instructions)).await?; - match negotiated_video_codec(&answer) { - Some(codec) => tracing::info!(%codec, "gpt_realtime_bot: OpenAI negotiated video"), - None => tracing::warn!( - "gpt_realtime_bot: OpenAI's answer carries no video m-line — the model will not see the caller" - ), - } pc.set_remote_description( RTCSessionDescription::answer(answer).context("parse OpenAI SDP answer")?, ) @@ -428,7 +386,6 @@ async fn configure_openai( pc, _events: events, mic, - camera, tasks, }) } @@ -443,19 +400,6 @@ fn spawn_rtcp_drain( }); } -/// The codec on the answer's video m-line, e.g. `H264/90000 (payload type 100)`. -/// `None` means the answer has no video at all. -fn negotiated_video_codec(sdp: &str) -> Option { - let m_line = sdp.lines().find(|l| l.starts_with("m=video"))?; - let payload_type = m_line.split_whitespace().nth(3)?; - let rtpmap = format!("a=rtpmap:{payload_type} "); - let codec = sdp - .lines() - .find_map(|l| l.strip_prefix(&rtpmap)) - .unwrap_or("unknown"); - Some(format!("{codec} (payload type {payload_type})")) -} - async fn send_event(channel: &Arc, event: &Value) -> Result<()> { channel .send_text(event.to_string()) @@ -483,9 +427,6 @@ fn handle_openai_event(payload: &[u8], stream_audio: &LocalAudioTrack) { #[derive(Default)] struct MediaProgress { audio_seen: AtomicBool, - frames_decoded: AtomicU64, - frames_encoded: AtomicU64, - latest_frame: StdMutex>, } async fn pump_audio_in( @@ -512,82 +453,6 @@ async fn pump_audio_in( } } -async fn pump_video_in( - remote: RemoteTrack, - progress: Arc, - mut cancel: watch::Receiver, -) { - let user = remote.participant().user_id.clone(); - tracing::info!(%user, codec = %remote.codec().mime_type, "gpt_realtime_bot: video track"); - - loop { - let frame = tokio::select! { - _ = cancel.changed() => return, - frame = remote.next_video_frame() => frame, - }; - let Some(frame) = frame else { break }; - let n = progress.frames_decoded.fetch_add(1, Ordering::Relaxed) + 1; - if n == 1 || n.is_multiple_of(30) { - tracing::info!( - %user, - frames = n, - width = frame.width, - height = frame.height, - "gpt_realtime_bot: decoded video" - ); - } - *progress - .latest_frame - .lock() - .unwrap_or_else(|e| e.into_inner()) = Some(frame); - } - tracing::info!(%user, "gpt_realtime_bot: video track ended"); -} - -async fn pump_video_encoder( - progress: Arc, - camera: LocalVideoTrack, - mut cancel: watch::Receiver, -) { - let mut interval = tokio::time::interval(VIDEO_SEND_INTERVAL); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - tokio::select! { - _ = cancel.changed() => return, - _ = interval.tick() => {} - } - let Some(frame) = progress - .latest_frame - .lock() - .unwrap_or_else(|e| e.into_inner()) - .take() - else { - continue; - }; - let frame = frame.downscale_to_fit(VIDEO_SEND_MAX_EDGE); - - match camera - .write_i420(&frame.data, frame.width, frame.height, VIDEO_SEND_INTERVAL) - .await - { - Ok(()) => { - let n = progress.frames_encoded.fetch_add(1, Ordering::Relaxed) + 1; - if n == 1 { - tracing::info!( - width = frame.width, - height = frame.height, - "gpt_realtime_bot: first H264 frame for OpenAI" - ); - } - } - Err(e) => { - tracing::warn!(error = %e, "gpt_realtime_bot: H264 encode failed"); - return; - } - } - } -} - struct MediaBridge { cancel: watch::Sender, task: JoinHandle<()>, @@ -598,14 +463,10 @@ impl MediaBridge { mut track_rx: mpsc::Receiver, progress: Arc, mic: LocalAudioTrack, - camera: LocalVideoTrack, ) -> Self { let (cancel, mut cancel_rx) = watch::channel(false); - let encoder_cancel = cancel_rx.clone(); - let encoder_progress = progress.clone(); let task = tokio::spawn(async move { let mut pumps = JoinSet::new(); - pumps.spawn(pump_video_encoder(encoder_progress, camera, encoder_cancel)); loop { tokio::select! { @@ -617,14 +478,9 @@ impl MediaBridge { progress.audio_seen.store(true, Ordering::Relaxed); pumps.spawn(pump_audio_in(track, mic.clone(), cancel_rx.clone())); } - TrackType::Video | TrackType::ScreenShare => { - pumps.spawn(pump_video_in( - track, - progress.clone(), - cancel_rx.clone(), - )); - } - TrackType::Unspecified => {} + TrackType::Video + | TrackType::ScreenShare + | TrackType::Unspecified => {} } } result = pumps.join_next(), if !pumps.is_empty() => { @@ -687,7 +543,7 @@ async fn publish_bot_audio(call: &Call) -> Result { } /// Join the call as `bot`, publish audio, connect the OpenAI PeerConnection, and -/// subscribe to audio + video. Returns a [`BotHandle`]; the caller stays +/// subscribe to audio. Returns a [`BotHandle`]; the caller stays /// connected and calls [`BotHandle::shutdown`] to leave. pub async fn start_bot( client: &Stream, @@ -743,26 +599,14 @@ pub async fn start_bot( }); if let Err(error) = call - .update_subscriptions(SubscriptionConfig { - default: TrackSubscriptionConfig { - track_types: vec![TrackType::Audio, TrackType::Video], - video_dimension: (640, 360), - ..Default::default() - }, - ..Default::default() - }) + .update_subscriptions(SubscriptionConfig::audio_all()) .await .context("update_subscriptions") { return Err(cleanup_failed_bot_setup(&call, Some(openai), error).await); } - let bridge = MediaBridge::spawn( - track_rx, - progress.clone(), - openai.mic.clone(), - openai.camera.clone(), - ); + let bridge = MediaBridge::spawn(track_rx, progress.clone(), openai.mic.clone()); Ok(BotHandle { call, diff --git a/src/rtc/codecs/h264.rs b/src/rtc/codecs/h264.rs deleted file mode 100644 index 874b428..0000000 --- a/src/rtc/codecs/h264.rs +++ /dev/null @@ -1,470 +0,0 @@ -//! In-process H264 encode/decode through the maintained `openh264` wrapper. -//! -//! OpenH264 is BSD-2-Clause, but H264 may be covered by patents in some -//! jurisdictions. Applications distributing H264 functionality must evaluate -//! their own patent-licensing obligations. - -use openh264::OpenH264API; -use openh264::decoder::Decoder; -use openh264::encoder::{ - BitRate, Complexity, Encoder, EncoderConfig, FrameRate, FrameType, IntraFramePeriod, Level, - Profile, RateControlMode, SpsPpsStrategy, UsageType, -}; -use openh264::formats::YUVSource; - -use crate::rtc::error::{Result, RtcError}; -use crate::rtc::video_frame::{VideoFrame, i420_len}; - -const MAX_H264_DECODE_DIMENSION: usize = 3_840; -const MAX_H264_DECODE_PIXELS: usize = 3_840 * 2_160; -const MAX_H264_ACCESS_UNIT_BYTES: usize = 16 * 1024 * 1024; -const H264_MACROBLOCK_EDGE: u32 = 16; -const H264_LEVEL_3_1_MAX_CODED_EDGE: u32 = 2_704; -const H264_LEVEL_3_1_MAX_FRAME_MACROBLOCKS: u32 = 3_600; -const H264_LEVEL_3_1_MAX_MACROBLOCKS_PER_SECOND: u32 = 108_000; -const H264_ENCODER_MAX_FRAME_RATE: u32 = 30; - -fn minimum_frame_duration_ns() -> u128 { - 1_000_000_000u128.div_ceil(u128::from(H264_ENCODER_MAX_FRAME_RATE)) -} - -fn minimum_mbps_duration_ns(macroblocks: u32) -> u128 { - (u128::from(macroblocks) * 1_000_000_000) - .div_ceil(u128::from(H264_LEVEL_3_1_MAX_MACROBLOCKS_PER_SECOND)) -} - -fn validate_h264_encode_dimensions(width: u32, height: u32) -> Result { - if width == 0 - || height == 0 - || !width.is_multiple_of(2) - || !height.is_multiple_of(2) - || width > H264_LEVEL_3_1_MAX_CODED_EDGE - || height > H264_LEVEL_3_1_MAX_CODED_EDGE - { - return Err(RtcError::Media(format!( - "H264 level 3.1 encode dimensions must be non-zero, even, and at most \ - {H264_LEVEL_3_1_MAX_CODED_EDGE} per edge (got {width}x{height})" - ))); - } - let macroblocks = width - .div_ceil(H264_MACROBLOCK_EDGE) - .checked_mul(height.div_ceil(H264_MACROBLOCK_EDGE)) - .ok_or_else(|| RtcError::Media("H264 macroblock count overflow".to_owned()))?; - if macroblocks > H264_LEVEL_3_1_MAX_FRAME_MACROBLOCKS { - return Err(RtcError::Media(format!( - "H264 level 3.1 allows at most {H264_LEVEL_3_1_MAX_FRAME_MACROBLOCKS} \ - macroblocks per frame (got {macroblocks} for {width}x{height})" - ))); - } - Ok(macroblocks) -} - -pub(crate) fn validate_h264_encode_request( - width: u32, - height: u32, - duration: std::time::Duration, -) -> Result<()> { - let macroblocks = validate_h264_encode_dimensions(width, height)?; - let duration_ns = duration.as_nanos(); - let minimum_duration_ns = - minimum_frame_duration_ns().max(minimum_mbps_duration_ns(macroblocks)); - if duration_ns < minimum_duration_ns { - return Err(RtcError::Media(format!( - "H264 level 3.1 encode duration is too short for {width}x{height}: \ - need at least {minimum_duration_ns} ns (got {duration_ns} ns)" - ))); - } - Ok(()) -} - -pub(crate) fn access_unit_has_idr(data: &[u8]) -> bool { - let mut offset = 0usize; - while offset + 4 < data.len() { - let start_len = if data[offset..].starts_with(&[0, 0, 0, 1]) { - 4 - } else if data[offset..].starts_with(&[0, 0, 1]) { - 3 - } else { - offset += 1; - continue; - }; - let nal_offset = offset + start_len; - if data - .get(nal_offset) - .is_some_and(|header| header & 0x1f == 5) - { - return true; - } - offset = nal_offset.saturating_add(1); - } - false -} - -struct PackedI420<'a> { - data: &'a [u8], - width: usize, - height: usize, - y_len: usize, - chroma_len: usize, -} - -impl<'a> PackedI420<'a> { - fn new(data: &'a [u8], width: u32, height: u32) -> Result { - let width = usize::try_from(width) - .map_err(|_| RtcError::Media("H264 width does not fit usize".to_owned()))?; - let height = usize::try_from(height) - .map_err(|_| RtcError::Media("H264 height does not fit usize".to_owned()))?; - if width == 0 - || height == 0 - || !width.is_multiple_of(2) - || !height.is_multiple_of(2) - || width > MAX_H264_DECODE_DIMENSION - || height > MAX_H264_DECODE_DIMENSION - { - return Err(RtcError::Media(format!( - "H264 I420 dimensions must be non-zero, even, and at most \ - {MAX_H264_DECODE_DIMENSION} per edge (got {width}x{height})" - ))); - } - let pixels = width - .checked_mul(height) - .filter(|pixels| *pixels <= MAX_H264_DECODE_PIXELS) - .ok_or_else(|| RtcError::Media("H264 frame dimensions are too large".to_owned()))?; - let chroma_len = pixels / 4; - let expected = pixels - .checked_add(chroma_len.saturating_mul(2)) - .ok_or_else(|| RtcError::Media("H264 I420 length overflow".to_owned()))?; - if data.len() < expected { - return Err(RtcError::Media(format!( - "H264 I420 buffer too small: {} bytes for {width}x{height} (need {expected})", - data.len() - ))); - } - Ok(Self { - data: &data[..expected], - width, - height, - y_len: pixels, - chroma_len, - }) - } -} - -impl YUVSource for PackedI420<'_> { - fn dimensions(&self) -> (usize, usize) { - (self.width, self.height) - } - - fn strides(&self) -> (usize, usize, usize) { - (self.width, self.width / 2, self.width / 2) - } - - fn y(&self) -> &[u8] { - &self.data[..self.y_len] - } - - fn u(&self) -> &[u8] { - &self.data[self.y_len..self.y_len + self.chroma_len] - } - - fn v(&self) -> &[u8] { - &self.data[self.y_len + self.chroma_len..] - } -} - -pub(crate) struct H264Encoder { - encoder: Encoder, - dimensions: Option<(u32, u32)>, -} - -impl H264Encoder { - pub(crate) fn new(bitrate_bps: u32) -> Result { - let config = EncoderConfig::new() - .bitrate(BitRate::from_bps(bitrate_bps)) - .max_frame_rate(FrameRate::from_hz(30.0)) - .rate_control_mode(RateControlMode::Bitrate) - .skip_frames(true) - .usage_type(UsageType::CameraVideoRealTime) - .sps_pps_strategy(SpsPpsStrategy::ConstantId) - .profile(Profile::Baseline) - .level(Level::Level_3_1) - .complexity(Complexity::Low) - .num_threads(2) - .intra_frame_period(IntraFramePeriod::from_num_frames(30)); - let encoder = Encoder::with_api_config(OpenH264API::from_source(), config) - .map_err(|error| RtcError::Media(format!("OpenH264 encoder init: {error}")))?; - Ok(Self { - encoder, - dimensions: None, - }) - } - - #[cfg(test)] - fn dimensions(&self) -> Option<(u32, u32)> { - self.dimensions - } - - pub(crate) fn encode_into( - &mut self, - i420: &[u8], - width: u32, - height: u32, - force_keyframe: bool, - output: &mut Vec, - ) -> Result { - validate_h264_encode_dimensions(width, height)?; - let source = PackedI420::new(i420, width, height)?; - let dimensions_changed = self.dimensions != Some((width, height)); - if force_keyframe && self.dimensions.is_some() && !dimensions_changed { - self.encoder.force_intra_frame(); - } - - let bitstream = self - .encoder - .encode(&source) - .map_err(|error| RtcError::Media(format!("OpenH264 encode: {error}")))?; - let keyframe = matches!(bitstream.frame_type(), FrameType::IDR | FrameType::I); - output.clear(); - bitstream.write_vec(output); - if output.len() > MAX_H264_ACCESS_UNIT_BYTES { - output.clear(); - return Err(RtcError::Media( - "OpenH264 encoded access unit exceeds size limit".to_owned(), - )); - } - self.dimensions = Some((width, height)); - Ok(keyframe) - } -} - -pub(crate) struct H264Decoder { - decoder: Decoder, -} - -impl H264Decoder { - pub(crate) fn new() -> Result { - let decoder = Decoder::new() - .map_err(|error| RtcError::Media(format!("OpenH264 decoder init: {error}")))?; - Ok(Self { decoder }) - } - - pub(crate) fn restart(&mut self) -> Result<()> { - self.decoder = Decoder::new() - .map_err(|error| RtcError::Media(format!("OpenH264 decoder restart: {error}")))?; - Ok(()) - } - - pub(crate) fn decode( - &mut self, - access_unit: &[u8], - rtp_timestamp: u32, - ) -> Result> { - if access_unit.is_empty() { - return Ok(Vec::new()); - } - if access_unit.len() > MAX_H264_ACCESS_UNIT_BYTES { - return Err(RtcError::Media( - "H264 access unit exceeds size limit".to_owned(), - )); - } - - let decoded = self - .decoder - .decode(access_unit) - .map_err(|error| RtcError::Media(format!("OpenH264 decode: {error}")))?; - decoded - .map(|image| copy_packed_i420(&image, rtp_timestamp).map(|frame| vec![frame])) - .transpose() - .map(Option::unwrap_or_default) - } -} - -fn copy_packed_i420(image: &impl YUVSource, rtp_timestamp: u32) -> Result { - let (width, height) = image.dimensions(); - if width == 0 - || height == 0 - || !width.is_multiple_of(2) - || !height.is_multiple_of(2) - || width > MAX_H264_DECODE_DIMENSION - || height > MAX_H264_DECODE_DIMENSION - || width - .checked_mul(height) - .is_none_or(|pixels| pixels > MAX_H264_DECODE_PIXELS) - { - return Err(RtcError::Media(format!( - "OpenH264 returned unsupported frame dimensions {width}x{height}" - ))); - } - - let (y_stride, u_stride, v_stride) = image.strides(); - let chroma_width = width / 2; - let chroma_height = height / 2; - let planes = [ - (image.y(), y_stride, width, height), - (image.u(), u_stride, chroma_width, chroma_height), - (image.v(), v_stride, chroma_width, chroma_height), - ]; - let width_u32 = u32::try_from(width) - .map_err(|_| RtcError::Media("decoded H264 width does not fit u32".to_owned()))?; - let height_u32 = u32::try_from(height) - .map_err(|_| RtcError::Media("decoded H264 height does not fit u32".to_owned()))?; - let mut data = Vec::with_capacity(i420_len(width_u32, height_u32)); - for (plane, stride, row_len, rows) in planes { - if stride < row_len - || rows - .checked_mul(stride) - .is_none_or(|required| plane.len() < required) - { - return Err(RtcError::Media( - "OpenH264 returned an invalid I420 plane layout".to_owned(), - )); - } - for row in 0..rows { - let offset = row * stride; - data.extend_from_slice(&plane[offset..offset + row_len]); - } - } - Ok(VideoFrame { - width: width_u32, - height: height_u32, - data, - rtp_timestamp, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ramp_i420(width: u32, height: u32) -> Vec { - let (width, height) = (width as usize, height as usize); - let mut frame = Vec::with_capacity(width * height * 3 / 2); - for _ in 0..height { - for x in 0..width { - frame.push((x * 219 / width.max(1) + 16) as u8); - } - } - frame.extend(std::iter::repeat_n(128, width * height / 4)); - frame.extend(std::iter::repeat_n(128, width * height / 4)); - frame - } - - fn round_trip( - encoder: &mut H264Encoder, - decoder: &mut H264Decoder, - width: u32, - height: u32, - timestamp: u32, - ) { - let source = ramp_i420(width, height); - let mut encoded = Vec::new(); - let keyframe = encoder - .encode_into(&source, width, height, true, &mut encoded) - .expect("encode H264"); - assert!(keyframe); - assert!(!encoded.is_empty()); - let frames = decoder.decode(&encoded, timestamp).expect("decode H264"); - let frame = frames.first().expect("decoded H264 frame"); - assert_eq!((frame.width, frame.height), (width, height)); - assert_eq!(frame.data.len(), i420_len(width, height)); - assert_eq!(frame.rtp_timestamp, timestamp); - let y = &frame.data[..(width * height) as usize]; - assert!(y[0] < 50); - assert!(y[width as usize - 1] > 190); - } - - #[test] - fn h264_round_trips_packed_i420() { - let mut encoder = H264Encoder::new(1_000_000).expect("encoder"); - let mut decoder = H264Decoder::new().expect("decoder"); - round_trip(&mut encoder, &mut decoder, 320, 240, 90_000); - } - - #[test] - fn h264_resolution_change_reinitializes_cleanly() { - let mut encoder = H264Encoder::new(1_000_000).expect("encoder"); - let mut decoder = H264Decoder::new().expect("decoder"); - round_trip(&mut encoder, &mut decoder, 320, 240, 90_000); - round_trip(&mut encoder, &mut decoder, 640, 360, 180_000); - assert_eq!(encoder.dimensions(), Some((640, 360))); - } - - #[test] - fn malformed_h264_is_an_error_or_no_frame_not_a_panic() { - let mut decoder = H264Decoder::new().expect("decoder"); - let result = decoder.decode(&[0, 0, 0, 1, 0x65, 0xff, 0xff, 0xff], 0); - assert!(result.is_err() || result.is_ok_and(|frames| frames.is_empty())); - } - - #[test] - fn h264_decoder_restarts_after_malformed_input() { - let source = ramp_i420(320, 240); - let mut encoder = H264Encoder::new(1_000_000).expect("encoder"); - let mut encoded = Vec::new(); - encoder - .encode_into(&source, 320, 240, true, &mut encoded) - .expect("encode recovery keyframe"); - - let mut decoder = H264Decoder::new().expect("decoder"); - let _ = decoder.decode(&[0, 0, 0, 1, 0x65, 0xff, 0xff, 0xff], 0); - decoder.restart().expect("restart decoder"); - let frames = decoder - .decode(&encoded, 90_000) - .expect("decode after restart"); - assert_eq!(frames.len(), 1); - assert_eq!( - (frames[0].width, frames[0].height), - (320, 240), - "the restarted decoder must accept a fresh SPS/PPS/IDR access unit" - ); - } - - #[test] - fn invalid_i420_dimensions_and_lengths_are_rejected() { - let mut encoder = H264Encoder::new(1_000_000).expect("encoder"); - let mut output = Vec::new(); - assert!( - encoder - .encode_into(&[], 321, 240, true, &mut output) - .is_err() - ); - assert!( - encoder - .encode_into(&[0; 10], 320, 240, true, &mut output) - .is_err() - ); - } - - #[test] - fn level_3_1_encoder_rejects_frames_beyond_max_fs() { - let mut encoder = H264Encoder::new(1_000_000).expect("encoder"); - let mut output = Vec::new(); - let error = encoder - .encode_into(&[], 1_920, 1_080, true, &mut output) - .expect_err("1080p exceeds level 3.1 MaxFS"); - assert!(error.to_string().contains("level 3.1")); - } - - #[test] - fn level_3_1_timing_uses_ceiling_division_at_30_fps_boundary() { - assert_eq!(minimum_frame_duration_ns(), 33_333_334); - assert_eq!( - minimum_mbps_duration_ns(H264_LEVEL_3_1_MAX_FRAME_MACROBLOCKS), - 33_333_334 - ); - assert!( - validate_h264_encode_request(1_280, 720, std::time::Duration::from_nanos(33_333_333)) - .is_err() - ); - validate_h264_encode_request(1_280, 720, std::time::Duration::from_nanos(33_333_334)) - .expect("ceiling-rounded level 3.1 boundary"); - } - - #[test] - fn idr_detection_accepts_three_and_four_byte_start_codes() { - assert!(access_unit_has_idr(&[0, 0, 0, 1, 0x65, 1])); - assert!(access_unit_has_idr(&[0, 0, 1, 0x65, 1])); - assert!(!access_unit_has_idr(&[0, 0, 0, 1, 0x41, 1])); - assert!(!access_unit_has_idr(&[0x65, 1])); - } -} diff --git a/src/rtc/codecs/mod.rs b/src/rtc/codecs/mod.rs index 29b8304..cdb30a8 100644 --- a/src/rtc/codecs/mod.rs +++ b/src/rtc/codecs/mod.rs @@ -1,6 +1,4 @@ //! Video codecs and their RTP payload formats. -pub(crate) mod h264; -pub(crate) mod rtp_h264; pub(crate) mod rtp_vpx; pub(crate) mod vpx; diff --git a/src/rtc/codecs/rtp_h264.rs b/src/rtc/codecs/rtp_h264.rs deleted file mode 100644 index 68c188d..0000000 --- a/src/rtc/codecs/rtp_h264.rs +++ /dev/null @@ -1,360 +0,0 @@ -//! RFC 6184 H264 RTP packetization and bounded, panic-free depacketization. - -use bytes::{BufMut, Bytes, BytesMut}; -use webrtc::rtp::Error as RtpError; -use webrtc::rtp::codecs::h264::H264Payloader; -use webrtc::rtp::packetizer::{Depacketizer, Payloader}; - -use crate::rtc::error::{Result, RtcError}; - -const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; -const NAL_TYPE_MASK: u8 = 0x1f; -const NAL_NRI_MASK: u8 = 0x60; -const STAP_A: u8 = 24; -const FU_A: u8 = 28; -const FU_START: u8 = 0x80; -const FU_END: u8 = 0x40; -const FU_RESERVED: u8 = 0x20; -const MAX_FRAGMENTED_NAL_BYTES: usize = 16 * 1024 * 1024; - -type RtpResult = std::result::Result; - -pub(crate) struct RtpPayload { - pub data: Bytes, - pub last: bool, -} - -#[derive(Default)] -pub(crate) struct H264RtpPacketizer { - payloader: H264Payloader, -} - -impl H264RtpPacketizer { - pub(crate) fn packetize( - &mut self, - access_unit: &[u8], - max_payload: usize, - ) -> Result> { - if access_unit.is_empty() { - return Ok(Vec::new()); - } - if max_payload <= 2 { - return Err(RtcError::Media( - "H264 RTP payload size must exceed the FU-A header".to_owned(), - )); - } - - let payloads = self - .payloader - .payload(max_payload, &Bytes::copy_from_slice(access_unit)) - .map_err(|error| RtcError::Media(format!("H264 packetize: {error}")))?; - let last_index = payloads.len().saturating_sub(1); - Ok(payloads - .into_iter() - .enumerate() - .map(|(index, data)| RtpPayload { - data, - last: index == last_index, - }) - .collect()) - } -} - -struct FragmentedNal { - indicator: u8, - nal_type: u8, - data: BytesMut, -} - -/// Stateful H264 depacketizer that rejects malformed STAP-A and FU-A payloads. -/// -/// The upstream depacketizer accepts FU-A fragments without a start packet and -/// can index past a malformed STAP-A length field. This implementation keeps -/// those network inputs fallible and caps fragmented-NAL growth. -#[derive(Default)] -pub(crate) struct H264Depacketizer { - fragmented: Option, -} - -impl H264Depacketizer { - fn malformed(message: impl Into) -> RtpError { - RtpError::Other(message.into()) - } - - fn depacketize_single(packet: &Bytes) -> RtpResult { - if packet.len() < 2 { - return Err(RtpError::ErrShortPacket); - } - let mut output = BytesMut::with_capacity(ANNEX_B_START_CODE.len() + packet.len()); - output.put_slice(ANNEX_B_START_CODE); - output.put_slice(packet); - Ok(output.freeze()) - } - - fn depacketize_stap_a(packet: &Bytes) -> RtpResult { - let mut offset = 1usize; - let mut output = BytesMut::new(); - while offset < packet.len() { - let remaining = packet.len() - offset; - if remaining < 2 { - return Err(Self::malformed("truncated H264 STAP-A length")); - } - let nal_len = usize::from(u16::from_be_bytes([packet[offset], packet[offset + 1]])); - offset += 2; - if nal_len == 0 { - return Err(Self::malformed("zero-length H264 STAP-A NAL unit")); - } - let available = packet.len() - offset; - if nal_len > available { - return Err(RtpError::StapASizeLargerThanBuffer(nal_len, available)); - } - let nal_type = packet[offset] & NAL_TYPE_MASK; - if !(1..=23).contains(&nal_type) { - return Err(Self::malformed(format!( - "nested or invalid H264 STAP-A NAL type {nal_type}" - ))); - } - if output - .len() - .saturating_add(ANNEX_B_START_CODE.len()) - .saturating_add(nal_len) - > MAX_FRAGMENTED_NAL_BYTES - { - return Err(Self::malformed("H264 STAP-A exceeds size limit")); - } - output.reserve(ANNEX_B_START_CODE.len() + nal_len); - output.put_slice(ANNEX_B_START_CODE); - output.put_slice(&packet[offset..offset + nal_len]); - offset += nal_len; - } - if output.is_empty() { - return Err(Self::malformed("empty H264 STAP-A payload")); - } - Ok(output.freeze()) - } - - fn depacketize_fu_a(&mut self, packet: &Bytes) -> RtpResult { - if packet.len() <= 2 { - self.fragmented = None; - return Err(RtpError::ErrShortPacket); - } - - let indicator = packet[0] & (0x80 | NAL_NRI_MASK); - let header = packet[1]; - let nal_type = header & NAL_TYPE_MASK; - let start = header & FU_START != 0; - let end = header & FU_END != 0; - if header & FU_RESERVED != 0 || nal_type == 0 || start && end { - self.fragmented = None; - return Err(Self::malformed("invalid H264 FU-A header")); - } - - if start { - if self.fragmented.is_some() { - self.fragmented = None; - return Err(Self::malformed( - "H264 FU-A start arrived before the prior fragment ended", - )); - } - if ANNEX_B_START_CODE - .len() - .saturating_add(1) - .saturating_add(packet.len() - 2) - > MAX_FRAGMENTED_NAL_BYTES - { - return Err(Self::malformed("H264 fragmented NAL exceeds size limit")); - } - let mut data = BytesMut::with_capacity(packet.len() + ANNEX_B_START_CODE.len()); - data.put_slice(ANNEX_B_START_CODE); - data.put_u8(indicator | nal_type); - data.put_slice(&packet[2..]); - self.fragmented = Some(FragmentedNal { - indicator, - nal_type, - data, - }); - return Ok(Bytes::new()); - } - - let Some(fragmented) = self.fragmented.as_mut() else { - return Err(Self::malformed("H264 FU-A continuation without a start")); - }; - if fragmented.indicator != indicator || fragmented.nal_type != nal_type { - self.fragmented = None; - return Err(Self::malformed( - "H264 FU-A continuation changed NRI or NAL type", - )); - } - if fragmented.data.len().saturating_add(packet.len() - 2) > MAX_FRAGMENTED_NAL_BYTES { - self.fragmented = None; - return Err(Self::malformed("H264 fragmented NAL exceeds size limit")); - } - fragmented.data.put_slice(&packet[2..]); - if end { - let completed = self - .fragmented - .take() - .ok_or_else(|| Self::malformed("H264 FU-A state disappeared"))?; - return Ok(completed.data.freeze()); - } - Ok(Bytes::new()) - } -} - -impl Depacketizer for H264Depacketizer { - fn depacketize(&mut self, packet: &Bytes) -> RtpResult { - let Some(first) = packet.first().copied() else { - self.fragmented = None; - return Err(RtpError::ErrShortPacket); - }; - match first & NAL_TYPE_MASK { - 1..=23 => { - if self.fragmented.take().is_some() { - return Err(Self::malformed( - "single H264 NAL interrupted a fragmented NAL", - )); - } - Self::depacketize_single(packet) - } - STAP_A => { - if self.fragmented.take().is_some() { - return Err(Self::malformed("H264 STAP-A interrupted a fragmented NAL")); - } - Self::depacketize_stap_a(packet) - } - FU_A => self.depacketize_fu_a(packet), - nal_type => { - self.fragmented = None; - Err(RtpError::NaluTypeIsNotHandled(nal_type)) - } - } - } - - fn is_partition_head(&self, payload: &Bytes) -> bool { - let Some(first) = payload.first() else { - return false; - }; - if first & NAL_TYPE_MASK == FU_A { - payload.get(1).is_some_and(|header| header & FU_START != 0) - } else { - true - } - } - - fn is_partition_tail(&self, marker: bool, _payload: &Bytes) -> bool { - marker - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn annex_b_access_unit_round_trips_through_stap_and_fu_a() { - let mut access_unit = Vec::new(); - access_unit.extend_from_slice(ANNEX_B_START_CODE); - access_unit.extend_from_slice(&[0x67, 0x42, 0xe0, 0x1f, 0x89]); - access_unit.extend_from_slice(ANNEX_B_START_CODE); - access_unit.extend_from_slice(&[0x68, 0xce, 0x06, 0xe2]); - access_unit.extend_from_slice(ANNEX_B_START_CODE); - access_unit.extend_from_slice(&[0x65]); - access_unit.extend(std::iter::repeat_n(0xab, 4_000)); - - let mut packetizer = H264RtpPacketizer::default(); - let payloads = packetizer - .packetize(&access_unit, 1_200) - .expect("packetize H264"); - assert!(payloads.len() >= 5); - assert_eq!(payloads[0].data[0] & NAL_TYPE_MASK, STAP_A); - assert!( - payloads - .iter() - .skip(1) - .any(|p| p.data[0] & NAL_TYPE_MASK == FU_A) - ); - assert!(payloads.last().is_some_and(|payload| payload.last)); - - let mut depacketizer = H264Depacketizer::default(); - let mut rebuilt = Vec::new(); - for payload in payloads { - rebuilt.extend( - depacketizer - .depacketize(&payload.data) - .expect("depacketize H264"), - ); - } - assert_eq!(rebuilt, access_unit); - } - - #[test] - fn single_nal_packetization_mode_round_trips() { - let packet = Bytes::from_static(&[0x61, 1, 2, 3]); - let mut depacketizer = H264Depacketizer::default(); - let rebuilt = depacketizer - .depacketize(&packet) - .expect("depacketize single NAL"); - assert_eq!(rebuilt.as_ref(), &[0, 0, 0, 1, 0x61, 1, 2, 3]); - } - - #[test] - fn stap_a_preserves_sps_pps_idr_order() { - let stap = Bytes::from_static(&[ - 0x78, // STAP-A with NRI=3 - 0, 3, 0x67, 0x42, 0xe0, // SPS - 0, 2, 0x68, 0xce, // PPS - 0, 3, 0x65, 0xaa, 0xbb, // IDR - ]); - let mut depacketizer = H264Depacketizer::default(); - let rebuilt = depacketizer - .depacketize(&stap) - .expect("depacketize SPS/PPS/IDR STAP-A"); - assert_eq!( - rebuilt.as_ref(), - &[ - 0, 0, 0, 1, 0x67, 0x42, 0xe0, 0, 0, 0, 1, 0x68, 0xce, 0, 0, 0, 1, 0x65, 0xaa, 0xbb, - ] - ); - } - - #[test] - fn malformed_stap_a_trailing_length_byte_is_an_error() { - let mut depacketizer = H264Depacketizer::default(); - let malformed = Bytes::from_static(&[STAP_A, 0, 1, 0x67, 0]); - assert!(depacketizer.depacketize(&malformed).is_err()); - } - - #[test] - fn fu_a_continuation_without_start_is_an_error() { - let mut depacketizer = H264Depacketizer::default(); - let continuation = Bytes::from_static(&[FU_A | 0x60, 0x05, 1, 2, 3]); - assert!(depacketizer.depacketize(&continuation).is_err()); - } - - #[test] - fn malformed_fu_a_clears_state_for_the_next_nal() { - let mut depacketizer = H264Depacketizer::default(); - let start = Bytes::from_static(&[FU_A | 0x60, FU_START | 0x05, 1, 2]); - assert!( - depacketizer - .depacketize(&start) - .expect("start fragmented IDR") - .is_empty() - ); - - let wrong_type = Bytes::from_static(&[FU_A | 0x60, 0x01, 3]); - assert!(depacketizer.depacketize(&wrong_type).is_err()); - - let single = Bytes::from_static(&[0x61, 9, 8, 7]); - let rebuilt = depacketizer - .depacketize(&single) - .expect("recover with a complete single NAL"); - assert_eq!(rebuilt.as_ref(), &[0, 0, 0, 1, 0x61, 9, 8, 7]); - } - - #[test] - fn invalid_packetizer_mtu_is_an_error() { - let mut packetizer = H264RtpPacketizer::default(); - assert!(packetizer.packetize(&[0x65, 1, 2], 2).is_err()); - } -} diff --git a/src/rtc/join/tests.rs b/src/rtc/join/tests.rs index e3af698..03b7cd4 100644 --- a/src/rtc/join/tests.rs +++ b/src/rtc/join/tests.rs @@ -3,8 +3,7 @@ use super::*; use crate::client::ClientConfig; use crate::rtc::{ - ClientPublishOptions, LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig, PcmFrame, - PreferredVideoCodec, publish_options::H264_FMTP, + ClientPublishOptions, LocalAudioTrack, LocalVideoTrack, PcmFrame, PreferredVideoCodec, }; use std::io::{Read, Write}; use std::net::TcpListener; @@ -289,23 +288,23 @@ fn preferred_codec(core: &RtcCore, generation: u64) -> Option { } #[test] -fn pre_join_h264_preference_builds_canonical_join_option() { +fn pre_join_vp8_preference_builds_canonical_join_option() { let core = test_core(); core.update_publish_options(ClientPublishOptions { - preferred_codec: Some(PreferredVideoCodec::H264), + preferred_codec: Some(PreferredVideoCodec::Vp8), }); let generation = core.begin_join().expect("join generation"); - let codec = preferred_codec(&core, generation).expect("H264 preference"); - assert_eq!(codec.name, "H264"); - assert_eq!(codec.fmtp, H264_FMTP); + let codec = preferred_codec(&core, generation).expect("VP8 preference"); + assert_eq!(codec.name, "VP8"); + assert_eq!(codec.fmtp, ""); } #[test] fn update_after_join_starts_has_no_effect() { let core = test_core(); core.update_publish_options(ClientPublishOptions { - preferred_codec: Some(PreferredVideoCodec::H264), + preferred_codec: Some(PreferredVideoCodec::Vp8), }); let generation = core.begin_join().expect("join generation"); assert!(core.set_state_if_current(generation, CallingState::Joined)); @@ -314,8 +313,8 @@ fn update_after_join_starts_has_no_effect() { preferred_codec: Some(PreferredVideoCodec::Vp9), }); - let codec = preferred_codec(&core, generation).expect("active H264 preference"); - assert_eq!(codec.name, "H264"); + let codec = preferred_codec(&core, generation).expect("active VP8 preference"); + assert_eq!(codec.name, "VP8"); let leave_generation = core.cancel_generation(); assert!(core.set_state_if_current(leave_generation, CallingState::Left)); @@ -328,15 +327,15 @@ fn update_after_join_starts_has_no_effect() { async fn publish_preference_persists_across_leave_and_rejoin() { let core = test_core(); core.update_publish_options(ClientPublishOptions { - preferred_codec: Some(PreferredVideoCodec::H264), + preferred_codec: Some(PreferredVideoCodec::Vp8), }); core.begin_join().expect("first generation"); core.leave("test leave").await.expect("leave"); let second = core.begin_join().expect("second generation"); - let codec = preferred_codec(&core, second).expect("retained H264 preference"); - assert_eq!(codec.name, "H264"); - assert_eq!(codec.fmtp, H264_FMTP); + let codec = preferred_codec(&core, second).expect("retained VP8 preference"); + assert_eq!(codec.name, "VP8"); + assert_eq!(codec.fmtp, ""); core.leave("test reset").await.expect("second leave"); core.update_publish_options(ClientPublishOptions::default()); @@ -1094,16 +1093,16 @@ async fn a_stopped_simulcast_publication_is_not_replaced() { let (mut connection, _sfu) = establish_fake(&core, generation).await; connection.publish_options = vec![models::PublishOption { id: 1, - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, codec: Some(models::Codec { - name: "h264".to_owned(), + name: "vp8".to_owned(), ..Default::default() }), ..Default::default() }]; let stopped = LocalTrack::Video { - track: LocalVideoTrack::h264_simulcast().expect("simulcast track"), - track_type: TrackType::Video, + track: LocalVideoTrack::vp8_simulcast().expect("simulcast track"), + track_type: TrackType::ScreenShare, }; let reader_tasks = crate::rtc::peer::publisher::add_transceiver_for_track( &connection.publisher, @@ -1116,7 +1115,7 @@ async fn a_stopped_simulcast_publication_is_not_replaced() { core.own_capabilities .lock() .unwrap_or_else(|error| error.into_inner()) - .insert("send-video".to_owned()); + .insert("screenshare".to_owned()); { let mut media = core.media.lock().await; media.begin_publish(stopped.clone(), 1); @@ -1125,20 +1124,20 @@ async fn a_stopped_simulcast_publication_is_not_replaced() { stopped.stop(); for track in [ - LocalVideoTrack::h264_simulcast().expect("new simulcast track"), - LocalVideoTrack::h264().expect("new single-layer track"), + LocalVideoTrack::vp8_simulcast().expect("new simulcast track"), + LocalVideoTrack::vp8().expect("new single-layer track"), ] { let result = core .publish(LocalTrack::Video { track, - track_type: TrackType::Video, + track_type: TrackType::ScreenShare, }) .await; assert!( matches!( result, Err(RtcError::SimulcastReplace { - track_type: TrackType::Video + track_type: TrackType::ScreenShare }) ), "{result:?}" @@ -1843,9 +1842,7 @@ async fn video_track_with_a_non_video_type_is_not_published() { .lock() .unwrap_or_else(|error| error.into_inner()) .insert("send-video".to_owned()); - let track = - LocalVideoTrack::h264_with_config(LocalVideoTrackConfig::default().server_managed()) - .expect("video track"); + let track = LocalVideoTrack::vp9_svc().expect("video track"); let result = core .publish(LocalTrack::Video { @@ -2366,19 +2363,18 @@ fn mute_state_builder_deduplicates_track_types() { #[test] fn publish_quality_routes_by_option_id_and_track_type() { - let config = LocalVideoTrackConfig::default().server_managed(); let first = LocalTrack::Video { - track: LocalVideoTrack::h264_with_config(config).expect("first layered track"), - track_type: TrackType::Video, + track: LocalVideoTrack::vp8_simulcast().expect("first layered track"), + track_type: TrackType::ScreenShare, }; let second = LocalTrack::Video { - track: LocalVideoTrack::h264_with_config(config).expect("second layered track"), - track_type: TrackType::Video, + track: LocalVideoTrack::vp8_simulcast().expect("second layered track"), + track_type: TrackType::ScreenShare, }; let option = models::PublishOption { - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, codec: Some(models::Codec { - name: "H264".to_owned(), + name: "VP8".to_owned(), ..Default::default() }), bitrate: 1_200_000, @@ -2398,7 +2394,7 @@ fn publish_quality_routes_by_option_id_and_track_type() { let matched = media.apply_publish_quality(&event::ChangePublishQuality { video_senders: vec![event::VideoSender { - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, publish_option_id: 11, layers: vec![event::VideoLayerSetting { name: "h".to_owned(), @@ -2417,22 +2413,19 @@ fn publish_quality_routes_by_option_id_and_track_type() { } #[test] -fn publish_quality_routes_to_vp9_without_mutating_an_h264_publication() { - let h264_track = - LocalVideoTrack::h264_with_config(LocalVideoTrackConfig::default().server_managed()) - .expect("layered H264"); - let vp9_track = LocalVideoTrack::vp9_svc().expect("VP9 SVC"); - let h264 = LocalTrack::Video { - track: h264_track, - track_type: TrackType::Video, +fn publish_quality_routes_to_vp9_without_mutating_a_vp8_publication() { + let vp8 = LocalTrack::Video { + track: LocalVideoTrack::vp8_simulcast().expect("layered VP8"), + track_type: TrackType::ScreenShare, }; + let vp9_track = LocalVideoTrack::vp9_svc().expect("VP9 SVC"); let vp9 = LocalTrack::Video { track: vp9_track.clone(), track_type: TrackType::Video, }; - let option = |id, codec: &str| models::PublishOption { + let option = |id, track_type: TrackType, codec: &str| models::PublishOption { id, - track_type: TrackType::Video as i32, + track_type: track_type as i32, codec: Some(models::Codec { name: codec.to_owned(), ..Default::default() @@ -2447,13 +2440,13 @@ fn publish_quality_routes_to_vp9_without_mutating_an_h264_publication() { }), ..Default::default() }; - h264.configure_for_publish(&option(10, "H264")) - .expect("H264 plan"); - vp9.configure_for_publish(&option(11, "VP9")) + vp8.configure_for_publish(&option(10, TrackType::ScreenShare, "VP8")) + .expect("VP8 plan"); + vp9.configure_for_publish(&option(11, TrackType::Video, "VP9")) .expect("VP9 plan"); - let h264_before = h264.video_layer_control_state("q").expect("H264 low layer"); + let vp8_before = vp8.video_layer_control_state("q").expect("VP8 low layer"); let mut media = MediaState::default(); - media.begin_publish(h264.clone(), 10); + media.begin_publish(vp8.clone(), 10); media.begin_publish(vp9.clone(), 11); let matched = media.apply_publish_quality(&event::ChangePublishQuality { @@ -2475,7 +2468,7 @@ fn publish_quality_routes_to_vp9_without_mutating_an_h264_publication() { }); assert_eq!(matched, 1); - assert_eq!(h264.video_layer_control_state("q"), Some(h264_before)); + assert_eq!(vp8.video_layer_control_state("q"), Some(vp8_before)); assert_eq!(vp9_track.svc_mode(), Some((1, 2))); assert_eq!( vp9.video_layer_control_state("q"), @@ -2573,9 +2566,8 @@ fn refreshed_vp9_publish_option_restores_cached_svc_quality() { fn refreshed_publish_options_preserve_pair_scoped_quality() { let audio = LocalTrack::Audio(LocalAudioTrack::opus().expect("audio track")); let video = LocalTrack::Video { - track: LocalVideoTrack::h264_with_config(LocalVideoTrackConfig::default().server_managed()) - .expect("layered video"), - track_type: TrackType::Video, + track: LocalVideoTrack::vp8_simulcast().expect("layered video"), + track_type: TrackType::ScreenShare, }; let options = vec![ models::PublishOption { @@ -2589,9 +2581,9 @@ fn refreshed_publish_options_preserve_pair_scoped_quality() { }, models::PublishOption { id: 41, - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, codec: Some(models::Codec { - name: "H264".to_owned(), + name: "VP8".to_owned(), ..Default::default() }), bitrate: 1_200_000, @@ -2613,7 +2605,7 @@ fn refreshed_publish_options_preserve_pair_scoped_quality() { assert_eq!( media.apply_publish_quality(&event::ChangePublishQuality { video_senders: vec![event::VideoSender { - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, publish_option_id: 41, layers: vec![event::VideoLayerSetting { name: "h".to_owned(), @@ -2646,7 +2638,7 @@ fn refreshed_publish_options_preserve_pair_scoped_quality() { assert!( media .publish_quality - .contains_key(&(41, TrackType::Video as i32)) + .contains_key(&(41, TrackType::ScreenShare as i32)) ); assert!( !media diff --git a/src/rtc/peer/connection.rs b/src/rtc/peer/connection.rs index e7e9279..ddc89b2 100644 --- a/src/rtc/peer/connection.rs +++ b/src/rtc/peer/connection.rs @@ -4,9 +4,7 @@ use std::sync::Arc; use serde_json::json; use webrtc::api::interceptor_registry::register_default_interceptors; -use webrtc::api::media_engine::{ - MIME_TYPE_H264, MIME_TYPE_OPUS, MIME_TYPE_VP8, MIME_TYPE_VP9, MediaEngine, -}; +use webrtc::api::media_engine::{MIME_TYPE_OPUS, MIME_TYPE_VP8, MIME_TYPE_VP9, MediaEngine}; use webrtc::api::{API, APIBuilder}; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; @@ -23,19 +21,18 @@ use webrtc::sdp::extmap::{ use crate::rtc::coordinator::IceServer; use crate::rtc::error::Result; -use crate::rtc::publish_options::H264_FMTP; use crate::rtc::tracer::Tracer; const OPUS_PAYLOAD_TYPE: u8 = 111; const VP8_PAYLOAD_TYPE: u8 = 96; const VP9_PAYLOAD_TYPE: u8 = 98; -const H264_PAYLOAD_TYPE: u8 = 125; /// Register exactly the codecs that the SDK can encode or decode. /// /// `MediaEngine::register_default_codecs` also advertises legacy audio, VP9 -/// profile 1, AV1, and HEVC. Negotiating any of those would deliver a track -/// that the decoded [`RemoteTrack`](crate::rtc::RemoteTrack) APIs cannot consume. +/// profile 1, H264, AV1, and HEVC. Negotiating any of those would deliver a +/// track that the decoded [`RemoteTrack`](crate::rtc::RemoteTrack) APIs cannot +/// consume. fn register_supported_codecs(media_engine: &mut MediaEngine) -> Result<()> { media_engine.register_codec( RTCRtpCodecParameters { @@ -73,7 +70,6 @@ fn register_supported_codecs(media_engine: &mut MediaEngine) -> Result<()> { for (mime_type, payload_type, fmtp) in [ (MIME_TYPE_VP8, VP8_PAYLOAD_TYPE, ""), (MIME_TYPE_VP9, VP9_PAYLOAD_TYPE, "profile-id=0"), - (MIME_TYPE_H264, H264_PAYLOAD_TYPE, H264_FMTP), ] { media_engine.register_codec( RTCRtpCodecParameters { @@ -386,13 +382,13 @@ mod tests { ); } - for supported in ["VP8/90000", "VP9/90000", "H264/90000"] { + for supported in ["VP8/90000", "VP9/90000"] { assert!( video.contains(supported), "missing supported video codec {supported}:\n{video}" ); } - for unsupported in ["AV1/90000", "H265/90000"] { + for unsupported in ["H264/90000", "AV1/90000", "H265/90000"] { assert!( !video.contains(unsupported), "advertised unsupported video codec {unsupported}:\n{video}" diff --git a/src/rtc/peer/publisher.rs b/src/rtc/peer/publisher.rs index 983b627..345880c 100644 --- a/src/rtc/peer/publisher.rs +++ b/src/rtc/peer/publisher.rs @@ -365,7 +365,7 @@ mod tests { use crate::rtc::peer; use crate::rtc::proto::event::VideoLayerSetting; use crate::rtc::proto::models::{Codec, VideoDimension}; - use crate::rtc::tracks::{LocalAudioTrack, LocalVideoTrack, LocalVideoTrackConfig}; + use crate::rtc::tracks::{LocalAudioTrack, LocalVideoTrack}; use std::time::Duration; use webrtc::rtp_transceiver::rtp_codec::RTPCodecType; @@ -591,16 +591,16 @@ mod tests { #[test] fn publish_codec_validation_accepts_an_exact_case_insensitive_match() { - let track: LocalTrack = LocalVideoTrack::h264().expect("H264 track").into(); - validate_publish_codecs(&[track], &[video_option("H264")]) - .expect("matching H264 publish option"); + let track: LocalTrack = LocalVideoTrack::vp9().expect("VP9 track").into(); + validate_publish_codecs(&[track], &[video_option("vp9")]) + .expect("matching VP9 publish option"); } #[test] fn publish_codec_validation_rejects_a_fallback_codec() { - let track: LocalTrack = LocalVideoTrack::h264().expect("H264 track").into(); + let track: LocalTrack = LocalVideoTrack::vp8().expect("VP8 track").into(); let error = validate_publish_codecs(&[track], &[video_option("VP9")]) - .expect_err("VP9 cannot carry an H264 bitstream"); + .expect_err("VP9 cannot carry a VP8 bitstream"); assert!( matches!(error, RtcError::Media(message) if message.contains("available codecs: VP9")) ); @@ -608,13 +608,13 @@ mod tests { #[test] fn duplicate_codec_publications_receive_distinct_server_option_ids() { - let first: LocalTrack = LocalVideoTrack::h264().expect("first H264").into(); - let second: LocalTrack = LocalVideoTrack::h264().expect("second H264").into(); + let first: LocalTrack = LocalVideoTrack::vp9().expect("first VP9").into(); + let second: LocalTrack = LocalVideoTrack::vp9().expect("second VP9").into(); let options = [ - video_option("H264"), + video_option("VP9"), PublishOption { id: 8, - ..video_option("H264") + ..video_option("VP9") }, ]; let mut used = HashSet::new(); @@ -636,18 +636,16 @@ mod tests { #[tokio::test] async fn layered_track_uses_one_mline_and_metadata_planning_is_read_only() { - let track = - LocalVideoTrack::h264_with_config(LocalVideoTrackConfig::default().server_managed()) - .expect("layered H264"); + let track = LocalVideoTrack::vp8_simulcast().expect("layered VP8"); let local = LocalTrack::Video { track, - track_type: TrackType::Video, + track_type: TrackType::ScreenShare, }; let option = PublishOption { id: 73, - track_type: TrackType::Video as i32, + track_type: TrackType::ScreenShare as i32, codec: Some(Codec { - name: "H264".to_owned(), + name: "VP8".to_owned(), ..Default::default() }), bitrate: 1_200_000, diff --git a/src/rtc/publish_options.rs b/src/rtc/publish_options.rs index 5b0362f..d6fe1bc 100644 --- a/src/rtc/publish_options.rs +++ b/src/rtc/publish_options.rs @@ -5,9 +5,6 @@ use std::str::FromStr; use super::error::RtcError; use super::proto::models::{Codec, PublishOption, TrackType}; -pub(crate) const H264_FMTP: &str = - "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"; - /// A video codec that the Rust media path can encode and decode. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -16,8 +13,6 @@ pub enum PreferredVideoCodec { Vp8, /// VP9 profile 0. Vp9, - /// H264 Constrained Baseline, packetization mode 1. - H264, } impl PreferredVideoCodec { @@ -25,7 +20,6 @@ impl PreferredVideoCodec { match self { Self::Vp8 => "VP8", Self::Vp9 => "VP9", - Self::H264 => "H264", } } @@ -33,7 +27,6 @@ impl PreferredVideoCodec { match self { Self::Vp8 => "", Self::Vp9 => "profile-id=0", - Self::H264 => H264_FMTP, } } } @@ -46,11 +39,9 @@ impl FromStr for PreferredVideoCodec { Ok(Self::Vp8) } else if value.eq_ignore_ascii_case("vp9") { Ok(Self::Vp9) - } else if value.eq_ignore_ascii_case("h264") { - Ok(Self::H264) } else { Err(RtcError::Media(format!( - "unsupported preferred video codec {value:?}; supported codecs are VP8, VP9, and H264" + "unsupported preferred video codec {value:?}; supported codecs are VP8 and VP9" ))) } } @@ -98,15 +89,15 @@ mod tests { use super::*; #[test] - fn h264_preference_uses_local_track_profile() { + fn vp9_preference_uses_profile_0() { let options = - ClientPublishOptions::new(PreferredVideoCodec::H264).preferred_publish_options(); + ClientPublishOptions::new(PreferredVideoCodec::Vp9).preferred_publish_options(); assert_eq!(options.len(), 1); assert_eq!(options[0].track_type, TrackType::Video as i32); - let codec = options[0].codec.as_ref().expect("H264 codec"); - assert_eq!(codec.name, "H264"); - assert_eq!(codec.fmtp, H264_FMTP); + let codec = options[0].codec.as_ref().expect("VP9 codec"); + assert_eq!(codec.name, "VP9"); + assert_eq!(codec.fmtp, "profile-id=0"); } #[test] @@ -120,9 +111,11 @@ mod tests { #[test] fn unsupported_codec_is_rejected() { - let error = "av1" - .parse::() - .expect_err("Rust media does not support AV1"); - assert!(matches!(error, RtcError::Media(message) if message.contains("av1"))); + for codec in ["av1", "h264"] { + let error = codec + .parse::() + .expect_err("Rust media supports only VP8 and VP9"); + assert!(matches!(error, RtcError::Media(message) if message.contains(codec))); + } } } diff --git a/src/rtc/tracks/local.rs b/src/rtc/tracks/local.rs index 6605ee1..b9fafe4 100644 --- a/src/rtc/tracks/local.rs +++ b/src/rtc/tracks/local.rs @@ -37,7 +37,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bytes::Bytes; use tokio::sync::{Notify, Semaphore}; use tokio::task::JoinHandle; -use webrtc::api::media_engine::{MIME_TYPE_H264, MIME_TYPE_OPUS, MIME_TYPE_VP8, MIME_TYPE_VP9}; +use webrtc::api::media_engine::{MIME_TYPE_OPUS, MIME_TYPE_VP8, MIME_TYPE_VP9}; use webrtc::rtp::extension::HeaderExtension; use webrtc::rtp::extension::audio_level_extension::AudioLevelExtension; use webrtc::rtp::packetizer::{Packetizer, new_packetizer}; @@ -48,8 +48,6 @@ use webrtc::track::track_local::TrackLocalWriter; use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP; use super::layers::{PlannedVideoLayer, simulcast_layers, single_layer}; -use crate::rtc::codecs::h264::{H264Encoder, validate_h264_encode_request}; -use crate::rtc::codecs::rtp_h264::H264RtpPacketizer; use crate::rtc::codecs::rtp_vpx::VpxRtpPacketizer; use crate::rtc::codecs::vpx::{Vp9SvcMode, VpxCodec, VpxEncoder, VpxSvcEncoder}; use crate::rtc::error::{Result, RtcError}; @@ -835,12 +833,6 @@ fn encode_opus_into(encoder: &mut opus::Encoder, pcm: &[i16], output: &mut [u8]) // Video -#[derive(Clone, Copy)] -enum VideoCodec { - Vpx(VpxCodec), - H264, -} - enum VideoCodecState { Vpx { encoder: VpxEncoder, @@ -850,11 +842,6 @@ enum VideoCodecState { encoder: VpxSvcEncoder, packetizer: VpxRtpPacketizer, }, - H264 { - encoder: Box, - packetizer: H264RtpPacketizer, - encoded: Vec, - }, } /// A video encoder plus its presentation clock, RTP packetizer, and outbound @@ -905,7 +892,7 @@ impl VideoEncoding { struct VideoInner { encodings: Vec, - codec_id: VideoCodec, + codec_id: VpxCodec, /// Serializes frame work before it enters Tokio's blocking pool. The permit /// moves into the worker so cancellation cannot build an unbounded queue. encode_gate: Arc, @@ -930,8 +917,8 @@ pub enum VideoLayering { Single, /// Build server-managed layering, optionally capped locally. /// - /// VP9 camera tracks use one-SSRC codec-native SVC. H264 camera and VP8 - /// screen-share tracks use independent RID simulcast encodings. + /// VP9 camera tracks use one-SSRC codec-native SVC. VP8 screen-share tracks + /// use independent RID simulcast encodings. ServerManaged { /// Maximum local spatial layers. `None` allows up to three. max_spatial_layers: Option, @@ -979,7 +966,7 @@ impl LocalVideoTrackConfig { } } -/// An outbound video track (VP8, VP9, or H264). +/// An outbound video track (VP8 or VP9). /// /// Feed raw frames via [`write_i420`](Self::write_i420) (the SDK encodes to the /// track's codec and packetizes), publish pre-encoded frames via @@ -1007,7 +994,7 @@ impl LocalVideoTrack { sdp_fmtp_line: String::new(), rtcp_feedback: vec![], }, - VideoCodec::Vpx(VpxCodec::Vp8), + VpxCodec::Vp8, config, ) } @@ -1034,7 +1021,7 @@ impl LocalVideoTrack { sdp_fmtp_line: "profile-id=0".to_owned(), rtcp_feedback: vec![], }, - VideoCodec::Vpx(VpxCodec::Vp9), + VpxCodec::Vp9, config, ) } @@ -1048,40 +1035,9 @@ impl LocalVideoTrack { Self::vp9_with_config(LocalVideoTrackConfig::default().server_managed()) } - /// Build an H264 Constrained Baseline, packetization-mode 1 video track. - /// - /// VP9 remains the SDK's default camera codec. Select H264 for peers that - /// require it, including Safari-origin media and OpenAI Realtime video. - /// H264 may be covered by patents in some jurisdictions; distributors must - /// evaluate their own licensing obligations. - pub fn h264() -> Result { - Self::h264_with_config(LocalVideoTrackConfig::default()) - } - - /// Build an H264 track with explicit local encoder settings. - pub fn h264_with_config(config: LocalVideoTrackConfig) -> Result { - Self::with_codec( - RTCRtpCodecCapability { - mime_type: MIME_TYPE_H264.to_owned(), - clock_rate: 90_000, - channels: 0, - sdp_fmtp_line: crate::rtc::publish_options::H264_FMTP.to_owned(), - rtcp_feedback: vec![], - }, - VideoCodec::H264, - config, - ) - } - - /// Build an H264 camera track configured for server-managed simulcast. - /// Publish it with [`crate::Call::publish_video`]. - pub fn h264_simulcast() -> Result { - Self::h264_with_config(LocalVideoTrackConfig::default().server_managed()) - } - fn with_codec( codec: RTCRtpCodecCapability, - codec_id: VideoCodec, + codec_id: VpxCodec, config: LocalVideoTrackConfig, ) -> Result { if config.target_bitrate_bps == 0 { @@ -1093,8 +1049,10 @@ impl LocalVideoTrack { let stream_id = "stream-rust-video".to_owned(); let rids: &[Option<&str>] = match (config.layering, codec_id) { (VideoLayering::Single, _) => &[None], - (VideoLayering::ServerManaged { .. }, VideoCodec::Vpx(VpxCodec::Vp9)) => &[Some("q")], - (VideoLayering::ServerManaged { .. }, _) => &[Some("q"), Some("h"), Some("f")], + (VideoLayering::ServerManaged { .. }, VpxCodec::Vp9) => &[Some("q")], + (VideoLayering::ServerManaged { .. }, VpxCodec::Vp8) => { + &[Some("q"), Some("h"), Some("f")] + } }; let bitrate_kbps = config.target_bitrate_bps.saturating_add(999) / 1_000; let mut encodings = Vec::with_capacity(rids.len()); @@ -1204,9 +1162,6 @@ impl LocalVideoTrack { and {MAX_LOCAL_VIDEO_PIXELS} pixels (got {width}x{height})" )) })?; - if matches!(self.inner.codec_id, VideoCodec::H264) { - validate_h264_encode_request(width, height, duration)?; - } let expected = usize::try_from(width) .ok() .and_then(|width| { @@ -1417,12 +1372,10 @@ impl LocalVideoTrack { max_spatial_layers, max_temporal_layers: _, } => { - let supported = (matches!(self.inner.codec_id, VideoCodec::H264) - && track_type == TrackType::Video) - || (matches!(self.inner.codec_id, VideoCodec::Vpx(VpxCodec::Vp8)) - && track_type == TrackType::ScreenShare) - || (matches!(self.inner.codec_id, VideoCodec::Vpx(VpxCodec::Vp9)) - && track_type == TrackType::Video); + let supported = matches!( + (self.inner.codec_id, track_type), + (VpxCodec::Vp8, TrackType::ScreenShare) | (VpxCodec::Vp9, TrackType::Video) + ); if !supported { return Err(RtcError::UnsupportedVideoLayering { codec: self.mime_type(), @@ -1532,7 +1485,7 @@ impl LocalVideoTrack { } fn is_vp9_svc(&self) -> bool { - matches!(self.inner.codec_id, VideoCodec::Vpx(VpxCodec::Vp9)) + matches!(self.inner.codec_id, VpxCodec::Vp9) && matches!(self.inner.layering, VideoLayering::ServerManaged { .. }) } @@ -1701,13 +1654,6 @@ fn encode_i420_layers( let scale = encoding.scale_resolution_down_by(); let layer_width = scaled_even(width, scale); let layer_height = scaled_even(height, scale); - if matches!(inner.codec_id, VideoCodec::H264) { - validate_h264_encode_request( - layer_width, - layer_height, - Duration::from_millis(dur_ms.max(1) as u64), - )?; - } let scaled; let layer_data = if layer_width == width && layer_height == height { data @@ -1724,7 +1670,7 @@ fn encode_i420_layers( pts, dur_ms, timestamp, - if matches!(inner.codec_id, VideoCodec::Vpx(VpxCodec::Vp9)) + if matches!(inner.codec_id, VpxCodec::Vp9) && matches!(inner.layering, VideoLayering::ServerManaged { .. }) { Some(Vp9SvcMode::new( @@ -1742,7 +1688,7 @@ fn encode_i420_layers( #[allow(clippy::too_many_arguments)] fn encode_layer_packets( - codec_id: VideoCodec, + codec_id: VpxCodec, encoding: &VideoEncoding, data: &[u8], width: u32, @@ -1758,7 +1704,7 @@ fn encode_layer_packets( Some(state) => { let current_svc_mode = match &state.codec { VideoCodecState::Vp9Svc { encoder, .. } => Some(encoder.mode()), - VideoCodecState::Vpx { .. } | VideoCodecState::H264 { .. } => None, + VideoCodecState::Vpx { .. } => None, }; state.width != width || state.height != height @@ -1774,24 +1720,19 @@ fn encode_layer_packets( .unwrap_or_else(seed_u16); let prior_vp9_packetizer = guard.as_ref().and_then(|state| match &state.codec { VideoCodecState::Vp9Svc { packetizer, .. } => Some(packetizer.clone()), - VideoCodecState::Vpx { .. } | VideoCodecState::H264 { .. } => None, + VideoCodecState::Vpx { .. } => None, }); let codec = match (codec_id, svc_mode) { - (VideoCodec::Vpx(VpxCodec::Vp9), Some(mode)) => VideoCodecState::Vp9Svc { + (VpxCodec::Vp9, Some(mode)) => VideoCodecState::Vp9Svc { encoder: VpxSvcEncoder::new(width, height, bitrate_kbps, mode)?, packetizer: prior_vp9_packetizer .unwrap_or_else(|| VpxRtpPacketizer::new(VpxCodec::Vp9)), }, - (VideoCodec::Vpx(codec), None) => VideoCodecState::Vpx { + (codec, None) => VideoCodecState::Vpx { encoder: VpxEncoder::new(codec, width, height, bitrate_kbps)?, packetizer: VpxRtpPacketizer::new(codec), }, - (VideoCodec::H264, None) => VideoCodecState::H264 { - encoder: Box::new(H264Encoder::new(bitrate_kbps.saturating_mul(1_000))?), - packetizer: H264RtpPacketizer::default(), - encoded: Vec::new(), - }, - (VideoCodec::Vpx(VpxCodec::Vp8), Some(_)) | (VideoCodec::H264, Some(_)) => { + (VpxCodec::Vp8, Some(_)) => { return Err(RtcError::Media( "VP9 SVC mode supplied for a non-VP9 encoder".to_owned(), )); @@ -1898,35 +1839,6 @@ fn encode_layer_packets( seq = seq.wrapping_add(1); } } - VideoCodecState::H264 { - encoder, - packetizer, - encoded, - } => { - let key = encoder.encode_into(data, width, height, force_key, encoded)?; - tracing::trace!( - bytes = encoded.len(), - key, - mime = %encoding.core.mime_type, - "stream.rtc.video.encoded_frame" - ); - for payload in packetizer.packetize(encoded, PACKET_MTU)? { - let header = webrtc::rtp::header::Header { - version: 2, - payload_type: PLACEHOLDER_PT, - sequence_number: seq, - timestamp, - ssrc: PLACEHOLDER_SSRC, - marker: payload.last, - ..Default::default() - }; - out.push(RtpPacket { - header, - payload: payload.data, - }); - seq = seq.wrapping_add(1); - } - } } state.seq = seq; Ok(out) @@ -2859,10 +2771,6 @@ mod tests { assert!(track.track_id().starts_with("video-")); } - fn layered_config() -> LocalVideoTrackConfig { - LocalVideoTrackConfig::default().server_managed() - } - fn layered_option(track_type: TrackType, codec: &str) -> PublishOption { PublishOption { id: 41, @@ -2883,29 +2791,6 @@ mod tests { } } - #[test] - fn layered_h264_camera_builds_three_rid_encodings() { - let track = LocalVideoTrack::h264_with_config(layered_config()).expect("layered H264"); - let layers = track - .configure_for_publish(TrackType::Video, &layered_option(TrackType::Video, "H264")) - .expect("supported H264 camera topology"); - assert_eq!( - layers - .iter() - .map(|layer| layer.rid.as_str()) - .collect::>(), - ["q", "h", "f"] - ); - assert_eq!( - track - .webrtc_tracks() - .iter() - .filter_map(|track| track.rid()) - .collect::>(), - ["q", "h", "f"] - ); - } - #[test] fn layered_vp8_screen_share_builds_three_rid_encodings() { let track = LocalVideoTrack::vp8_simulcast().expect("layered VP8"); @@ -3130,9 +3015,12 @@ mod tests { #[test] fn publish_quality_updates_only_the_named_rid_and_forces_keyframe_on_resume() { - let track = LocalVideoTrack::h264_with_config(layered_config()).expect("layered H264"); + let track = LocalVideoTrack::vp8_simulcast().expect("layered VP8"); track - .configure_for_publish(TrackType::Video, &layered_option(TrackType::Video, "H264")) + .configure_for_publish( + TrackType::ScreenShare, + &layered_option(TrackType::ScreenShare, "VP8"), + ) .expect("configure layers"); track.apply_layer_setting(&VideoLayerSetting { name: "h".to_owned(), @@ -3227,40 +3115,6 @@ mod tests { } } - #[tokio::test] - async fn h264_write_i420_encodes_blue_frame() { - let track = LocalVideoTrack::h264().expect("H264 track"); - let (w, h) = (320u32, 240u32); - let mut buf = vec![41u8; (w * h) as usize]; - buf.extend(std::iter::repeat_n(240u8, ((w / 2) * (h / 2)) as usize)); - buf.extend(std::iter::repeat_n(110u8, ((w / 2) * (h / 2)) as usize)); - track - .write_i420(&buf, w, h, Duration::from_millis(100)) - .await - .expect("write_i420 H264 blue frame"); - assert_eq!(track.mime_type(), MIME_TYPE_H264); - } - - #[tokio::test] - async fn h264_write_i420_rejects_frames_beyond_level_3_1_before_copy() { - let track = LocalVideoTrack::h264().expect("H264 track"); - let max_fs_error = track - .write_i420(&[], 1_920, 1_080, Duration::from_millis(100)) - .await - .expect_err("1080p exceeds level 3.1 MaxFS"); - assert!(max_fs_error.to_string().contains("level 3.1")); - - let frame_rate_error = track - .write_i420(&[], 1_280, 720, Duration::from_millis(16)) - .await - .expect_err("720p60 exceeds the configured level 3.1 rate"); - assert!( - frame_rate_error - .to_string() - .contains("duration is too short") - ); - } - #[tokio::test] async fn vp9_write_i420_rejects_unbounded_dimensions_before_copy() { let track = LocalVideoTrack::vp9().expect("VP9 track"); diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index 72f13c1..ceb5d5c 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -9,7 +9,7 @@ //! - [`RemoteTrack::next_pcm`] — decoded 48 kHz mono [`PcmFrame`] (audio only; //! Opus decode, for the PCM bridge / bots). //! - [`RemoteTrack::next_video_frame`] — decoded packed-I420 [`VideoFrame`] -//! (VP8/VP9/H264 video, for bots that need to *see* the call). +//! (VP8/VP9 video, for bots that need to *see* the call). //! //! Read operations on one track are serialized. Do not mix raw and decoded //! reads: each RTP packet is consumed by whichever read operation acquires the @@ -33,8 +33,6 @@ use webrtc::rtp::codecs::vp9::Vp9Packet; use webrtc::track::track_remote::TrackRemote; use super::local::RtpPacket; -use crate::rtc::codecs::h264::{H264Decoder, access_unit_has_idr}; -use crate::rtc::codecs::rtp_h264::H264Depacketizer; use crate::rtc::codecs::vpx::{VpxCodec, VpxDecoder}; use crate::rtc::error::{Result, RtcError}; use crate::rtc::pcm::{FRAME_SAMPLES_20MS, OPUS_SAMPLE_RATE, PcmFrame}; @@ -146,7 +144,6 @@ pub struct Codec { enum VideoSamples { Vp8(SampleBuilder), Vp9(SampleBuilder), - H264(SampleBuilder), } impl VideoSamples { @@ -162,11 +159,6 @@ impl VideoSamples { Vp9Packet::default(), VIDEO_CLOCK_RATE, )), - VideoCodec::H264 => Self::H264(SampleBuilder::new( - VIDEO_MAX_LATE, - H264Depacketizer::default(), - VIDEO_CLOCK_RATE, - )), } } @@ -174,7 +166,6 @@ impl VideoSamples { match self { Self::Vp8(b) => b.push(packet), Self::Vp9(b) => b.push(packet), - Self::H264(b) => b.push(packet), } } @@ -182,7 +173,6 @@ impl VideoSamples { match self { Self::Vp8(b) => b.pop(), Self::Vp9(b) => b.pop(), - Self::H264(b) => b.pop(), } } } @@ -191,22 +181,15 @@ impl VideoSamples { enum VideoCodec { Vp8, Vp9, - H264, -} - -enum VideoDecoder { - Vpx(VpxDecoder), - H264(H264Decoder), } /// Inbound video reassembly + decode state, plus frames already decoded but not /// yet handed to the caller (one sample can yield more than one frame). struct VideoDecode { samples: VideoSamples, - decoder: VideoDecoder, + decoder: VpxDecoder, ready: VecDeque, last_resolution: Option<(u32, u32)>, - awaiting_h264_idr: bool, } /// Inbound audio decode state, plus frames already decoded but not yet handed to @@ -227,7 +210,7 @@ struct AudioDecode { enum Decode { /// Opus → 48 kHz mono PCM. Audio(StdMutex), - /// VP8/VP9/H264 RTP → packed I420 frames. Shared with the bounded blocking + /// VP8/VP9 RTP → packed I420 frames. Shared with the bounded blocking /// decode work; a [`SampleBuilder`] carries a full sequence-number window. Video(Arc>), /// No decoder: a codec we cannot decode (for example AV1), or a decoder that @@ -438,7 +421,7 @@ impl RemoteTrack { let packet = self.read_rtp_inner().await?; // Packet reordering/depacketization is cheap and stays on the async - // task. Native VPx/OpenH264 decode is measured in milliseconds at + // task. Native VPx decode is measured in milliseconds at // 720p, so only complete samples cross into Tokio's blocking pool. let samples = { let mut s = state.lock().unwrap_or_else(|e| e.into_inner()); @@ -653,25 +636,10 @@ impl VideoDecode { "stream.rtc.remote.video_packets_dropped" ); needs_keyframe = true; - self.restart_h264_after_discontinuity(); } - if self.awaiting_h264_idr && !access_unit_has_idr(&sample.data) { - needs_keyframe = true; - continue; - } - - let decoded = match &mut self.decoder { - VideoDecoder::Vpx(decoder) => decoder.decode(&sample.data, sample.packet_timestamp), - VideoDecoder::H264(decoder) => { - decoder.decode(&sample.data, sample.packet_timestamp) - } - }; - match decoded { + match self.decoder.decode(&sample.data, sample.packet_timestamp) { Ok(frames) => { - if self.awaiting_h264_idr && !frames.is_empty() { - self.awaiting_h264_idr = false; - } for frame in frames { let resolution = (frame.width, frame.height); if self.last_resolution != Some(resolution) { @@ -688,25 +656,11 @@ impl VideoDecode { Err(e) => { tracing::debug!(error = %e, "stream.rtc.remote.video_decode_failed"); needs_keyframe = true; - self.restart_h264_after_discontinuity(); } } } needs_keyframe } - - fn restart_h264_after_discontinuity(&mut self) { - let VideoDecoder::H264(decoder) = &mut self.decoder else { - return; - }; - self.awaiting_h264_idr = true; - if let Err(error) = decoder.restart() { - tracing::warn!( - error = %error, - "stream.rtc.remote.h264_decoder_restart_failed" - ); - } - } } impl Drop for RemoteTrack { @@ -745,22 +699,20 @@ fn build_decoder(track_type: TrackType, codec: &Codec) -> Decode { tracing::warn!( mime_type = %codec.mime_type, "stream.rtc.remote.video_codec_not_decodable: next_video_frame will return None; \ - VP8, VP9, and H264 are decodable (read_rtp still works)" + VP8 and VP9 are decodable (read_rtp still works)" ); return Decode::None; }; - let decoder = match video_codec { - VideoCodec::Vp8 => VpxDecoder::new(VpxCodec::Vp8).map(VideoDecoder::Vpx), - VideoCodec::Vp9 => VpxDecoder::new(VpxCodec::Vp9).map(VideoDecoder::Vpx), - VideoCodec::H264 => H264Decoder::new().map(VideoDecoder::H264), - }; + let decoder = VpxDecoder::new(match video_codec { + VideoCodec::Vp8 => VpxCodec::Vp8, + VideoCodec::Vp9 => VpxCodec::Vp9, + }); match decoder { Ok(decoder) => Decode::Video(Arc::new(StdMutex::new(VideoDecode { samples: VideoSamples::new(video_codec), decoder, ready: VecDeque::new(), last_resolution: None, - awaiting_h264_idr: video_codec == VideoCodec::H264, }))), Err(e) => { tracing::warn!( @@ -779,7 +731,6 @@ fn video_codec_for(mime_type: &str) -> Option { match () { () if mime.ends_with("/vp8") => Some(VideoCodec::Vp8), () if mime.ends_with("/vp9") => Some(VideoCodec::Vp9), - () if mime.ends_with("/h264") => Some(VideoCodec::H264), () => None, } } @@ -792,7 +743,7 @@ mod tests { fn supported_video_mime_types_map_to_a_decoder() { assert_eq!(video_codec_for("video/VP8"), Some(VideoCodec::Vp8)); assert_eq!(video_codec_for("video/vp9"), Some(VideoCodec::Vp9)); - assert_eq!(video_codec_for("video/H264"), Some(VideoCodec::H264)); + assert_eq!(video_codec_for("video/H264"), None); } /// One 20 ms frame of 440 Hz tone, loud enough that a silent or badly diff --git a/tests/gpt_realtime_bot.rs b/tests/gpt_realtime_bot.rs index a9702b6..64f0db4 100644 --- a/tests/gpt_realtime_bot.rs +++ b/tests/gpt_realtime_bot.rs @@ -1,8 +1,7 @@ //! Live end-to-end test for the `gpt_realtime_bot` example. //! -//! A second SDK session publishes bursty audio and a blue video frame, then -//! verifies that the bot processes both media tracks and publishes an audible -//! response from OpenAI Realtime. +//! A second SDK session publishes bursty audio, then verifies that the bot +//! receives it and publishes an audible response from OpenAI Realtime. //! //! Skips cleanly without `STREAM_API_*` (no client) or without `OPENAI_API_KEY` //! (no OpenAI bridge). Nothing is mocked. @@ -21,9 +20,7 @@ use getstream::models::{ CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest, UserRequest, }; use getstream::rtc::proto::models::TrackType; -use getstream::rtc::{ - JoinCallData, LocalAudioTrack, LocalVideoTrack, PcmFrame, RemoteTrack, SubscriptionConfig, -}; +use getstream::rtc::{JoinCallData, LocalAudioTrack, PcmFrame, RemoteTrack, SubscriptionConfig}; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; use tokio::task::JoinHandle; @@ -35,8 +32,6 @@ const TONE_HZ: f64 = 300.0; const TONE_AMP: f64 = 12_000.0; /// A comfortably non-silent RMS floor (silence is 0; our tone is ~0.26). const NON_SILENT_RMS: f64 = 0.02; -const VIDEO_W: u32 = 320; -const VIDEO_H: u32 = 240; fn init_tracing() { let _ = tracing_subscriber::fmt() @@ -82,33 +77,6 @@ fn spawn_speech_tone(track: LocalAudioTrack) -> JoinHandle<()> { }) } -/// A solid-blue frame in packed I420 (BT.601 limited-range). -fn solid_blue_i420(width: u32, height: u32) -> Vec { - let (w, h) = (width as usize, height as usize); - let mut buf = vec![41u8; w * h]; - buf.extend(std::iter::repeat_n(240u8, (w / 2) * (h / 2))); - buf.extend(std::iter::repeat_n(110u8, (w / 2) * (h / 2))); - buf -} - -/// Publish a solid-blue I420 frame at ~10 fps until the track is stopped. -fn spawn_blue_video(track: LocalVideoTrack) -> JoinHandle<()> { - tokio::spawn(async move { - let frame = solid_blue_i420(VIDEO_W, VIDEO_H); - let mut interval = tokio::time::interval(Duration::from_millis(100)); - loop { - interval.tick().await; - if track - .write_i420(&frame, VIDEO_W, VIDEO_H, Duration::from_millis(100)) - .await - .is_err() - { - return; - } - } - }) -} - /// Register an `on_track` sink forwarding each `RemoteTrack` to an mpsc channel. fn track_sink(call: &Call) -> Receiver { let (tx, rx) = channel(8); @@ -192,7 +160,7 @@ async fn max_block_rms(remote: &RemoteTrack, overall: Duration) -> f64 { } #[tokio::test] -async fn gpt_bot_hears_audio_video_and_replies() { +async fn gpt_bot_hears_audio_and_replies() { let Some(client) = common::client_or_skip() else { return; }; @@ -233,7 +201,6 @@ async fn gpt_bot_hears_audio_video_and_replies() { let call_s = client.video().call("default", &call_id); let mut bot_handle = None; let mut tone = None; - let mut blue = None; let outcome: Result<()> = tokio::time::timeout(Duration::from_secs(180), async { bot_handle = Some( bot::start_bot(&client, &cfg, &bot_user, "default", &call_id) @@ -254,15 +221,8 @@ async fn gpt_bot_hears_audio_video_and_replies() { .context("speaker publish audio")?; tone = Some(spawn_speech_tone(audio)); - let video = LocalVideoTrack::vp9().context("speaker VP9 track")?; - call_s - .publish_video(video.clone()) - .await - .context("speaker publish video")?; - blue = Some(spawn_blue_video(video)); - call_s - .update_subscriptions(SubscriptionConfig::audio_video()) + .update_subscriptions(SubscriptionConfig::audio_all()) .await .context("speaker subscriptions")?; @@ -271,19 +231,6 @@ async fn gpt_bot_hears_audio_video_and_replies() { "bot on_track never fired for AUDIO (subscription/ICE/RTP stage)" ); - ensure!( - wait_until(Duration::from_secs(60), || bot.video_frames_decoded() >= 3).await, - "bot received video but decoded only {} frames (reassembly/keyframe/decode stage)", - bot.video_frames_decoded() - ); - - ensure!( - wait_until(Duration::from_secs(30), || bot.video_frames_encoded() >= 2).await, - "bot decoded video but only H264-encoded {} frames for OpenAI \ - (downscale/OpenH264/track-write stage)", - bot.video_frames_encoded() - ); - let bot_audio = recv_track( &mut rx_s, &bot_user, @@ -297,15 +244,6 @@ async fn gpt_bot_hears_audio_video_and_replies() { rms > NON_SILENT_RMS, "bot produced only silence (max block rms={rms:.4})" ); - - let codec = bot - .openai_video_codec() - .await - .context("OpenAI's SDP answer carried no video m-line")?; - ensure!( - codec.contains("H264"), - "expected OpenAI to negotiate H264 video, got {codec}" - ); ensure!( bot.openai_connection_state() == RTCPeerConnectionState::Connected, "the OpenAI PeerConnection is not connected: {:?}", @@ -318,7 +256,6 @@ async fn gpt_bot_hears_audio_video_and_replies() { .and_then(|result| result); stop_task(tone).await; - stop_task(blue).await; let speaker_cleanup = call_s.leave().await; let bot_cleanup = match bot_handle { Some(bot) => bot.shutdown().await, diff --git a/tests/rtc_media.rs b/tests/rtc_media.rs index e85930b..a0a861b 100644 --- a/tests/rtc_media.rs +++ b/tests/rtc_media.rs @@ -21,9 +21,9 @@ use getstream::models::UserRequest; use getstream::models::{CallRequest, DeleteCallRequest, GetOrCreateCallRequest, MemberRequest}; use getstream::rtc::proto::models::TrackType; use getstream::rtc::{ - CallingState, ClientCallEvent, ClientPublishOptions, JoinCallData, LocalAudioTrack, LocalTrack, - LocalVideoTrack, PcmFrame, PreferredVideoCodec, RemoteTrack, RtcError, SfuCallEvent, - SubscriptionConfig, SubscriptionTarget, TrackSubscriptionConfig, VideoFrame, + CallingState, ClientCallEvent, JoinCallData, LocalAudioTrack, LocalTrack, LocalVideoTrack, + PcmFrame, RemoteTrack, RtcError, SfuCallEvent, SubscriptionConfig, SubscriptionTarget, + TrackSubscriptionConfig, VideoFrame, }; use getstream::video::Call; use tokio::sync::mpsc::{Receiver, channel}; @@ -1121,93 +1121,6 @@ async fn vp9_svc_preserves_one_ssrc_and_adapts_all_spatial_layers() { } } -#[tokio::test] -async fn publish_h264_video_b_decodes_i420_frame() { - let Some(client) = common::client_or_skip() else { - return; - }; - init_tracing(); - - let user_a = common::unique_id("h264-a"); - let user_b = common::unique_id("h264-b"); - let (admin, call_id) = setup_call(&client, &[&user_a, &user_b]).await; - - let outcome = tokio::time::timeout(Duration::from_secs(160), async { - let call_a = client.video().call("default", &call_id); - let call_b = client.video().call("default", &call_id); - - call_a.update_publish_options(ClientPublishOptions::new(PreferredVideoCodec::H264)); - call_a - .join(JoinCallData::new(&user_a)) - .await - .expect("H264 publisher join"); - let video_a = LocalVideoTrack::h264().expect("H264 track"); - - // No public accessor exposes the join response's advertised - // publish_options, so attempting the publish is the structured skip - // signal the API gives us: `publish_video` validates the requested codec - // against the SFU's advertised options before any SetPublisher RPC, so a - // VP-only edge (all app edges are `-vp`) returns a narrow "did not - // advertise" media error. Skip cleanly only on that, surface anything - // else, and run the full publish -> subscribe -> decode assertion path - // on an H264-advertising edge. - match call_a.publish_video(video_a.clone()).await { - Ok(()) => {} - Err(RtcError::Media(message)) if message.contains("did not advertise") => { - eprintln!( - "SKIP: publish_h264_video_b_decodes_i420_frame: SFU edge does not \ - advertise H264 ({message})" - ); - call_a - .leave() - .await - .expect("H264 publisher leave after skip"); - return; - } - Err(error) => panic!("unexpected H264 publish_video failure: {error}"), - } - let feeder = spawn_blue_video(video_a); - - let mut rx_b = track_sink(&call_b); - call_b - .join(JoinCallData::new(&user_b)) - .await - .expect("H264 subscriber join"); - call_b - .update_subscriptions(SubscriptionConfig::audio_video()) - .await - .expect("subscribe to H264 video"); - - let remote = recv_track( - &mut rx_b, - &user_a, - TrackType::Video, - Duration::from_secs(60), - ) - .await - .expect("subscriber did not receive the H264 track"); - assert!( - remote.codec().mime_type.eq_ignore_ascii_case("video/h264"), - "expected H264, negotiated {}", - remote.codec().mime_type - ); - - let frame = tokio::time::timeout(Duration::from_secs(45), remote.next_video_frame()) - .await - .expect("timed out reassembling and decoding H264") - .expect("H264 track ended before a frame decoded"); - assert_packed_blue_frame(&frame); - - feeder.abort(); - call_a.leave().await.expect("H264 publisher leave"); - call_b.leave().await.expect("H264 subscriber leave"); - }) - .await; - - let _ = admin.delete(DeleteCallRequest { hard: Some(true) }).await; - outcome.expect("H264 live media test timed out"); -} - // Test 8: the SFU reports A taking over as speaking / dominant // // Verified through **signaling**, not inbound RTP: the SFU's subscriber From 0b521ac1a472092169b789a37fbf9919a882957e Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Tue, 6 Oct 2026 22:54:21 +0200 Subject: [PATCH 36/37] fix: decode every inter frame of a flexible-mode VP9 stream Browsers send VP9 SVC in flexible mode, with a reference index in each inter-frame packet. The webrtc-rs Vp9Packet keeps the reference indices of earlier packets and fails when three accumulate. The SampleBuilder uses one Vp9Packet for the stream, so after the third reference index no inter frame completed and next_video_frame gave no more frames. Vp9Depacketizer resets the Vp9Packet before each packet. Add debug logs for inbound video: the codec of a new track, a payload type change, and receive counts every 5 s (packets, sequence gaps, markers, samples, frames, decode errors). --- src/rtc/tracks/remote.rs | 187 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 2 deletions(-) diff --git a/src/rtc/tracks/remote.rs b/src/rtc/tracks/remote.rs index ceb5d5c..9e2709a 100644 --- a/src/rtc/tracks/remote.rs +++ b/src/rtc/tracks/remote.rs @@ -23,6 +23,7 @@ use std::sync::Mutex as StdMutex; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; +use bytes::Bytes; use tokio::sync::{Mutex as AsyncMutex, Semaphore}; use webrtc::media::io::sample_builder::SampleBuilder; use webrtc::peer_connection::RTCPeerConnection; @@ -30,6 +31,7 @@ use webrtc::rtcp::packet::Packet as RtcpPacket; use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; use webrtc::rtp::codecs::vp8::Vp8Packet; use webrtc::rtp::codecs::vp9::Vp9Packet; +use webrtc::rtp::packetizer::Depacketizer; use webrtc::track::track_remote::TrackRemote; use super::local::RtpPacket; @@ -54,6 +56,8 @@ const VIDEO_CLOCK_RATE: u32 = 90_000; /// one is in flight for at least a round trip, so asking faster only wastes /// uplink. const KEYFRAME_REQUEST_INTERVAL: Duration = Duration::from_secs(1); +/// Interval between `stream.rtc.remote.video_receive_stats` debug logs. +const VIDEO_STATS_INTERVAL: Duration = Duration::from_secs(5); /// The publishing participant a [`RemoteTrack`] belongs to. #[derive(Debug, Clone, Default, PartialEq)] @@ -143,7 +147,28 @@ pub struct Codec { /// decoder corrupts any frame that arrives out of order. enum VideoSamples { Vp8(SampleBuilder), - Vp9(SampleBuilder), + Vp9(SampleBuilder), +} + +/// [`Vp9Packet`] reset before each packet. `Vp9Packet::depacketize` keeps the +/// reference indices of earlier packets and fails once three accumulate, which +/// rejects every flexible-mode inter frame after the third. +#[derive(Default)] +struct Vp9Depacketizer(Vp9Packet); + +impl Depacketizer for Vp9Depacketizer { + fn depacketize(&mut self, packet: &Bytes) -> std::result::Result { + self.0 = Vp9Packet::default(); + self.0.depacketize(packet) + } + + fn is_partition_head(&self, payload: &Bytes) -> bool { + self.0.is_partition_head(payload) + } + + fn is_partition_tail(&self, marker: bool, payload: &Bytes) -> bool { + self.0.is_partition_tail(marker, payload) + } } impl VideoSamples { @@ -156,7 +181,7 @@ impl VideoSamples { )), VideoCodec::Vp9 => Self::Vp9(SampleBuilder::new( VIDEO_MAX_LATE, - Vp9Packet::default(), + Vp9Depacketizer::default(), VIDEO_CLOCK_RATE, )), } @@ -190,6 +215,26 @@ struct VideoDecode { decoder: VpxDecoder, ready: VecDeque, last_resolution: Option<(u32, u32)>, + /// Payload type of the last packet, starting at the negotiated one. + payload_type: u8, + last_sequence_number: Option, + stats: VideoStats, + stats_since: Instant, +} + +/// Inbound video counts since the last stats log. [`SampleBuilder`] discards +/// its build errors, so packets without completed samples are the only sign of +/// a frame that could not be assembled. +#[derive(Default)] +struct VideoStats { + packets: usize, + /// Packets whose sequence number does not follow the previous packet's. + sequence_gaps: usize, + markers: usize, + samples: usize, + frames: usize, + decode_errors: usize, + last_payload_byte: Option, } /// Inbound audio decode state, plus frames already decoded but not yet handed to @@ -268,6 +313,14 @@ impl RemoteTrack { clock_rate: params.capability.clock_rate, channels: params.capability.channels, }; + tracing::debug!( + user_id = %participant.user_id, + ?track_type, + mime_type = %codec.mime_type, + payload_type = codec.payload_type, + ssrc = track.ssrc(), + "stream.rtc.remote.track_codec" + ); let decode = build_decoder(track_type, &codec); @@ -615,11 +668,54 @@ impl VideoDecode { /// Feed one RTP packet into the reassembler and return every sample it /// completes. This path performs no native decode work. fn push_packet(&mut self, packet: RtpPacket) -> Vec { + let ssrc = packet.header.ssrc; + let payload_type = packet.header.payload_type; + if payload_type != self.payload_type { + tracing::debug!( + ssrc, + from = self.payload_type, + to = payload_type, + "stream.rtc.remote.video_payload_type_changed" + ); + self.payload_type = payload_type; + } + self.stats.packets += 1; + let sequence_number = packet.header.sequence_number; + if self + .last_sequence_number + .is_some_and(|last| sequence_number != last.wrapping_add(1)) + { + self.stats.sequence_gaps += 1; + } + self.last_sequence_number = Some(sequence_number); + if packet.header.marker { + self.stats.markers += 1; + } + self.stats.last_payload_byte = packet.payload.first().copied(); + let mut completed = Vec::with_capacity(1); self.samples.push(packet); while let Some(sample) = self.samples.pop() { completed.push(sample); } + self.stats.samples += completed.len(); + + if self.stats_since.elapsed() >= VIDEO_STATS_INTERVAL { + let stats = std::mem::take(&mut self.stats); + tracing::debug!( + ssrc, + payload_type, + packets = stats.packets, + sequence_gaps = stats.sequence_gaps, + markers = stats.markers, + samples = stats.samples, + frames = stats.frames, + decode_errors = stats.decode_errors, + last_payload_byte = ?stats.last_payload_byte.map(|b| format!("{b:#04x}")), + "stream.rtc.remote.video_receive_stats" + ); + self.stats_since = Instant::now(); + } completed } @@ -640,6 +736,7 @@ impl VideoDecode { match self.decoder.decode(&sample.data, sample.packet_timestamp) { Ok(frames) => { + self.stats.frames += frames.len(); for frame in frames { let resolution = (frame.width, frame.height); if self.last_resolution != Some(resolution) { @@ -654,6 +751,7 @@ impl VideoDecode { } } Err(e) => { + self.stats.decode_errors += 1; tracing::debug!(error = %e, "stream.rtc.remote.video_decode_failed"); needs_keyframe = true; } @@ -713,6 +811,10 @@ fn build_decoder(track_type: TrackType, codec: &Codec) -> Decode { decoder, ready: VecDeque::new(), last_resolution: None, + payload_type: codec.payload_type, + last_sequence_number: None, + stats: VideoStats::default(), + stats_since: Instant::now(), }))), Err(e) => { tracing::warn!( @@ -738,6 +840,7 @@ fn video_codec_for(mime_type: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::rtc::codecs::vpx::{EncodedFrame, VpxEncoder}; #[test] fn supported_video_mime_types_map_to_a_decoder() { @@ -746,6 +849,86 @@ mod tests { assert_eq!(video_codec_for("video/H264"), None); } + /// RTP packets for one VP9 frame in flexible mode, as browsers send VP9 + /// SVC: a 15-bit picture ID and, on an inter frame, one reference index. + fn vp9_flexible_mode_packets( + frame: &EncodedFrame, + picture_id: u16, + first_sequence_number: u16, + ) -> Vec { + let chunks: Vec<&[u8]> = frame.data.chunks(200).collect(); + let last = chunks.len() - 1; + chunks + .into_iter() + .enumerate() + .map(|(index, chunk)| { + // I (picture ID) + F (flexible mode). + let mut payload = vec![0x90, 0x80 | (picture_id >> 8) as u8, picture_id as u8]; + if !frame.key { + payload[0] |= 0x40; + // P_DIFF = 1, N = 0. + payload.push(1 << 1); + } + if index == 0 { + payload[0] |= 0x08; + } + if index == last { + payload[0] |= 0x04; + } + payload.extend_from_slice(chunk); + RtpPacket { + header: webrtc::rtp::header::Header { + version: 2, + payload_type: 98, + sequence_number: first_sequence_number.wrapping_add(index as u16), + timestamp: u32::from(picture_id) * 3_000, + marker: index == last, + ..Default::default() + }, + payload: Bytes::from(payload), + } + }) + .collect() + } + + #[test] + fn every_frame_of_a_flexible_mode_vp9_stream_is_decoded() { + let (width, height) = (160, 120); + let mut encoder = VpxEncoder::new(VpxCodec::Vp9, width, height, 400).expect("vp9 encoder"); + let image = vec![128u8; (width * height * 3 / 2) as usize]; + let codec = Codec { + mime_type: "video/VP9".to_owned(), + payload_type: 98, + clock_rate: VIDEO_CLOCK_RATE, + channels: 0, + }; + let Decode::Video(state) = build_decoder(TrackType::Video, &codec) else { + panic!("VP9 must have a decoder"); + }; + let mut state = state.lock().expect("video decode state"); + + let mut pictures: u16 = 0; + let mut sequence_number: u16 = 0; + for index in 0..10 { + let encoded = encoder + .encode(&image, index * 33, 33, index == 0) + .expect("encode"); + for frame in &encoded { + let packets = vp9_flexible_mode_packets(frame, pictures, sequence_number); + pictures += 1; + sequence_number = sequence_number.wrapping_add(packets.len() as u16); + for packet in packets { + let samples = state.push_packet(packet); + state.decode_samples(samples); + } + } + } + + // The last frame completes only when a packet of a later frame arrives. + assert!(pictures > 3, "the encoder must emit inter frames"); + assert_eq!(state.ready.len(), usize::from(pictures) - 1); + } + /// One 20 ms frame of 440 Hz tone, loud enough that a silent or badly /// rebuilt frame is obvious. fn tone_20ms() -> Vec { From ef8694d3e376a8f45fb6ec90cbe05fb5f6a2afc8 Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Tue, 6 Oct 2026 22:54:21 +0200 Subject: [PATCH 37/37] docs: add the VP9 receive fix to the changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d70857d..6284b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -184,6 +184,10 @@ on `Call` (`get_call_participant_session_metrics`, - While a track is muted, its RTP clock keeps running, for audio and video. The first packet after an unmute shows the length of the mute (RFC 3550). Before, the timestamps continued from the last packet before the mute. +- `RemoteTrack::next_video_frame` decodes the VP9 SVC that browsers send in + flexible mode. Before, it gave the first frames and then no more frames: the + depacketizer kept the reference indices of earlier packets and rejected each + inter frame after the third reference index. # v0.1.0-preview.2