gemini_adk_rs/live/
input_vad.rs

1//! Backend input VAD for browser microphone PCM.
2
3use std::time::Instant;
4
5/// A per-frame processor applied to outgoing microphone audio inside
6/// [`LiveHandle::send_audio`](super::LiveHandle::send_audio) — the L1 seam
7/// the L2 `voice` chain (denoiser, noise gate) plugs into so hosted
8/// surfaces (web bridge, API server) get the same hardened path as native
9/// pumps. Runs on the send path: keep it fast and allocation-light.
10pub trait InputAudioProcessor: Send {
11    /// Process one PCM16 frame in place (the frame may change length).
12    fn process_frame(&mut self, frame: &mut Vec<i16>);
13}
14
15/// Who decides when user speech interrupts the model.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
17pub enum ActivityAuthority {
18    /// The Live API's automatic activity detection decides (default).
19    #[default]
20    Server,
21    /// This client's input VAD decides: `send_audio` emits
22    /// `activityStart`/`activityEnd` marks on speech edges. Only meaningful
23    /// when the session was configured with automatic activity detection
24    /// disabled — otherwise the server ignores the marks.
25    Client,
26}
27
28use gemini_genai_rs::prelude::{VadConfig, VadEvent, VoiceActivityDetector};
29use gemini_genai_rs::vad::VadState;
30use serde::Serialize;
31
32/// Snapshot of backend VAD state for devtools and diagnostics.
33#[derive(Debug, Clone, Serialize)]
34pub struct BackendVadSnapshot {
35    /// Active detector backend name.
36    pub backend: &'static str,
37    /// Input sample rate in Hz.
38    pub sample_rate: u32,
39    /// Detector frame duration in milliseconds.
40    pub frame_duration_ms: u32,
41    /// Detector frame size in samples.
42    pub frame_size: usize,
43    /// Current detector state.
44    pub state: &'static str,
45    /// Whether the detector is currently in a speech state.
46    pub speaking: bool,
47    /// Last normalized speech probability or binary decision.
48    pub last_probability: Option<f32>,
49    /// Number of complete frames processed by the backend detector.
50    pub frames_processed: u64,
51    /// Milliseconds since the last speech start/end transition.
52    pub last_transition_ms_ago: Option<u64>,
53}
54
55/// Incremental VAD over arbitrary PCM16 byte chunks.
56pub struct BackendInputVad {
57    detector: VoiceActivityDetector,
58    config: VadConfig,
59    pending_samples: Vec<i16>,
60    frames_processed: u64,
61    last_transition_at: Option<Instant>,
62}
63
64impl BackendInputVad {
65    /// Create a backend input VAD with explicit detector configuration.
66    pub fn new(config: VadConfig) -> Self {
67        Self {
68            detector: VoiceActivityDetector::new(config.clone()),
69            config,
70            pending_samples: Vec::new(),
71            frames_processed: 0,
72            last_transition_at: None,
73        }
74    }
75
76    /// Process arbitrary little-endian PCM16 bytes and return speech edge events.
77    pub fn process_pcm_bytes(&mut self, bytes: &[u8]) -> Vec<VadEvent> {
78        self.pending_samples
79            .extend(bytes.chunks_exact(2).map(|pair| {
80                let raw = [pair[0], pair[1]];
81                i16::from_le_bytes(raw)
82            }));
83
84        let frame_size = self.config.frame_size();
85        if frame_size == 0 {
86            return Vec::new();
87        }
88
89        let mut events = Vec::new();
90        while self.pending_samples.len() >= frame_size {
91            let frame: Vec<i16> = self.pending_samples.drain(..frame_size).collect();
92            self.frames_processed += 1;
93            if let Some(event) = self.detector.process_frame(&frame) {
94                self.last_transition_at = Some(Instant::now());
95                events.push(event);
96            }
97        }
98        events
99    }
100
101    /// The detector's configured input sample rate in Hz.
102    pub fn sample_rate(&self) -> u32 {
103        self.config.sample_rate
104    }
105
106    /// Return a diagnostics snapshot suitable for UI/devtools display.
107    pub fn snapshot(&self) -> BackendVadSnapshot {
108        BackendVadSnapshot {
109            backend: self.detector.backend_name(),
110            sample_rate: self.config.sample_rate,
111            frame_duration_ms: self.config.frame_duration_ms,
112            frame_size: self.config.frame_size(),
113            state: state_name(self.detector.state()),
114            speaking: self.detector.is_speaking(),
115            last_probability: self.detector.last_probability(),
116            frames_processed: self.frames_processed,
117            last_transition_ms_ago: self
118                .last_transition_at
119                .map(|instant| instant.elapsed().as_millis() as u64),
120        }
121    }
122
123    #[cfg(test)]
124    /// Whether the detector is currently speaking.
125    pub fn is_speaking(&self) -> bool {
126        self.detector.is_speaking()
127    }
128}
129
130impl Default for BackendInputVad {
131    fn default() -> Self {
132        Self::new(VadConfig {
133            sample_rate: 16000,
134            frame_duration_ms: 30,
135            min_speech_frames: 2,
136            hangover_frames: 8,
137            ..VadConfig::default()
138        })
139    }
140}
141
142fn state_name(state: VadState) -> &'static str {
143    match state {
144        VadState::Silence => "silence",
145        VadState::PendingSpeech => "pending_speech",
146        VadState::Speech => "speech",
147        VadState::Hangover => "hangover",
148    }
149}
150
151#[cfg(test)]
152mod tests {
153    use super::*;
154
155    fn speech_frame(len: usize, amplitude: i16) -> Vec<i16> {
156        (0..len)
157            .map(|i| if i % 4 < 2 { amplitude } else { -amplitude })
158            .collect()
159    }
160
161    fn bytes(samples: &[i16]) -> Vec<u8> {
162        samples
163            .iter()
164            .flat_map(|sample| sample.to_le_bytes())
165            .collect()
166    }
167
168    #[test]
169    fn buffers_arbitrary_chunks_into_vad_frames() {
170        let mut vad = BackendInputVad::new(VadConfig {
171            sample_rate: 16000,
172            frame_duration_ms: 20,
173            min_speech_frames: 2,
174            hangover_frames: 2,
175            speech_zcr_range: (0.01, 0.9),
176            ..VadConfig::default()
177        });
178        let speech = speech_frame(640, 10000);
179        let half = bytes(&speech[..100]);
180        assert!(vad.process_pcm_bytes(&half).is_empty());
181
182        let rest = bytes(&speech[100..]);
183        let events = vad.process_pcm_bytes(&rest);
184        assert!(events.contains(&VadEvent::SpeechStart));
185        assert!(vad.is_speaking());
186    }
187}
gemini_adk_rs/live/input_vad.rs

gemini_adk_rs/live/
input_vad.rs
