1use std::collections::HashMap;
27
28use base64::Engine as _;
29use serde::{Deserialize, Serialize};
30use tokio::sync::{mpsc, watch};
31use tokio::task::JoinHandle;
32
33use gemini_adk_rs::live::LiveHandle;
34
35use super::g711;
36use crate::voice::{Playback, VoicePump, pump};
37
38pub const TWILIO_HZ: u32 = 8_000;
40
41pub use super::bridge::{KEY_CALL_SID, KEY_DTMF, KEY_DTMF_HISTORY, KEY_STREAM_SID};
44
45#[derive(Debug, Clone, Deserialize, Default, PartialEq)]
49#[serde(rename_all = "camelCase", default)]
50pub struct StartMeta {
51 pub stream_sid: String,
53 pub call_sid: String,
55 pub account_sid: String,
57 pub tracks: Vec<String>,
59 pub media_format: MediaFormat,
61 pub custom_parameters: HashMap<String, String>,
63}
64
65#[derive(Debug, Clone, Deserialize, Default, PartialEq)]
67#[serde(rename_all = "camelCase", default)]
68pub struct MediaFormat {
69 pub encoding: String,
71 pub sample_rate: u32,
73 pub channels: u32,
75}
76
77#[derive(Debug, Clone, Deserialize)]
78#[serde(rename_all = "camelCase")]
79struct MediaPayload {
80 #[serde(default)]
81 track: String,
82 payload: String,
83}
84
85#[derive(Debug, Clone, Deserialize)]
86struct DtmfPayload {
87 digit: String,
88}
89
90#[derive(Debug, Clone, Deserialize)]
91struct MarkPayload {
92 name: String,
93}
94
95#[derive(Debug, Clone, Deserialize)]
96#[serde(tag = "event", rename_all = "lowercase")]
97enum RawInbound {
98 Connected {},
99 Start { start: StartMeta },
100 Media { media: MediaPayload },
101 Dtmf { dtmf: DtmfPayload },
102 Mark { mark: MarkPayload },
103 Stop {},
104}
105
106#[derive(Debug, Clone, PartialEq)]
108pub enum Inbound {
109 Connected,
111 Started(StartMeta),
113 Audio(Vec<i16>),
115 Dtmf(char),
117 Mark(String),
119 Stopped,
121 Ignored,
123}
124
125pub fn parse_inbound(text: &str) -> Result<Inbound, TwilioError> {
130 let raw: RawInbound = match serde_json::from_str(text) {
131 Ok(raw) => raw,
132 Err(_) => {
133 let value: serde_json::Value =
135 serde_json::from_str(text).map_err(TwilioError::Malformed)?;
136 return if value.get("event").is_some() {
137 Ok(Inbound::Ignored)
138 } else {
139 Err(TwilioError::NotAFrame)
140 };
141 }
142 };
143 Ok(match raw {
144 RawInbound::Connected {} => Inbound::Connected,
145 RawInbound::Start { start } => Inbound::Started(start),
146 RawInbound::Media { media } => {
147 if !media.track.is_empty() && media.track != "inbound" {
150 return Ok(Inbound::Ignored);
151 }
152 let mulaw = base64::engine::general_purpose::STANDARD
153 .decode(media.payload.as_bytes())
154 .map_err(TwilioError::BadPayload)?;
155 Inbound::Audio(g711::decode_ulaw(&mulaw))
156 }
157 RawInbound::Dtmf { dtmf } => match dtmf.digit.chars().next() {
158 Some(digit) => Inbound::Dtmf(digit),
159 None => Inbound::Ignored,
160 },
161 RawInbound::Mark { mark } => Inbound::Mark(mark.name),
162 RawInbound::Stop {} => Inbound::Stopped,
163 })
164}
165
166#[derive(Debug)]
168pub enum TwilioError {
169 Malformed(serde_json::Error),
171 NotAFrame,
173 BadPayload(base64::DecodeError),
175}
176
177impl std::fmt::Display for TwilioError {
178 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
179 match self {
180 Self::Malformed(e) => write!(f, "malformed Twilio frame: {e}"),
181 Self::NotAFrame => write!(f, "JSON without an event field"),
182 Self::BadPayload(e) => write!(f, "invalid base64 media payload: {e}"),
183 }
184 }
185}
186
187impl std::error::Error for TwilioError {}
188
189#[derive(Serialize)]
192#[serde(rename_all = "camelCase")]
193struct OutMedia<'a> {
194 event: &'static str,
195 stream_sid: &'a str,
196 media: OutMediaPayload,
197}
198
199#[derive(Serialize)]
200struct OutMediaPayload {
201 payload: String,
202}
203
204#[derive(Serialize)]
205#[serde(rename_all = "camelCase")]
206struct OutClear<'a> {
207 event: &'static str,
208 stream_sid: &'a str,
209}
210
211#[derive(Serialize)]
212#[serde(rename_all = "camelCase")]
213struct OutMark<'a> {
214 event: &'static str,
215 stream_sid: &'a str,
216 mark: OutMarkPayload<'a>,
217}
218
219#[derive(Serialize)]
220struct OutMarkPayload<'a> {
221 name: &'a str,
222}
223
224pub fn media_frame(stream_sid: &str, samples: &[i16]) -> String {
226 let payload = base64::engine::general_purpose::STANDARD.encode(g711::encode_ulaw(samples));
227 serde_json::to_string(&OutMedia {
228 event: "media",
229 stream_sid,
230 media: OutMediaPayload { payload },
231 })
232 .expect("media frame serializes")
233}
234
235pub fn clear_frame(stream_sid: &str) -> String {
238 serde_json::to_string(&OutClear {
239 event: "clear",
240 stream_sid,
241 })
242 .expect("clear frame serializes")
243}
244
245pub fn mark_frame(stream_sid: &str, name: &str) -> String {
247 serde_json::to_string(&OutMark {
248 event: "mark",
249 stream_sid,
250 mark: OutMarkPayload { name },
251 })
252 .expect("mark frame serializes")
253}
254
255const FRAME_SAMPLES: usize = 160;
259const FRAME_DURATION: std::time::Duration = std::time::Duration::from_millis(20);
260const TAIL_WAIT: std::time::Duration = std::time::Duration::from_millis(40);
262
263pub const KEY_UNPLAYED_MS: &str = "telephony:unplayed_ms";
267
268#[derive(Debug, Default)]
270struct Framer {
271 pending: Vec<i16>,
273 plays_until: Option<std::time::Instant>,
275}
276
277impl Framer {
278 fn is_idle(&self) -> bool {
279 self.pending.is_empty()
280 }
281
282 fn push(&mut self, samples: &[i16], now: std::time::Instant) -> Vec<Vec<i16>> {
284 self.pending.extend_from_slice(samples);
285 let mut frames = Vec::new();
286 while self.pending.len() >= FRAME_SAMPLES {
287 frames.push(self.pending.drain(..FRAME_SAMPLES).collect());
288 let from = self.plays_until.map_or(now, |t| t.max(now));
289 self.plays_until = Some(from + FRAME_DURATION);
290 }
291 frames
292 }
293
294 fn pad_tail(&mut self) {
296 if !self.pending.is_empty() {
297 self.pending.resize(FRAME_SAMPLES, 0);
298 }
299 }
300
301 fn flush(&mut self, now: std::time::Instant) -> std::time::Duration {
304 self.pending.clear();
305 let unplayed = self.plays_until.map_or(std::time::Duration::ZERO, |t| {
306 t.saturating_duration_since(now)
307 });
308 self.plays_until = Some(now);
309 unplayed
310 }
311}
312
313#[derive(Clone, Default)]
315pub struct CallOptions {
316 pub recorder: Option<std::sync::Arc<super::recorder::CallRecorder>>,
318 pub in_band_dtmf: bool,
321}
322
323pub struct TwilioCall {
348 pub from_twilio: mpsc::Sender<String>,
350 pub to_twilio: mpsc::Receiver<String>,
352 pump: VoicePump,
353 inbound_task: JoinHandle<()>,
354 outbound_task: JoinHandle<()>,
355}
356
357impl TwilioCall {
358 pub fn attach(handle: &LiveHandle) -> TwilioCall {
366 Self::attach_with(handle, CallOptions::default())
367 }
368
369 pub fn attach_with(handle: &LiveHandle, options: CallOptions) -> TwilioCall {
382 let (from_tx, mut from_rx) = mpsc::channel::<String>(64);
383 let (to_tx, to_rx) = mpsc::channel::<String>(64);
384 let (mic_tx, mic_rx) = mpsc::channel::<Vec<i16>>(64);
385 let (speaker_tx, mut speaker_rx) = mpsc::channel::<Playback>(64);
386 let (sid_tx, sid_rx) = watch::channel::<Option<String>>(None);
387
388 let voice_pump = pump(handle, mic_rx, TWILIO_HZ, speaker_tx, TWILIO_HZ);
389
390 let state = handle.state().clone();
391 let mut guard = super::bridge::KeypadGuard::new(state.clone(), TWILIO_HZ)
392 .record_in_band(options.in_band_dtmf);
393 let inbound_recorder = options.recorder.clone();
394 let inbound_task = tokio::spawn(async move {
395 use crate::voice::InputAudioProcessor as _;
396 while let Some(text) = from_rx.recv().await {
397 match parse_inbound(&text) {
398 Ok(Inbound::Audio(mut samples)) => {
399 guard.process_frame(&mut samples);
400 if let Some(recorder) = &inbound_recorder {
401 recorder.caller(&samples);
402 }
403 if mic_tx.send(samples).await.is_err() {
404 break;
405 }
406 }
407 Ok(Inbound::Started(meta)) => {
408 let _ = state.set(KEY_CALL_SID, meta.call_sid.clone());
409 let _ = state.set(KEY_STREAM_SID, meta.stream_sid.clone());
410 let _ = sid_tx.send(Some(meta.stream_sid));
411 }
412 Ok(Inbound::Dtmf(digit)) => super::bridge::record_dtmf(&state, digit),
413 Ok(Inbound::Stopped) => break,
414 Ok(Inbound::Connected | Inbound::Mark(_) | Inbound::Ignored) => {}
415 Err(err) => tracing::warn!("dropping unparseable Twilio frame: {err}"),
416 }
417 }
418 });
419
420 let out_state = handle.state().clone();
421 let outbound_recorder = options.recorder;
422 let outbound_task = tokio::spawn(async move {
423 let mut framer = Framer::default();
424 loop {
425 let next = if framer.is_idle() {
426 speaker_rx.recv().await
427 } else {
428 match tokio::time::timeout(TAIL_WAIT, speaker_rx.recv()).await {
429 Ok(next) => next,
430 Err(_) => {
432 framer.pad_tail();
433 Some(Playback::Chunk(Vec::new()))
434 }
435 }
436 };
437 let Some(playback) = next else { break };
438 let Some(sid) = sid_rx.borrow().clone() else {
440 continue;
441 };
442 let now = std::time::Instant::now();
443 let mut frames = Vec::new();
444 match playback {
445 Playback::Chunk(samples) => {
446 for frame in framer.push(&samples, now) {
447 if let Some(recorder) = &outbound_recorder {
448 recorder.agent(&frame);
449 }
450 frames.push(media_frame(&sid, &frame));
451 }
452 }
453 Playback::Flush => {
454 let unplayed = framer.flush(now);
455 let _ = out_state.set(KEY_UNPLAYED_MS, unplayed.as_millis() as u64);
456 if let Some(recorder) = &outbound_recorder {
457 recorder.flush();
458 }
459 frames.push(clear_frame(&sid));
460 }
461 }
462 for frame in frames {
463 if to_tx.send(frame).await.is_err() {
464 return;
465 }
466 }
467 }
468 });
469
470 TwilioCall {
471 from_twilio: from_tx,
472 to_twilio: to_rx,
473 pump: voice_pump,
474 inbound_task,
475 outbound_task,
476 }
477 }
478
479 pub async fn join(self) {
482 let _ = self.inbound_task.await;
483 let _ = self.outbound_task.await;
484 self.pump.join().await;
485 }
486
487 pub fn abort(&self) {
489 self.inbound_task.abort();
490 self.outbound_task.abort();
491 self.pump.abort();
492 }
493}
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498
499 #[test]
500 fn agent_audio_goes_out_in_20ms_frames_on_a_playout_clock() {
501 let t0 = std::time::Instant::now();
502 let mut framer = Framer::default();
503 let frames = framer.push(&[1; 400], t0);
505 assert_eq!(frames.len(), 2);
506 assert!(frames.iter().all(|f| f.len() == FRAME_SAMPLES));
507 assert!(!framer.is_idle());
508 framer.pad_tail();
510 let tail = framer.push(&[], t0);
511 assert_eq!(tail.len(), 1);
512 assert_eq!(&tail[0][..80], &[1; 80]);
513 assert_eq!(&tail[0][80..], &[0; 80]);
514 let unplayed = framer.flush(t0 + std::time::Duration::from_millis(25));
516 assert_eq!(unplayed, std::time::Duration::from_millis(35));
517 assert!(framer.is_idle());
518 }
519
520 #[test]
521 fn parses_the_start_frame() {
522 let text = r#"{
523 "event": "start", "sequenceNumber": "1", "streamSid": "MZxyz",
524 "start": {
525 "accountSid": "ACabc", "streamSid": "MZxyz", "callSid": "CAdef",
526 "tracks": ["inbound"],
527 "mediaFormat": {"encoding": "audio/x-mulaw", "sampleRate": 8000, "channels": 1},
528 "customParameters": {"agent": "front-desk"}
529 }
530 }"#;
531 match parse_inbound(text).unwrap() {
532 Inbound::Started(meta) => {
533 assert_eq!(meta.stream_sid, "MZxyz");
534 assert_eq!(meta.call_sid, "CAdef");
535 assert_eq!(meta.media_format.sample_rate, 8000);
536 assert_eq!(meta.custom_parameters["agent"], "front-desk");
537 }
538 other => panic!("expected Started, got {other:?}"),
539 }
540 }
541
542 #[test]
543 fn media_frames_decode_to_pcm() {
544 let payload = base64::engine::general_purpose::STANDARD.encode([0xFFu8; 4]);
546 let text = format!(
547 r#"{{"event":"media","streamSid":"MZ1","media":{{"track":"inbound","chunk":"1","timestamp":"5","payload":"{payload}"}}}}"#
548 );
549 assert_eq!(parse_inbound(&text).unwrap(), Inbound::Audio(vec![0i16; 4]));
550 }
551
552 #[test]
553 fn outbound_track_media_is_skipped() {
554 let payload = base64::engine::general_purpose::STANDARD.encode([0xFFu8; 4]);
555 let text = format!(
556 r#"{{"event":"media","streamSid":"MZ1","media":{{"track":"outbound","payload":"{payload}"}}}}"#
557 );
558 assert_eq!(parse_inbound(&text).unwrap(), Inbound::Ignored);
559 }
560
561 #[test]
562 fn dtmf_marks_stop_and_unknown_events() {
563 assert_eq!(
564 parse_inbound(
565 r#"{"event":"dtmf","streamSid":"MZ1","dtmf":{"track":"inbound_track","digit":"7"}}"#
566 )
567 .unwrap(),
568 Inbound::Dtmf('7')
569 );
570 assert_eq!(
571 parse_inbound(r#"{"event":"mark","streamSid":"MZ1","mark":{"name":"m1"}}"#).unwrap(),
572 Inbound::Mark("m1".into())
573 );
574 assert_eq!(
575 parse_inbound(r#"{"event":"stop","streamSid":"MZ1","stop":{}}"#).unwrap(),
576 Inbound::Stopped
577 );
578 assert_eq!(
579 parse_inbound(r#"{"event":"connected","protocol":"Call","version":"1.0.0"}"#).unwrap(),
580 Inbound::Connected
581 );
582 assert_eq!(
584 parse_inbound(r#"{"event":"totally-new-thing"}"#).unwrap(),
585 Inbound::Ignored
586 );
587 assert!(parse_inbound("not json").is_err());
588 assert!(parse_inbound(r#"{"no_event": true}"#).is_err());
589 }
590
591 #[test]
592 fn media_frame_round_trips_through_the_codec() {
593 let samples = vec![0i16, 1000, -1000, 8000];
594 let frame = media_frame("MZ1", &samples);
595 let value: serde_json::Value = serde_json::from_str(&frame).unwrap();
596 assert_eq!(value["event"], "media");
597 assert_eq!(value["streamSid"], "MZ1");
598 let mulaw = base64::engine::general_purpose::STANDARD
599 .decode(value["media"]["payload"].as_str().unwrap())
600 .unwrap();
601 let decoded = g711::decode_ulaw(&mulaw);
602 assert_eq!(decoded.len(), samples.len());
603 for (orig, rt) in samples.iter().zip(&decoded) {
605 assert!(((orig - rt) as i32).abs() <= (orig.unsigned_abs() as i32 / 16).max(16));
606 }
607 }
608
609 #[test]
610 fn clear_and_mark_frames_have_the_wire_shape() {
611 let clear: serde_json::Value = serde_json::from_str(&clear_frame("MZ9")).unwrap();
612 assert_eq!(
613 clear,
614 serde_json::json!({"event": "clear", "streamSid": "MZ9"})
615 );
616 let mark: serde_json::Value = serde_json::from_str(&mark_frame("MZ9", "done")).unwrap();
617 assert_eq!(
618 mark,
619 serde_json::json!({"event": "mark", "streamSid": "MZ9", "mark": {"name": "done"}})
620 );
621 }
622}