gemini_genai_rs/protocol/messages/
client.rs

1//! Client → Server message types for the Gemini Live wire protocol.
2
3use serde::{Deserialize, Serialize};
4
5use crate::protocol::types::*;
6
7/// Top-level setup message sent immediately after WebSocket connect.
8#[derive(Debug, Clone, Serialize)]
9pub struct SetupMessage {
10    /// The setup payload.
11    pub setup: SetupPayload,
12}
13
14/// Payload of the setup message.
15#[derive(Debug, Clone, Serialize)]
16#[serde(rename_all = "camelCase")]
17pub struct SetupPayload {
18    /// Model URI string (e.g. `"models/gemini-2.0-flash-live-001"`).
19    pub model: String,
20    /// Generation parameters (modalities, temperature, etc.).
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub generation_config: Option<GenerationConfig>,
23    /// System instruction content.
24    #[serde(skip_serializing_if = "Option::is_none")]
25    pub system_instruction: Option<Content>,
26    /// Tool declarations for function calling, search, etc.
27    #[serde(skip_serializing_if = "Vec::is_empty")]
28    pub tools: Vec<Tool>,
29    /// Tool usage configuration.
30    #[serde(skip_serializing_if = "Option::is_none")]
31    pub tool_config: Option<ToolConfig>,
32    /// Enable input audio transcription.
33    #[serde(skip_serializing_if = "Option::is_none")]
34    pub input_audio_transcription: Option<InputAudioTranscription>,
35    /// Enable output audio transcription.
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub output_audio_transcription: Option<OutputAudioTranscription>,
38    /// Realtime input configuration (VAD, activity handling).
39    #[serde(skip_serializing_if = "Option::is_none")]
40    pub realtime_input_config: Option<RealtimeInputConfig>,
41    /// Session resumption configuration.
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub session_resumption: Option<SessionResumptionConfig>,
44    /// Context window compression configuration.
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub context_window_compression: Option<ContextWindowCompressionConfig>,
47    /// Proactivity configuration.
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub proactivity: Option<ProactivityConfig>,
50    /// Ask for explicit `voiceActivity` events (Vertex AI).
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub explicit_vad_signal: Option<bool>,
53    /// How `clientContent` history is treated.
54    #[serde(skip_serializing_if = "Option::is_none")]
55    pub history_config: Option<HistoryConfig>,
56    /// Live Avatar video output.
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub avatar_config: Option<AvatarConfig>,
59}
60
61impl SessionConfig {
62    /// Build the setup message from this configuration.
63    ///
64    /// Settings the target does not accept are left off the wire rather
65    /// than failing the handshake; [`SessionConfig::ignored_settings`] lists
66    /// exactly what this drops:
67    ///
68    /// - `behavior` on function declarations, on Vertex AI models without
69    ///   async tool calling;
70    /// - `thinkingConfig` on Vertex AI and on models without thinking;
71    /// - `enableAffectiveDialog` and `proactivity` on models where both are
72    ///   always on (Gemini 3.8 Live);
73    /// - on Google AI, which has no such fields: `proactivity`,
74    ///   `explicitVadSignal`, `sessionResumption.transparent`, and
75    ///   `avatarConfig.avatarName` / `customizedAvatar`.
76    pub fn to_setup_message(&self) -> SetupMessage {
77        let profile = self.model_profile();
78        let tools = if self.supports_async_tools() {
79            self.tools.clone()
80        } else {
81            self.tools
82                .iter()
83                .map(|tool| {
84                    let mut t = tool.clone();
85                    if let Some(ref mut decls) = t.function_declarations {
86                        for d in decls.iter_mut() {
87                            d.behavior = None;
88                        }
89                    }
90                    t
91                })
92                .collect()
93        };
94
95        let mut generation_config = self.generation_config.clone();
96        let mut output_audio_transcription = self.output_audio_transcription.clone();
97        if self.text_via_transcription() {
98            generation_config.response_modalities = Some(vec![Modality::Audio]);
99            output_audio_transcription.get_or_insert_with(Default::default);
100        }
101        if !self.supports_thinking() {
102            generation_config.thinking_config = None;
103        }
104        if !profile.affective_dialog_flag {
105            generation_config.enable_affective_dialog = None;
106        }
107
108        SetupMessage {
109            setup: SetupPayload {
110                model: self.model_uri(),
111                generation_config: Some(generation_config),
112                system_instruction: self.system_instruction.clone(),
113                tools,
114                tool_config: self.tool_config.clone(),
115                input_audio_transcription: self.input_audio_transcription.clone(),
116                output_audio_transcription,
117                realtime_input_config: self.realtime_input_config.clone(),
118                session_resumption: self.session_resumption.clone().map(|mut r| {
119                    if !self.is_vertex() {
120                        r.transparent = None;
121                    }
122                    r
123                }),
124                context_window_compression: self.context_window_compression.clone(),
125                proactivity: self
126                    .proactivity
127                    .clone()
128                    .filter(|_| self.is_vertex() && profile.proactivity_flag),
129                explicit_vad_signal: self.explicit_vad_signal.filter(|_| self.is_vertex()),
130                history_config: self.history_config.clone(),
131                avatar_config: self.avatar_config.clone().map(|mut a| {
132                    if !self.is_vertex() {
133                        a.avatar_name = None;
134                        a.customized_avatar = None;
135                    }
136                    a
137                }),
138            },
139        }
140    }
141
142    /// Pre-serialize the setup message to JSON. Called once at connection time.
143    pub fn to_setup_json(&self) -> String {
144        serde_json::to_string(&self.to_setup_message())
145            .expect("setup message serialization is infallible for valid config")
146    }
147}
148
149/// Realtime audio input sent as a stream of chunks.
150#[derive(Debug, Clone, Serialize)]
151#[serde(rename_all = "camelCase")]
152pub struct RealtimeInputMessage {
153    /// The realtime input payload.
154    pub realtime_input: RealtimeInputPayload,
155}
156
157/// Payload for realtime audio input.
158#[derive(Debug, Clone, Serialize)]
159#[serde(rename_all = "camelCase")]
160pub struct RealtimeInputPayload {
161    /// Deprecated: use `audio` instead. Kept for backward compatibility.
162    #[serde(skip_serializing_if = "Vec::is_empty")]
163    pub media_chunks: Vec<MediaChunk>,
164    /// Audio input blob (preferred over media_chunks).
165    #[serde(skip_serializing_if = "Option::is_none")]
166    pub audio: Option<Blob>,
167    /// Video input blob.
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub video: Option<Blob>,
170    /// Signal end of audio stream.
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub audio_stream_end: Option<bool>,
173    /// Realtime text input (streamed inline, distinct from clientContent).
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub text: Option<String>,
176}
177
178/// A single chunk of media data (audio). Deprecated — use Blob in `audio` field.
179#[derive(Debug, Clone, Serialize)]
180#[serde(rename_all = "camelCase")]
181pub struct MediaChunk {
182    /// MIME type of the media (e.g. `"audio/pcm"`).
183    pub mime_type: String,
184    /// Base64-encoded media data.
185    pub data: String, // base64-encoded
186}
187
188/// Client content message for sending text or conversation history.
189#[derive(Debug, Clone, Serialize)]
190#[serde(rename_all = "camelCase")]
191pub struct ClientContentMessage {
192    /// The client content payload.
193    pub client_content: ClientContentPayload,
194}
195
196/// Payload for client content.
197#[derive(Debug, Clone, Serialize)]
198#[serde(rename_all = "camelCase")]
199pub struct ClientContentPayload {
200    /// Conversation turns to send.
201    pub turns: Vec<Content>,
202    /// Whether this completes the client's turn.
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub turn_complete: Option<bool>,
205}
206
207/// Tool response message sent after executing function calls.
208#[derive(Debug, Clone, Serialize)]
209#[serde(rename_all = "camelCase")]
210pub struct ToolResponseMessage {
211    /// The tool response payload.
212    pub tool_response: ToolResponsePayload,
213}
214
215/// Payload for tool response.
216#[derive(Debug, Clone, Serialize)]
217#[serde(rename_all = "camelCase")]
218pub struct ToolResponsePayload {
219    /// Function call responses to return to the model.
220    pub function_responses: Vec<FunctionResponse>,
221}
222
223/// Activity signal for client-side VAD events.
224#[derive(Debug, Clone, Serialize)]
225#[serde(rename_all = "camelCase")]
226pub struct ActivitySignalMessage {
227    /// The activity signal payload.
228    pub realtime_input: ActivitySignalPayload,
229}
230
231/// Payload for activity signals.
232#[derive(Debug, Clone, Serialize)]
233#[serde(rename_all = "camelCase")]
234pub struct ActivitySignalPayload {
235    /// Present when signaling activity start.
236    #[serde(skip_serializing_if = "Option::is_none")]
237    pub activity_start: Option<ActivityStart>,
238    /// Present when signaling activity end.
239    #[serde(skip_serializing_if = "Option::is_none")]
240    pub activity_end: Option<ActivityEnd>,
241}
242
243/// Marker for speech activity start.
244#[derive(Debug, Clone, Serialize, Deserialize)]
245pub struct ActivityStart {}
246
247/// Marker for speech activity end.
248#[derive(Debug, Clone, Serialize, Deserialize)]
249pub struct ActivityEnd {}
gemini_genai_rs/protocol/messages/client.rs

gemini_genai_rs/protocol/messages/
client.rs
