gemini_adk_rs/evaluation/
rubric_evaluator.rs

1//! Rubric-based evaluator — evaluate agent responses against rubric criteria.
2//!
3//! Uses an LLM-as-judge to score agent outputs against one or more free-text
4//! rubric criteria. Supports both final-response quality and tool-use quality
5//! evaluation modes.
6
7use std::sync::Arc;
8
9use async_trait::async_trait;
10
11use super::eval_case::Invocation;
12use super::eval_result::{EvalMetric, EvalResult, PerInvocationResult};
13use super::evaluator::{EvalError, Evaluator};
14use crate::llm::BaseLlm;
15
16/// Evaluation mode for rubric evaluation.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum RubricMode {
19    /// Evaluate the final response quality.
20    FinalResponse,
21    /// Evaluate tool use quality (selection, arguments, sequencing).
22    ToolUse,
23}
24
25/// Evaluator that scores agent outputs against free-text rubric criteria
26/// using an LLM as judge.
27pub struct RubricEvaluator {
28    /// The rubric criteria to evaluate against.
29    rubrics: Vec<String>,
30    /// Optional override for the judge model.
31    judge_model: Option<String>,
32    /// The evaluation mode (response vs tool use).
33    mode: RubricMode,
34    /// Optional LLM for performing evaluations.
35    llm: Option<Arc<dyn BaseLlm>>,
36}
37
38impl RubricEvaluator {
39    /// Create a new rubric evaluator with the given rubric criteria.
40    pub fn new(rubrics: Vec<String>) -> Self {
41        Self {
42            rubrics,
43            judge_model: None,
44            mode: RubricMode::FinalResponse,
45            llm: None,
46        }
47    }
48
49    /// Create a rubric evaluator for final response quality.
50    ///
51    /// Uses the `rubric_based_final_response_quality_v1` evaluation strategy.
52    pub fn for_response(rubrics: Vec<String>) -> Self {
53        Self {
54            rubrics,
55            judge_model: None,
56            mode: RubricMode::FinalResponse,
57            llm: None,
58        }
59    }
60
61    /// Create a rubric evaluator for tool use quality.
62    ///
63    /// Uses the `rubric_based_tool_use_quality_v1` evaluation strategy.
64    pub fn for_tool_use(rubrics: Vec<String>) -> Self {
65        Self {
66            rubrics,
67            judge_model: None,
68            mode: RubricMode::ToolUse,
69            llm: None,
70        }
71    }
72
73    /// Set an override judge model name.
74    pub fn with_judge_model(mut self, model: impl Into<String>) -> Self {
75        self.judge_model = Some(model.into());
76        self
77    }
78
79    /// Provide an LLM instance for performing evaluations.
80    pub fn with_llm(mut self, llm: Arc<dyn BaseLlm>) -> Self {
81        self.llm = Some(llm);
82        self
83    }
84
85    /// Build the evaluation prompt for a single invocation.
86    fn build_prompt(&self, actual: &Invocation, expected: Option<&Invocation>) -> String {
87        let mode_label = match self.mode {
88            RubricMode::FinalResponse => "FINAL RESPONSE QUALITY",
89            RubricMode::ToolUse => "TOOL USE QUALITY",
90        };
91
92        let mut prompt = format!(
93            "You are an expert evaluator assessing {mode_label}.\n\n\
94             Score the agent's performance on a scale of 0.0 to 1.0 for EACH rubric criterion.\n\n"
95        );
96
97        // Add rubrics
98        prompt.push_str("RUBRIC CRITERIA:\n");
99        for (i, rubric) in self.rubrics.iter().enumerate() {
100            prompt.push_str(&format!("{}. {}\n", i + 1, rubric));
101        }
102        prompt.push('\n');
103
104        // Add actual conversation
105        prompt.push_str("ACTUAL AGENT CONVERSATION:\n");
106        for turn in &actual.turns {
107            prompt.push_str(&format!("[{}]: {}\n", turn.role, turn.content));
108            if !turn.tool_calls.is_empty() {
109                prompt.push_str(&format!(
110                    "  Tool calls: {}\n",
111                    serde_json::json!(turn.tool_calls)
112                ));
113            }
114            if !turn.tool_results.is_empty() {
115                prompt.push_str(&format!(
116                    "  Tool results: {}\n",
117                    serde_json::json!(turn.tool_results)
118                ));
119            }
120        }
121
122        // Add expected conversation if available
123        if let Some(expected) = expected {
124            prompt.push_str("\nEXPECTED CONVERSATION:\n");
125            for turn in &expected.turns {
126                prompt.push_str(&format!("[{}]: {}\n", turn.role, turn.content));
127                if !turn.tool_calls.is_empty() {
128                    prompt.push_str(&format!(
129                        "  Tool calls: {}\n",
130                        serde_json::json!(turn.tool_calls)
131                    ));
132                }
133            }
134        }
135
136        prompt.push_str(
137            "\nRespond with ONLY a JSON object:\n\
138             {\"scores\": [<float per rubric criterion>], \
139             \"overall_score\": <float average>, \
140             \"explanation\": \"<text>\"}\n",
141        );
142
143        prompt
144    }
145
146    /// Parse the LLM judge response to extract rubric scores.
147    fn parse_response(text: &str, num_rubrics: usize) -> (f64, String) {
148        // Try full JSON parse first
149        if let Some((score, explanation)) = try_parse_json(text) {
150            return (score, explanation);
151        }
152
153        // Try to find JSON embedded in text
154        if let Some(start) = text.find('{')
155            && let Some(end) = text[start..].rfind('}')
156        {
157            let json_str = &text[start..=start + end];
158            if let Some((score, explanation)) = try_parse_json(json_str) {
159                return (score, explanation);
160            }
161        }
162
163        // Fallback: try to find individual scores
164        let _ = num_rubrics; // Used in full implementation
165        (
166            0.0,
167            format!("Failed to parse rubric judge response: {text}"),
168        )
169    }
170}
171
172/// Try to parse a JSON string into a score and explanation.
173fn try_parse_json(text: &str) -> Option<(f64, String)> {
174    let v: serde_json::Value = serde_json::from_str(text).ok()?;
175
176    let score = if let Some(overall) = v["overall_score"].as_f64() {
177        overall.clamp(0.0, 1.0)
178    } else if let Some(scores) = v["scores"].as_array() {
179        // Average only the entries that are actually numbers. Dividing by
180        // `scores.len()` counted entries the numerator never received — a judge
181        // emitting `[0.9, "n/a"]` for a criterion it could not score scored the
182        // candidate 0.45 — and an empty array averaged to a confident 0.0, a
183        // failing grade for a response nobody graded. Both are "unparseable",
184        // which is what `None` means here, as it does in the sibling evaluator.
185        let graded: Vec<f64> = scores
186            .iter()
187            .filter_map(serde_json::Value::as_f64)
188            .map(|s| s.clamp(0.0, 1.0))
189            .collect();
190        if graded.is_empty() {
191            return None;
192        }
193        graded.iter().sum::<f64>() / graded.len() as f64
194    } else {
195        return None;
196    };
197
198    let explanation = v["explanation"]
199        .as_str()
200        .unwrap_or("No explanation")
201        .to_string();
202
203    Some((score, explanation))
204}
205
206#[async_trait]
207impl Evaluator for RubricEvaluator {
208    async fn evaluate(
209        &self,
210        actual: &[Invocation],
211        expected: Option<&[Invocation]>,
212    ) -> Result<EvalResult, EvalError> {
213        let llm = self.llm.as_ref().ok_or_else(|| {
214            EvalError::Llm(
215                "RubricEvaluator requires an LLM instance — call .with_llm() before evaluating"
216                    .into(),
217            )
218        })?;
219
220        let mut per_invocation = Vec::new();
221        let mut total_score = 0.0;
222
223        for (i, actual_inv) in actual.iter().enumerate() {
224            let expected_inv = expected.and_then(|e| e.get(i));
225            let prompt = self.build_prompt(actual_inv, expected_inv);
226
227            let request = crate::llm::LlmRequest::from_text(&prompt);
228            let response = llm
229                .generate(request)
230                .await
231                .map_err(|e| EvalError::Llm(e.to_string()))?;
232
233            let (score, explanation) = Self::parse_response(&response.text(), self.rubrics.len());
234            total_score += score;
235
236            per_invocation.push(PerInvocationResult {
237                invocation_id: if actual_inv.id.is_empty() {
238                    format!("inv-{i}")
239                } else {
240                    actual_inv.id.clone()
241                },
242                score,
243                explanation: Some(explanation),
244            });
245        }
246
247        let overall_score = if actual.is_empty() {
248            0.0
249        } else {
250            total_score / actual.len() as f64
251        };
252
253        let metric_name = match self.mode {
254            RubricMode::FinalResponse => "rubric_based_final_response_quality_v1",
255            RubricMode::ToolUse => "rubric_based_tool_use_quality_v1",
256        };
257
258        Ok(EvalResult {
259            overall_score,
260            metrics: vec![EvalMetric {
261                name: metric_name.into(),
262                score: overall_score,
263                per_invocation,
264            }],
265        })
266    }
267}
268
269#[cfg(test)]
270mod tests {
271    use super::*;
272
273    #[test]
274    fn parse_valid_response() {
275        let json = r#"{"scores": [0.8, 0.9], "overall_score": 0.85, "explanation": "Good"}"#;
276        let (score, explanation) = RubricEvaluator::parse_response(json, 2);
277        assert!((score - 0.85).abs() < f64::EPSILON);
278        assert_eq!(explanation, "Good");
279    }
280
281    #[test]
282    fn parse_scores_only() {
283        let json = r#"{"scores": [0.8, 0.6]}"#;
284        let (score, _) = RubricEvaluator::parse_response(json, 2);
285        assert!((score - 0.7).abs() < f64::EPSILON);
286    }
287
288    #[test]
289    fn ungradeable_entries_do_not_dilute_the_average() {
290        // The denominator is the number of criteria actually scored. Counting
291        // the ones the judge could not score halved a good response's grade.
292        let json = r#"{"scores": [0.9, "n/a", null]}"#;
293        let (score, _) = RubricEvaluator::parse_response(json, 3);
294        assert!(
295            (score - 0.9).abs() < f64::EPSILON,
296            "one criterion scored 0.9 and two were ungradeable, so the score is \
297             0.9, not 0.9 spread over three: got {score}"
298        );
299    }
300
301    #[test]
302    fn an_empty_score_array_is_unparseable_not_zero() {
303        // Zero is a grade. A judge that graded nothing has not awarded it, so
304        // this must fall through to the parse failure rather than report 0.0 as
305        // if it were the judge's verdict.
306        let (score, explanation) = RubricEvaluator::parse_response(r#"{"scores": []}"#, 2);
307        assert!((score - 0.0).abs() < f64::EPSILON);
308        assert!(
309            explanation.contains("Failed to parse"),
310            "an empty score array must read as unparseable, not as a zero \
311             verdict: {explanation}"
312        );
313    }
314
315    #[test]
316    fn parse_embedded_json() {
317        let text = r#"Here is my evaluation: {"overall_score": 0.9, "explanation": "Great"}"#;
318        let (score, _) = RubricEvaluator::parse_response(text, 1);
319        assert!((score - 0.9).abs() < f64::EPSILON);
320    }
321
322    #[test]
323    fn parse_invalid() {
324        let (score, explanation) = RubricEvaluator::parse_response("no json here", 1);
325        assert!((score - 0.0).abs() < f64::EPSILON);
326        assert!(explanation.contains("Failed to parse"));
327    }
328
329    #[test]
330    fn for_response_mode() {
331        let eval = RubricEvaluator::for_response(vec!["Accuracy".into()]);
332        assert_eq!(eval.mode, RubricMode::FinalResponse);
333    }
334
335    #[test]
336    fn for_tool_use_mode() {
337        let eval = RubricEvaluator::for_tool_use(vec!["Tool selection".into()]);
338        assert_eq!(eval.mode, RubricMode::ToolUse);
339    }
340
341    #[test]
342    fn build_prompt_includes_rubrics() {
343        use crate::evaluation::eval_case::InvocationTurn;
344
345        let eval = RubricEvaluator::new(vec![
346            "Is the response accurate?".into(),
347            "Is it well-formatted?".into(),
348        ]);
349        let inv = Invocation {
350            id: "test".into(),
351            turns: vec![InvocationTurn {
352                role: "user".into(),
353                content: "Hello".into(),
354                tool_calls: vec![],
355                tool_results: vec![],
356            }],
357            metadata: serde_json::Value::Null,
358        };
359        let prompt = eval.build_prompt(&inv, None);
360        assert!(prompt.contains("Is the response accurate?"));
361        assert!(prompt.contains("Is it well-formatted?"));
362        assert!(prompt.contains("FINAL RESPONSE QUALITY"));
363    }
364
365    #[test]
366    fn with_judge_model() {
367        let eval = RubricEvaluator::new(vec!["test".into()]).with_judge_model("gemini-2.0-flash");
368        assert_eq!(eval.judge_model.as_deref(), Some("gemini-2.0-flash"));
369    }
370
371    #[test]
372    fn score_clamped() {
373        let json = r#"{"overall_score": 1.5, "explanation": "Over"}"#;
374        let (score, _) = RubricEvaluator::parse_response(json, 1);
375        assert!((score - 1.0).abs() < f64::EPSILON);
376    }
377}
gemini_adk_rs/evaluation/rubric_evaluator.rs

gemini_adk_rs/evaluation/
rubric_evaluator.rs
