gemini_adk_rs/evaluation/
rubric_evaluator.rs

1//! Rubric-based evaluator — evaluate agent responses against rubric criteria.
2//!
3//! Uses an LLM-as-judge to score agent outputs against one or more free-text
4//! rubric criteria. Supports both final-response quality and tool-use quality
5//! evaluation modes.
6
7use std::sync::Arc;
8
9use async_trait::async_trait;
10
11use super::eval_case::Invocation;
12use super::eval_result::{EvalMetric, EvalResult, PerInvocationResult};
13use super::evaluator::{EvalError, Evaluator};
14use crate::llm::BaseLlm;
15
16/// Evaluation mode for rubric evaluation.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum RubricMode {
19    /// Evaluate the final response quality.
20    FinalResponse,
21    /// Evaluate tool use quality (selection, arguments, sequencing).
22    ToolUse,
23}
24
25/// Evaluator that scores agent outputs against free-text rubric criteria
26/// using an LLM as judge.
27pub struct RubricEvaluator {
28    /// The rubric criteria to evaluate against.
29    rubrics: Vec<String>,
30    /// Optional override for the judge model.
31    judge_model: Option<String>,
32    /// The evaluation mode (response vs tool use).
33    mode: RubricMode,
34    /// Optional LLM for performing evaluations.
35    llm: Option<Arc<dyn BaseLlm>>,
36}
37
38impl RubricEvaluator {
39    /// Create a new rubric evaluator with the given rubric criteria.
40    pub fn new(rubrics: Vec<String>) -> Self {
41        Self {
42            rubrics,
43            judge_model: None,
44            mode: RubricMode::FinalResponse,
45            llm: None,
46        }
47    }
48
49    /// Create a rubric evaluator for final response quality.
50    ///
51    /// Uses the `rubric_based_final_response_quality_v1` evaluation strategy.
52    pub fn for_response(rubrics: Vec<String>) -> Self {
53        Self {
54            rubrics,
55            judge_model: None,
56            mode: RubricMode::FinalResponse,
57            llm: None,
58        }
59    }
60
61    /// Create a rubric evaluator for tool use quality.
62    ///
63    /// Uses the `rubric_based_tool_use_quality_v1` evaluation strategy.
64    pub fn for_tool_use(rubrics: Vec<String>) -> Self {
65        Self {
66            rubrics,
67            judge_model: None,
68            mode: RubricMode::ToolUse,
69            llm: None,
70        }
71    }
72
73    /// Set an override judge model name.
74    pub fn with_judge_model(mut self, model: impl Into<String>) -> Self {
75        self.judge_model = Some(model.into());
76        self
77    }
78
79    /// Provide an LLM instance for performing evaluations.
80    pub fn with_llm(mut self, llm: Arc<dyn BaseLlm>) -> Self {
81        self.llm = Some(llm);
82        self
83    }
84
85    /// Build the evaluation prompt for a single invocation.
86    fn build_prompt(&self, actual: &Invocation, expected: Option<&Invocation>) -> String {
87        let mode_label = match self.mode {
88            RubricMode::FinalResponse => "FINAL RESPONSE QUALITY",
89            RubricMode::ToolUse => "TOOL USE QUALITY",
90        };
91
92        let mut prompt = format!(
93            "You are an expert evaluator assessing {mode_label}.\n\n\
94             Score the agent's performance on a scale of 0.0 to 1.0 for EACH rubric criterion.\n\n"
95        );
96
97        // Add rubrics
98        prompt.push_str("RUBRIC CRITERIA:\n");
99        for (i, rubric) in self.rubrics.iter().enumerate() {
100            prompt.push_str(&format!("{}. {}\n", i + 1, rubric));
101        }
102        prompt.push('\n');
103
104        // Add actual conversation
105        prompt.push_str("ACTUAL AGENT CONVERSATION:\n");
106        for turn in &actual.turns {
107            prompt.push_str(&format!("[{}]: {}\n", turn.role, turn.content));
108            if !turn.tool_calls.is_empty() {
109                prompt.push_str(&format!(
110                    "  Tool calls: {}\n",
111                    serde_json::json!(turn.tool_calls)
112                ));
113            }
114            if !turn.tool_results.is_empty() {
115                prompt.push_str(&format!(
116                    "  Tool results: {}\n",
117                    serde_json::json!(turn.tool_results)
118                ));
119            }
120        }
121
122        // Add expected conversation if available
123        if let Some(expected) = expected {
124            prompt.push_str("\nEXPECTED CONVERSATION:\n");
125            for turn in &expected.turns {
126                prompt.push_str(&format!("[{}]: {}\n", turn.role, turn.content));
127                if !turn.tool_calls.is_empty() {
128                    prompt.push_str(&format!(
129                        "  Tool calls: {}\n",
130                        serde_json::json!(turn.tool_calls)
131                    ));
132                }
133            }
134        }
135
136        prompt.push_str(
137            "\nRespond with ONLY a JSON object:\n\
138             {\"scores\": [<float per rubric criterion>], \
139             \"overall_score\": <float average>, \
140             \"explanation\": \"<text>\"}\n",
141        );
142
143        prompt
144    }
145
146    /// Parse the LLM judge response to extract rubric scores.
147    fn parse_response(text: &str, num_rubrics: usize) -> (f64, String) {
148        // Try full JSON parse first
149        if let Some((score, explanation)) = try_parse_json(text) {
150            return (score, explanation);
151        }
152
153        // Try to find JSON embedded in text
154        if let Some(start) = text.find('{')
155            && let Some(end) = text[start..].rfind('}')
156        {
157            let json_str = &text[start..=start + end];
158            if let Some((score, explanation)) = try_parse_json(json_str) {
159                return (score, explanation);
160            }
161        }
162
163        // Fallback: try to find individual scores
164        let _ = num_rubrics; // Used in full implementation
165        (
166            0.0,
167            format!("Failed to parse rubric judge response: {text}"),
168        )
169    }
170}
171
172/// Try to parse a JSON string into a score and explanation.
173fn try_parse_json(text: &str) -> Option<(f64, String)> {
174    let v: serde_json::Value = serde_json::from_str(text).ok()?;
175
176    let score = if let Some(overall) = v["overall_score"].as_f64() {
177        overall.clamp(0.0, 1.0)
178    } else {
179        let scores = v["scores"].as_array()?;
180        // Average only the entries that are actually numbers. Dividing by
181        // `scores.len()` counted entries the numerator never received — a judge
182        // emitting `[0.9, "n/a"]` for a criterion it could not score scored the
183        // candidate 0.45 — and an empty array averaged to a confident 0.0, a
184        // failing grade for a response nobody graded. Both are "unparseable",
185        // which is what `None` means here, as it does in the sibling evaluator.
186        let graded: Vec<f64> = scores
187            .iter()
188            .filter_map(serde_json::Value::as_f64)
189            .map(|s| s.clamp(0.0, 1.0))
190            .collect();
191        if graded.is_empty() {
192            return None;
193        }
194        graded.iter().sum::<f64>() / graded.len() as f64
195    };
196
197    let explanation = v["explanation"]
198        .as_str()
199        .unwrap_or("No explanation")
200        .to_string();
201
202    Some((score, explanation))
203}
204
205#[async_trait]
206impl Evaluator for RubricEvaluator {
207    async fn evaluate(
208        &self,
209        actual: &[Invocation],
210        expected: Option<&[Invocation]>,
211    ) -> Result<EvalResult, EvalError> {
212        let llm = self.llm.as_ref().ok_or_else(|| {
213            EvalError::Llm(
214                "RubricEvaluator requires an LLM instance — call .with_llm() before evaluating"
215                    .into(),
216            )
217        })?;
218
219        let mut per_invocation = Vec::new();
220        let mut total_score = 0.0;
221
222        for (i, actual_inv) in actual.iter().enumerate() {
223            let expected_inv = expected.and_then(|e| e.get(i));
224            let prompt = self.build_prompt(actual_inv, expected_inv);
225
226            let request = crate::llm::LlmRequest::from_text(&prompt);
227            let response = llm
228                .generate(request)
229                .await
230                .map_err(|e| EvalError::Llm(e.to_string()))?;
231
232            let (score, explanation) = Self::parse_response(&response.text(), self.rubrics.len());
233            total_score += score;
234
235            per_invocation.push(PerInvocationResult {
236                invocation_id: if actual_inv.id.is_empty() {
237                    format!("inv-{i}")
238                } else {
239                    actual_inv.id.clone()
240                },
241                score,
242                explanation: Some(explanation),
243            });
244        }
245
246        let overall_score = if actual.is_empty() {
247            0.0
248        } else {
249            total_score / actual.len() as f64
250        };
251
252        let metric_name = match self.mode {
253            RubricMode::FinalResponse => "rubric_based_final_response_quality_v1",
254            RubricMode::ToolUse => "rubric_based_tool_use_quality_v1",
255        };
256
257        Ok(EvalResult {
258            overall_score,
259            metrics: vec![EvalMetric {
260                name: metric_name.into(),
261                score: overall_score,
262                per_invocation,
263            }],
264        })
265    }
266}
267
268#[cfg(test)]
269mod tests {
270    use super::*;
271
272    #[test]
273    fn parse_valid_response() {
274        let json = r#"{"scores": [0.8, 0.9], "overall_score": 0.85, "explanation": "Good"}"#;
275        let (score, explanation) = RubricEvaluator::parse_response(json, 2);
276        assert!((score - 0.85).abs() < f64::EPSILON);
277        assert_eq!(explanation, "Good");
278    }
279
280    #[test]
281    fn parse_scores_only() {
282        let json = r#"{"scores": [0.8, 0.6]}"#;
283        let (score, _) = RubricEvaluator::parse_response(json, 2);
284        assert!((score - 0.7).abs() < f64::EPSILON);
285    }
286
287    #[test]
288    fn ungradeable_entries_do_not_dilute_the_average() {
289        // The denominator is the number of criteria actually scored. Counting
290        // the ones the judge could not score halved a good response's grade.
291        let json = r#"{"scores": [0.9, "n/a", null]}"#;
292        let (score, _) = RubricEvaluator::parse_response(json, 3);
293        assert!(
294            (score - 0.9).abs() < f64::EPSILON,
295            "one criterion scored 0.9 and two were ungradeable, so the score is \
296             0.9, not 0.9 spread over three: got {score}"
297        );
298    }
299
300    #[test]
301    fn an_empty_score_array_is_unparseable_not_zero() {
302        // Zero is a grade. A judge that graded nothing has not awarded it, so
303        // this must fall through to the parse failure rather than report 0.0 as
304        // if it were the judge's verdict.
305        let (score, explanation) = RubricEvaluator::parse_response(r#"{"scores": []}"#, 2);
306        assert!((score - 0.0).abs() < f64::EPSILON);
307        assert!(
308            explanation.contains("Failed to parse"),
309            "an empty score array must read as unparseable, not as a zero \
310             verdict: {explanation}"
311        );
312    }
313
314    #[test]
315    fn parse_embedded_json() {
316        let text = r#"Here is my evaluation: {"overall_score": 0.9, "explanation": "Great"}"#;
317        let (score, _) = RubricEvaluator::parse_response(text, 1);
318        assert!((score - 0.9).abs() < f64::EPSILON);
319    }
320
321    #[test]
322    fn parse_invalid() {
323        let (score, explanation) = RubricEvaluator::parse_response("no json here", 1);
324        assert!((score - 0.0).abs() < f64::EPSILON);
325        assert!(explanation.contains("Failed to parse"));
326    }
327
328    #[test]
329    fn for_response_mode() {
330        let eval = RubricEvaluator::for_response(vec!["Accuracy".into()]);
331        assert_eq!(eval.mode, RubricMode::FinalResponse);
332    }
333
334    #[test]
335    fn for_tool_use_mode() {
336        let eval = RubricEvaluator::for_tool_use(vec!["Tool selection".into()]);
337        assert_eq!(eval.mode, RubricMode::ToolUse);
338    }
339
340    #[test]
341    fn build_prompt_includes_rubrics() {
342        use crate::evaluation::eval_case::InvocationTurn;
343
344        let eval = RubricEvaluator::new(vec![
345            "Is the response accurate?".into(),
346            "Is it well-formatted?".into(),
347        ]);
348        let inv = Invocation {
349            id: "test".into(),
350            turns: vec![InvocationTurn {
351                role: "user".into(),
352                content: "Hello".into(),
353                tool_calls: vec![],
354                tool_results: vec![],
355            }],
356            metadata: serde_json::Value::Null,
357        };
358        let prompt = eval.build_prompt(&inv, None);
359        assert!(prompt.contains("Is the response accurate?"));
360        assert!(prompt.contains("Is it well-formatted?"));
361        assert!(prompt.contains("FINAL RESPONSE QUALITY"));
362    }
363
364    #[test]
365    fn with_judge_model() {
366        let eval = RubricEvaluator::new(vec!["test".into()]).with_judge_model("gemini-2.0-flash");
367        assert_eq!(eval.judge_model.as_deref(), Some("gemini-2.0-flash"));
368    }
369
370    #[test]
371    fn score_clamped() {
372        let json = r#"{"overall_score": 1.5, "explanation": "Over"}"#;
373        let (score, _) = RubricEvaluator::parse_response(json, 1);
374        assert!((score - 1.0).abs() < f64::EPSILON);
375    }
376}
gemini_adk_rs/evaluation/rubric_evaluator.rs

gemini_adk_rs/evaluation/
rubric_evaluator.rs
