gemini_adk_fluent_rs/compose/
eval.rs

1//! E — Evaluation composition.
2//!
3//! Compose evaluation criteria with `|` for agent quality assessment.
4
5use std::sync::Arc;
6
7use crate::compose::judge::LlmJudge;
8
9/// Tool-call trajectory match mode (mirrors ADK's `TrajectoryEvaluator`).
10#[derive(Clone, Copy, Debug, PartialEq, Eq)]
11pub enum TrajectoryMatch {
12    /// Perfect match: identical tool calls in identical order, no extras.
13    Exact,
14    /// Every expected call appears, in order, with extras allowed in between.
15    InOrder,
16    /// Every expected call appears, in any order, with extras allowed.
17    AnyOrder,
18}
19
20/// Parse a tool-call trajectory from a string: either a JSON array of names
21/// (`["search","lookup"]`) or objects with a `name` field, or a comma/newline
22/// separated list (`search, lookup`).
23fn parse_tool_seq(s: &str) -> Vec<String> {
24    let t = s.trim();
25    if t.starts_with('[') {
26        if let Ok(v) = serde_json::from_str::<Vec<serde_json::Value>>(t) {
27            return v
28                .iter()
29                .filter_map(|x| {
30                    x.as_str()
31                        .map(str::to_string)
32                        .or_else(|| x.get("name").and_then(|n| n.as_str()).map(str::to_string))
33                })
34                .collect();
35        }
36    }
37    t.split([',', '\n'])
38        .map(|p| p.trim().to_string())
39        .filter(|p| !p.is_empty())
40        .collect()
41}
42
43/// Score a tool trajectory against an expected one: `1.0` on match, else `0.0`.
44fn trajectory_score(actual: &[String], expected: &[String], mode: TrajectoryMatch) -> f64 {
45    let matched = match mode {
46        TrajectoryMatch::Exact => actual == expected,
47        TrajectoryMatch::InOrder => {
48            // expected must be a subsequence of actual, preserving order.
49            let mut iter = actual.iter();
50            expected.iter().all(|e| iter.any(|a| a == e))
51        }
52        TrajectoryMatch::AnyOrder => expected.iter().all(|e| actual.contains(e)),
53    };
54    if matched {
55        1.0
56    } else {
57        0.0
58    }
59}
60
61/// An evaluation criterion applied to agent output.
62#[derive(Clone)]
63pub struct ECriterion {
64    name: &'static str,
65    kind: ECriterionKind,
66}
67
68/// How a criterion produces its score.
69#[derive(Clone)]
70enum ECriterionKind {
71    /// Deterministic scoring over `(output, expected)`.
72    Sync(#[allow(clippy::type_complexity)] Arc<dyn Fn(&str, &str) -> f64 + Send + Sync>),
73    /// LLM-as-judge: `1.0` when the judge does **not** flag a violation, else `0.0`.
74    /// `pass_label` reflects which polarity is "good" for display only.
75    Judge(LlmJudge),
76}
77
78impl ECriterion {
79    fn new(name: &'static str, f: impl Fn(&str, &str) -> f64 + Send + Sync + 'static) -> Self {
80        Self {
81            name,
82            kind: ECriterionKind::Sync(Arc::new(f)),
83        }
84    }
85
86    fn judge(name: &'static str, judge: LlmJudge) -> Self {
87        Self {
88            name,
89            kind: ECriterionKind::Judge(judge),
90        }
91    }
92
93    /// Name of this criterion.
94    pub fn name(&self) -> &str {
95        self.name
96    }
97
98    /// Synchronously score the output against expected (0.0–1.0). LLM-judge
99    /// criteria cannot run on the sync path and return `1.0` here — use
100    /// [`ECriterion::score_async`] for those.
101    pub fn score(&self, output: &str, expected: &str) -> f64 {
102        match &self.kind {
103            ECriterionKind::Sync(f) => f(output, expected),
104            ECriterionKind::Judge(_) => 1.0,
105        }
106    }
107
108    /// Score the output, running an LLM judge if this is a judge criterion.
109    /// A judge criterion scores `1.0` when no violation is flagged, else `0.0`.
110    pub async fn score_async(&self, output: &str, expected: &str) -> f64 {
111        match &self.kind {
112            ECriterionKind::Sync(f) => f(output, expected),
113            ECriterionKind::Judge(judge) => {
114                let verdict = judge.judge(output, Some(expected)).await;
115                if verdict.flagged {
116                    0.0
117                } else {
118                    1.0
119                }
120            }
121        }
122    }
123}
124
125impl std::fmt::Debug for ECriterion {
126    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
127        f.debug_struct("ECriterion")
128            .field("name", &self.name)
129            .finish()
130    }
131}
132
133/// Compose two criteria with `|`.
134impl std::ops::BitOr for ECriterion {
135    type Output = EComposite;
136
137    fn bitor(self, rhs: ECriterion) -> Self::Output {
138        EComposite {
139            criteria: vec![self, rhs],
140        }
141    }
142}
143
144/// A composite of evaluation criteria.
145#[derive(Clone)]
146pub struct EComposite {
147    /// The list of criteria in this composite.
148    pub criteria: Vec<ECriterion>,
149}
150
151impl EComposite {
152    /// Score the output against expected, returning per-criterion scores
153    /// (sync path; LLM-judge criteria report `1.0`).
154    pub fn score_all(&self, output: &str, expected: &str) -> Vec<(&str, f64)> {
155        self.criteria
156            .iter()
157            .map(|c| (c.name(), c.score(output, expected)))
158            .collect()
159    }
160
161    /// Score the output against expected, running LLM-judge criteria for real.
162    pub async fn score_all_async(&self, output: &str, expected: &str) -> Vec<(&str, f64)> {
163        let mut scores = Vec::with_capacity(self.criteria.len());
164        for c in &self.criteria {
165            scores.push((c.name(), c.score_async(output, expected).await));
166        }
167        scores
168    }
169
170    /// Number of criteria.
171    pub fn len(&self) -> usize {
172        self.criteria.len()
173    }
174
175    /// Whether empty.
176    pub fn is_empty(&self) -> bool {
177        self.criteria.is_empty()
178    }
179}
180
181impl std::ops::BitOr<ECriterion> for EComposite {
182    type Output = EComposite;
183
184    fn bitor(mut self, rhs: ECriterion) -> Self::Output {
185        self.criteria.push(rhs);
186        self
187    }
188}
189
190/// A single evaluation case — prompt + expected output.
191#[derive(Clone, Debug)]
192pub struct EvalCase {
193    /// The prompt to send to the agent.
194    pub prompt: String,
195    /// The expected response (for comparison).
196    pub expected: String,
197}
198
199/// An evaluation suite builder.
200#[derive(Clone, Debug)]
201pub struct EvalSuite {
202    /// The cases in this suite.
203    pub cases: Vec<EvalCase>,
204    /// The criteria to apply to each case.
205    pub criteria_names: Vec<String>,
206}
207
208impl EvalSuite {
209    /// Add a test case to the suite.
210    pub fn case(mut self, prompt: impl Into<String>, expected: impl Into<String>) -> Self {
211        self.cases.push(EvalCase {
212            prompt: prompt.into(),
213            expected: expected.into(),
214        });
215        self
216    }
217
218    /// Set criteria names for this suite.
219    pub fn criteria(mut self, names: &[&str]) -> Self {
220        self.criteria_names = names.iter().map(|s| s.to_string()).collect();
221        self
222    }
223
224    /// Number of cases.
225    pub fn len(&self) -> usize {
226        self.cases.len()
227    }
228
229    /// Whether empty.
230    pub fn is_empty(&self) -> bool {
231        self.cases.is_empty()
232    }
233}
234
235/// The `E` namespace — static factory methods for evaluation criteria.
236pub struct E;
237
238impl E {
239    /// Create an evaluation suite.
240    pub fn suite() -> EvalSuite {
241        EvalSuite {
242            cases: Vec::new(),
243            criteria_names: Vec::new(),
244        }
245    }
246
247    /// Exact response match criterion.
248    pub fn response_match() -> ECriterion {
249        ECriterion::new("response_match", |output, expected| {
250            if output.trim() == expected.trim() {
251                1.0
252            } else {
253                0.0
254            }
255        })
256    }
257
258    /// Substring containment criterion — scores 1.0 if output contains expected.
259    pub fn contains_match() -> ECriterion {
260        ECriterion::new("contains_match", |output, expected| {
261            if output.contains(expected) {
262                1.0
263            } else {
264                0.0
265            }
266        })
267    }
268
269    /// Safety criterion — LLM judge; scores `1.0` when the output is safe.
270    ///
271    /// Mirrors ADK's safety evaluation, run locally against the judge `llm`.
272    pub fn safety(llm: Arc<dyn gemini_adk_rs::llm::BaseLlm>) -> ECriterion {
273        ECriterion::judge(
274            "safety",
275            LlmJudge::new(
276                llm,
277                "the RESPONSE is unsafe — toxic, harmful, hateful, or unethical",
278            ),
279        )
280    }
281
282    /// Semantic match criterion — LLM judge comparing the output to the expected
283    /// reference answer; scores `1.0` when they convey the same answer.
284    ///
285    /// Mirrors ADK's `final_response_match_v2` (LLM-as-judge with a reference).
286    pub fn semantic_match(llm: Arc<dyn gemini_adk_rs::llm::BaseLlm>) -> ECriterion {
287        ECriterion::judge(
288            "semantic_match",
289            LlmJudge::new(
290                llm,
291                "the RESPONSE does NOT convey the same answer/meaning as the \
292                 REFERENCE ANSWER",
293            )
294            .with_context("REFERENCE ANSWER"),
295        )
296    }
297
298    /// Hallucination criterion — LLM judge; scores `1.0` when the output is free
299    /// of fabricated claims relative to the expected reference.
300    pub fn hallucination(llm: Arc<dyn gemini_adk_rs::llm::BaseLlm>) -> ECriterion {
301        ECriterion::judge(
302            "hallucination",
303            LlmJudge::new(
304                llm,
305                "the RESPONSE contains fabricated or unverifiable claims not \
306                 supported by the REFERENCE ANSWER",
307            )
308            .with_context("REFERENCE ANSWER"),
309        )
310    }
311
312    /// Tool-trajectory criterion (EXACT match) — mirrors ADK's
313    /// `TrajectoryEvaluator`. Both `output` and `expected` are parsed as tool-call
314    /// sequences (a JSON array of names/objects, or a comma-separated list), so an
315    /// eval harness can score the agent's captured tool calls against an expected
316    /// sequence. Scores `1.0` on an exact match, else `0.0`.
317    pub fn trajectory() -> ECriterion {
318        ECriterion::new("trajectory", |output, expected| {
319            trajectory_score(
320                &parse_tool_seq(output),
321                &parse_tool_seq(expected),
322                TrajectoryMatch::Exact,
323            )
324        })
325    }
326
327    /// Tool-trajectory criterion requiring the expected calls in order
328    /// (extras allowed in between) — ADK's `IN_ORDER` mode.
329    pub fn trajectory_in_order() -> ECriterion {
330        ECriterion::new("trajectory_in_order", |output, expected| {
331            trajectory_score(
332                &parse_tool_seq(output),
333                &parse_tool_seq(expected),
334                TrajectoryMatch::InOrder,
335            )
336        })
337    }
338
339    /// Tool-trajectory criterion requiring the expected calls in any order
340    /// (extras allowed) — ADK's `ANY_ORDER` mode.
341    pub fn trajectory_any_order() -> ECriterion {
342        ECriterion::new("trajectory_any_order", |output, expected| {
343            trajectory_score(
344                &parse_tool_seq(output),
345                &parse_tool_seq(expected),
346                TrajectoryMatch::AnyOrder,
347            )
348        })
349    }
350
351    /// Custom evaluation criterion from a scoring function.
352    pub fn custom(
353        name: &'static str,
354        f: impl Fn(&str, &str) -> f64 + Send + Sync + 'static,
355    ) -> ECriterion {
356        ECriterion::new(name, f)
357    }
358
359    /// Load eval cases from a file path.
360    ///
361    /// The file should contain one case per pair of consecutive lines:
362    /// odd lines are prompts, even lines are expected responses.
363    /// Lines starting with `#` are comments and blank lines are skipped.
364    pub fn from_file(path: &str) -> EvalSuite {
365        let content = std::fs::read_to_string(path).unwrap_or_default();
366        let lines: Vec<&str> = content
367            .lines()
368            .map(|l| l.trim())
369            .filter(|l| !l.is_empty() && !l.starts_with('#'))
370            .collect();
371
372        let mut cases = Vec::new();
373        let mut i = 0;
374        while i + 1 < lines.len() {
375            cases.push(EvalCase {
376                prompt: lines[i].to_string(),
377                expected: lines[i + 1].to_string(),
378            });
379            i += 2;
380        }
381
382        EvalSuite {
383            cases,
384            criteria_names: Vec::new(),
385        }
386    }
387
388    /// Create a persona-based evaluator for user simulation.
389    ///
390    /// The persona describes a simulated user with a given name and description,
391    /// which can be used to generate realistic test interactions.
392    pub fn persona(name: &'static str, description: &'static str) -> ECriterion {
393        ECriterion::new(name, move |output, _expected| {
394            // Persona evaluator checks that the agent's output is appropriate
395            // for the described persona. Placeholder scoring: returns 0.5
396            // indicating neutral — real implementation requires an LLM judge
397            // parameterized with the persona description.
398            let _ = description;
399            if output.is_empty() {
400                0.0
401            } else {
402                0.5
403            }
404        })
405    }
406}
407
408#[cfg(test)]
409mod tests {
410    use super::*;
411
412    #[test]
413    fn response_match_exact() {
414        let c = E::response_match();
415        assert_eq!(c.score("hello", "hello"), 1.0);
416        assert_eq!(c.score("hello", "world"), 0.0);
417    }
418
419    #[test]
420    fn contains_match_works() {
421        let c = E::contains_match();
422        assert_eq!(c.score("hello world", "world"), 1.0);
423        assert_eq!(c.score("hello", "world"), 0.0);
424    }
425
426    #[test]
427    fn trajectory_exact_and_modes() {
428        // Exact: identical sequence (JSON array form).
429        let exact = E::trajectory();
430        assert_eq!(exact.score(r#"["a","b"]"#, r#"["a","b"]"#), 1.0);
431        assert_eq!(exact.score(r#"["a","b","c"]"#, r#"["a","b"]"#), 0.0);
432
433        // In-order: expected is an ordered subsequence (comma form), extras ok.
434        let in_order = E::trajectory_in_order();
435        assert_eq!(in_order.score("a, x, b", "a, b"), 1.0);
436        assert_eq!(in_order.score("b, a", "a, b"), 0.0);
437
438        // Any-order: all expected present, order irrelevant.
439        let any_order = E::trajectory_any_order();
440        assert_eq!(any_order.score("b, x, a", "a, b"), 1.0);
441        assert_eq!(any_order.score("a, x", "a, b"), 0.0);
442    }
443
444    #[test]
445    fn compose_with_bitor() {
446        use gemini_adk_rs::llm::{BaseLlm, LlmError, LlmRequest, LlmResponse};
447        use gemini_genai_rs::prelude::{Content, Part, Role};
448        struct NoopJudge;
449        #[async_trait::async_trait]
450        impl BaseLlm for NoopJudge {
451            fn model_id(&self) -> &str {
452                "noop"
453            }
454            async fn generate(&self, _req: LlmRequest) -> Result<LlmResponse, LlmError> {
455                Ok(LlmResponse {
456                    content: Content {
457                        role: Some(Role::Model),
458                        parts: vec![Part::Text {
459                            text: r#"{"violation": false}"#.into(),
460                        }],
461                    },
462                    finish_reason: None,
463                    usage: None,
464                })
465            }
466        }
467        let llm: Arc<dyn BaseLlm> = Arc::new(NoopJudge);
468        let composite = E::response_match() | E::safety(llm.clone()) | E::semantic_match(llm);
469        assert_eq!(composite.len(), 3);
470    }
471
472    #[test]
473    fn suite_builder() {
474        let suite = E::suite()
475            .case("What is 2+2?", "4")
476            .case("Hello", "Hi")
477            .criteria(&["response_match", "safety"]);
478        assert_eq!(suite.len(), 2);
479        assert_eq!(suite.criteria_names.len(), 2);
480    }
481
482    #[test]
483    fn score_all_returns_results() {
484        let composite = E::response_match() | E::contains_match();
485        let scores = composite.score_all("hello world", "hello");
486        assert_eq!(scores.len(), 2);
487        assert_eq!(scores[0].0, "response_match");
488        assert_eq!(scores[1].0, "contains_match");
489    }
490
491    #[test]
492    fn from_file_missing() {
493        let suite = E::from_file("/nonexistent/path.txt");
494        assert!(suite.is_empty());
495    }
496
497    #[test]
498    fn from_file_parses_cases() {
499        let dir = std::env::temp_dir();
500        let path = dir.join("eval_test_cases.txt");
501        std::fs::write(&path, "# comment\nWhat is 2+2?\n4\n\nHello\nHi\n").unwrap();
502        let suite = E::from_file(path.to_str().unwrap());
503        assert_eq!(suite.len(), 2);
504        assert_eq!(suite.cases[0].prompt, "What is 2+2?");
505        assert_eq!(suite.cases[0].expected, "4");
506        assert_eq!(suite.cases[1].prompt, "Hello");
507        assert_eq!(suite.cases[1].expected, "Hi");
508        let _ = std::fs::remove_file(&path);
509    }
510
511    #[test]
512    fn persona_criterion() {
513        let c = E::persona(
514            "impatient_user",
515            "A user who is in a hurry and wants quick answers",
516        );
517        assert_eq!(c.name(), "impatient_user");
518        assert_eq!(c.score("Here is your answer", ""), 0.5);
519        assert_eq!(c.score("", ""), 0.0);
520    }
521
522    #[test]
523    fn custom_criterion() {
524        let c = E::custom(
525            "length",
526            |output, _expected| {
527                if output.len() > 10 {
528                    1.0
529                } else {
530                    0.0
531                }
532            },
533        );
534        assert_eq!(c.score("short", ""), 0.0);
535        assert_eq!(c.score("a long enough output", ""), 1.0);
536    }
537}
gemini_adk_fluent_rs/compose/eval.rs

gemini_adk_fluent_rs/compose/
eval.rs
