gemini_memory_rs/evals/
fixtures.rs

1//! The evaluation corpus and cases.
2//!
3//! A small, hand-written corpus that states what the engine is *supposed* to
4//! do. The cases are the specification the thresholds in [`super::harness`] are
5//! judged against; changing one should be a deliberate decision about product
6//! behaviour, not a way to make a run go green.
7
8use chrono::{Duration, Utc};
9
10use crate::core::{
11    CanonicalMemory, CanonicalPredicate, EntityRef, EvidenceCounters, Explicitness, MemoryId,
12    MemoryKind, MemorySource, MemoryStatus, MemoryValue, PrivacyMetadata, RetrievalMetadata,
13    SessionId, SpeakerAttribution, TemporalMetadata, TemporalScope, TurnId, UserId,
14};
15
16/// One retrieval expectation.
17#[derive(Debug, Clone)]
18pub struct RetrievalCase {
19    /// Case name, for failure messages.
20    pub name: &'static str,
21    /// What the user said.
22    pub query: &'static str,
23    /// Whether memory should be consulted at all.
24    pub expects_memory: bool,
25    /// Records that would be reasonable to return.
26    pub relevant: &'static [&'static str],
27    /// Records that must never be returned.
28    pub forbidden: &'static [&'static str],
29}
30
31/// What should happen to one utterance at ingestion.
32#[derive(Debug, Clone)]
33pub struct IngestionCase {
34    /// Case name.
35    pub name: &'static str,
36    /// What was said.
37    pub utterance: &'static str,
38    /// Who said it.
39    pub speaker: SpeakerAttribution,
40    /// Whether a candidate should be created at all.
41    pub stores: bool,
42    /// The kind expected, when one is stored.
43    pub kind: Option<MemoryKind>,
44    /// The explicitness expected, when one is stored.
45    pub explicitness: Option<Explicitness>,
46}
47
48/// The evaluation user.
49pub fn eval_user() -> UserId {
50    UserId::new("usr_eval")
51}
52
53fn record(
54    id: &str,
55    kind: MemoryKind,
56    predicate: &str,
57    subject: EntityRef,
58    statement: &str,
59    tags: &[&str],
60    aliases: &[&str],
61) -> CanonicalMemory {
62    let now = Utc::now();
63    let subject_form = crate::core::normalize_token(&subject.display);
64    let entities = subject.surface_forms();
65    CanonicalMemory {
66        id: MemoryId::new(id),
67        owner: eval_user(),
68        kind,
69        predicate: CanonicalPredicate::new(predicate),
70        status: MemoryStatus::Active,
71        confidence: 0.92,
72        subject,
73        value: MemoryValue::Text(statement.to_string()),
74        statement: statement.to_string(),
75        evidence_summary: "Explicitly stated by the user.".into(),
76        source: MemorySource::from_explicitness(
77            Explicitness::ExplicitStatement,
78            SessionId::new("ses_eval"),
79            TurnId(1),
80        ),
81        temporal: TemporalMetadata::created_at(now),
82        retrieval: RetrievalMetadata {
83            subject: subject_form,
84            tags: tags.iter().map(|t| (*t).to_string()).collect(),
85            aliases: aliases.iter().map(|a| (*a).to_string()).collect(),
86            entities,
87            location: None,
88        },
89        evidence: EvidenceCounters {
90            count: 2,
91            distinct_sessions: 2,
92            distinct_days: 2,
93        },
94        privacy: PrivacyMetadata::default(),
95        temporal_scope: TemporalScope::Persistent,
96        supersedes: Vec::new(),
97        superseded_by: None,
98        qualifier: None,
99    }
100}
101
102/// The evaluation corpus.
103pub fn corpus() -> Vec<CanonicalMemory> {
104    let rhea = EntityRef::named("Rhea")
105        .with_alias("my wife")
106        .with_alias("wife");
107
108    let mut episode = record(
109        "mem_bandra",
110        MemoryKind::Episodic,
111        "outing_outcome",
112        EntityRef::user(),
113        "Dinner at a noisy restaurant in Bandra went badly.",
114        &["restaurant", "bandra", "noise", "dinner"],
115        &["the noisy dinner"],
116    );
117    episode.temporal_scope = TemporalScope::RecentHistory;
118    episode.temporal.valid_from = Utc::now() - Duration::days(3);
119    episode.temporal.expires_at = Some(Utc::now() + Duration::days(4));
120
121    let mut superseded = record(
122        "mem_old_vegetarian",
123        MemoryKind::Preference,
124        "dietary_identity",
125        EntityRef::user(),
126        "The user is vegetarian.",
127        &["diet", "food", "vegetarian"],
128        &["does not eat meat"],
129    );
130    superseded.status = MemoryStatus::Superseded;
131    superseded.superseded_by = Some(MemoryId::new("mem_diet"));
132
133    vec![
134        record(
135            "mem_diet",
136            MemoryKind::Preference,
137            "dietary_identity",
138            EntityRef::user(),
139            "The user is pescatarian.",
140            &["diet", "food", "pescatarian"],
141            &["does not eat meat", "eats fish"],
142        ),
143        record(
144            "mem_rhea",
145            MemoryKind::Relationship,
146            "spouse",
147            rhea.clone(),
148            "Rhea is the user's wife.",
149            &["family", "wife", "spouse"],
150            &["married to Rhea"],
151        ),
152        record(
153            "mem_rhea_quiet",
154            MemoryKind::RelationshipPreference,
155            "venue_preference",
156            rhea,
157            "Rhea prefers quiet restaurants.",
158            &["restaurant", "quiet", "noise", "venue"],
159            &["dislikes loud places"],
160        ),
161        record(
162            "mem_music",
163            MemoryKind::Preference,
164            "venue_preference",
165            EntityRef::user(),
166            "The user enjoys live music venues with friends.",
167            &["music", "venue", "friends"],
168            &["likes gigs"],
169        ),
170        record(
171            "mem_gym",
172            MemoryKind::Routine,
173            "exercise_routine",
174            EntityRef::user(),
175            "The user goes to the gym before work.",
176            &["gym", "exercise", "morning", "routine"],
177            &["works out in the morning"],
178        ),
179        record(
180            "mem_coffee",
181            MemoryKind::Preference,
182            "beverage_preference",
183            EntityRef::user(),
184            "The user drinks flat white coffee.",
185            &["coffee", "beverage", "flat white"],
186            &["usual order"],
187        ),
188        episode,
189        superseded,
190    ]
191}
192
193/// Retrieval cases (§37.1).
194pub fn retrieval_cases() -> Vec<RetrievalCase> {
195    vec![
196        RetrievalCase {
197            name: "generic world knowledge skips memory",
198            query: "what is the capital of France",
199            expects_memory: false,
200            relevant: &[],
201            forbidden: &["mem_diet", "mem_rhea", "mem_coffee"],
202        },
203        RetrievalCase {
204            name: "visual question skips memory",
205            query: "what does this label say",
206            expects_memory: false,
207            relevant: &[],
208            forbidden: &["mem_diet"],
209        },
210        RetrievalCase {
211            name: "explicit recall of diet",
212            query: "what do you remember about my diet and food preferences",
213            expects_memory: true,
214            relevant: &["mem_diet"],
215            forbidden: &["mem_old_vegetarian"],
216        },
217        RetrievalCase {
218            name: "recommendation for a relationship",
219            query: "find a quiet restaurant for my wife",
220            expects_memory: true,
221            relevant: &["mem_rhea_quiet", "mem_rhea", "mem_bandra"],
222            forbidden: &["mem_old_vegetarian"],
223        },
224        RetrievalCase {
225            name: "prior event reference",
226            query: "how did that noisy dinner in Bandra go last week",
227            expects_memory: true,
228            relevant: &["mem_bandra", "mem_rhea_quiet"],
229            forbidden: &["mem_old_vegetarian", "mem_coffee"],
230        },
231        RetrievalCase {
232            name: "routine recall",
233            query: "remind me about my gym routine",
234            expects_memory: true,
235            relevant: &["mem_gym"],
236            forbidden: &["mem_coffee", "mem_old_vegetarian"],
237        },
238        RetrievalCase {
239            name: "beverage preference",
240            query: "do you remember what coffee I like",
241            expects_memory: true,
242            relevant: &["mem_coffee"],
243            forbidden: &["mem_old_vegetarian"],
244        },
245        RetrievalCase {
246            name: "superseded facts never resurface",
247            query: "do you remember whether I eat vegetarian food",
248            expects_memory: true,
249            relevant: &["mem_diet"],
250            forbidden: &["mem_old_vegetarian"],
251        },
252    ]
253}
254
255/// Ingestion cases (§37.2).
256pub fn ingestion_cases() -> Vec<IngestionCase> {
257    vec![
258        IngestionCase {
259            name: "explicit preference",
260            utterance: "I am pescatarian",
261            speaker: SpeakerAttribution::User,
262            stores: true,
263            kind: Some(MemoryKind::Identity),
264            explicitness: Some(Explicitness::ExplicitStatement),
265        },
266        IngestionCase {
267            name: "explicit memory command",
268            utterance: "please remember that I am allergic to shellfish",
269            speaker: SpeakerAttribution::User,
270            stores: true,
271            kind: None,
272            explicitness: Some(Explicitness::ExplicitCommand),
273        },
274        IngestionCase {
275            name: "time-bounded plan is episodic",
276            utterance: "I am meeting Kushal for dinner tonight",
277            speaker: SpeakerAttribution::User,
278            stores: true,
279            kind: Some(MemoryKind::Episodic),
280            explicitness: Some(Explicitness::ExplicitStatement),
281        },
282        IngestionCase {
283            name: "routine statement",
284            utterance: "I always go to the gym before work",
285            speaker: SpeakerAttribution::User,
286            stores: true,
287            kind: Some(MemoryKind::Routine),
288            explicitness: Some(Explicitness::ExplicitStatement),
289        },
290        IngestionCase {
291            name: "small talk stores nothing",
292            utterance: "the weather is lovely today",
293            speaker: SpeakerAttribution::User,
294            stores: false,
295            kind: None,
296            explicitness: None,
297        },
298        IngestionCase {
299            name: "a question is not a statement",
300            utterance: "what i am asking is whether the place is open",
301            speaker: SpeakerAttribution::User,
302            stores: false,
303            kind: None,
304            explicitness: None,
305        },
306        IngestionCase {
307            name: "bystander speech is refused",
308            utterance: "I am vegetarian",
309            speaker: SpeakerAttribution::Bystander,
310            stores: false,
311            kind: None,
312            explicitness: None,
313        },
314        IngestionCase {
315            name: "assistant speech is refused",
316            utterance: "I am a helpful assistant",
317            speaker: SpeakerAttribution::Assistant,
318            stores: false,
319            kind: None,
320            explicitness: None,
321        },
322        IngestionCase {
323            name: "unattributed speech is refused",
324            utterance: "I prefer window seats",
325            speaker: SpeakerAttribution::Unknown,
326            stores: false,
327            kind: None,
328            explicitness: None,
329        },
330        IngestionCase {
331            name: "forget command is recognised",
332            utterance: "forget that I like sushi",
333            speaker: SpeakerAttribution::User,
334            stores: true,
335            kind: None,
336            explicitness: Some(Explicitness::ExplicitCommand),
337        },
338    ]
339}
340
341#[cfg(test)]
342mod tests {
343    use super::*;
344
345    #[test]
346    fn the_corpus_is_internally_consistent() {
347        let corpus = corpus();
348        let ids: Vec<String> = corpus.iter().map(|m| m.id.to_string()).collect();
349        let unique: std::collections::HashSet<_> = ids.iter().collect();
350        assert_eq!(unique.len(), ids.len(), "duplicate record ids");
351        assert!(corpus.iter().all(|m| m.owner == eval_user()));
352        assert!(
353            corpus.iter().any(|m| m.status == MemoryStatus::Superseded),
354            "the corpus must include a superseded record to test against"
355        );
356    }
357
358    #[test]
359    fn every_case_names_records_that_exist() {
360        let ids: Vec<String> = corpus().iter().map(|m| m.id.to_string()).collect();
361        for case in retrieval_cases() {
362            for id in case.relevant.iter().chain(case.forbidden.iter()) {
363                assert!(
364                    ids.contains(&(*id).to_string()),
365                    "case `{}` names unknown record `{id}`",
366                    case.name
367                );
368            }
369        }
370    }
371
372    #[test]
373    fn the_case_set_covers_both_skip_and_recall() {
374        let cases = retrieval_cases();
375        assert!(cases.iter().any(|c| !c.expects_memory));
376        assert!(cases.iter().any(|c| c.expects_memory));
377        assert!(ingestion_cases().iter().any(|c| !c.stores));
378        assert!(ingestion_cases().iter().any(|c| c.stores));
379    }
380}
gemini_memory_rs/evals/fixtures.rs

gemini_memory_rs/evals/
fixtures.rs
