🧭 Benchmarks query at a time their data gives: LongMemEval question dates parsed, LoCoMo as of its last turn

crates/cmem-eval-locomo/src/lib.rs

@@ -17,6 +17,9 @@
 //! `<sample_id>:qa:<one-based position>`. Effective QA IDs are unique across the file.
 //! `answer`, `category`, `speaker`, `img_url`, `blip_caption`, `query` and dates
 //! remain optional annotations. Raw dates are retained alongside normalized timestamps.
+//! The conventional runner uses the latest parseable source timestamp, inclusive,
+//! as the scene time for every QA. Official dates without an offset mean UTC.
+//! It rejects a conversation with no parseable timestamp before creating run state.
 //! QA `evidence` admits scalar references and arrays of scalars or objects with
 //! `dia_id`. References to absent turns do not reject the sample. Missing
 //! annotations do not determine abstention status.

crates/cmem-eval-longmemeval/Cargo.toml

@@ -9,6 +9,7 @@ chrono.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tracing.workspace = true
+tiktoken-rs.workspace = true
 cmem-eval = { path = "../cmem-eval" }
 
 [dev-dependencies]

crates/cmem-eval-longmemeval/src/identity.rs

@@ -38,13 +38,24 @@ impl Identities {
             *ordinal += 1;
             identities.session_ids.insert(assigned.clone(), raw.clone());
             for turn in &session.turns {
-                identities.turn_ids.insert(
-                    format!("{assigned}:turn:{}", turn.index),
-                    format!("{raw}:turn:{}", turn.index),
-                );
+                for (part, _) in turn.observation_texts().enumerate() {
+                    identities.turn_ids.insert(
+                        observation_id(&assigned, turn.index, part),
+                        format!("{raw}:turn:{}", turn.index),
+                    );
+                }
             }
             identities.episodes.push(assigned);
         }
         identities
     }
 }
+
+pub(crate) fn observation_id(episode: &str, turn: usize, part: usize) -> String {
+    let original = format!("{episode}:turn:{turn}");
+    if part == 0 {
+        original
+    } else {
+        format!("{original}:part:{}", part + 1)
+    }
+}

crates/cmem-eval-longmemeval/src/ingest.rs

@@ -42,19 +42,21 @@ pub fn to_memory_inputs(instance: &LongMemEvalInstance) -> LongMemEvalMemoryInpu
             }),
         });
         for turn in &session.turns {
-            observations.push(ObservationInput {
-                external_id: format!("{}:turn:{}", assigned, turn.index),
-                episode_external_id: assigned.clone(),
-                namespace: namespace.clone(),
-                speaker: turn.speaker.clone(),
-                text: turn.text.clone(),
-                observed_at: session.date.clone(),
-                metadata: serde_json::json!({
-                    "source": "longmemeval_s",
-                    "raw_date": session.raw_date.clone(),
-                    "normalized_date": session.date.clone()
-                }),
-            });
+            for (part, text) in turn.observation_texts().enumerate() {
+                observations.push(ObservationInput {
+                    external_id: crate::identity::observation_id(assigned, turn.index, part),
+                    episode_external_id: assigned.clone(),
+                    namespace: namespace.clone(),
+                    speaker: turn.speaker.clone(),
+                    text: text.to_string(),
+                    observed_at: session.date.clone(),
+                    metadata: serde_json::json!({
+                        "source": "longmemeval_s",
+                        "raw_date": session.raw_date.clone(),
+                        "normalized_date": session.date.clone()
+                    }),
+                });
+            }
         }
     }
 

crates/cmem-eval-longmemeval/src/lib.rs

@@ -16,10 +16,14 @@
 //! `haystack_dates` is optional: absent or null is admitted; otherwise it must be
 //! an array matching the session count. Each null or non-string slot means no
 //! date at that position, without shifting later dates. Raw dates are retained
-//! alongside normalized timestamps. `question_type`, `answer`, `question_date`,
-//! `role` and `answer_session_ids` remain optional. `has_answer` defaults to false
-//! when absent or not boolean; references to absent sessions do not reject the
-//! item. Optional annotations do not determine abstention status.
+//! alongside normalized timestamps. `question_date` is optional when absent or
+//! null; supplied dates must parse as an official timestamp or RFC3339 and are
+//! normalized to RFC3339 UTC, treating official timestamps without an offset as
+//! UTC. Invalid dates reject the item with its question identity.
+//! `question_type`, `answer`, `role` and `answer_session_ids` remain optional.
+//! `has_answer` defaults to false when absent or not boolean; references to absent
+//! sessions do not reject the item. Optional annotations do not determine
+//! abstention status.
 //!
 //! Ingest assigns each occurrence its own episode identity: the first keeps the
 //! raw benchmark session ID, later copies receive `#2`, `#3`, ... while skipping
@@ -28,6 +32,17 @@
 //! raw ID. Scoring recomputes the same identity tables from the item to map both
 //! session and turn rankings back to benchmark IDs, crediting each ID only at
 //! its first retrieved rank. Retrieved row IDs are assigned; gold IDs are raw.
+//!
+//! Oversized turns retain their original text/index but gain observation parts
+//! at greedy sentence/blank-paragraph boundaries, each native prefixed surface
+//! at most 8192 cl100k tokens. If none fits, use the last newline, then the last
+//! whitespace under the limit; an unbreakable oversized span is an admission
+//! error. The manifest records each part's boundary kind; source end closes its
+//! final paragraph.
+//! Ingest keeps the original ID for the first part and uses `:part:2`, ... for
+//! later parts in the same episode/speaker/time. Scoring maps every part to the
+//! original turn; full-history text and converter source text stay unchanged.
+//! Conventional run headers record affected identities and the split policy.
 
 mod error;
 mod identity;

crates/cmem-eval-longmemeval/src/loader.rs

@@ -51,12 +51,30 @@ fn parse_instance(raw: Value, index: usize) -> Result<LongMemEvalInstance, LoadE
     let question = nonblank_string_field(&raw, "question")
         .ok_or_else(|| location.error("question", "expected a non-blank string"))?;
     let sessions = parse_sessions(&raw, &location)?;
+    let question_date = match raw.get("question_date") {
+        None | Some(Value::Null) => None,
+        Some(value) => Some(
+            value
+                .as_str()
+                .and_then(|value| {
+                    let value = value.trim();
+                    DateTime::parse_from_rfc3339(value)
+                        .map(|timestamp| timestamp.with_timezone(&Utc))
+                        .ok()
+                        .or_else(|| parse_official_longmemeval_timestamp(value))
+                })
+                .map(|timestamp| timestamp.to_rfc3339_opts(SecondsFormat::AutoSi, true))
+                .ok_or_else(|| {
+                    location.error("question_date", "expected an official timestamp or RFC3339")
+                })?,
+        ),
+    };
     Ok(LongMemEvalInstance {
         question_id,
         question_type: string_field(&raw, "question_type"),
         question,
         answer: string_field(&raw, "answer"),
-        question_date: string_field(&raw, "question_date"),
+        question_date,
         sessions,
         answer_session_ids: string_array(raw.get("answer_session_ids")),
     })
@@ -195,12 +213,20 @@ fn parse_turns(
             if !turn.is_object() {
                 return Err(location.error(format!("{field}[{idx}]"), "expected a turn object"));
             }
+            let text = string_field(turn, "content").ok_or_else(|| {
+                location.error(format!("{field}[{idx}].content"), "expected a string")
+            })?;
+            let observation_parts = split_observation(&text).ok_or_else(|| {
+                location.error(
+                    format!("{field}[{idx}].content"),
+                    "a span without a usable boundary exceeds the 8192-token observation limit",
+                )
+            })?;
             Ok(LongMemEvalTurn {
                 index: idx + 1,
                 speaker: string_field(turn, "role"),
-                text: string_field(turn, "content").ok_or_else(|| {
-                    location.error(format!("{field}[{idx}].content"), "expected a string")
-                })?,
+                text,
+                observation_parts,
                 has_answer: turn
                     .get("has_answer")
                     .and_then(Value::as_bool)
@@ -210,6 +236,78 @@ fn parse_turns(
         .collect()
 }
 
+pub(crate) const OBSERVATION_TOKEN_LIMIT: usize = 8192;
+
+fn observation_tokens(text: &str) -> usize {
+    // Include the native surface prefix in the provider's token limit.
+    let clean = text.split_whitespace().collect::<Vec<_>>().join(" ");
+    let surface = if clean.is_empty() {
+        "Observation excerpt".to_string()
+    } else {
+        format!("Observation excerpt: {clean}")
+    };
+    tiktoken_rs::cl100k_base_singleton()
+        .encode_ordinary(&surface)
+        .len()
+}
+
+fn split_observation(text: &str) -> Option<Vec<crate::LongMemEvalObservationPart>> {
+    use cmem_eval::ObservationBoundary::{Newline, Paragraph, Sentence, Whitespace};
+    if text.len() + "Observation excerpt: ".len() <= OBSERVATION_TOKEN_LIMIT
+        || observation_tokens(text) <= OBSERVATION_TOKEN_LIMIT
+    {
+        return Some(Vec::new());
+    }
+    let ends = text
+        .char_indices()
+        .filter_map(|(index, ch)| {
+            let end = index + ch.len_utf8();
+            let sentence = matches!(ch, '.' | '!' | '?')
+                && text[end..].chars().next().is_none_or(char::is_whitespace);
+            let kind = if ch == '\n'
+                && (text[end..].starts_with('\n') || text[end..].starts_with("\r\n"))
+            {
+                Paragraph
+            } else if sentence {
+                Sentence
+            } else if ch == '\n' {
+                Newline
+            } else if ch.is_whitespace() {
+                Whitespace
+            } else {
+                return None;
+            };
+            Some((end, kind))
+        })
+        .collect::<Vec<_>>();
+    let mut parts = Vec::new();
+    let mut start = 0;
+    while observation_tokens(&text[start..]) > OBSERVATION_TOKEN_LIMIT {
+        let last_fit = |kinds: &[cmem_eval::ObservationBoundary]| {
+            ends.iter().rev().copied().find(|(end, kind)| {
+                *end > start
+                    && kinds.contains(kind)
+                    && !text[start..*end].trim().is_empty()
+                    && observation_tokens(&text[start..*end]) <= OBSERVATION_TOKEN_LIMIT
+            })
+        };
+        let (end, boundary_kind) = last_fit(&[Paragraph, Sentence])
+            .or_else(|| last_fit(&[Newline]))
+            .or_else(|| last_fit(&[Whitespace]))?;
+        parts.push(crate::LongMemEvalObservationPart {
+            text: text[start..end].to_string(),
+            boundary_kind,
+        });
+        start = end;
+    }
+    // End of the source turn closes its final paragraph without another cut.
+    parts.push(crate::LongMemEvalObservationPart {
+        text: text[start..].to_string(),
+        boundary_kind: Paragraph,
+    });
+    Some(parts)
+}
+
 fn string_field(value: &Value, key: &str) -> Option<String> {
     value
         .get(key)
@@ -234,6 +332,59 @@ fn string_array(value: Option<&Value>) -> Vec<String> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn records_preferred_boundary_then_newline_and_whitespace_fallbacks() {
+        use cmem_eval::ObservationBoundary::{Newline, Paragraph, Sentence, Whitespace};
+        for (separator, expected) in [
+            (". ", Sentence),
+            ("\n\n", Paragraph),
+            ("\n", Newline),
+            (" ", Whitespace),
+        ] {
+            let text = format!(
+                "{}{separator}{}",
+                "alpha ".repeat(4500),
+                "beta ".repeat(4500)
+            );
+            let parts = split_observation(&text).unwrap();
+            assert_eq!(parts[0].boundary_kind, expected);
+            assert_eq!(parts.last().unwrap().boundary_kind, Paragraph);
+            assert_eq!(
+                parts
+                    .iter()
+                    .map(|part| part.text.as_str())
+                    .collect::<String>(),
+                text
+            );
+            assert!(
+                parts
+                    .iter()
+                    .all(|part| observation_tokens(&part.text) <= OBSERVATION_TOKEN_LIMIT)
+            );
+        }
+    }
+
+    #[test]
+    fn paragraph_split_preserves_unicode_and_crlf_bytes_under_native_token_limit() {
+        let text = format!("{}\r\n\r\n{}", "猫 ".repeat(2200), "猫 ".repeat(2200));
+        assert!(observation_tokens(&text) > OBSERVATION_TOKEN_LIMIT);
+        let parts = split_observation(&text).unwrap();
+        assert_eq!(parts.len(), 2);
+        assert_eq!(
+            parts
+                .iter()
+                .map(|part| part.text.as_str())
+                .collect::<String>(),
+            text
+        );
+        assert!(
+            parts
+                .iter()
+                .all(|part| observation_tokens(&part.text) <= OBSERVATION_TOKEN_LIMIT)
+        );
+        assert!(split_observation(&" \n".repeat(9000)).unwrap().is_empty());
+    }
+
     #[test]
     fn parses_official_fixture() {
         let value = serde_json::json!([{

crates/cmem-eval-longmemeval/src/types.rs

@@ -28,6 +28,49 @@ impl LongMemEvalInstance {
             })
             .collect()
     }
+
+    pub fn observation_splits(&self) -> Vec<cmem_eval::ObservationSplit> {
+        if !self
+            .sessions
+            .iter()
+            .flat_map(|session| &session.turns)
+            .any(|turn| !turn.observation_parts.is_empty())
+        {
+            return Vec::new();
+        }
+        let identities = crate::identity::Identities::new(self);
+        self.sessions
+            .iter()
+            .zip(&identities.episodes)
+            .flat_map(|(session, episode)| {
+                session
+                    .turns
+                    .iter()
+                    .filter(|turn| !turn.observation_parts.is_empty())
+                    .map(|turn| cmem_eval::ObservationSplit {
+                        dataset_item_id: self.question_id.clone(),
+                        source_observation_id: crate::identity::observation_id(
+                            episode, turn.index, 0,
+                        ),
+                        parts: turn
+                            .observation_parts
+                            .iter()
+                            .enumerate()
+                            .map(|(index, part)| cmem_eval::ObservationSplitPart {
+                                observation_id: crate::identity::observation_id(
+                                    episode, turn.index, index,
+                                ),
+                                boundary_kind: part.boundary_kind,
+                            })
+                            .collect(),
+                        boundary_policy: "greedy_sentence_paragraph_then_newline_whitespace_v1"
+                            .into(),
+                        tokenizer: "cl100k_base".into(),
+                        max_input_tokens: crate::loader::OBSERVATION_TOKEN_LIMIT,
+                    })
+            })
+            .collect()
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -43,9 +86,26 @@ pub struct LongMemEvalTurn {
     pub index: usize,
     pub speaker: Option<String>,
     pub text: String,
+    pub observation_parts: Vec<LongMemEvalObservationPart>,
     pub has_answer: bool,
 }
 
+impl LongMemEvalTurn {
+    pub fn observation_texts(&self) -> impl Iterator<Item = &str> {
+        self.observation_parts
+            .is_empty()
+            .then_some(self.text.as_str())
+            .into_iter()
+            .chain(self.observation_parts.iter().map(|part| part.text.as_str()))
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LongMemEvalObservationPart {
+    pub text: String,
+    pub boundary_kind: cmem_eval::ObservationBoundary,
+}
+
 #[derive(Debug, Clone)]
 pub struct LongMemEvalMemoryInputs {
     pub episodes: Vec<cmem_eval::EpisodeInput>,