Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/pr_validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,6 @@ jobs:

- name: Run service parity tests
run: cargo test --verbose --test vector_port_contract_tests service_and_embedded_

- name: Verify service deletes preserve other typed objects
run: cargo test --verbose --lib adapters::qdrant::store::tests::port_contract::service_delete_removes_every_surface_of_only_the_requested_object -- --exact
5 changes: 4 additions & 1 deletion src/adapters/qdrant/payload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ pub(crate) const OBJECT_ID_FIELD: &str = QdrantPayloadField::ObjectId.name();
pub(crate) const OBJECT_TYPE_FIELD: &str = QdrantPayloadField::ObjectType.name();

pub(crate) fn qdrant_point_id(record: &VectorRecord) -> MemoryId {
MemoryId::new_v5(&record.object_id, record.surface.to_string().as_bytes())
MemoryId::new_v5(
&record.object_id,
format!("{}:{}", record.object_type, record.surface).as_bytes(),
)
}

pub(crate) fn read_candidate_match<'a>(
Expand Down
75 changes: 54 additions & 21 deletions src/adapters/qdrant/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use qdrant_client::qdrant::{
};
use qdrant_client::{config::QdrantConfig, Qdrant, QdrantError};

use crate::domain::MemoryId;
use crate::domain::MemoryObjectRef;
use crate::errors::{
CollectionCompatibilityError, CollectionMismatch, CustomError, IoErrorKind, TransportStatus,
VectorDatabaseError, VectorDatabaseErrorKind,
Expand Down Expand Up @@ -316,14 +316,19 @@ impl VectorCandidateStore for QdrantVectorCandidateStore {
})
}

async fn delete_candidates(&self, object_ids: &[MemoryId]) -> Result<(), CustomError> {
if object_ids.is_empty() {
async fn delete_candidates(&self, objects: &[MemoryObjectRef]) -> Result<(), CustomError> {
if objects.is_empty() {
return Ok(());
}

let conditions: Vec<_> = object_ids
let conditions: Vec<_> = objects
.iter()
.map(|id| Condition::matches(OBJECT_ID_FIELD, id.to_string()))
.map(|object| {
Condition::from(Filter::must([
Condition::matches(OBJECT_ID_FIELD, object.id.to_string()),
Condition::matches(OBJECT_TYPE_FIELD, object.object_type.to_string()),
]))
})
.collect();
let selector = PointsSelectorOneOf::Filter(Filter::should(conditions));
let request = DeletePointsBuilder::new(&self.collection_name)
Expand Down Expand Up @@ -574,7 +579,7 @@ mod tests {
use super::*;
use crate::adapters::qdrant::payload::QdrantPayloadField;
use crate::api::types::retrieval::VectorRecallCompleteness;
use crate::domain::{ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION};
use crate::domain::{MemoryId, ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION};
use crate::models::vector::{VectorRecord, VectorRecordEmbedding};
use qdrant_client::qdrant::{
point_id::PointIdOptions, value::Kind, vector, vectors, DeleteCollectionBuilder, PointId,
Expand Down Expand Up @@ -705,14 +710,16 @@ mod tests {
store: &dyn VectorCandidateStore,
) -> Result<(VectorCandidateRecall, VectorCandidateRecall), CustomError> {
let records = [
(MemoryId::from_u128(1), VectorSurface::Summary),
(MemoryId::from_u128(1), VectorSurface::Text),
(MemoryId::from_u128(2), VectorSurface::Summary),
(1, ObjectType::Episode, VectorSurface::Summary),
(1, ObjectType::Episode, VectorSurface::Text),
(2, ObjectType::Episode, VectorSurface::Summary),
(1, ObjectType::MemoryThread, VectorSurface::Summary),
(2, ObjectType::MemoryThread, VectorSurface::Summary),
]
.map(|(id, surface)| {
.map(|(id, object_type, surface)| {
VectorRecord::new(
id,
ObjectType::Episode,
MemoryId::from_u128(id),
object_type,
surface,
DEFAULT_SCHEMA_VERSION,
"surface",
Expand All @@ -723,9 +730,18 @@ mod tests {
.map(|record| VectorRecordEmbedding::new(record, &[1.0, 0.0]))
.collect::<Vec<_>>();
store.upsert_vector_records(&embeddings).await?;
let query = VectorCandidateSearch::new(vec![1.0, 0.0], 10, vec![ObjectType::Episode]);
let query = VectorCandidateSearch::new(
vec![1.0, 0.0],
10,
vec![ObjectType::Episode, ObjectType::MemoryThread],
);
let before = store.search_candidates(&query).await?;
store.delete_candidates(&[MemoryId::from_u128(1)]).await?;
store
.delete_candidates(&[
MemoryObjectRef::new(ObjectType::Episode, MemoryId::from_u128(1)),
MemoryObjectRef::new(ObjectType::MemoryThread, MemoryId::from_u128(2)),
])
.await?;
let after = store.search_candidates(&query).await?;
Ok((before, after))
}
Expand All @@ -737,20 +753,37 @@ mod tests {
recall
.candidates
.iter()
.map(|candidate| (candidate.object_id, candidate.surface))
.map(|candidate| {
(
candidate.object_id,
candidate.object_type,
candidate.surface,
)
})
.collect::<HashSet<_>>()
};
assert_eq!(
surfaces(&before),
HashSet::from([
(MemoryId::from_u128(1), VectorSurface::Summary),
(MemoryId::from_u128(1), VectorSurface::Text),
(MemoryId::from_u128(2), VectorSurface::Summary),
])
[
(1, ObjectType::Episode, VectorSurface::Summary),
(1, ObjectType::Episode, VectorSurface::Text),
(2, ObjectType::Episode, VectorSurface::Summary),
(1, ObjectType::MemoryThread, VectorSurface::Summary),
(2, ObjectType::MemoryThread, VectorSurface::Summary),
]
.map(|(id, object_type, surface)| (MemoryId::from_u128(id), object_type, surface))
.into_iter()
.collect()
);
assert_eq!(
surfaces(&after),
HashSet::from([(MemoryId::from_u128(2), VectorSurface::Summary)])
[
(2, ObjectType::Episode, VectorSurface::Summary),
(1, ObjectType::MemoryThread, VectorSurface::Summary),
]
.map(|(id, object_type, surface)| (MemoryId::from_u128(id), object_type, surface))
.into_iter()
.collect()
);
}
}
Expand Down
63 changes: 40 additions & 23 deletions src/adapters/qdrant_edge/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ use std::time::Duration;
use async_trait::async_trait;
use qdrant_edge::{
Condition, CountRequest, CreateIndex, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard,
EdgeVectorParams, FieldCondition, FieldIndexOperations, Filter, Match, MatchValue, NamedQuery,
PayloadFieldSchema, PayloadSchemaType, PointInsertOperations, PointOperations, PointStruct,
QueryEnum, ScoredPoint, ScrollRequest, SearchParams, SearchRequestBuilder, UpdateOperation,
ValueVariants, WithPayloadInterface, DEFAULT_VECTOR_NAME,
EdgeVectorParams, FieldCondition, FieldIndexOperations, Filter, NamedQuery, PayloadFieldSchema,
PayloadSchemaType, PointInsertOperations, PointOperations, PointStruct, QueryEnum, ScoredPoint,
ScrollRequest, SearchParams, SearchRequestBuilder, UpdateOperation, WithPayloadInterface,
DEFAULT_VECTOR_NAME,
};
use serde::{Deserialize, Serialize};
use tokio::sync::{oneshot, Mutex};
Expand All @@ -21,7 +21,7 @@ use crate::adapters::qdrant::payload::{
QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD,
};
use crate::adapters::qdrant::tie_closure::close_tie_cohort;
use crate::domain::{MemoryId, DEFAULT_SCHEMA_VERSION};
use crate::domain::{MemoryObjectRef, DEFAULT_SCHEMA_VERSION};
use crate::errors::{
CollectionCompatibilityError, CollectionMismatch, ConfigValidationError,
ConfigValidationReason, CustomError, IoErrorKind, VectorDatabaseError, VectorDatabaseErrorKind,
Expand Down Expand Up @@ -59,7 +59,7 @@ enum Command {
reply: Reply<()>,
},
Delete {
object_ids: Vec<String>,
filter: Filter,
reply: Reply<()>,
},
Search {
Expand Down Expand Up @@ -263,14 +263,29 @@ impl VectorCandidateStore for QdrantEdgeVectorCandidateStore {
})
}

async fn delete_candidates(&self, object_ids: &[MemoryId]) -> Result<(), CustomError> {
if object_ids.is_empty() {
async fn delete_candidates(&self, objects: &[MemoryObjectRef]) -> Result<(), CustomError> {
if objects.is_empty() {
return Ok(());
}
let _operation = self.operation.lock().await;
let (reply, receiver) = oneshot::channel();
let conditions = objects
.iter()
.map(|object| {
Condition::Filter(Filter {
must: Some(vec![
string_condition(OBJECT_ID_FIELD, object.id.to_string()),
string_condition(OBJECT_TYPE_FIELD, object.object_type.to_string()),
]),
..Filter::new()
})
})
.collect();
self.send(Command::Delete {
object_ids: object_ids.iter().map(ToString::to_string).collect(),
filter: Filter {
should: Some(conditions),
..Filter::new()
},
reply,
})?;
receive(receiver).await
Expand Down Expand Up @@ -319,8 +334,7 @@ fn owner_loop(shard: EdgeShard, commands: mpsc::Receiver<Command>) {
.and_then(|_| shard.flush().map_err(edge_error));
let _ = reply.send(result);
}
Command::Delete { object_ids, reply } => {
let filter = string_filter(OBJECT_ID_FIELD, object_ids);
Command::Delete { filter, reply } => {
let result = shard
.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(filter),
Expand Down Expand Up @@ -586,23 +600,23 @@ fn payload_to_match(
fn string_filter(field: &str, values: Vec<String>) -> Filter {
let conditions = values
.into_iter()
.map(|value| {
Condition::Field(FieldCondition::new_match(
field
.try_into()
.expect("adapter fields are valid JSON paths"),
Match::Value(MatchValue {
value: ValueVariants::String(value),
}),
))
})
.map(|value| string_condition(field, value))
.collect();
Filter {
should: Some(conditions),
..Filter::new()
}
}

fn string_condition(field: &str, value: String) -> Condition {
Condition::Field(FieldCondition::new_match(
field
.try_into()
.expect("adapter fields are valid JSON paths"),
value.into(),
))
}

fn object_type_tokens(query: &VectorCandidateSearch) -> Vec<String> {
query.object_types.iter().map(ToString::to_string).collect()
}
Expand Down Expand Up @@ -738,7 +752,7 @@ mod tests {

use super::*;
use crate::api::types::retrieval::VectorRecallCompleteness;
use crate::domain::{ObjectType, VectorSurface};
use crate::domain::{MemoryId, ObjectType, VectorSurface};
use crate::models::vector::VectorRecord;

const CHILD_MODE: &str = "CM_QDRANT_EDGE_CHILD_MODE";
Expand Down Expand Up @@ -889,7 +903,10 @@ mod tests {
assert_eq!(store.search_candidates(&query(3)).await.unwrap(), first);

store
.delete_candidates(&[records[0].object_id])
.delete_candidates(&[MemoryObjectRef::new(
records[0].object_type,
records[0].object_id,
)])
.await
.unwrap();
store.close().await.unwrap();
Expand Down
96 changes: 95 additions & 1 deletion src/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,100 @@ mod tests {
memory.close().await.unwrap();
}

#[tokio::test]
async fn episode_and_thread_with_the_same_id_keep_distinct_vectors_on_write_and_forget() {
let memory = injected_memory().await;
let shared_id = MemoryId::from_u128(1901);
let episode_text = "A heron crossed the lake at dawn.";
let thread_text = "Planning the next observatory visit.";
let mut episode = EpisodeDraft::new(episode_text);
episode.id = Some(shared_id);
let mut thread = MemoryThreadDraft::new("Observatory visit", thread_text);
thread.id = Some(shared_id);
let plan = memory
.prepare(
RememberInput::new(episode_text)
.with_episode(episode)
.with_memory_thread(thread),
PrepareOptions::default(),
)
.await
.unwrap();
let committed = memory.commit(plan, CommitOptions::default()).await.unwrap();
assert!(committed.vector_indexing_failure.is_none());
let mut recalled = Vec::new();
for (object_type, text) in [
(ObjectType::Episode, episode_text),
(ObjectType::MemoryThread, thread_text),
] {
let mut context = RetrievalContext::new(text).with_trace();
context.object_type_defaults = vec![object_type];
context.graph_limits.max_depth = 0;
let outcome = memory.retrieve(context).await.unwrap();
let candidates = outcome
.trace
.unwrap()
.vector_candidates
.into_iter()
.map(|candidate| candidate.object)
.collect::<Vec<_>>();
let summaries = if object_type == ObjectType::Episode {
outcome
.pack
.relevant_episodes
.into_iter()
.map(|episode| (episode.id, episode.summary))
.collect::<Vec<_>>()
} else {
outcome
.pack
.active_threads
.into_iter()
.map(|thread| (thread.id, thread.summary))
.collect::<Vec<_>>()
};
recalled.push((candidates, summaries));
}
assert_eq!(
recalled,
vec![
(
vec![MemoryObjectRef::new(ObjectType::Episode, shared_id)],
vec![(shared_id, episode_text.to_owned())]
),
(
vec![MemoryObjectRef::new(ObjectType::MemoryThread, shared_id)],
vec![(shared_id, thread_text.to_owned())]
),
]
);
let forgotten = memory
.forget(ForgetMemoryDraft::suppress(
LifecycleTargetRef::episode(shared_id),
"Forget the episode, preserving the separate thread.",
))
.await
.unwrap();
assert!(forgotten.vector_maintenance_failure.is_none());
let mut context = RetrievalContext::new(thread_text).with_trace();
context.object_type_defaults = vec![ObjectType::MemoryThread];
context.graph_limits.max_depth = 0;
let outcome = memory.retrieve(context).await.unwrap();
memory.close().await.unwrap();
assert_eq!(
outcome
.trace
.unwrap()
.vector_candidates
.into_iter()
.map(|candidate| candidate.object)
.collect::<Vec<_>>(),
vec![MemoryObjectRef::new(ObjectType::MemoryThread, shared_id)]
);
assert_eq!(outcome.pack.active_threads.len(), 1);
assert_eq!(outcome.pack.active_threads[0].summary, thread_text);
}

#[tokio::test]
async fn retrieve_rejects_an_empty_configured_object_type_scope_at_the_boundary() {
let memory = injected_memory().await;
Expand Down Expand Up @@ -1258,7 +1352,7 @@ mod tests {
})
}

async fn delete_candidates(&self, _object_ids: &[MemoryId]) -> Result<(), CustomError> {
async fn delete_candidates(&self, _objects: &[MemoryObjectRef]) -> Result<(), CustomError> {
Ok(())
}
}
Expand Down
Loading
Loading