From f48f3cd420ed256edc9eb5e91d01217fc814e0c7 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Tue, 25 Aug 2026 09:59:51 -0500 Subject: [PATCH 1/2] perf: bound session reads to selected sessions --- src/synapt/recall/cli.py | 74 ++++++++---- src/synapt/recall/core.py | 47 ++++++-- src/synapt/recall/resume.py | 86 ++++++++++++++ src/synapt/recall/server.py | 17 ++- src/synapt/recall/sharded_db.py | 83 +++++++++++++ src/synapt/recall/storage.py | 167 ++++++++++++++++++++++++++- tests/recall/test_cli_sessions.py | 48 ++++++++ tests/recall/test_core.py | 31 +++++ tests/recall/test_resume.py | 102 ++++++++++++++++ tests/recall/test_server_sessions.py | 23 ++++ tests/recall/test_sharded_db.py | 90 +++++++++++++++ tests/recall/test_storage.py | 14 +++ 12 files changed, 741 insertions(+), 41 deletions(-) create mode 100644 tests/recall/test_cli_sessions.py create mode 100644 tests/recall/test_server_sessions.py diff --git a/src/synapt/recall/cli.py b/src/synapt/recall/cli.py index d3594041..23bd2466 100644 --- a/src/synapt/recall/cli.py +++ b/src/synapt/recall/cli.py @@ -1255,18 +1255,31 @@ def cmd_stats(args: argparse.Namespace) -> None: def cmd_sessions(args: argparse.Namespace) -> None: """List recent sessions with date, turn count, and first message.""" + from synapt.recall.sharding import is_sharded + index_dir = _resolve_index_dir(args) - if not (index_dir / "recall.db").exists() and not (index_dir / "chunks.jsonl").exists(): + if ( + not (index_dir / "recall.db").exists() + and not (index_dir / "chunks.jsonl").exists() + and not is_sharded(index_dir) + ): print(f"Error: no index found at {index_dir}", file=sys.stderr) print("Run 'synapt build' or 'synapt setup' first.", file=sys.stderr) sys.exit(1) - index = TranscriptIndex.load(index_dir, use_embeddings=False) - sessions = index.list_sessions( - max_sessions=args.max_sessions, - after=args.after, - before=args.before, - ) + from synapt.recall.resume import load_resume_index + + index = load_resume_index(index_dir) + try: + sessions = index.list_sessions( + max_sessions=args.max_sessions, + after=args.after, + before=args.before, + ) + finally: + db = getattr(index, "_db", None) + if db is not None: + db.close() if not sessions: print("No sessions found.") @@ -1290,30 +1303,45 @@ def cmd_resume(args: argparse.Namespace) -> None: request was wrong). Collapsing them would send the reader down the wrong path. """ from synapt.recall.journal import _journal_path - from synapt.recall.resume import ResumeError, build_resume_view, format_resume + from synapt.recall.resume import ( + ResumeError, + build_resume_view, + format_resume, + load_resume_index, + ) + from synapt.recall.sharding import is_sharded index_dir = _resolve_index_dir(args) - if not (index_dir / "recall.db").exists() and not (index_dir / "chunks.jsonl").exists(): + if ( + not (index_dir / "recall.db").exists() + and not (index_dir / "chunks.jsonl").exists() + and not is_sharded(index_dir) + ): print(f"Error: no index found at {index_dir}", file=sys.stderr) print("Run 'synapt recall build' or 'synapt init' first.", file=sys.stderr) sys.exit(1) - index = TranscriptIndex.load(index_dir, use_embeddings=False) + index = load_resume_index(index_dir) try: - view = build_resume_view( - index, - session_id=getattr(args, "session", None), - limit=getattr(args, "turns", None) or 10, - journal_path=_journal_path(), - ) - except ResumeError as exc: - # An empty index is an honest empty state, not a failure to act on. - if not index._session_order: - print("No sessions indexed yet. Nothing to resume.") - return - print(f"Error: {exc}", file=sys.stderr) - sys.exit(1) + try: + view = build_resume_view( + index, + session_id=getattr(args, "session", None), + limit=getattr(args, "turns", None) or 10, + journal_path=_journal_path(), + ) + except ResumeError as exc: + # An empty index is an honest empty state, not a failure to act on. + if not index._session_order: + print("No sessions indexed yet. Nothing to resume.") + return + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + finally: + db = getattr(index, "_db", None) + if db is not None: + db.close() # Freshness is attached AFTER the view is built, so build_resume_view keeps # its no-implicit-I/O contract and the check can never change what is shown diff --git a/src/synapt/recall/core.py b/src/synapt/recall/core.py index 4f1523ca..2564c8f6 100644 --- a/src/synapt/recall/core.py +++ b/src/synapt/recall/core.py @@ -4185,7 +4185,7 @@ def list_sessions( before: str | None = None, ) -> list[dict]: """Return recent sessions with summary info, newest-first.""" - results = [] + candidates = [] for session_id in self._session_order: chunks = self.sessions[session_id] @@ -4201,15 +4201,43 @@ def list_sessions( if before and earliest_ts >= before: continue - # First user message as summary (skip journal chunks) transcript_chunks = [c for c in chunks if c.turn_index >= 0] - sorted_chunks = sorted( - transcript_chunks or chunks, - key=lambda c: c.turn_index, - ) + candidates.append(( + session_id, + chunks, + transcript_chunks, + earliest_ts, + )) + if len(candidates) >= max_sessions: + break + + hydrated_by_id: dict[str, TranscriptChunk] = {} + if self._lazy_chunks and self._db is not None: + rowids = [] + for _, chunks, _, _ in candidates: + for chunk in chunks: + idx = self._id_to_idx.get(chunk.id) + rowid = self._idx_to_rowid.get(idx) if idx is not None else None + if rowid is not None: + rowids.append(rowid) + hydrated_by_id = { + chunk.id: chunk + for chunk in self._db.load_chunks_by_rowids(rowids).values() + } + else: + hydrated_by_id = { + chunk.id: chunk + for _, chunks, _, _ in candidates + for chunk in chunks + } + + results = [] + for session_id, chunks, transcript_chunks, earliest_ts in candidates: + # First user message as summary (skip journal chunks) + sorted_chunks = sorted(transcript_chunks or chunks, key=lambda c: c.turn_index) first_msg = "" for c in sorted_chunks: - full = self._get_chunk(self._id_to_idx[c.id]) + full = hydrated_by_id.get(c.id, c) if full.user_text: first_msg = full.user_text[:120] if len(full.user_text) > 120: @@ -4218,7 +4246,7 @@ def list_sessions( all_files = set() for c in chunks: - full = self._get_chunk(self._id_to_idx[c.id]) + full = hydrated_by_id.get(c.id, c) all_files.update(full.files_touched) results.append({ @@ -4229,9 +4257,6 @@ def list_sessions( "files_count": len(all_files), }) - if len(results) >= max_sessions: - break - return results def session_tail(self, session_id: str) -> list[TranscriptChunk]: diff --git a/src/synapt/recall/resume.py b/src/synapt/recall/resume.py index 68aea7e1..489b1036 100644 --- a/src/synapt/recall/resume.py +++ b/src/synapt/recall/resume.py @@ -29,6 +29,7 @@ from synapt.recall.freshness import IndexFreshness from synapt.recall.core import TranscriptChunk, TranscriptIndex from synapt.recall.journal import JournalEntry, read_entries +from synapt.recall.sharded_db import ShardedRecallDB DEFAULT_TURNS = 10 @@ -106,6 +107,91 @@ class ResumeView: freshness: "IndexFreshness | None" = None +class BoundedResumeIndex: + """The small TranscriptIndex surface resume needs over a read-only store.""" + + def __init__(self, db: ShardedRecallDB): + self._db = db + self._overview = db.session_overview() + self.sessions = {session_id: [] for session_id in self._overview} + self._session_order = sorted( + self._overview, + key=lambda session_id: self._overview[session_id]["activity"], + reverse=True, + ) + + def session_tail(self, session_id: str) -> list[TranscriptChunk]: + return self._db.load_session_chunks(session_id) + + def list_sessions( + self, + max_sessions: int = 20, + after: str | None = None, + before: str | None = None, + ) -> list[dict]: + """Return recent session summaries while hydrating only candidates.""" + session_ids = [] + for session_id in self._session_order: + overview = self._overview[session_id] + earliest_ts = overview["earliest_ts"] + latest_ts = overview["latest_ts"] + if not earliest_ts or not latest_ts: + continue + if after and latest_ts < after: + continue + if before and earliest_ts >= before: + continue + session_ids.append(session_id) + if len(session_ids) >= max_sessions: + break + + hydrated = self._db.load_session_listing(session_ids) + results = [] + for session_id in session_ids: + overview = self._overview[session_id] + chunks = hydrated.get(session_id, []) + transcript_chunks = [ + chunk for chunk in chunks if chunk["turn_index"] >= 0 + ] + first_message = "" + for chunk in sorted( + transcript_chunks or chunks, + key=lambda item: item["turn_index"], + ): + if chunk["user_text"]: + first_message = chunk["user_text"][:120] + if len(chunk["user_text"]) > 120: + first_message += "..." + break + files = { + file_path + for chunk in chunks + for file_path in chunk["files_touched"] + } + results.append( + { + "session_id": session_id, + "date": overview["earliest_ts"][:10], + "turn_count": overview["turn_count"], + "first_message": first_message, + "files_count": len(files), + } + ) + return results + + def close(self) -> None: + self._db.close() + + +def load_resume_index(directory: Path) -> TranscriptIndex | BoundedResumeIndex: + """Load only session routing metadata until one session is selected.""" + from synapt.recall.sharding import is_sharded + + if (directory / "recall.db").exists() or is_sharded(directory): + return BoundedResumeIndex(ShardedRecallDB.open_readonly(directory)) + return TranscriptIndex.load(directory, use_embeddings=False) + + # --------------------------------------------------------------------------- # Discrimination # --------------------------------------------------------------------------- diff --git a/src/synapt/recall/server.py b/src/synapt/recall/server.py index 0f254f95..f50ccbac 100644 --- a/src/synapt/recall/server.py +++ b/src/synapt/recall/server.py @@ -527,11 +527,18 @@ def recall_sessions( after: Only sessions with activity after this date (ISO 8601). before: Only sessions with activity before this date (ISO 8601). """ - index = _get_index() - if index is None: - index_dir = project_index_dir() + from synapt.recall.resume import load_resume_index + from synapt.recall.sharding import is_sharded + + index_dir = project_index_dir() + if ( + not (index_dir / "recall.db").exists() + and not (index_dir / "chunks.jsonl").exists() + and not is_sharded(index_dir) + ): return f"No index found at {index_dir}. Run `synapt recall setup` first." + index = load_resume_index(index_dir) try: sessions = index.list_sessions( max_sessions=max_sessions, @@ -540,6 +547,10 @@ def recall_sessions( ) except Exception as exc: return f"Session listing failed: {exc}" + finally: + db = getattr(index, "_db", None) + if db is not None: + db.close() if not sessions: return "No sessions found." diff --git a/src/synapt/recall/sharded_db.py b/src/synapt/recall/sharded_db.py index 45fa3622..9fc5fe4a 100644 --- a/src/synapt/recall/sharded_db.py +++ b/src/synapt/recall/sharded_db.py @@ -89,6 +89,20 @@ def open(cls, index_dir: Path) -> ShardedRecallDB: db = RecallDB(index_dir / "recall.db") return cls(db, []) + @classmethod + def open_readonly(cls, index_dir: Path) -> ShardedRecallDB: + """Open an existing layout without DDL, migrations, or write access.""" + if is_sharded(index_dir): + index_db = RecallDB.open_readonly(index_dir / "index.db") + try: + data_dbs = [RecallDB.open_readonly(p) for p in list_shards(index_dir)] + except Exception: + index_db.close() + raise + return cls(index_db, data_dbs) + + return cls(RecallDB.open_readonly(index_dir / "recall.db"), []) + # -- Delegated methods (index DB) -------------------------------------- def load_manifest(self) -> dict: @@ -180,6 +194,75 @@ def load_chunks_by_rowids(self, rowids: list[int]): # noqa: ANN201 }) return loaded return self._index.load_chunks_by_rowids(rowids) + + def session_overview(self) -> dict[str, dict]: + """Return merged session metadata across all chunk shards.""" + result: dict[str, dict] = {} + for _, db in self._iter_data_shards(): + for session_id, overview in db.session_overview().items(): + current = result.get(session_id) + if current is None: + result[session_id] = dict(overview) + continue + if overview["has_real_activity"] and not current["has_real_activity"]: + current["activity"] = overview["activity"] + elif overview["has_real_activity"] == current["has_real_activity"]: + current["activity"] = max( + current["activity"], overview["activity"] + ) + current["has_real_activity"] = ( + current["has_real_activity"] or overview["has_real_activity"] + ) + if overview["earliest_ts"]: + current["earliest_ts"] = min( + filter(None, (current["earliest_ts"], overview["earliest_ts"])) + ) + if overview["latest_ts"]: + current["latest_ts"] = max( + current["latest_ts"], overview["latest_ts"] + ) + current["turn_count"] += overview["turn_count"] + return result + + def session_activity(self) -> dict[str, tuple[int, str]]: + """Return newest activity per session across all chunk shards.""" + return { + session_id: overview["activity"] + for session_id, overview in self.session_overview().items() + } + + def load_session_chunks(self, session_id: str): # noqa: ANN201 + """Load one session across all shards, oldest turn first.""" + return self.load_session_chunks_many( + [session_id], include_journal=False + ).get(session_id, []) + + def load_session_chunks_many( + self, + session_ids: list[str], + include_journal: bool = True, + ): # noqa: ANN201 + """Load a bounded set of sessions across all shards.""" + grouped = {session_id: [] for session_id in session_ids} + for _, db in self._iter_data_shards(): + partial = db.load_session_chunks_many(session_ids, include_journal) + for session_id, chunks in partial.items(): + grouped.setdefault(session_id, []).extend(chunks) + for chunks in grouped.values(): + chunks.sort(key=lambda chunk: chunk.turn_index) + return grouped + + def load_session_listing(self, session_ids: list[str]) -> dict[str, list[dict]]: + """Load summary fields for a bounded set of sessions across shards.""" + grouped: dict[str, list[dict]] = {session_id: [] for session_id in session_ids} + for _, db in self._iter_data_shards(): + partial = db.load_session_listing(session_ids) + for session_id, rows in partial.items(): + grouped.setdefault(session_id, []).extend(rows) + for rows in grouped.values(): + rows.sort(key=lambda row: row["turn_index"]) + return grouped + def sample_chunk_texts(self, limit: int = 100) -> list[str]: """Return representative chunk text samples across all shards.""" if self._data_dbs: diff --git a/src/synapt/recall/storage.py b/src/synapt/recall/storage.py index 4bad8b9f..b4fa2c9d 100644 --- a/src/synapt/recall/storage.py +++ b/src/synapt/recall/storage.py @@ -1047,15 +1047,174 @@ def load_chunk_by_rowid(self, rowid: int) -> TranscriptChunk | None: ) def load_chunks_by_rowids(self, rowids: list[int]) -> dict[int, TranscriptChunk]: - """Load multiple chunks by rowid.""" + """Load multiple chunks by rowid with bounded SQL query count.""" if not rowids: return {} + + from synapt.recall.core import TranscriptChunk + + loaded: dict[int, TranscriptChunk] = {} + # Stay below SQLite's commonly configured 999-variable limit. + for offset in range(0, len(rowids), 900): + batch = rowids[offset:offset + 900] + placeholders = ",".join("?" for _ in batch) + rows = self._conn.execute( + "SELECT rowid, id, session_id, timestamp, turn_index, " + "user_text, assistant_text, tools_used, files_touched, " + "tool_content, date_text, transcript_path, byte_offset, byte_length, " + "agent_id FROM chunks WHERE rowid IN (" + placeholders + ")", + batch, + ).fetchall() + for r in rows: + loaded[r["rowid"]] = TranscriptChunk( + id=r["id"], + session_id=r["session_id"], + timestamp=r["timestamp"], + turn_index=r["turn_index"], + user_text=r["user_text"], + assistant_text=r["assistant_text"], + tools_used=json.loads(r["tools_used"]) if r["tools_used"] else [], + files_touched=( + json.loads(r["files_touched"]) if r["files_touched"] else [] + ), + tool_content=r["tool_content"] or "", + date_text=r["date_text"] or "", + transcript_path=r["transcript_path"] or "", + byte_offset=r["byte_offset"] if r["byte_offset"] is not None else -1, + byte_length=r["byte_length"] if r["byte_length"] is not None else 0, + agent_id=r["agent_id"], + ) + return loaded + + def session_overview(self) -> dict[str, dict]: + """Return routing and listing metadata without materializing chunks.""" + rows = self._conn.execute( + "SELECT session_id, " + "MIN(NULLIF(timestamp, '')) AS earliest_ts, " + "MAX(NULLIF(timestamp, '')) AS latest_ts, " + "SUM(CASE WHEN turn_index >= 0 THEN 1 ELSE 0 END) AS turn_count, " + "SUM(CASE WHEN turn_index != -1 AND timestamp IS NOT NULL " + " AND timestamp != '' THEN 1 ELSE 0 END) AS activity_count, " + "MAX(CASE WHEN turn_index != -1 THEN julianday(timestamp) END) AS activity_jd, " + "MAX(CASE WHEN turn_index != -1 AND julianday(timestamp) IS NULL " + " THEN timestamp END) AS activity_raw, " + "MAX(julianday(timestamp)) AS fallback_jd, " + "MAX(CASE WHEN julianday(timestamp) IS NULL THEN timestamp END) AS fallback_raw " + "FROM chunks GROUP BY session_id" + ).fetchall() + + result: dict[str, dict] = {} + for row in rows: + parsed = row["activity_jd"] + raw = row["activity_raw"] + if parsed is None and raw is None: + parsed = row["fallback_jd"] + raw = row["fallback_raw"] + if parsed is not None: + unix_seconds = (float(parsed) - 2440587.5) * 86400 + activity = (1, f"{unix_seconds:020.6f}") + else: + activity = (0, raw or "") + result[row["session_id"]] = { + "activity": activity, + "earliest_ts": row["earliest_ts"] or "", + "latest_ts": row["latest_ts"] or "", + "turn_count": int(row["turn_count"] or 0), + "has_real_activity": bool(row["activity_count"]), + } + return result + + def session_activity(self) -> dict[str, tuple[int, str]]: + """Return each session's newest activity without materializing chunks.""" return { - rowid: chunk - for rowid in rowids - if (chunk := self.load_chunk_by_rowid(rowid)) is not None + session_id: overview["activity"] + for session_id, overview in self.session_overview().items() } + def load_session_chunks(self, session_id: str) -> list[TranscriptChunk]: + """Load conversation chunks for one session, oldest turn first.""" + return self.load_session_chunks_many([session_id], include_journal=False).get( + session_id, [] + ) + + def load_session_chunks_many( + self, + session_ids: list[str], + include_journal: bool = True, + ) -> dict[str, list[TranscriptChunk]]: + """Load full chunks for a bounded set of sessions.""" + from synapt.recall.core import TranscriptChunk + + grouped: dict[str, list[TranscriptChunk]] = {sid: [] for sid in session_ids} + if not session_ids: + return grouped + for offset in range(0, len(session_ids), 900): + batch = session_ids[offset:offset + 900] + placeholders = ",".join("?" for _ in batch) + journal_clause = "" if include_journal else " AND turn_index >= 0" + rows = self._conn.execute( + "SELECT id, session_id, timestamp, turn_index, " + "user_text, assistant_text, tools_used, files_touched, " + "tool_content, date_text, transcript_path, byte_offset, byte_length, " + "agent_id FROM chunks WHERE session_id IN (" + placeholders + ")" + + journal_clause + " ORDER BY session_id, turn_index", + batch, + ).fetchall() + for r in rows: + grouped.setdefault(r["session_id"], []).append( + TranscriptChunk( + id=r["id"], + session_id=r["session_id"], + timestamp=r["timestamp"], + turn_index=r["turn_index"], + user_text=r["user_text"], + assistant_text=r["assistant_text"], + tools_used=( + json.loads(r["tools_used"]) if r["tools_used"] else [] + ), + files_touched=( + json.loads(r["files_touched"]) + if r["files_touched"] else [] + ), + tool_content=r["tool_content"] or "", + date_text=r["date_text"] or "", + transcript_path=r["transcript_path"] or "", + byte_offset=( + r["byte_offset"] if r["byte_offset"] is not None else -1 + ), + byte_length=( + r["byte_length"] if r["byte_length"] is not None else 0 + ), + agent_id=r["agent_id"], + ) + ) + return grouped + + def load_session_listing(self, session_ids: list[str]) -> dict[str, list[dict]]: + """Load only the fields needed to render session summaries.""" + grouped: dict[str, list[dict]] = {sid: [] for sid in session_ids} + for offset in range(0, len(session_ids), 900): + batch = session_ids[offset:offset + 900] + if not batch: + continue + placeholders = ",".join("?" for _ in batch) + rows = self._conn.execute( + "SELECT session_id, turn_index, user_text, files_touched " + "FROM chunks WHERE session_id IN (" + placeholders + ") " + "ORDER BY session_id, turn_index", + batch, + ).fetchall() + for row in rows: + grouped.setdefault(row["session_id"], []).append({ + "turn_index": row["turn_index"], + "user_text": row["user_text"] or "", + "files_touched": ( + json.loads(row["files_touched"]) + if row["files_touched"] else [] + ), + }) + return grouped + def chunk_count(self) -> int: """Number of chunks in the database.""" row = self._conn.execute("SELECT COUNT(*) FROM chunks").fetchone() diff --git a/tests/recall/test_cli_sessions.py b/tests/recall/test_cli_sessions.py new file mode 100644 index 00000000..c36ac224 --- /dev/null +++ b/tests/recall/test_cli_sessions.py @@ -0,0 +1,48 @@ +from argparse import Namespace +from unittest import mock + +from synapt.recall.cli import cmd_sessions +from synapt.recall.core import TranscriptChunk +from synapt.recall.sharded_db import ShardedRecallDB +from synapt.recall.storage import RecallDB + + +def test_sessions_accepts_a_sharded_only_index(tmp_path, capsys): + RecallDB(tmp_path / "index.db").close() + shard = RecallDB(tmp_path / "data_001.db") + shard.save_chunks( + [ + TranscriptChunk( + id="session-a:t0", + session_id="session-a", + timestamp="2026-08-25T10:00:00Z", + turn_index=0, + user_text="bounded session browsing", + assistant_text="working", + ) + ] + ) + shard.close() + + with ( + mock.patch( + "synapt.recall.core.TranscriptIndex.load", + side_effect=AssertionError("session browsing constructed the full index"), + ), + mock.patch.object( + ShardedRecallDB, + "load_session_chunks_many", + side_effect=AssertionError("session listing loaded full chunk bodies"), + ), + ): + cmd_sessions( + Namespace( + index=str(tmp_path), + out=None, + max_sessions=20, + after=None, + before=None, + ) + ) + + assert "session-" in capsys.readouterr().out diff --git a/tests/recall/test_core.py b/tests/recall/test_core.py index 857ee928..0aff512e 100644 --- a/tests/recall/test_core.py +++ b/tests/recall/test_core.py @@ -1086,6 +1086,37 @@ def test_list_sessions_empty_index(): assert index.list_sessions() == [] +def test_list_sessions_batch_hydrates_candidate_sessions(tmp_path, monkeypatch): + """Lazy session listing must not issue one hydration query per header.""" + from unittest.mock import Mock + + from synapt.recall.storage import RecallDB + + directory = tmp_path / "index" + db = RecallDB(directory / "recall.db") + try: + db.save_chunks(make_test_chunks()) + finally: + db.close() + + index = TranscriptIndex.load(directory, use_embeddings=False) + batch = Mock(wraps=index._db.load_chunks_by_rowids) + monkeypatch.setattr(index._db, "load_chunks_by_rowids", batch) + monkeypatch.setattr( + index, + "_get_chunk", + lambda _idx: (_ for _ in ()).throw( + AssertionError("session listing hydrated one chunk at a time") + ), + ) + + sessions = index.list_sessions() + + assert len(sessions) == 2 + batch.assert_called_once() + index._db.close() + + # --------------------------------------------------------------------------- # Tests: build_index incremental change detection # --------------------------------------------------------------------------- diff --git a/tests/recall/test_resume.py b/tests/recall/test_resume.py index 54ac3d92..f7161771 100644 --- a/tests/recall/test_resume.py +++ b/tests/recall/test_resume.py @@ -39,6 +39,7 @@ build_resume_view, format_resume, is_harness_authored, + load_resume_index, resolve_session, ) @@ -515,6 +516,107 @@ def test_hydrated_turns_survive_the_noise_filter(self): self.assertEqual(len(view.turns), 2) +class TestBoundedResumeLoad(unittest.TestCase): + """The CLI read should scale with sessions plus the selected tail, not all chunks.""" + + def test_sqlite_loader_does_not_construct_the_full_transcript_index(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + _save_sqlite_index([ + _chunk(SESSION_A, 0, "older question", "older answer", + timestamp="2026-08-01T10:00:00Z"), + _chunk(SESSION_B, 0, "newer question", "newer answer", + timestamp="2026-08-05T10:00:00Z"), + ], directory) + + with mock.patch.object( + TranscriptIndex, + "load", + side_effect=AssertionError("full index load should not run"), + ): + index = load_resume_index(directory) + try: + view = build_resume_view(index, limit=10, journal_path=None) + finally: + index.close() + + self.assertEqual(view.session_id, SESSION_B) + self.assertEqual(view.turns[0].user_text, "newer question") + + def test_session_order_ignores_newer_journal_timestamp(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + _save_sqlite_index([ + _chunk(SESSION_A, 0, "live", "work", + timestamp="2026-08-05T10:00:00Z"), + _chunk(SESSION_B, 0, "old", "work", + timestamp="2026-08-01T10:00:00Z"), + _chunk(SESSION_B, -1, "journal", "", + timestamp="2026-08-25T10:00:00Z"), + ], directory) + + index = load_resume_index(directory) + try: + self.assertEqual(resolve_session(index, None), SESSION_A) + finally: + index.close() + + def test_only_selected_session_is_hydrated(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + _save_sqlite_index([ + _chunk(SESSION_A, 0, "secret older question", "older answer", + timestamp="2026-08-01T10:00:00Z"), + _chunk(SESSION_B, 0, "selected question", "selected answer", + timestamp="2026-08-05T10:00:00Z"), + ], directory) + + index = load_resume_index(directory) + try: + with mock.patch.object( + index._db, + "load_session_chunks", + wraps=index._db.load_session_chunks, + ) as load: + view = build_resume_view(index, limit=10, journal_path=None) + load.assert_called_once_with(SESSION_B) + finally: + index.close() + + rendered = format_resume(view) + self.assertIn("selected question", rendered) + self.assertNotIn("secret older question", rendered) + + def test_session_without_a_timestamp_does_not_disappear(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + _save_sqlite_index([ + _chunk(SESSION_A, 0, "question", "answer", timestamp=""), + ], directory) + + index = load_resume_index(directory) + try: + self.assertEqual(resolve_session(index, None), SESSION_A) + finally: + index.close() + + def test_timestamp_spelling_does_not_decide_bounded_order(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + _save_sqlite_index([ + _chunk(SESSION_A, 0, "later", "answer", + timestamp="2026-08-06T03:00:00.500000+00:00"), + _chunk(SESSION_B, 0, "earlier", "answer", + timestamp="2026-08-06T03:00:00Z"), + ], directory) + + index = load_resume_index(directory) + try: + self.assertEqual(resolve_session(index, None), SESSION_A) + finally: + index.close() + + # --------------------------------------------------------------------------- # Continuation segments # --------------------------------------------------------------------------- diff --git a/tests/recall/test_server_sessions.py b/tests/recall/test_server_sessions.py new file mode 100644 index 00000000..152e50bd --- /dev/null +++ b/tests/recall/test_server_sessions.py @@ -0,0 +1,23 @@ +from unittest.mock import Mock, patch + + +def test_recall_sessions_uses_the_bounded_no_embedding_surface(monkeypatch, tmp_path): + from synapt.recall import server + + index = Mock() + index.list_sessions.return_value = [] + index._db = Mock() + (tmp_path / "recall.db").touch() + monkeypatch.setattr(server, "project_index_dir", lambda: tmp_path) + with ( + patch("synapt.recall.resume.load_resume_index", return_value=index) as load, + patch.object( + server, + "_get_index", + side_effect=AssertionError("session browsing constructed the full index"), + ), + ): + assert server.recall_sessions() == "No sessions found." + + load.assert_called_once_with(tmp_path) + index._db.close.assert_called_once_with() diff --git a/tests/recall/test_sharded_db.py b/tests/recall/test_sharded_db.py index f7aeac0d..20acc0eb 100644 --- a/tests/recall/test_sharded_db.py +++ b/tests/recall/test_sharded_db.py @@ -3,6 +3,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock from synapt.recall.core import TranscriptChunk, TranscriptIndex from synapt.recall.sharded_db import ShardedRecallDB @@ -29,6 +30,28 @@ def test_open_existing_recall_db(self): self.assertTrue(db.is_monolithic) db.close() + def test_open_readonly_uses_query_only_connection(self): + writer = RecallDB(self.index_dir / "recall.db") + writer.close() + + db = ShardedRecallDB.open_readonly(self.index_dir) + self.assertTrue(db.is_monolithic) + with self.assertRaises(Exception): + db._index._conn.execute("DELETE FROM chunks") + db.close() + + def test_open_readonly_skips_schema_work(self): + RecallDB(self.index_dir / "recall.db").close() + + with mock.patch.object( + RecallDB, + "_ensure_schema", + side_effect=AssertionError("read-only open ran schema work"), + ): + db = ShardedRecallDB.open_readonly(self.index_dir) + self.assertEqual(db.chunk_count(), 0) + db.close() + def test_knowledge_roundtrip(self): db = ShardedRecallDB.open(self.index_dir) node = { @@ -121,6 +144,73 @@ def test_open_detects_sharded_layout(self): self.assertEqual(db.shard_count, 1) db.close() + def test_open_readonly_detects_sharded_layout(self): + self._create_two_shard_layout().close() + + db = ShardedRecallDB.open_readonly(self.index_dir) + self.assertFalse(db.is_monolithic) + self.assertEqual(db.shard_count, 2) + self.assertEqual(db.chunk_count(), 2) + db.close() + + def test_bounded_session_reads_merge_across_shards(self): + RecallDB(self.index_dir / "index.db").close() + first = RecallDB(self.index_dir / "data_001.db") + second = RecallDB(self.index_dir / "data_002.db") + first.save_chunks([ + self._make_chunk("shared:t0", "shared", "2026-01-01T00:00:00Z", "first"), + ]) + second.save_chunks([ + TranscriptChunk( + id="shared:t1", + session_id="shared", + timestamp="2026-01-02T00:00:00Z", + turn_index=1, + user_text="second", + assistant_text="assistant", + ), + ]) + first.close() + second.close() + + db = ShardedRecallDB.open_readonly(self.index_dir) + try: + self.assertIn("shared", db.session_activity()) + self.assertEqual( + [chunk.id for chunk in db.load_session_chunks("shared")], + ["shared:t0", "shared:t1"], + ) + finally: + db.close() + + def test_journal_in_a_later_shard_does_not_replace_real_activity(self): + RecallDB(self.index_dir / "index.db").close() + first = RecallDB(self.index_dir / "data_001.db") + second = RecallDB(self.index_dir / "data_002.db") + first.save_chunks([ + self._make_chunk("dead:t0", "dead", "2026-01-01T00:00:00Z", "old"), + self._make_chunk("live:t0", "live", "2026-01-02T00:00:00Z", "new"), + ]) + second.save_chunks([ + TranscriptChunk( + id="dead:journal", + session_id="dead", + timestamp="2026-08-25T00:00:00Z", + turn_index=-1, + user_text="journal", + assistant_text="", + ), + ]) + first.close() + second.close() + + db = ShardedRecallDB.open_readonly(self.index_dir) + try: + activity = db.session_activity() + self.assertGreater(activity["live"], activity["dead"]) + finally: + db.close() + def test_multiple_shards(self): RecallDB(self.index_dir / "index.db").close() RecallDB(self.index_dir / "data_001.db").close() diff --git a/tests/recall/test_storage.py b/tests/recall/test_storage.py index fef76191..3bb97116 100644 --- a/tests/recall/test_storage.py +++ b/tests/recall/test_storage.py @@ -184,6 +184,20 @@ def test_load_chunk_by_rowid_and_headers(self, db, sample_chunks): assert set(batch) == {1, 2} assert batch[2].id == sample_chunks[1].id + def test_batch_load_does_not_fall_back_to_one_query_per_row( + self, db, sample_chunks, monkeypatch + ): + db.save_chunks(sample_chunks) + + def individual_load_is_a_failure(_rowid): + raise AssertionError("batch loading issued an individual row query") + + monkeypatch.setattr(db, "load_chunk_by_rowid", individual_load_is_a_failure) + batch = db.load_chunks_by_rowids([1, 2, 999]) + + assert set(batch) == {1, 2} + assert batch[1].id == sample_chunks[0].id + def test_save_replaces_existing(self, db, sample_chunks): db.save_chunks(sample_chunks) assert db.chunk_count() == 3 From 8b5528c819a8a050485fd80cd282723a4686e3fe Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Tue, 25 Aug 2026 11:06:23 -0500 Subject: [PATCH 2/2] feat: package automatic session continuity hooks --- .claude-plugin/marketplace.json | 15 +++++ README.md | 21 +++++++ claude-plugin/.claude-plugin/plugin.json | 4 +- claude-plugin/hooks/hooks.json | 17 ++++++ claude-plugin/project-settings.json | 15 +++++ claude-plugin/skills/recall/SKILL.md | 6 +- codex-plugin/.codex-plugin/plugin.json | 19 ++++++- codex-plugin/AGENTS.md | 1 + codex-plugin/hooks/hooks.json | 18 ++++++ tests/test_agent_plugin_packages.py | 71 ++++++++++++++++++++++++ 10 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 .claude-plugin/marketplace.json create mode 100644 claude-plugin/hooks/hooks.json create mode 100644 claude-plugin/project-settings.json create mode 100644 codex-plugin/hooks/hooks.json create mode 100644 tests/test_agent_plugin_packages.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 00000000..2d095193 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,15 @@ +{ + "name": "synapt-plugins", + "owner": { + "name": "synapt" + }, + "description": "Local-first memory and continuity plugins from synapt.", + "plugins": [ + { + "name": "synapt-recall", + "source": "./claude-plugin", + "description": "Persistent recall and automatic bounded session context for Claude Code", + "category": "Productivity" + } + ] +} diff --git a/README.md b/README.md index 7a798bbe..6ead3fa9 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,22 @@ synapt init `synapt init` installs session hooks for automatic transcript archiving. +The repository's `claude-plugin/` package also owns a bounded `SessionStart` +hook. Install it from this repository's marketplace: + +```text +/plugin marketplace add synapt-dev/recall +/plugin install synapt-recall@synapt-plugins +``` + +Current recall context is then injected automatically without resuming an old +conversation. Do not also install the legacy global `SessionStart` hook because +Claude Code runs both registrations. + +For shared workspaces, link `claude-plugin/project-settings.json` to +`.claude/settings.json`. After the folder is trusted, Claude Code prompts once +for the marketplace and plugin consent, then keeps the plugin updated. + ### Codex CLI Install synapt and register the MCP server: @@ -404,6 +420,11 @@ synapt init `synapt init` installs the `dev-loop` skill automatically, giving Codex recall search, channel coordination, and journal access. +The repository's `codex-plugin/` package owns the equivalent bounded +`SessionStart` hook. A gripspace can link that package's `hooks/hooks.json` to +workspace `.codex/hooks.json`, keeping the capability in recall while making +continuity automatic for every Codex session in the workspace. + ## What `synapt init` does Run from a project root: diff --git a/claude-plugin/.claude-plugin/plugin.json b/claude-plugin/.claude-plugin/plugin.json index 7eb12082..0df3a011 100644 --- a/claude-plugin/.claude-plugin/plugin.json +++ b/claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "synapt-recall", - "version": "0.1.0", - "description": "Persistent memory for Claude Code sessions via synapt recall", + "version": "0.2.0", + "description": "Persistent memory and automatic session context for Claude Code via synapt recall", "author": { "name": "synapt", "url": "https://github.com/synapt-dev" diff --git a/claude-plugin/hooks/hooks.json b/claude-plugin/hooks/hooks.json new file mode 100644 index 00000000..910c027a --- /dev/null +++ b/claude-plugin/hooks/hooks.json @@ -0,0 +1,17 @@ +{ + "description": "Load bounded synapt recall context when a Claude Code session starts.", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "synapt recall hook session-start", + "statusMessage": "Loading current synapt context", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/claude-plugin/project-settings.json b/claude-plugin/project-settings.json new file mode 100644 index 00000000..ba3dd7d7 --- /dev/null +++ b/claude-plugin/project-settings.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "enabledPlugins": { + "synapt-recall@synapt-plugins": true + }, + "extraKnownMarketplaces": { + "synapt-plugins": { + "source": { + "source": "github", + "repo": "synapt-dev/recall" + }, + "autoUpdate": true + } + } +} diff --git a/claude-plugin/skills/recall/SKILL.md b/claude-plugin/skills/recall/SKILL.md index fa30de51..c5098631 100644 --- a/claude-plugin/skills/recall/SKILL.md +++ b/claude-plugin/skills/recall/SKILL.md @@ -5,14 +5,16 @@ description: Persistent memory across Claude Code sessions. Search before you ac # synapt recall -Persistent memory for Claude Code. Search past sessions, save durable knowledge, and maintain context across conversations. +Persistent memory for Claude Code. The plugin loads bounded current context at +session start. Search past sessions for deeper context, save durable knowledge, +and maintain context across conversations. ## When to use (do this automatically, without being asked) - **Before making a design decision**: `recall_search` for prior discussion - **When debugging an error**: `recall_search` for past fixes - **When user references past work**: `recall_search` immediately -- **Starting a session**: `recall_journal` to read recent entries +- **Starting a session**: use the injected current context first, then `recall_journal` when more detail is needed - **When unsure if something was discussed**: `recall_quick` (fast, cheap) - **When you need file history**: `recall_files` for who changed what and why diff --git a/codex-plugin/.codex-plugin/plugin.json b/codex-plugin/.codex-plugin/plugin.json index 02e135b7..1a8a5394 100644 --- a/codex-plugin/.codex-plugin/plugin.json +++ b/codex-plugin/.codex-plugin/plugin.json @@ -1,7 +1,15 @@ { "name": "synapt-recall", - "version": "0.1.0", - "description": "Codex plugin scaffold for synapt recall MCP access.", + "version": "0.2.0", + "description": "Persistent memory and automatic session context for Codex via synapt recall.", + "author": { + "name": "synapt", + "url": "https://github.com/synapt-dev" + }, + "homepage": "https://synapt.dev", + "repository": "https://github.com/synapt-dev/recall", + "license": "MIT", + "keywords": ["memory", "recall", "persistent", "context", "sessions"], "interface": { "displayName": "synapt Recall", "shortDescription": "Local-first memory and recall for Codex.", @@ -11,7 +19,12 @@ "capabilities": [ "MCP", "Recall", - "Local Memory" + "Local Memory", + "Automatic Session Context" + ], + "defaultPrompt": [ + "Find the most relevant context from our past work.", + "Save this decision so a future session inherits it." ] } } diff --git a/codex-plugin/AGENTS.md b/codex-plugin/AGENTS.md index 63dd24f4..7258863b 100644 --- a/codex-plugin/AGENTS.md +++ b/codex-plugin/AGENTS.md @@ -5,6 +5,7 @@ Use this plugin when Codex should have direct access to synapt recall over MCP. ## What it provides - local `synapt server` MCP access +- bounded recall context on every session start - recall search, save, context, journal, and channel tools - no identity or org behavior; this is OSS-only packaging of existing recall primitives diff --git a/codex-plugin/hooks/hooks.json b/codex-plugin/hooks/hooks.json new file mode 100644 index 00000000..9abd0095 --- /dev/null +++ b/codex-plugin/hooks/hooks.json @@ -0,0 +1,18 @@ +{ + "description": "Load bounded synapt recall context when a Codex session starts.", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "synapt recall hook session-start", + "statusMessage": "Loading current synapt context", + "timeout": 30, + "additionalContextLimit": 3000 + } + ] + } + ] + } +} diff --git a/tests/test_agent_plugin_packages.py b/tests/test_agent_plugin_packages.py new file mode 100644 index 00000000..bc1a90b0 --- /dev/null +++ b/tests/test_agent_plugin_packages.py @@ -0,0 +1,71 @@ +"""Contract tests for the Codex and Claude Code plugin packages.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +HOOK_COMMAND = "synapt recall hook session-start" + + +def _hooks(runtime: str) -> dict: + path = REPO_ROOT / f"{runtime}-plugin" / "hooks" / "hooks.json" + return json.loads(path.read_text()) + + +def _json(path: Path) -> dict: + return json.loads(path.read_text()) + + +def _session_start_command(runtime: str) -> dict: + hooks = _hooks(runtime)["hooks"] + assert set(hooks) == {"SessionStart"} + groups = hooks["SessionStart"] + assert len(groups) == 1 + commands = groups[0]["hooks"] + assert len(commands) == 1 + return commands[0] + + +def test_codex_plugin_loads_bounded_session_context() -> None: + command = _session_start_command("codex") + + assert command["type"] == "command" + assert command["command"] == HOOK_COMMAND + assert command["timeout"] == 30 + assert command["additionalContextLimit"] == 3000 + + +def test_claude_plugin_loads_bounded_session_context() -> None: + command = _session_start_command("claude") + + assert command["type"] == "command" + assert command["command"] == HOOK_COMMAND + assert command["timeout"] == 30 + assert "additionalContextLimit" not in command + + +def test_claude_marketplace_distributes_repo_plugin() -> None: + marketplace = _json(REPO_ROOT / ".claude-plugin" / "marketplace.json") + + assert marketplace["name"] == "synapt-plugins" + plugins = marketplace["plugins"] + assert len(plugins) == 1 + assert plugins[0]["name"] == "synapt-recall" + assert plugins[0]["source"] == "./claude-plugin" + + +def test_claude_project_settings_enable_recall_marketplace() -> None: + settings = _json(REPO_ROOT / "claude-plugin" / "project-settings.json") + + assert settings["enabledPlugins"] == { + "synapt-recall@synapt-plugins": True, + } + marketplace = settings["extraKnownMarketplaces"]["synapt-plugins"] + assert marketplace["source"] == { + "source": "github", + "repo": "synapt-dev/recall", + } + assert marketplace["autoUpdate"] is True