diff --git a/codebase_rag/constants/graph.py b/codebase_rag/constants/graph.py index 696aa5efbc..666f7f4585 100644 --- a/codebase_rag/constants/graph.py +++ b/codebase_rag/constants/graph.py @@ -1648,6 +1648,25 @@ class AuditCheck(StrEnum): KEY_ANCHOR_STATE = "anchor_state" KEY_MENTIONS = "mentions" KEY_MENTION_QNS = "mention_qns" +# Each mention's anchors, recorded at write time beside `mention_qns` and in +# its order, so a renamed or moved mention is followed the way the subject is +# (issue #3230). An empty string stands for "none": the wire carries string +# lists, not lists with nulls. `mentions_lost` names the mentions the repair +# could not place; they stay in `mention_qns` with no edge. One name can hold a +# live entry and lost ones (`a` deleted, then `b` renamed to `a`): each keeps +# its own slot and anchors, the live one first, and `mentions_lost` repeats +# the name once per lost entry. Before that, names were unique and read the +# same way. +KEY_MENTION_HASHES = "mention_hashes" +KEY_MENTION_QUOTES = "mention_quotes" +KEY_MENTIONS_LOST = "mentions_lost" +KEY_ATTACHED = "attached" +KEY_ATTACH_QNS = "attach_qns" +KEY_ATTACH_KEYS = "attach_keys" +KEY_KEY_SEPARATOR = "key_separator" +# Joins a followed mention's new name to the hash it was followed by, in one +# string the write compares as a whole; a qualified name never holds it. +GLOSS_MENTION_KEY_SEPARATOR = "\x1f" KEY_ANCHOR_HASH = "anchor_hash" # Written by the repair tiers (issue #1808, stage four). `moved_from` is the # name a MOVED gloss was written against, kept so the move stays visible; diff --git a/codebase_rag/cypher_queries.py b/codebase_rag/cypher_queries.py index 4560b0d1bd..0192461101 100644 --- a/codebase_rag/cypher_queries.py +++ b/codebase_rag/cypher_queries.py @@ -1151,6 +1151,8 @@ def build_missing_rel_endpoints_query( g.anchor_quote = $anchor_quote, g.anchor_prefix = $anchor_prefix, g.anchor_suffix = $anchor_suffix, g.write_id = $write_id, g.mention_qns = $mention_qns, + g.mention_hashes = $mention_hashes, g.mention_quotes = $mention_quotes, + g.mentions_lost = null, g.project = $project_name, g.moved_from = null, g.candidate_qns = null WITH g, t, mentioned OPTIONAL MATCH (g)-[old:{_ANNOTATES}]->(prev) @@ -1294,11 +1296,58 @@ def build_missing_rel_endpoints_query( MERGE (g)-[:{_ANNOTATES}]->(t)""" CYPHER_GLOSS_MARK = f"""MATCH (g:{_GLOSS} {{qualified_name: $qn}}) SET g.anchor_state = $anchor_state, g.candidate_qns = $candidate_qns""" -CYPHER_REANCHOR_GLOSS_MENTIONS = f"""MATCH (g:{_GLOSS})-[:{_ANNOTATES}]->() -WHERE g.mention_qns IS NOT NULL AND {_GLOSS_OF_SYNCING_PROJECT} -UNWIND g.mention_qns AS mention_qn -MATCH (m {{qualified_name: mention_qn}}) -WHERE {_any_label("m", _GRAPH_DEFINITION_LABELS)} +# A note's mentions are placed after its subject (issue #3230), by +# `gloss_repair.repair_mentions`: the attached notes that mention anything, +# with each mention's recorded anchors and the edges it has now; the +# definitions currently under the mentioned names; and one statement per note +# whose mentions changed. A mention is never bound by its name alone once +# that name holds other code and the recorded code is found elsewhere, and +# one that cannot be placed is listed in `mentions_lost` rather than dropped. +CYPHER_GLOSS_MENTION_ANCHORS = f"""MATCH (g:{_GLOSS})-[:{_ANNOTATES}]->() +WITH DISTINCT g +WHERE g.mention_qns IS NOT NULL AND size(g.mention_qns) > 0 +OPTIONAL MATCH (g)-[:{_MENTIONS}]->(m) +RETURN g.qualified_name AS qualified_name, g.project AS project, + g.target_qn AS target_qn, g.mention_qns AS mention_qns, + g.mention_hashes AS mention_hashes, g.mention_quotes AS mention_quotes, + g.mentions_lost AS mentions_lost, g.write_id AS write_id, + collect(m.qualified_name) AS attached""" +CYPHER_DEFINITIONS_BY_QNS = f"""UNWIND $qns AS qn +MATCH (t {{qualified_name: qn}}) +WHERE {_any_label("t", _GRAPH_DEFINITION_LABELS)} +RETURN t.qualified_name AS qualified_name, t.anchor_hash AS anchor_hash""" +# The note's mentions as the repair placed them, and its MENTIONS edges +# rebuilt to match. A mention kept on its name is bound by `attach_qns`; one +# followed to a new name by `attach_keys`, its name and the hash the pass +# read joined by `key_separator`, so a definition replaced between the read +# and this write is not bound on the strength of its name. The targets are +# matched BEFORE any edge is deleted, and only an edge to a node that is no +# longer a target is deleted: with the definition indexes in place, Memgraph +# 3 lost the edge when one statement deleted it and then merged it again. +# The note must still carry the `write_id` the pass read: an `annotate` that +# landed in between wrote newer mentions, which this result would overwrite. +# And every followed key must still match a target: a definition replaced +# under its new name would otherwise have that name saved without an edge, +# and the next pass would read the newcomer as an in-place edit and bind it. +# Either way nothing is written, and the next sync places the note afresh. +CYPHER_GLOSS_SET_MENTIONS = f"""MATCH (g:{_GLOSS} {{qualified_name: $qn}}) +WHERE coalesce(g.write_id, '') = $write_id +OPTIONAL MATCH (m) +WHERE {_any_label("m", _GRAPH_DEFINITION_LABELS)} AND m.qualified_name STARTS WITH $project_prefix + AND (m.qualified_name IN $attach_qns + OR (m.qualified_name + $key_separator + m.anchor_hash) IN $attach_keys) +WITH g, collect(DISTINCT m) AS targets +WHERE all(key IN $attach_keys WHERE any(target IN targets + WHERE target.qualified_name + $key_separator + target.anchor_hash = key)) +SET g.mention_qns = $mention_qns, g.mention_hashes = $mention_hashes, + g.mention_quotes = $mention_quotes, g.mentions_lost = $mentions_lost +WITH g, targets +OPTIONAL MATCH (g)-[old:{_MENTIONS}]->(t) +WHERE NOT t IN targets +WITH g, targets, collect(old) AS stale_edges +FOREACH (edge IN stale_edges | DELETE edge) +WITH g, targets +UNWIND targets AS m MERGE (g)-[:{_MENTIONS}]->(m)""" # Staleness (issue #1808): a gloss recorded its subject's `anchor_hash` as # `target_hash` when it was written; after a sync the two are compared and the @@ -1325,7 +1374,8 @@ def build_missing_rel_endpoints_query( "g.commit_sha AS commit_sha, g.target_qn AS target_qn, " "g.target_hash AS target_hash, g.anchor_state AS anchor_state, " "g.moved_from AS moved_from, g.candidate_qns AS candidate_qns, " - "g.write_id AS write_id, collect(m.qualified_name) AS mentions" + "g.write_id AS write_id, g.mentions_lost AS mentions_lost, " + "collect(m.qualified_name) AS mentions" ) CYPHER_GLOSS_READ = f"""MATCH (g:{_GLOSS} {{qualified_name: $qn}}) OPTIONAL MATCH (g)-[:{_MENTIONS}]->(m) diff --git a/codebase_rag/gloss.py b/codebase_rag/gloss.py index 80e3816496..d1c1870132 100644 --- a/codebase_rag/gloss.py +++ b/codebase_rag/gloss.py @@ -69,6 +69,11 @@ class GlossRow(TypedDict): moved_from: str | None candidate_qns: list[str] mentions: list[str] + # Mentions whose definition the repair could not place after a rename, + # move or delete: still named, with no MENTIONS edge (issue #3230). A + # name `mentions` also lists is a lost mention whose name another one + # has since taken. + mentions_lost: list[str] class GlossRefusal(TypedDict, total=False): @@ -274,6 +279,25 @@ class _TargetFacts(NamedTuple): anchor: TextAnchor | None +def _mention_anchor_props( + fetch_all: QueryFn, + project_name: str, + mention_qns: list[str], + read_source: SourceReader | None, +) -> PropertyDict: + # Each mention records the anchors the subject does, in `mention_qns` + # order, so the repair can follow a renamed or moved mention instead of + # dropping it or handing it to a newcomer with the old name (#3230). + facts = [ + _target_facts(fetch_all, project_name, qn, read_source) for qn in mention_qns + ] + return { + cs.KEY_MENTION_QNS: mention_qns, + cs.KEY_MENTION_HASHES: [f.target_hash or "" for f in facts], + cs.KEY_MENTION_QUOTES: [f.anchor.quote if f.anchor else "" for f in facts], + } + + def _target_facts( fetch_all: QueryFn, project_name: str, @@ -338,6 +362,7 @@ def _gloss_row(row: ResultRow) -> GlossRow: moved_from=_opt_str(row.get(cs.KEY_MOVED_FROM)), candidate_qns=_str_list(row.get(cs.KEY_CANDIDATE_QNS)), mentions=mentions, + mentions_lost=_str_list(row.get(cs.KEY_MENTIONS_LOST)), ) @@ -419,7 +444,12 @@ def write_gloss( cs.KEY_QN: key, cs.KEY_PROJECT_PREFIX: _prefix(project_name), cs.KEY_TARGET_QN: target_qn, - cs.KEY_MENTION_QNS: sorted({m["qualified_name"] for m in mentioned}), + **_mention_anchor_props( + fetch_all, + project_name, + sorted({m["qualified_name"] for m in mentioned}), + read_source, + ), cs.KEY_KIND: kind, cs.KEY_STATUS: cs.GLOSS_STATUS_ACCEPTED, cs.KEY_BODY: text, diff --git a/codebase_rag/gloss_repair.py b/codebase_rag/gloss_repair.py index cfa2d13c2a..410b3668aa 100644 --- a/codebase_rag/gloss_repair.py +++ b/codebase_rag/gloss_repair.py @@ -34,9 +34,9 @@ from __future__ import annotations -from collections import defaultdict -from collections.abc import Callable -from typing import NamedTuple +from collections import Counter, defaultdict +from collections.abc import Callable, Sequence +from typing import NamedTuple, Protocol from . import constants as cs from . import cypher_queries as cq @@ -48,7 +48,7 @@ text_anchor, ) from .graph_query import QueryFn -from .types_defs import PropertyDict, ResultRow +from .types_defs import PropertyDict, ResultRow, ResultValue WriteFn = Callable[[str, PropertyDict | None], None] @@ -108,7 +108,20 @@ def _unanchored(row: ResultRow) -> _Unanchored: ) -def _projects_of(fetch_all: QueryFn, notes: list[_Unanchored]) -> dict[str, str | None]: +class _Recorded(Protocol): + """What `_projects_of` reads off a note: its key, subject and project.""" + + @property + def qualified_name(self) -> str: ... + @property + def target_qn(self) -> str: ... + @property + def project(self) -> str | None: ... + + +def _projects_of( + fetch_all: QueryFn, notes: Sequence[_Recorded] +) -> dict[str, str | None]: """Each note's project, by note key. The recorded `project` wins. A note without one (written before the @@ -141,6 +154,30 @@ def _projects_of(fetch_all: QueryFn, notes: list[_Unanchored]) -> dict[str, str return projects +def _owned_by[N: _Recorded]( + fetch_all: QueryFn, notes: list[N], project_name: str | None +) -> tuple[list[N], dict[str, str | None]]: + """The notes `project_name` owns, and each kept note's project. + + The owner is the one `_projects_of` assigns: the recorded project, or for + a legacy note the longest registered project prefixing its target. None + keeps every note. + """ + if project_name is not None: + prefix = f"{project_name}{cs.SEPARATOR_DOT}" + notes = [ + note + for note in notes + if note.project == project_name + or (note.project is None and note.target_qn.startswith(prefix)) + ] + projects = _projects_of(fetch_all, notes) + if project_name is not None: + # A legacy note under this prefix may belong to a longer project. + notes = [n for n in notes if projects.get(n.qualified_name) == project_name] + return notes, projects + + def _candidates_by_hash( fetch_all: QueryFn, notes: list[_Unanchored], projects: dict[str, str | None] ) -> dict[tuple[str, str], list[str]]: @@ -207,6 +244,18 @@ def candidates(self, project: str, quote: str) -> list[_QuoteCandidate]: self._by_project[project] = self._build_quote_index(project) return self._by_project[project].get(quote, []) + def anchor_of(self, project: str, qualified_name: str) -> TextAnchor | None: + """The current anchor of the one definition under a name, if any.""" + if project not in self._by_project: + self._by_project[project] = self._build_quote_index(project) + found = [ + row.anchor + for rows in self._by_project[project].values() + for row in rows + if row.qualified_name == qualified_name + ] + return found[0] if len(found) == 1 else None + def _build_quote_index(self, project: str) -> dict[str, list[_QuoteCandidate]]: rows = self._fetch_all( cq.CYPHER_DEFINITION_SPANS, @@ -334,18 +383,7 @@ def repair_unanchored( (_unanchored(row) for row in fetch_all(cq.CYPHER_UNANCHORED_GLOSSES, None)), key=lambda n: n.qualified_name, ) - if project_name is not None: - prefix = f"{project_name}{cs.SEPARATOR_DOT}" - notes = [ - note - for note in notes - if note.project == project_name - or (note.project is None and note.target_qn.startswith(prefix)) - ] - projects = _projects_of(fetch_all, notes) - if project_name is not None: - # A legacy note under this prefix may belong to a longer project. - notes = [n for n in notes if projects.get(n.qualified_name) == project_name] + notes, projects = _owned_by(fetch_all, notes, project_name) candidates = _candidates_by_hash(fetch_all, notes, projects) quotes = _QuoteIndex(fetch_all, read_source) if read_source is not None else None report = RepairReport(moved=[], ambiguous=[], lost=[]) @@ -455,3 +493,343 @@ def _moved_to(fetch_all: QueryFn, key: str, candidate: str) -> bool: """ rows = fetch_all(cq.CYPHER_GLOSS_READ, {cs.KEY_QN: key}) return bool(rows) and rows[0].get(cs.KEY_TARGET_QN) == candidate + + +# --- mentions (issue #3230) --------------------------------------------------- + + +class MentionReport(NamedTuple): + """What one mention pass did, by note key, each list sorted.""" + + # Notes with a mention followed to a new name. + moved: list[str] + # Notes with a mention that could not be placed. + lost: list[str] + + +class _Mention(NamedTuple): + qualified_name: str + # The anchors recorded for it; None where none was (a note from before + # mentions recorded them, a definition without a hash, a file the writer + # could not read). An anchor in an older format simply matches nothing, + # and a mention whose name still holds code then has its anchors renewed. + anchor_hash: str | None + quote: str | None + lost: bool + + +class _MentionNote(NamedTuple): + qualified_name: str + target_qn: str + project: str | None + mentions: list[_Mention] + # As stored, to tell whether the pass changed anything. + stored: tuple[list[str], list[str], list[str], list[str]] + attached: frozenset[str] + # The nonce of the write the pass read ("" before nonces): the repair is + # applied only while the note still carries it. + write_id: str + + +def _strings(value: ResultValue | None) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if item is not None] + + +def _lost_flags(qns: list[str], lost: list[str]) -> list[bool]: + # A name holds at most one live entry, written before its lost ones, and + # `mentions_lost` repeats a name once per lost entry: so a name's lost + # entries are its last that-many. A note from before a name could hold + # both has unique names and reads as it always did. + lost_left = Counter(lost) + remaining = Counter(qns) + flags = [] + for qn in qns: + remaining[qn] -= 1 + flags.append(remaining[qn] < lost_left[qn]) + return flags + + +def _mention_note(row: ResultRow) -> _MentionNote: + qns = _strings(row.get(cs.KEY_MENTION_QNS)) + hashes = _strings(row.get(cs.KEY_MENTION_HASHES)) + quotes = _strings(row.get(cs.KEY_MENTION_QUOTES)) + lost = _strings(row.get(cs.KEY_MENTIONS_LOST)) + # Anchors are positional: a list that does not line up with the names + # (a note from before they were recorded) is read as no anchors at all. + if len(hashes) != len(qns): + hashes = [""] * len(qns) + if len(quotes) != len(qns): + quotes = [""] * len(qns) + project = row.get(cs.KEY_PROJECT) + write_id = row.get(cs.KEY_WRITE_ID) + return _MentionNote( + qualified_name=str(row.get(cs.KEY_QUALIFIED_NAME, "")), + target_qn=str(row.get(cs.KEY_TARGET_QN, "")), + project=project if isinstance(project, str) and project else None, + mentions=[ + _Mention( + qualified_name=qn, + anchor_hash=h or None, + quote=q or None, + lost=is_lost, + ) + for qn, h, q, is_lost in zip( + qns, hashes, quotes, _lost_flags(qns, lost), strict=True + ) + ], + stored=(qns, hashes, quotes, sorted(lost)), + attached=frozenset(_strings(row.get(cs.KEY_ATTACHED))), + write_id=write_id if isinstance(write_id, str) else "", + ) + + +class _Placed(NamedTuple): + """A mention after the pass: where it is, and how to bind it.""" + + qualified_name: str + anchor_hash: str + quote: str + lost: bool + # The hash the write re-validates a followed binding by; "" binds by name. + expect_hash: str = "" + moved: bool = False + # For a mention renewed on its name: the lost record it becomes when + # another mention's code is found under that name instead. + fallback: _Placed | None = None + + +class _MentionPlacer: + """Places one note's mentions: by name while the name holds the recorded + code, else by hash, then by quote, else marked lost.""" + + def __init__( + self, + fetch_all: QueryFn, + current: dict[str, list[str | None]], + quotes: _QuoteIndex | None, + ) -> None: + self._fetch_all = fetch_all + self._current = current + self._quotes = quotes + self._by_hash: dict[tuple[str, str], list[str]] = {} + + def place(self, mention: _Mention, project: str | None) -> _Placed: + hashes = self._current.get(mention.qualified_name, []) + if hashes and mention.anchor_hash in hashes: + # The name still holds the recorded code (or, for a definition + # with no hash, there was none to record): bound by name. + return _Placed( + mention.qualified_name, + mention.anchor_hash or "", + mention.quote or "", + lost=False, + ) + found = self._elsewhere(mention, project) if project is not None else [] + if len(found) == 1: + qn, anchor_hash = found[0] + return _Placed( + qn, + anchor_hash, + mention.quote or "", + lost=False, + expect_hash=anchor_hash, + moved=qn != mention.qualified_name, + ) + if ( + hashes + and not mention.lost + and (not found or any(qn == mention.qualified_name for qn, _ in found)) + ): + # The name holds other code and the recorded code went nowhere + # else: the definition was edited in place, or the note is from + # before mentions recorded anchors. Its anchors are renewed so a + # later move is followed from here. Code found in several places + # other than the name did go somewhere, so the name keeps nothing. + return self._renewed(mention, project, hashes) + return _Placed( + mention.qualified_name, + mention.anchor_hash or "", + mention.quote or "", + lost=True, + ) + + def _renewed( + self, mention: _Mention, project: str | None, hashes: list[str | None] + ) -> _Placed: + anchor_hash = hashes[0] if len(hashes) == 1 else None + anchor = ( + self._quotes.anchor_of(project, mention.qualified_name) + if self._quotes is not None and project is not None + else None + ) + return _Placed( + mention.qualified_name, + anchor_hash or mention.anchor_hash or "", + anchor.quote if anchor is not None else mention.quote or "", + lost=False, + fallback=_Placed( + mention.qualified_name, + mention.anchor_hash or "", + mention.quote or "", + lost=True, + ), + ) + + def _elsewhere(self, mention: _Mention, project: str) -> list[tuple[str, str]]: + """Where the recorded code is now, one entry per physical node.""" + if mention.anchor_hash is not None: + rows = self._with_hash(project, mention.anchor_hash) + if rows: + return [(qn, mention.anchor_hash) for qn in rows] + if mention.quote is None or self._quotes is None: + return [] + # A candidate the write cannot re-validate (no hash) is no home. + return [ + (row.qualified_name, row.anchor_hash) + for row in self._quotes.candidates(project, mention.quote) + if row.anchor_hash is not None + ] + + def _with_hash(self, project: str, anchor_hash: str) -> list[str]: + key = (project, anchor_hash) + if key not in self._by_hash: + rows = self._fetch_all( + cq.CYPHER_DEFINITIONS_BY_ANCHOR_HASH, + { + cs.KEY_HASHES: [anchor_hash], + cs.KEY_PROJECT_PREFIX: f"{project}{cs.SEPARATOR_DOT}", + }, + ) + self._by_hash[key] = [ + qn + for row in rows + if isinstance(qn := row.get(cs.KEY_QUALIFIED_NAME), str) + ] + return self._by_hash[key] + + +def repair_mentions( + fetch_all: QueryFn, + execute_write: WriteFn, + read_source: SourceReader | None = None, + project_name: str | None = None, +) -> MentionReport: + """Place every attached note's mentions, then rebuild their edges. + + A mention stays on its name while that name holds the code recorded when + the note was written. When + the name is gone, or holds other code, the recorded code is looked for + in the note's project, by content hash and then by text quote, as the + subject is: exactly one home is followed, renaming the mention. Failing + that, a name that is still there, with the recorded code in no other + place, was edited in place and keeps the mention, with its anchors + renewed; otherwise the mention goes to `mentions_lost`, with no edge, + until its code has one home again, even if another mention takes its + name meanwhile. A + name reused by unrelated code therefore never takes a mention by name + alone: not while the recorded code is elsewhere, and not once the + mention is lost. + + A note is written only when its mentions or edges differ from what the + pass read, so an unchanged graph costs one read. Deterministic: notes + and mentions are visited in key order. + + `project_name` scopes the pass to one project's notes, by the owner rule + `repair_unanchored` uses: a sync changed that project's definitions + only, so another project's mentions move only on its own sync (issue + #3236). + """ + notes = sorted( + ( + _mention_note(row) + for row in fetch_all(cq.CYPHER_GLOSS_MENTION_ANCHORS, None) + ), + key=lambda n: n.qualified_name, + ) + notes, projects = _owned_by(fetch_all, notes, project_name) + report = MentionReport(moved=[], lost=[]) + if not notes: + return report + names = sorted({m.qualified_name for note in notes for m in note.mentions}) + current: dict[str, list[str | None]] = defaultdict(list) + for row in fetch_all(cq.CYPHER_DEFINITIONS_BY_QNS, {cs.KEY_QNS: names}): + qn = row.get(cs.KEY_QUALIFIED_NAME) + anchor_hash = row.get(cs.KEY_ANCHOR_HASH) + if isinstance(qn, str): + current[qn].append(anchor_hash if isinstance(anchor_hash, str) else None) + quotes = _QuoteIndex(fetch_all, read_source) if read_source is not None else None + placer = _MentionPlacer(fetch_all, current, quotes) + for note in notes: + project = projects.get(note.qualified_name) + placed = _settled([placer.place(m, project) for m in note.mentions]) + _write_mentions(execute_write, note, project, placed) + if any(p.moved for p in placed): + report.moved.append(note.qualified_name) + if any(p.lost for p in placed): + report.lost.append(note.qualified_name) + return report + + +def _settled(placements: list[_Placed]) -> list[_Placed]: + """One live entry per name, every lost entry kept, live before lost. + + Two mentions can land on one name: `a` deleted, then `b` renamed to `a`. + The one followed there by its code is live, and a mention renewed there + by name alone was replaced, not edited, so it goes back to its recorded + anchors, lost. A lost entry keeps its own anchors beside the live one + under the same name, so its code is followed if it comes back (bot review + on PR #3329). Two live entries under one name share one edge; first come + wins. + """ + found = {p.qualified_name for p in placements if not p.lost and p.fallback is None} + kept: list[_Placed] = [] + live: set[str] = set() + for placed in placements: + if placed.fallback is not None and placed.qualified_name in found: + placed = placed.fallback + if not placed.lost: + if placed.qualified_name in live: + continue + live.add(placed.qualified_name) + kept.append(placed) + return sorted(kept, key=lambda p: (p.qualified_name, p.lost)) + + +def _write_mentions( + execute_write: WriteFn, + note: _MentionNote, + project: str | None, + ordered: list[_Placed], +) -> None: + bound = [p for p in ordered if not p.lost] + lists = ( + [p.qualified_name for p in ordered], + [p.anchor_hash for p in ordered], + [p.quote for p in ordered], + [p.qualified_name for p in ordered if p.lost], + ) + if lists == note.stored and note.attached == {p.qualified_name for p in bound}: + return + # A note no project claims keeps its names unscoped, as before. + prefix = f"{project}{cs.SEPARATOR_DOT}" if project is not None else "" + execute_write( + cq.CYPHER_GLOSS_SET_MENTIONS, + { + cs.KEY_QN: note.qualified_name, + cs.KEY_WRITE_ID: note.write_id, + cs.KEY_MENTION_QNS: lists[0], + cs.KEY_MENTION_HASHES: lists[1], + cs.KEY_MENTION_QUOTES: lists[2], + cs.KEY_MENTIONS_LOST: lists[3] or None, + cs.KEY_ATTACH_QNS: [p.qualified_name for p in bound if not p.expect_hash], + cs.KEY_ATTACH_KEYS: [ + f"{p.qualified_name}{cs.GLOSS_MENTION_KEY_SEPARATOR}{p.expect_hash}" + for p in bound + if p.expect_hash + ], + cs.KEY_KEY_SEPARATOR: cs.GLOSS_MENTION_KEY_SEPARATOR, + cs.KEY_PROJECT_PREFIX: prefix, + }, + ) diff --git a/codebase_rag/graph_updater.py b/codebase_rag/graph_updater.py index 92847d8deb..566cabe6ae 100644 --- a/codebase_rag/graph_updater.py +++ b/codebase_rag/graph_updater.py @@ -31,7 +31,7 @@ from .config import settings from .function_registry import FunctionRegistryTrie from .gloss_anchor import ParsedSource, parse_source -from .gloss_repair import repair_unanchored +from .gloss_repair import repair_mentions, repair_unanchored from .graph_query import _longer_project_prefixes from .interface_skip import ( callers_to_reparse, @@ -9717,7 +9717,15 @@ def _reanchor_glosses(self) -> None: self._read_project_source, project_name=self.project_name, ) - self.ingestor.execute_write(cq.CYPHER_REANCHOR_GLOSS_MENTIONS, scope) + # Mentions are placed the way the subject was: by name while the + # name holds the recorded code, else by hash or quote, else + # listed as lost rather than dropped or handed to a newcomer. + repair_mentions( + self.ingestor.fetch_all, + self.ingestor.execute_write, + self._read_project_source, + project_name=self.project_name, + ) # Then grade: the subject's `anchor_hash` was just re-emitted by # the parse, so comparing it with the note's recorded hash here # is what makes a note about changed code read STALE. diff --git a/codebase_rag/tests/integration/test_gloss_mentions_e2e.py b/codebase_rag/tests/integration/test_gloss_mentions_e2e.py new file mode 100644 index 0000000000..901fe91a96 --- /dev/null +++ b/codebase_rag/tests/integration/test_gloss_mentions_e2e.py @@ -0,0 +1,283 @@ +# Real-Memgraph check of issue #3230: a gloss's mentions were tracked by name +# alone. Renaming or moving a mentioned definition dropped its MENTIONS edge +# while the note stayed EXACT, and an unrelated definition that later took +# the old name inherited the mention. Mentions now carry the mentioned +# definition's content hash and text quote, the anchors the subject has. +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from codebase_rag import constants as cs +from codebase_rag import gloss +from codebase_rag.graph_updater import GraphUpdater +from codebase_rag.parser_loader import load_parsers + +if TYPE_CHECKING: + from codebase_rag.services.graph_service import MemgraphIngestor + +pytestmark = [pytest.mark.integration] + +P = "gm" +HELPER = f"{P}.a.helper" +A = "def helper(x):\n return x + 1\n" +B = "from a import helper\n\n\ndef run():\n return helper(2)\n" +B_EMPTY = "from a import helper\n" +NEWCOMER = '\n\ndef run():\n return "unrelated"\n' + + +class _Repo: + def __init__(self, ingestor: MemgraphIngestor, root: Path) -> None: + self.ingestor = ingestor + self.root = root + for rel, text in {"a.py": A, "b.py": B}.items(): + (root / rel).write_text(text, encoding="utf-8") + # The indexes `cgr start` creates: they change how Memgraph plans the + # mention statement, and a plan that deleted and re-merged the same + # edge in one statement lost it. + ingestor.ensure_constraints() + parsers, queries = load_parsers() + self.updater = GraphUpdater( + ingestor=ingestor, + repo_path=root, + parsers=parsers, + queries=queries, + project_name=P, + ) + self.updater.run(force=True) + + def annotate( + self, readable: bool = True, mentions: tuple[str, ...] = (f"{P}.b.run",) + ) -> None: + row = gloss.write_gloss( + self.ingestor.fetch_all, + self.ingestor.execute_write, + P, + HELPER, + "run() relies on the +1", + cs.GlossKind.INVARIANT, + mentions=",".join(mentions), + read_source=self.updater._read_project_source if readable else None, + ) + assert row.get("mentions") == sorted(mentions), row + + def edit(self, files: dict[str, str]) -> None: + for rel, text in files.items(): + (self.root / rel).write_text(text, encoding="utf-8") + self.updater.reingest(sorted(files)) + + def note(self) -> dict[str, object]: + result = gloss.glosses_for(self.ingestor.fetch_all, P, HELPER) + (row,) = result["annotating"] + return { + "anchor_state": row["anchor_state"], + "mentions": row["mentions"], + "mentions_lost": row.get("mentions_lost"), + } + + def mentioning(self, target: str) -> list[str]: + result = gloss.glosses_for(self.ingestor.fetch_all, P, target) + return [row["body"] for row in result.get("mentioning", [])] + + +@pytest.fixture +def repo(memgraph_ingestor: MemgraphIngestor, tmp_path: Path) -> _Repo: + repo = _Repo(memgraph_ingestor, tmp_path) + repo.annotate() + return repo + + +def test_a_renamed_mention_is_followed(repo: _Repo) -> None: + repo.edit({"b.py": B.replace("def run", "def run_job")}) + + assert repo.note() == { + "anchor_state": "EXACT", + "mentions": [f"{P}.b.run_job"], + "mentions_lost": [], + } + assert repo.mentioning(f"{P}.b.run_job") == ["run() relies on the +1"] + + +def test_a_mention_moved_unchanged_to_another_file_is_followed(repo: _Repo) -> None: + repo.edit({"b.py": B_EMPTY, "c.py": B}) + + assert repo.note()["mentions"] == [f"{P}.c.run"] + assert repo.note()["mentions_lost"] == [] + + +def test_a_newcomer_reusing_the_name_does_not_take_the_mention(repo: _Repo) -> None: + # One edit renames `run` and adds an unrelated `run`: the mention follows + # the renamed body, and the newcomer gets nothing. + repo.edit({"b.py": B.replace("def run", "def run_job") + NEWCOMER}) + + assert repo.note()["mentions"] == [f"{P}.b.run_job"] + assert repo.mentioning(f"{P}.b.run") == [] + + +def test_a_mention_that_cannot_be_placed_is_reported(repo: _Repo) -> None: + repo.edit({"b.py": B_EMPTY}) + + assert repo.note() == { + "anchor_state": "EXACT", + "mentions": [], + "mentions_lost": [f"{P}.b.run"], + } + + +def test_a_lost_mention_is_not_given_to_a_later_newcomer(repo: _Repo) -> None: + repo.edit({"b.py": B_EMPTY}) + repo.edit({"b.py": B_EMPTY + NEWCOMER}) + + assert repo.note()["mentions"] == [] + assert repo.note()["mentions_lost"] == [f"{P}.b.run"] + assert repo.mentioning(f"{P}.b.run") == [] + + +def test_a_lost_mention_whose_code_comes_back_is_restored(repo: _Repo) -> None: + # Negative: the same body under the same name is the same definition. + repo.edit({"b.py": B_EMPTY}) + repo.edit({"b.py": B}) + + assert repo.note()["mentions"] == [f"{P}.b.run"] + assert repo.note()["mentions_lost"] == [] + + +def test_a_mention_whose_code_went_to_two_places_is_not_guessed( + repo: _Repo, +) -> None: + # Negative: two identical copies and no `run` left; neither is the one. + repo.edit({"b.py": B_EMPTY, "c.py": B, "d.py": B}) + + assert repo.note()["mentions"] == [] + assert repo.note()["mentions_lost"] == [f"{P}.b.run"] + + +def test_a_mention_edited_in_place_keeps_its_edge(repo: _Repo) -> None: + # Negative: a changed body under the same name, with nowhere else the old + # body went, is the same definition edited. + repo.edit({"b.py": B.replace("helper(2)", "helper(3)")}) + + assert repo.note()["mentions"] == [f"{P}.b.run"] + assert repo.note()["mentions_lost"] == [] + # And it is followed from its edited body on, by a later rename. + repo.edit({"b.py": B.replace("helper(2)", "helper(3)").replace("run", "go")}) + + assert repo.note()["mentions"] == [f"{P}.b.go"] + + +def test_an_untouched_mention_stays_attached(repo: _Repo) -> None: + # Negative: a sync that re-parses the mentioned file unchanged. + repo.edit({"b.py": B}) + + assert repo.note() == { + "anchor_state": "EXACT", + "mentions": [f"{P}.b.run"], + "mentions_lost": [], + } + + +def test_a_note_from_before_mention_anchors_still_attaches_by_name( + repo: _Repo, +) -> None: + # A note written before mentions recorded anchors has only the name to go + # on: it re-attaches by name, as it did, and a gone name is reported. + repo.ingestor.execute_write( + "MATCH (g:Gloss) REMOVE g.mention_hashes, g.mention_quotes", None + ) + repo.edit({"b.py": B}) + + assert repo.note()["mentions"] == [f"{P}.b.run"] + repo.edit({"b.py": B_EMPTY}) + + assert repo.note()["mentions"] == [] + assert repo.note()["mentions_lost"] == [f"{P}.b.run"] + # Its first pass recorded the code under the name, so the same code + # coming back is the same definition. + repo.edit({"b.py": B}) + + assert repo.note()["mentions"] == [f"{P}.b.run"] + assert repo.note()["mentions_lost"] == [] + + +def test_a_note_from_before_mention_anchors_gains_them(repo: _Repo) -> None: + # Its first pass records the mentioned definition's hash, so a later + # move is followed. + repo.ingestor.execute_write( + "MATCH (g:Gloss) REMOVE g.mention_hashes, g.mention_quotes", None + ) + repo.edit({"b.py": B}) + repo.edit({"b.py": B_EMPTY, "c.py": B}) + + assert repo.note()["mentions"] == [f"{P}.c.run"] + + +STOP = "\n\ndef stop():\n return helper(9)\n" + + +@pytest.fixture +def two(memgraph_ingestor: MemgraphIngestor, tmp_path: Path) -> _Repo: + # A note that mentions `run` and `stop`. + repo = _Repo(memgraph_ingestor, tmp_path) + repo.edit({"b.py": B + STOP}) + repo.annotate(mentions=(f"{P}.b.run", f"{P}.b.stop")) + return repo + + +def test_a_lost_mention_survives_another_mention_taking_its_name(two: _Repo) -> None: + two.edit({"b.py": B_EMPTY + STOP}) + + assert two.note()["mentions_lost"] == [f"{P}.b.run"] + # `stop` is renamed to the lost mention's name and followed there by its + # quote; the lost `run` is still reported, under the same name. + two.edit({"b.py": B_EMPTY + STOP.replace("def stop", "def run")}) + + assert two.note() == { + "anchor_state": "EXACT", + "mentions": [f"{P}.b.run"], + "mentions_lost": [f"{P}.b.run"], + } + # `run`'s own code comes back in another file: its record was kept, so + # it is followed there, and `stop`'s code keeps the name it took. + two.edit({"c.py": B}) + + assert two.note()["mentions"] == [f"{P}.b.run", f"{P}.c.run"] + assert two.note()["mentions_lost"] == [] + + +def test_a_mention_replaced_in_the_same_sync_is_lost_not_renewed(two: _Repo) -> None: + # Delete `run` and rename `stop` to `run` in one edit: `run`'s name holds + # `stop`'s code, which is no edit of `run`, so `run` is lost as above. + two.edit({"b.py": B_EMPTY + STOP.replace("def stop", "def run")}) + + assert two.note()["mentions"] == [f"{P}.b.run"] + assert two.note()["mentions_lost"] == [f"{P}.b.run"] + two.edit({"c.py": B}) + + assert two.note()["mentions"] == [f"{P}.b.run", f"{P}.c.run"] + assert two.note()["mentions_lost"] == [] + + +@pytest.fixture +def unread(memgraph_ingestor: MemgraphIngestor, tmp_path: Path) -> _Repo: + # A note written away from the checkout records hashes but no quotes. + repo = _Repo(memgraph_ingestor, tmp_path) + repo.annotate(readable=False) + return repo + + +def test_a_moved_mention_is_followed_by_its_hash_alone(unread: _Repo) -> None: + unread.edit({"b.py": B_EMPTY, "c.py": B}) + + assert unread.note()["mentions"] == [f"{P}.c.run"] + + +def test_an_edited_mention_is_followed_from_its_renewed_anchors( + unread: _Repo, +) -> None: + edited = B.replace("helper(2)", "helper(3)") + unread.edit({"b.py": edited}) + unread.edit({"b.py": B_EMPTY, "c.py": edited}) + + assert unread.note()["mentions"] == [f"{P}.c.run"] diff --git a/codebase_rag/tests/test_gloss_mention_repair.py b/codebase_rag/tests/test_gloss_mention_repair.py new file mode 100644 index 0000000000..555a07327c --- /dev/null +++ b/codebase_rag/tests/test_gloss_mention_repair.py @@ -0,0 +1,479 @@ +"""The mention pass's writes, against a store that answers only its queries. + +A gloss's mentions were restored by name alone after every sync, so a +renamed or moved mention lost its edge and a newcomer with the old name took +it (issue #3230). The real-database behaviour is pinned in +`integration/test_gloss_mentions_e2e.py`; this pins what the pass writes: +nothing when nothing changed, and a move bound only while its target still +carries the hash the pass read. +""" + +from __future__ import annotations + +from codebase_rag import constants as cs +from codebase_rag import cypher_queries as cq +from codebase_rag.embedded_graph.connection import EmbeddedConnection +from codebase_rag.gloss_anchor import TextAnchor +from codebase_rag.gloss_repair import ( + _Mention, + _MentionPlacer, + _Placed, + _QuoteCandidate, + repair_mentions, +) +from codebase_rag.types_defs import PropertyDict, ResultRow + +P = "proj" +RUN = f"{P}.b.run" +RUN_HASH = f"{cs.ANCHOR_HASH_VERSION}run" + + +class _Store: + def __init__( + self, note: ResultRow, current: list[ResultRow], by_hash: list[ResultRow] + ) -> None: + self.note = note + self.current = current + self.by_hash = by_hash + self.writes: list[tuple[str, PropertyDict | None]] = [] + + def fetch_all(self, query: str, params: PropertyDict | None = None) -> list: + if query == cq.CYPHER_GLOSS_MENTION_ANCHORS: + return [self.note] + if query == cq.CYPHER_DEFINITIONS_BY_QNS: + return self.current + if query == cq.CYPHER_DEFINITIONS_BY_ANCHOR_HASH: + hashes = (params or {}).get(cs.KEY_HASHES) or [] + return [r for r in self.by_hash if r[cs.KEY_ANCHOR_HASH] in hashes] + raise AssertionError(f"unexpected query: {query}") + + def execute_write(self, query: str, params: PropertyDict | None = None) -> None: + self.writes.append((query, params)) + + +def _note(attached: list[str]) -> ResultRow: + return { + cs.KEY_QUALIFIED_NAME: "gloss:n", + cs.KEY_TARGET_QN: f"{P}.a.helper", + cs.KEY_PROJECT: P, + cs.KEY_MENTION_QNS: [RUN], + cs.KEY_MENTION_HASHES: [RUN_HASH], + cs.KEY_MENTION_QUOTES: [""], + cs.KEY_ATTACHED: attached, + } + + +def _definition(qn: str, anchor_hash: str) -> ResultRow: + return {cs.KEY_QUALIFIED_NAME: qn, cs.KEY_ANCHOR_HASH: anchor_hash} + + +def test_an_unchanged_mention_writes_nothing() -> None: + store = _Store(_note([RUN]), [_definition(RUN, RUN_HASH)], []) + + report = repair_mentions(store.fetch_all, store.execute_write) + + assert store.writes == [] + assert report.moved == [] + assert report.lost == [] + + +def test_a_mention_whose_edge_was_dropped_is_rebound_by_name() -> None: + store = _Store(_note([]), [_definition(RUN, RUN_HASH)], []) + + repair_mentions(store.fetch_all, store.execute_write) + + ((query, params),) = store.writes + assert query == cq.CYPHER_GLOSS_SET_MENTIONS + assert params is not None + # Kept by name: no hash to re-validate. + assert params[cs.KEY_ATTACH_QNS] == [RUN] + assert params[cs.KEY_ATTACH_KEYS] == [] + assert params[cs.KEY_MENTIONS_LOST] is None + + +def test_a_moved_mention_is_bound_only_while_it_carries_the_hash() -> None: + moved = f"{P}.c.run" + store = _Store(_note([]), [], [_definition(moved, RUN_HASH)]) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + assert params is not None + assert params[cs.KEY_MENTION_QNS] == [moved] + assert params[cs.KEY_ATTACH_QNS] == [] + assert params[cs.KEY_ATTACH_KEYS] == [ + f"{moved}{cs.GLOSS_MENTION_KEY_SEPARATOR}{RUN_HASH}" + ] + assert params[cs.KEY_KEY_SEPARATOR] == cs.GLOSS_MENTION_KEY_SEPARATOR + assert params[cs.KEY_PROJECT_PREFIX] == f"{P}." + assert report.moved == ["gloss:n"] + assert report.lost == [] + + +def test_a_mention_with_no_home_is_recorded_lost() -> None: + store = _Store(_note([]), [], []) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + assert params is not None + assert params[cs.KEY_MENTION_QNS] == [RUN] + assert params[cs.KEY_MENTIONS_LOST] == [RUN] + assert params[cs.KEY_ATTACH_QNS] == [] + assert params[cs.KEY_ATTACH_KEYS] == [] + assert report.lost == ["gloss:n"] + + +def test_a_mention_edited_in_place_renews_its_recorded_hash() -> None: + # The old code is nowhere, so the name keeps the mention, and the hash + # recorded from now on is the edited one (no reader, so the quote stays). + edited = f"{cs.ANCHOR_HASH_VERSION}edited" + store = _Store(_note([RUN]), [_definition(RUN, edited)], []) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + assert params is not None + assert params[cs.KEY_MENTION_HASHES] == [edited] + assert params[cs.KEY_ATTACH_QNS] == [RUN] + assert params[cs.KEY_ATTACH_KEYS] == [] + assert report.moved == [] + assert report.lost == [] + + +def test_a_mention_whose_code_has_two_homes_leaves_the_reused_name() -> None: + # `run`'s recorded code is now in c and in d, and b.run holds other code. + # The code went somewhere, just not to one place, so this is no edit in + # place: the name must not keep the mention, and neither copy is a guess + # worth binding (bot review on PR #3329). + edited = f"{cs.ANCHOR_HASH_VERSION}edited" + store = _Store( + _note([RUN]), + [_definition(RUN, edited)], + [_definition(f"{P}.c.run", RUN_HASH), _definition(f"{P}.d.run", RUN_HASH)], + ) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + assert params is not None + assert params[cs.KEY_MENTION_QNS] == [RUN] + assert params[cs.KEY_MENTION_HASHES] == [RUN_HASH] + assert params[cs.KEY_MENTIONS_LOST] == [RUN] + assert params[cs.KEY_ATTACH_QNS] == [] + assert params[cs.KEY_ATTACH_KEYS] == [] + assert report.lost == ["gloss:n"] + + +class _Quotes: + """The quote tier's answers, without a source reader behind them.""" + + def __init__(self, rows: list[_QuoteCandidate]) -> None: + self.rows = rows + + def candidates(self, project: str, quote: str) -> list[_QuoteCandidate]: + return [r for r in self.rows if r.anchor.quote == quote] + + def anchor_of(self, project: str, qualified_name: str) -> TextAnchor | None: + found = [r.anchor for r in self.rows if r.qualified_name == qualified_name] + return found[0] if len(found) == 1 else None + + +def test_a_name_among_several_quote_homes_keeps_its_mention() -> None: + # A decorator change moves the hash but not the quote, so b.run itself is + # one of the quote's homes, with an identical twin in c. The recorded code + # is still under the name: renewed there, not lost. + anchor = TextAnchor(quote="q-run", prefix="", suffix="") + decorated = f"{cs.ANCHOR_HASH_VERSION}decorated" + twin = f"{P}.c.run" + quotes = _Quotes( + [ + _QuoteCandidate(RUN, anchor, "b.py", 1, 2, decorated), + _QuoteCandidate(twin, anchor, "c.py", 1, 2, f"{cs.ANCHOR_HASH_VERSION}c"), + ] + ) + placer = _MentionPlacer( + _Store(_note([RUN]), [], []).fetch_all, {RUN: [decorated]}, quotes + ) + + placed = placer.place(_Mention(RUN, RUN_HASH, "q-run", lost=False), P) + + assert placed == _Placed( + RUN, + decorated, + "q-run", + lost=False, + fallback=_Placed(RUN, RUN_HASH, "q-run", lost=True), + ) + + +def test_the_set_statement_re_validates_a_moved_binding() -> None: + q = cq.CYPHER_GLOSS_SET_MENTIONS + assert "(m.qualified_name + $key_separator + m.anchor_hash) IN $attach_keys" in q + assert "m.qualified_name STARTS WITH $project_prefix" in q + # Targets are matched before any edge goes, and only stale edges go. + assert q.index("collect(DISTINCT m) AS targets") < q.index("DELETE edge") + assert "WHERE NOT t IN targets" in q + + +def test_the_pass_carries_the_write_id_it_read() -> None: + # The write is skipped when the note was rewritten after this read, so + # the pass hands back the nonce it saw; a note from before nonces has "". + note = {**_note([]), cs.KEY_WRITE_ID: "w-read"} + store = _Store(note, [_definition(RUN, RUN_HASH)], []) + + repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + assert params is not None + assert params[cs.KEY_WRITE_ID] == "w-read" + + unversioned = _Store(_note([]), [_definition(RUN, RUN_HASH)], []) + repair_mentions(unversioned.fetch_all, unversioned.execute_write) + ((_query, params),) = unversioned.writes + assert params is not None + assert params[cs.KEY_WRITE_ID] == "" + + +# Where the pass reads and where it writes are two statements, and an +# `annotate` from another server, or a sync of another checkout, can land in +# between. The statement itself is what refuses a stale result, so these run +# it on the embedded engine. +_MOVED = f"{P}.c.run" +_OTHER = f"{P}.c.other" +_NOTE_EDGES = """MATCH (g:Gloss {qualified_name: 'gloss:n'}) +OPTIONAL MATCH (g)-[:MENTIONS]->(m) +RETURN g.mention_qns AS qns, g.mentions_lost AS lost, + collect(m.qualified_name) AS edges""" + + +def _set_params( + write_id: str, + mention_qns: list[str], + attach_qns: list[str], + attach_keys: list[str], +) -> PropertyDict: + return { + cs.KEY_QN: "gloss:n", + cs.KEY_WRITE_ID: write_id, + cs.KEY_MENTION_QNS: mention_qns, + cs.KEY_MENTION_HASHES: [RUN_HASH] * len(mention_qns), + cs.KEY_MENTION_QUOTES: [""] * len(mention_qns), + cs.KEY_MENTIONS_LOST: None, + cs.KEY_ATTACH_QNS: attach_qns, + cs.KEY_ATTACH_KEYS: attach_keys, + cs.KEY_KEY_SEPARATOR: cs.GLOSS_MENTION_KEY_SEPARATOR, + cs.KEY_PROJECT_PREFIX: f"{P}.", + } + + +def _graph(conn: EmbeddedConnection, write_id: str) -> None: + conn.run( + "CREATE (:Function {qualified_name: $run, anchor_hash: $h}), " + "(:Function {qualified_name: $other, anchor_hash: $h})", + {"run": RUN, "other": _OTHER, "h": RUN_HASH}, + ) + conn.run( + "MATCH (r:Function {qualified_name: $run}) " + "CREATE (g:Gloss {qualified_name: 'gloss:n', write_id: $w, " + "mention_qns: [$run]})-[:MENTIONS]->(r)", + {"run": RUN, "w": write_id}, + ) + + +def test_the_set_statement_skips_a_note_rewritten_since_the_read() -> None: + with EmbeddedConnection(":memory:") as conn: + # An `annotate` replaced the note's nonce after the pass read "w-read". + _graph(conn, "w-newer") + conn.run( + cq.CYPHER_GLOSS_SET_MENTIONS, + _set_params("w-read", [_OTHER], [_OTHER], []), + ) + assert conn.run(_NOTE_EDGES).rows == [([RUN], None, [RUN])] + + +def test_the_set_statement_applies_while_the_note_is_as_read() -> None: + with EmbeddedConnection(":memory:") as conn: + _graph(conn, "w-read") + conn.run( + cq.CYPHER_GLOSS_SET_MENTIONS, + _set_params("w-read", [_OTHER], [_OTHER], []), + ) + assert conn.run(_NOTE_EDGES).rows == [([_OTHER], None, [_OTHER])] + + +def test_a_followed_target_replaced_since_the_read_changes_nothing() -> None: + # The pass followed the lost RUN to _MOVED by its hash; before the write, + # another sync replaced _MOVED with unrelated code. Saving the new name + # anyway would let the next pass read that code as an in-place edit and + # bind the mention to it, so the whole result is dropped instead. + with EmbeddedConnection(":memory:") as conn: + conn.run( + "CREATE (:Function {qualified_name: $moved, anchor_hash: 'replaced'})", + {"moved": _MOVED}, + ) + conn.run( + "CREATE (:Gloss {qualified_name: 'gloss:n', write_id: 'w-read', " + "mention_qns: [$run], mentions_lost: [$run]})", + {"run": RUN}, + ) + conn.run( + cq.CYPHER_GLOSS_SET_MENTIONS, + _set_params( + "w-read", + [_MOVED], + [], + [f"{_MOVED}{cs.GLOSS_MENTION_KEY_SEPARATOR}{RUN_HASH}"], + ), + ) + assert conn.run(_NOTE_EDGES).rows == [([RUN], [RUN], [])] + + +def test_a_followed_target_still_carrying_the_hash_is_bound() -> None: + with EmbeddedConnection(":memory:") as conn: + conn.run( + "CREATE (:Function {qualified_name: $moved, anchor_hash: $h})", + {"moved": _MOVED, "h": RUN_HASH}, + ) + conn.run( + "CREATE (:Gloss {qualified_name: 'gloss:n', write_id: 'w-read', " + "mention_qns: [$run], mentions_lost: [$run]})", + {"run": RUN}, + ) + conn.run( + cq.CYPHER_GLOSS_SET_MENTIONS, + _set_params( + "w-read", + [_MOVED], + [], + [f"{_MOVED}{cs.GLOSS_MENTION_KEY_SEPARATOR}{RUN_HASH}"], + ), + ) + assert conn.run(_NOTE_EDGES).rows == [([_MOVED], None, [_MOVED])] + + +# The note mentions `a` and `run`; `a` is deleted, then `run` is renamed to +# `a`. Both now place under `a`: `run`, followed there by its code, is live, +# and `a` is still lost. Neither may displace the other (bot review on PR +# #3329): the lost record keeps its own hash and quote, so `a`'s code is +# followed if it comes back. +_A = f"{P}.b.a" +_A_HASH = f"{cs.ANCHOR_HASH_VERSION}a" +_A_BACK = f"{P}.c.a" + + +def _two_mentions(**stored: object) -> ResultRow: + return { + **_note([]), + cs.KEY_MENTION_QNS: [_A, RUN], + cs.KEY_MENTION_HASHES: [_A_HASH, RUN_HASH], + cs.KEY_MENTION_QUOTES: ["q-a", ""], + **stored, + } + + +def _as_stored(note: ResultRow, params: PropertyDict) -> ResultRow: + """The note as the set statement leaves it, for the next pass to read.""" + keys = [str(k) for k in params[cs.KEY_ATTACH_KEYS]] # type: ignore[union-attr] + attached = [str(q) for q in params[cs.KEY_ATTACH_QNS]] # type: ignore[union-attr] + return { + **note, + cs.KEY_MENTION_QNS: params[cs.KEY_MENTION_QNS], + cs.KEY_MENTION_HASHES: params[cs.KEY_MENTION_HASHES], + cs.KEY_MENTION_QUOTES: params[cs.KEY_MENTION_QUOTES], + cs.KEY_MENTIONS_LOST: params[cs.KEY_MENTIONS_LOST], + cs.KEY_ATTACHED: attached + + [k.split(cs.GLOSS_MENTION_KEY_SEPARATOR)[0] for k in keys], + } + + +def _assert_both_kept_under_a(params: PropertyDict | None) -> None: + assert params is not None + # The live entry first, then the lost one, each with its own anchors. + assert params[cs.KEY_MENTION_QNS] == [_A, _A] + assert params[cs.KEY_MENTION_HASHES] == [RUN_HASH, _A_HASH] + assert params[cs.KEY_MENTION_QUOTES] == ["", "q-a"] + assert params[cs.KEY_MENTIONS_LOST] == [_A] + assert params[cs.KEY_ATTACH_QNS] == [] + assert params[cs.KEY_ATTACH_KEYS] == [ + f"{_A}{cs.GLOSS_MENTION_KEY_SEPARATOR}{RUN_HASH}" + ] + + +def test_a_lost_mention_keeps_its_record_when_another_takes_its_name() -> None: + note = _two_mentions(**{cs.KEY_MENTIONS_LOST: [_A]}) + store = _Store(note, [_definition(_A, RUN_HASH)], [_definition(_A, RUN_HASH)]) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + _assert_both_kept_under_a(params) + assert report.moved == ["gloss:n"] + assert report.lost == ["gloss:n"] + + # Read back as stored, the pair is stable: the next sync writes nothing. + assert params is not None + stored = _as_stored(note, params) + again = _Store(stored, [_definition(_A, RUN_HASH)], [_definition(_A, RUN_HASH)]) + repair_mentions(again.fetch_all, again.execute_write) + assert again.writes == [] + + # `a`'s code comes back in another module: the lost record follows it, + # and `run`'s code stays on `a`'s name. + back = _Store( + stored, + [_definition(_A, RUN_HASH)], + [_definition(_A, RUN_HASH), _definition(_A_BACK, _A_HASH)], + ) + report = repair_mentions(back.fetch_all, back.execute_write) + + ((_query, params),) = back.writes + assert params is not None + assert params[cs.KEY_MENTION_QNS] == [_A, _A_BACK] + assert params[cs.KEY_MENTION_HASHES] == [RUN_HASH, _A_HASH] + assert params[cs.KEY_MENTION_QUOTES] == ["", "q-a"] + assert params[cs.KEY_MENTIONS_LOST] is None + assert params[cs.KEY_ATTACH_QNS] == [_A] + assert params[cs.KEY_ATTACH_KEYS] == [ + f"{_A_BACK}{cs.GLOSS_MENTION_KEY_SEPARATOR}{_A_HASH}" + ] + assert report.moved == ["gloss:n"] + assert report.lost == [] + + +def test_a_mention_does_not_renew_onto_code_another_mention_was_followed_to() -> None: + # The same two changes in ONE sync: `a` is not yet lost, so its name, + # holding other code, would read as an edit in place. But that code is + # `run`'s, found by `run`'s own hash: `a` was replaced, not edited, and + # is lost with its own anchors, as if the changes came one sync apart. + store = _Store( + _two_mentions(), + [_definition(_A, RUN_HASH)], + [_definition(_A, RUN_HASH)], + ) + + report = repair_mentions(store.fetch_all, store.execute_write) + + ((_query, params),) = store.writes + _assert_both_kept_under_a(params) + assert report.lost == ["gloss:n"] + + +def test_a_note_from_before_repeated_names_reads_its_lost_mention_by_name() -> None: + # Before a name could hold a live and a lost entry, `mentions_lost` named + # each lost mention once, beside a unique `mention_qns`. Read that way it + # is unchanged: nothing moved, so nothing is written. + note = { + **_note([]), + cs.KEY_MENTION_QNS: [_A, RUN], + cs.KEY_MENTION_HASHES: [_A_HASH, RUN_HASH], + cs.KEY_MENTION_QUOTES: ["", ""], + cs.KEY_MENTIONS_LOST: [_A], + cs.KEY_ATTACHED: [RUN], + } + store = _Store(note, [_definition(RUN, RUN_HASH)], []) + + report = repair_mentions(store.fetch_all, store.execute_write) + + assert store.writes == [] + assert report.lost == ["gloss:n"] diff --git a/codebase_rag/tests/test_gloss_tools.py b/codebase_rag/tests/test_gloss_tools.py index 27471d769f..7a731680a1 100644 --- a/codebase_rag/tests/test_gloss_tools.py +++ b/codebase_rag/tests/test_gloss_tools.py @@ -19,6 +19,7 @@ from codebase_rag import constants as cs from codebase_rag import cypher_queries as cq from codebase_rag import gloss +from codebase_rag.gloss_repair import repair_mentions from codebase_rag.graph_updater import GraphUpdater from codebase_rag.mcp.tools import MCPToolsRegistry from codebase_rag.tools import tool_descriptions as td @@ -159,6 +160,31 @@ def names_target(props: PropertyDict) -> bool: for g, props in self.glosses.items() if in_project(props) and names_target(props) and g not in attached ] + elif query == cq.CYPHER_GLOSS_MENTION_ANCHORS: + # The mention pass (issue #3230): attached notes that mention + # anything, with the edges they have now. + attached = {g for g, _qn in self.annotates} + out = [ + {**row, cs.KEY_ATTACHED: row[cs.KEY_MENTIONS]} + for g, props in self.glosses.items() + if g in attached and props.get(cs.KEY_MENTION_QNS) + for row in [self._gloss_row(g)] + ] + elif query in ( + cq.CYPHER_DEFINITIONS_BY_QNS, + cq.CYPHER_DEFINITIONS_BY_ANCHOR_HASH, + ): + by_qn = query == cq.CYPHER_DEFINITIONS_BY_QNS + wanted = {str(v) for v in p[cs.KEY_QNS if by_qn else cs.KEY_HASHES]} # type: ignore[union-attr] + out = [ + { + cs.KEY_QUALIFIED_NAME: qn, + cs.KEY_ANCHOR_HASH: n.get(cs.KEY_ANCHOR_HASH), + } + for qn, n in self.nodes.items() + if (qn if by_qn else n.get(cs.KEY_ANCHOR_HASH)) in wanted + and (by_qn or qn.startswith(str(p[cs.KEY_PROJECT_PREFIX]))) + ] else: raise AssertionError(f"unexpected query: {query[:60]}") # Deliberately unsorted: the tools must sort. @@ -174,6 +200,9 @@ def execute_write(self, query: str, params: PropertyDict | None = None) -> None: """ p = params or {} self.writes.append((query, p)) + if query == cq.CYPHER_GLOSS_SET_MENTIONS: + self._set_mentions(p) + return if query != cq.CYPHER_GLOSS_WRITE: raise AssertionError(f"unexpected write: {query[:60]}") key = str(p[cs.KEY_QN]) @@ -222,6 +251,34 @@ def execute_write(self, query: str, params: PropertyDict | None = None) -> None: for qn in wanted: self.mentions.add((key, qn)) + def _set_mentions(self, p: PropertyDict) -> None: + """The mention pass's write, with the real statement's two gates: the + note still carries the nonce the pass read, and every followed key + still names a definition carrying that hash. Else nothing changes.""" + key = str(p[cs.KEY_QN]) + props = self.glosses[key] + if props.get(cs.KEY_WRITE_ID, "") != p[cs.KEY_WRITE_ID]: + return + separator = str(p[cs.KEY_KEY_SEPARATOR]) + targets = [str(q) for q in p[cs.KEY_ATTACH_QNS] if str(q) in self.nodes] # type: ignore[union-attr] + for attach_key in p[cs.KEY_ATTACH_KEYS]: # type: ignore[union-attr] + qn, _sep, anchor_hash = str(attach_key).partition(separator) + if self.nodes.get(qn, {}).get(cs.KEY_ANCHOR_HASH) != anchor_hash: + return + targets.append(qn) + for name in ( + cs.KEY_MENTION_QNS, + cs.KEY_MENTION_HASHES, + cs.KEY_MENTION_QUOTES, + cs.KEY_MENTIONS_LOST, + ): + if p[name] is None: + props.pop(name, None) + else: + props[name] = p[name] + self.mentions = {(g, qn) for g, qn in self.mentions if g != key} + self.mentions |= {(key, qn) for qn in targets} + def _write(graph: FakeGraph, target: str, **kw: Any) -> Any: args = {"body": BODY, "kind": cs.GlossKind.SAFETY_PRECONDITION.value} @@ -747,6 +804,50 @@ async def test_mcp_annotate_writes_and_glosses_reads_it_back(tmp_path: Path) -> assert [g["qualified_name"] for g in read["mentioning"]] == [row["qualified_name"]] +@pytest.mark.anyio +async def test_mcp_glosses_keeps_a_lost_mention_beside_one_that_took_its_name( + tmp_path: Path, +) -> None: + # The note mentions Store.get and validate. One sync deletes Store.get; a + # later one moves validate's code under Store.get's name. `glosses` shows + # the live mention and still lists the lost one, whose own anchors are + # kept, so its code is followed when it comes back (bot review, #3329). + graph = FakeGraph() + registry = _registry(graph, tmp_path) + row = await registry.annotate( + target=RUN, + body=BODY, + kind=cs.GlossKind.INVARIANT.value, + mentions=f"{STORE_GET},{VALIDATE}", + project=P, + ) + assert not _is_refusal(row) + + async def synced() -> tuple[list[str], list[str]]: + repair_mentions(graph.fetch_all, graph.execute_write, project_name=P) + read = await registry.glosses(target=RUN, project=P) + (note,) = read["annotating"] + return note["mentions"], note["mentions_lost"] + + del graph.nodes[STORE_GET] + assert await synced() == ([VALIDATE], [STORE_GET]) + + graph.nodes[STORE_GET] = { + **graph.nodes.pop(VALIDATE), + cs.KEY_QUALIFIED_NAME: STORE_GET, + } + assert await synced() == ([STORE_GET], [STORE_GET]) + stored = graph.glosses[row["qualified_name"]] + assert stored[cs.KEY_MENTION_QNS] == [STORE_GET, STORE_GET] + assert stored[cs.KEY_MENTION_HASHES] == ["fp-validate", "fp-store-get"] + + cache_get = f"{P}.app.Cache.get" + graph.nodes[cache_get] = _node( + "Method", cache_get, "app.py", 21, 23, anchor_hash="fp-store-get" + ) + assert await synced() == ([cache_get, STORE_GET], []) + + @pytest.mark.anyio async def test_mcp_stamps_the_commit_only_for_the_servers_own_checkout( tmp_path: Path, @@ -941,10 +1042,14 @@ def __init__(self) -> None: # When set, the store reports one note the name tier left unattached # (no comparable hash), so the repair tier has something to mark. self.unanchored: list[ResultRow] = [] + # When set, attached notes whose mentions the mention pass places. + self.mentioning: list[ResultRow] = [] def fetch_all(self, query: str, params: PropertyDict | None = None) -> list: if query == cq.CYPHER_UNANCHORED_GLOSSES: return list(self.unanchored) + if query == cq.CYPHER_GLOSS_MENTION_ANCHORS: + return list(self.mentioning) return [] def execute_write(self, query: str, params: PropertyDict | None = None) -> None: @@ -976,7 +1081,19 @@ def test_reanchoring_rebuilds_edges_from_the_notes_own_record(tmp_path: Path) -> cs.KEY_PROJECT: P, } ] - # The repair covers the syncing project's notes only (issue #3236). + # An attached note whose one mention has no edge now: the mention pass + # rebuilds its edges (here, marks the mention lost: nothing carries it). + store.mentioning = [ + { + cs.KEY_QUALIFIED_NAME: "gloss:attached", + cs.KEY_TARGET_QN: RUN, + cs.KEY_PROJECT: P, + cs.KEY_MENTION_QNS: [VALIDATE], + cs.KEY_ATTACHED: [], + } + ] + # The repair and the mention pass cover the syncing project's notes only + # (issue #3236). updater = GraphUpdater( ingestor=store, # type: ignore[arg-type] repo_path=tmp_path, @@ -987,16 +1104,16 @@ def test_reanchoring_rebuilds_edges_from_the_notes_own_record(tmp_path: Path) -> updater._reanchor_glosses() # Name tier first; then the repair tier places or marks what the name # tier left unattached (here: one note with no comparable hash, marked - # LOST); then the mentions restore, so a note that has just MOVED gets - # its MENTIONS edges back in the same run; then grading. + # LOST); then the mentions, so a note that has just MOVED gets its + # MENTIONS edges back in the same run; then grading. assert store.writes == [ cq.CYPHER_REANCHOR_GLOSSES, cq.CYPHER_GLOSS_MARK, - cq.CYPHER_REANCHOR_GLOSS_MENTIONS, + cq.CYPHER_GLOSS_SET_MENTIONS, cq.CYPHER_GRADE_GLOSS_ANCHORS, ] assert "g.target_qn" in cq.CYPHER_REANCHOR_GLOSSES - assert "g.mention_qns" in cq.CYPHER_REANCHOR_GLOSS_MENTIONS + assert "g.mention_qns" in cq.CYPHER_GLOSS_MENTION_ANCHORS # Only an unattached note is re-anchored; an attached one is left alone. assert "WHERE subjects = 0" in cq.CYPHER_REANCHOR_GLOSSES @@ -1167,6 +1284,36 @@ def test_a_note_records_its_subjects_text_quote_when_the_source_is_readable() -> assert calls == [(P, "app.py")] +def test_a_note_records_each_mentions_anchors_beside_its_name() -> None: + # A mention carries the anchors the subject does, in `mention_qns` order, + # so the repair can follow it when it is renamed or moved (issue #3230). + # An empty string stands for "none": a definition without a hash, or a + # file the reader cannot supply. + from codebase_rag.gloss_anchor import text_anchor + + graph = FakeGraph() + _calls, read = _reader({"app.py": APP_PY}) + row = _write( + graph, STORE_GET, mentions=f"{VALIDATE},{RUN},{UTIL_GET}", read_source=read + ) + assert not _is_refusal(row) + stored = graph.glosses[row["qualified_name"]] + parsed = read(P, "app.py") + assert parsed is not None + run_anchor = text_anchor(parsed, "run", 3, 8) + assert run_anchor is not None + assert stored[cs.KEY_MENTION_QNS] == [RUN, UTIL_GET, VALIDATE] + assert stored[cs.KEY_MENTION_HASHES] == ["fp-run", "", "fp-validate"] + assert stored[cs.KEY_MENTION_QUOTES] == [run_anchor.quote, "", ""] + + +def test_the_write_statement_records_the_mention_anchors() -> None: + # A fresh write is a fresh reading: no mention starts out lost. + assert "g.mention_hashes = $mention_hashes" in cq.CYPHER_GLOSS_WRITE + assert "g.mention_quotes = $mention_quotes" in cq.CYPHER_GLOSS_WRITE + assert "g.mentions_lost = null" in cq.CYPHER_GLOSS_WRITE + + def test_a_note_without_a_reader_or_without_the_file_carries_no_quote() -> None: graph = FakeGraph() row = _write(graph, RUN) diff --git a/codebase_rag/tests/test_protobuf_node_property_parity.py b/codebase_rag/tests/test_protobuf_node_property_parity.py index de38c9469b..f0f0a3048e 100644 --- a/codebase_rag/tests/test_protobuf_node_property_parity.py +++ b/codebase_rag/tests/test_protobuf_node_property_parity.py @@ -57,8 +57,21 @@ # redundant on the wire (issue #1808). `candidate_qns` is the repair # pass's record of where an AMBIGUOUS note could belong, recomputed from # the graph after every sync, so exporting it would only freeze a verdict - # the next run replaces (stage four of #1808). - "Gloss": frozenset({"write_id", "mention_qns", "candidate_qns"}), + # the next run replaces (stage four of #1808). `mention_hashes` and + # `mention_quotes` are each mention's anchors, positional beside + # `mention_qns` and meaningless without it; `mentions_lost` is the + # mention pass's verdict, recomputed after every sync like + # `candidate_qns` (issue #3230). + "Gloss": frozenset( + { + "write_id", + "mention_qns", + "candidate_qns", + "mention_hashes", + "mention_quotes", + "mentions_lost", + } + ), "Package": frozenset({"absolute_path"}), "Folder": frozenset({"absolute_path"}), "File": frozenset({"absolute_path"}), diff --git a/codebase_rag/tools/tool_descriptions.py b/codebase_rag/tools/tool_descriptions.py index 1ed423bcf8..e0d3b6ea48 100644 --- a/codebase_rag/tools/tool_descriptions.py +++ b/codebase_rag/tools/tool_descriptions.py @@ -319,6 +319,11 @@ class AgenticToolName(StrEnum): "hash to compare) and is LOST if its name disappears (a module has no span " "to quote); a class, interface, enum, type or union carries a hash like a " "function and is graded and repaired the same way. " + "A note's `mentions` are anchored the same way: a mentioned definition " + "that is renamed or moved is followed by its hash or quote, one that " + "cannot be placed is listed in `mentions_lost` with no edge (still, once " + "another mention has taken its name), and a new definition that reuses " + "its name does not inherit it. " "A target that no longer resolves returns the error plus `orphaned`: the " "unattached notes written against that name. " + _MCP_DETERMINISTIC_NOTE ) diff --git a/codebase_rag/types_defs.py b/codebase_rag/types_defs.py index af093975a7..077f660044 100644 --- a/codebase_rag/types_defs.py +++ b/codebase_rag/types_defs.py @@ -1473,7 +1473,9 @@ class RelationshipSchema(NamedTuple): "target_qn: string, target_hash: string?, anchor_quote: string?, " "anchor_prefix: string?, anchor_suffix: string?, anchor_state: string, " "moved_from: string?, candidate_qns: list[string]?, project: string?, " - "write_id: string?, mention_qns: list[string]?}" + "write_id: string?, mention_qns: list[string]?, " + "mention_hashes: list[string]?, mention_quotes: list[string]?, " + "mentions_lost: list[string]?}" ) NODE_SCHEMAS: tuple[NodeSchema, ...] = ( diff --git a/docs/architecture/graph-schema.md b/docs/architecture/graph-schema.md index 8af0324c91..c79f219dca 100644 --- a/docs/architecture/graph-schema.md +++ b/docs/architecture/graph-schema.md @@ -34,7 +34,7 @@ A label marked opt-in belongs to a [capture group](#capture-groups) that a defau | Pattern (opt-in: [`findings`](#capture-groups)) | `{qualified_name: string, name: string, message: string, start_line: int, end_line: int, path: string, snippet: string?}` | | CodeSmell (opt-in: [`findings`](#capture-groups)) | `{qualified_name: string, name: string, message: string, start_line: int, end_line: int, path: string, snippet: string?}` | | SecurityIssue (opt-in: [`findings`](#capture-groups)) | `{qualified_name: string, name: string, message: string, start_line: int, end_line: int, path: string, snippet: string?}` | -| Gloss (opt-in: [`glosses`](#capture-groups)) | `{qualified_name: string, kind: string, status: string, body: string, created_by: string, created_at: string, commit_sha: string?, target_qn: string, target_hash: string?, anchor_quote: string?, anchor_prefix: string?, anchor_suffix: string?, anchor_state: string, moved_from: string?, candidate_qns: list[string]?, project: string?, write_id: string?, mention_qns: list[string]?}` | +| Gloss (opt-in: [`glosses`](#capture-groups)) | `{qualified_name: string, kind: string, status: string, body: string, created_by: string, created_at: string, commit_sha: string?, target_qn: string, target_hash: string?, anchor_quote: string?, anchor_prefix: string?, anchor_suffix: string?, anchor_state: string, moved_from: string?, candidate_qns: list[string]?, project: string?, write_id: string?, mention_qns: list[string]?, mention_hashes: list[string]?, mention_quotes: list[string]?, mentions_lost: list[string]?}` | | Parameter (opt-in: [`parameters`](#capture-groups)) | `{qualified_name: string, name: string, index: int, path: string, absolute_path: string, start_line: int?, start_col: int?, type_name: string?, is_variadic: boolean?, has_default: boolean?}` | | Field (opt-in: [`fields`](#capture-groups)) | `{qualified_name: string, name: string, path: string, absolute_path: string, start_line: int?, start_col: int?, type_name: string?, modifiers: list[string]?, is_static: boolean?, docstring: string?}` | | EnumVariant (opt-in: [`enum_variants`](#capture-groups)) | `{qualified_name: string, name: string, path: string, absolute_path: string, start_line: int?, start_col: int?, index: int, value: string?, docstring: string?}` | diff --git a/docs/guide/mcp-server.md b/docs/guide/mcp-server.md index c30bd163e1..688a1f3c90 100644 --- a/docs/guide/mcp-server.md +++ b/docs/guide/mcp-server.md @@ -94,7 +94,7 @@ cgr daemon up | `endpoint_callers` | Call sites, in any indexed project, that reach one endpoint: `target` is the handler's qualified name or the endpoint identity (`GET /users/{id}`), whichever parameter syntax the route declares (`:id`, ``); a concrete request (`GET /users/42`) matches the routes it is served by. A target that names no endpoint the project exposes is refused with the close identities and handlers, so an empty list means nothing reaches it. Each row names the caller, its file, the URL it accesses and the direction (READS_FROM or WRITES_TO); an RPC or dispatch caller reaches the endpoint directly, so its `url` is the endpoint identity. Literal URLs resolve; dynamic ones do not and are absent here. Deterministic: fixed graph queries, no LLM, same graph gives the same JSON. Use this instead of query_code_graph whenever you know the exact name or location. | | `remote_dependencies` | Every network access a project makes, one row per call site and URL, with the endpoint, handler and project it resolves to when one is indexed; a row with no endpoint is a dependency the graph cannot place (a dynamic URL, or a service not indexed). Deterministic: fixed graph queries, no LLM, same graph gives the same JSON. Use this instead of query_code_graph whenever you know the exact name or location. | | `annotate` | Attach a durable note (a Gloss) to one definition in the graph, never to the source file. `target` names the definition the way `resolve` does, optionally with `#getter`, `#setter` or `#deleter` after it to name one member of a property (a `#` opening a name, as in a JS private member, is part of the name); a name matching several definitions is refused with the candidates, so pass a qualified name to disambiguate. `kind` types the claim (invariant, mirrors, platform-conditional, safety-precondition); `mentions` lists other definitions the note refers to, each becoming a MENTIONS edge. Write the WHY that is not in the code ("safe because validate() runs first"), not a restatement of what the code does. Returns the stored gloss, or an error and nothing written. | -| `glosses` | Notes (Glosses) about one definition: `annotating` are filed on it, `mentioning` are filed on other definitions and refer to it. `target` is resolved as `annotate` resolves it. Each note's `anchor_state` is graded after each sync of its project: EXACT when its definition's content hash still equals the one recorded at writing, STALE when it differs (EXACT again on revert), MOVED when the name is gone and exactly one definition in the project carries the recorded hash (the note follows it; `moved_from` keeps the old name), AMBIGUOUS when several do (`candidate_qns` lists them). When no definition carries the hash (a rename changes it), the note's text-quote anchor is tried: a digest of the definition's own text with its name masked, so exactly one definition in the project whose body still matches is followed as MOVED (then graded STALE, since the signature changed), several with the same body are told apart by the lines around them or listed as AMBIGUOUS, and none is LOST. The quote is recorded only when the note was written against this server's own checkout. A note on a module, or one written before hashes were recorded, is not graded while attached (there is no hash to compare) and is LOST if its name disappears (a module has no span to quote); a class, interface, enum, type or union carries a hash like a function and is graded and repaired the same way. A target that no longer resolves returns the error plus `orphaned`: the unattached notes written against that name. Deterministic: fixed graph queries, no LLM, same graph gives the same JSON. Use this instead of query_code_graph whenever you know the exact name or location. | +| `glosses` | Notes (Glosses) about one definition: `annotating` are filed on it, `mentioning` are filed on other definitions and refer to it. `target` is resolved as `annotate` resolves it. Each note's `anchor_state` is graded after each sync of its project: EXACT when its definition's content hash still equals the one recorded at writing, STALE when it differs (EXACT again on revert), MOVED when the name is gone and exactly one definition in the project carries the recorded hash (the note follows it; `moved_from` keeps the old name), AMBIGUOUS when several do (`candidate_qns` lists them). When no definition carries the hash (a rename changes it), the note's text-quote anchor is tried: a digest of the definition's own text with its name masked, so exactly one definition in the project whose body still matches is followed as MOVED (then graded STALE, since the signature changed), several with the same body are told apart by the lines around them or listed as AMBIGUOUS, and none is LOST. The quote is recorded only when the note was written against this server's own checkout. A note on a module, or one written before hashes were recorded, is not graded while attached (there is no hash to compare) and is LOST if its name disappears (a module has no span to quote); a class, interface, enum, type or union carries a hash like a function and is graded and repaired the same way. A note's `mentions` are anchored the same way: a mentioned definition that is renamed or moved is followed by its hash or quote, one that cannot be placed is listed in `mentions_lost` with no edge (still, once another mention has taken its name), and a new definition that reuses its name does not inherit it. A target that no longer resolves returns the error plus `orphaned`: the unattached notes written against that name. Deterministic: fixed graph queries, no LLM, same graph gives the same JSON. Use this instead of query_code_graph whenever you know the exact name or location. | | `rename` | Rename a function, method, class or other definition everywhere the graph knows it is referenced: the definition, every call, reference and instantiation site, import statements (aliases kept), overrides in both directions, and Python `__all__` entries. Refuses when any site was resolved heuristically, by overload fan-out, or only by a trace, unless `allow_heuristic` is true. It also refuses when the source names the symbol where the graph has no site (a call the index missed), listing those occurrences as `unplanned`; `allow_heuristic` rewrites them as guessed sites. A class referenced by an inheritance or type annotation edge that carries no rewrite location refuses regardless of `allow_heuristic`. Every rewritten file must still parse; otherwise nothing is written. Set `dry_run` to see the plan and diff without touching the tree. Applied edits are recorded for `cgr edits undo`, and Markdown mentions of the old name are listed for a human to review. | | `context` | A graph-ranked minimal context slice for a task, within a token budget. `target` is a qualified name, a bare name, `path:line`, or free text (matched by embedding similarity when the semantic extra is installed). Returns the target's source, its direct callers' call lines, its direct callees' signatures, the types it accepts and returns, the tests that reach it, and the documentation sections whose file links to it, ranked by graph distance (trace hotness and similarity break ties) and trimmed to `budget_tokens`. Every piece says why it is included. Use this before reading whole files. | | `query_code_graph` | Prefer the deterministic tools (resolve, definition, callers, callees, implementors, overrides, importers, tests_reaching) when you know the exact name or location: they run fixed queries with no LLM. Use this for open-ended questions. Query the codebase knowledge graph using natural language. Ask questions like 'What functions call UserService.create_user?' or 'Show me all classes that implement the Repository interface'. Pass `project` to restrict results to one indexed project; use list_projects for the available names. Omit it to search them all. The scope is enforced on the results before they are capped, so it holds regardless of the query generated and the result limit is spent on rows from the requested project. | diff --git a/evals/cgr_graph.py b/evals/cgr_graph.py index 7b94f4c334..d7073b942e 100644 --- a/evals/cgr_graph.py +++ b/evals/cgr_graph.py @@ -167,11 +167,16 @@ def execute_write(self, query: str, params: PropertyDict | None = None) -> None: # `CYPHER_QUERY_PROJECT_NODE_IDS` reads the internal ids a retired project's # vectors are keyed by (issue #2412). The double has no internal ids and no # vector store, so "no vectors to delete" is its true answer. +# +# `CYPHER_GLOSS_MENTION_ANCHORS` opens the mention pass that follows the +# repair (issue #3230). With no Gloss nodes there is no note that mentions +# anything, so the lookups it would lead to are never issued. _NOT_MODELLED: frozenset[str] = frozenset( { cs.CYPHER_QUERY_EMBEDDINGS, cs.CYPHER_QUERY_PROJECT_NODE_IDS, cq.CYPHER_UNANCHORED_GLOSSES, + cq.CYPHER_GLOSS_MENTION_ANCHORS, } ) _MODULE_QN_LABELS = frozenset(