Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 206 additions & 0 deletions codebase_rag/tests/test_change_signature_regressions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
from __future__ import annotations

from pathlib import Path

import pytest

from codebase_rag.editing import SignatureRefused, change_signature
from codebase_rag.graph_updater import GraphUpdater
from codebase_rag.tests.change_signature_helpers import (
PROJECT,
_index,
_qn,
_write,
repo_fixture,
)
from evals.cgr_graph import _StatefulIngestor


@pytest.fixture
def repo(temp_repo: Path):
return repo_fixture(temp_repo)


# --- review findings: rewrites that committed a broken program ------------------


def test_two_parameters_cannot_share_one_source(
repo: tuple[Path, _StatefulIngestor, GraphUpdater],
) -> None:
"""A source holds one value per site, so it cannot feed two parameters.

`_map_arguments` consumes the value: the first mapping took it and every
later one read as omitted, so the definition gained a parameter no caller
passed. The edit reported success and the callers raised `TypeError`.
"""
root, store, updater = repo
with pytest.raises(SignatureRefused) as excinfo:
change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.util.helper"),
["left@0", "right@0", "b@1"],
reingest=updater.reingest,
)
assert "left, right" in str(excinfo.value)
assert (root / "pkg/util.py").read_text().startswith("def helper(a: int")


def test_distinct_sources_are_not_refused_as_duplicates(temp_repo: Path) -> None:
"""The guard must not catch an ordinary rename of every parameter.

Its own fixture, with no defaulted parameter: the shared one has
`b: str = 'x'` and a `helper(2)` site relying on it, which a rename leaves
possibly-missing and the postcondition rightly rolls back. That rollback is
the operation working correctly, so the shared fixture would be testing
Comment on lines +53 to +56
something other than this guard.
"""
root = temp_repo / PROJECT
root.mkdir()
_write(root, "pkg/__init__.py", "")
_write(
root, "pkg/util.py", "def helper(a: int, b: str) -> str:\n return b * a\n"
)
_write(
root,
"pkg/app.py",
"from pkg.util import helper\n\n\ndef run():\n return helper(2, 'x')\n",
)
store, updater = _index(root)
report = change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.util.helper"),
["first@0", "second@1"],
reingest=updater.reingest,
)
assert report.applied, report.message
assert "def helper(first: int, second: str)" in (root / "pkg/util.py").read_text()


def test_several_new_parameters_are_not_duplicate_sources(temp_repo: Path) -> None:
"""Two parameters with literals share no source: both have none."""
root = temp_repo / PROJECT
root.mkdir()
_write(root, "pkg/__init__.py", "")
_write(
root, "pkg/util.py", "def helper(a: int, b: str) -> str:\n return b * a\n"
)
_write(
root,
"pkg/app.py",
"from pkg.util import helper\n\n\ndef run():\n return helper(2, 'x')\n",
)
store, updater = _index(root)
report = change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.util.helper"),
["a@0", "b@1", "m:int=1", "n:int=2"],
reingest=updater.reingest,
)
assert report.applied, report.message
assert report.new_params == ("a", "b", "m", "n")


def test_override_keyword_callers_use_the_override_s_own_names(
temp_repo: Path,
) -> None:
"""A caller binds by the name the method it calls declares.

`old_names` was derived once from the selected definition and reused for
every hierarchy member's call sites, so an override spelling a parameter
differently had its keyword callers read as unknown keywords and left
unrewritten -- against a definition that had been rewritten. The committed
program then raised `TypeError` for the stale keyword.

The existing hierarchy test spells the parameter `scale` in both classes,
which is why this went unseen: the names have to differ for the bug to
show.
"""
root = temp_repo / PROJECT
root.mkdir()
_write(root, "pkg/__init__.py", "")
_write(
root,
"pkg/shapes.py",
"class Base:\n def area(self, scale):\n return scale\n\n\n"
"class Square(Base):\n def area(self, factor):\n return factor * 4\n"
"\n\ndef use_square():\n return Square().area(factor=2)\n",
)
store, updater = _index(root)
report = change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.shapes.Base.area"),
["size@0"],
reingest=updater.reingest,
)
assert report.applied, report.message
text = (root / "pkg/shapes.py").read_text()
# Both definitions are rewritten, so no caller may still say `factor`.
assert "factor=2" not in text, text

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Broken calls can pass

This assertion only verifies that factor=2 disappeared. It also passes if the call is removed or changed to an invalid keyword, so the regression test can succeed while the rewritten program is no longer callable. Assert the rewritten signature and Square().area(size=2), and preferably execute the rewritten fixture to check its result. This is a non-blocking test-coverage concern; the practical cost is that future regressions can silently ship.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Artifacts

Command output from the check

  • Executed the prepared uv pytest command against the changed regression module; all six tests passed, showing the current weak assertion is accepted.

Evidence from the check

  • Authored and executed the focused Python harness that invokes the real rewrite flow and tests two broken rewritten-call mutants, providing the reproducible validation method.

Command output from the check

  • Captured the command and complete contents of the exact mutation harness that was executed, making the oracle test reproducible.

Command output from the check

  • Executed the mutation harness: line 146 accepted the invalid-keyword and removed-call results while their runtime behavior was broken, confirming the test-oracle weakness.

View artifacts

T-Rex Ran code and verified through T-Rex



def test_positional_only_marker_survives_a_rewrite(temp_repo: Path) -> None:
"""`/` is load-bearing on rebuild even though it is not a parameter.

It was skipped on parse and never re-emitted, so `def target(a, b, /, c)`
came back as `def target(a, b, c)` and started accepting `target(a=1, b=2,
c=3)` -- a call the original rejects. Dropping it widens the callable
contract silently.
"""
root = temp_repo / PROJECT
root.mkdir()
_write(root, "pkg/__init__.py", "")
_write(
root,
"pkg/only.py",
"def target(a, b, /, c):\n return a + b + c\n\n\n"
"def call():\n return target(1, 2, 3)\n",
)
store, updater = _index(root)
report = change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.only.target"),
["a@0", "b@1", "c@2", "d:int=4"],
reingest=updater.reingest,
)
assert report.applied, report.message
assert "/" in (root / "pkg/only.py").read_text().split("\n")[0]


def test_a_rewrite_across_the_positional_only_boundary_is_refused(
temp_repo: Path,
) -> None:
"""Reordering across `/` changes which arguments may be named.

That is a question this operation is not asked to answer, so it refuses
rather than emitting a signature whose contract differs from the one the
caller requested.
"""
root = temp_repo / PROJECT
root.mkdir()
_write(root, "pkg/__init__.py", "")
_write(
root,
"pkg/only.py",
"def target(a, b, /, c):\n return a + b + c\n\n\n"
"def call():\n return target(1, 2, 3)\n",
)
store, updater = _index(root)
with pytest.raises(SignatureRefused):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Refusal reason is unchecked

The bare pytest.raises(SignatureRefused) accepts any refusal, including one caused by an unrelated malformed parameter specification. Capture the exception and assert the positional-only-boundary diagnostic so this test specifically proves that moving the boundary is rejected. This is a non-blocking test-coverage concern; the practical cost is that the intended guard could regress without this test detecting it.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Artifacts

Evidence from the check

  • Authored and executed script constructs the same temporary indexed repository as the changed test and runs the intended and malformed-spec requests, demonstrating what the bare assertion accepts.

Command output from the check

  • Executed repro with the exact changed-test request; it raises SignatureRefused with the positional-only boundary message, establishing the intended behavior.

Command output from the check

  • Executed repro adds malformed `invalid@`; it still raises SignatureRefused but with a parameter-spec message, proving the bare assertion lacks diagnostic specificity.

Command output from the check

  • Executed the changed test node under pytest; it passed while asserting only the broad exception class, confirming the weak assertion is active.

View artifacts

T-Rex Ran code and verified through T-Rex

change_signature(
root,
store.fetch_all,
PROJECT,
_qn("pkg.only.target"),
["b@1", "a@0", "c@2"],
reingest=updater.reingest,
)
Loading