diff --git a/scripts/check-review-threads.sh b/scripts/check-review-threads.sh index c2164700d..f9968cc43 100755 --- a/scripts/check-review-threads.sh +++ b/scripts/check-review-threads.sh @@ -621,7 +621,11 @@ TRIGGER_RE='\A[[:space:]]*@(codex[[:space:]]+(security[[:space:]]+)?review|coder # sign-offs, then "**Reviewed commit:** ``", then a folded "About Codex in GitHub" # block. Since 2026-10-15 the comment opens with "", an # HTML comment on its own line, which verdict_lines drops like any hidden comment -# (strip_hidden_comments below). A CodeRabbit review that produces no comments replies +# (strip_hidden_comments below). That receipt can also name no commit: on #1282 +# (2026-10-25) it was the phrase and sign-off alone, and the summary comment's row read "โ€”" +# where the commit goes. A receipt in that shape is a verdict that covers nothing, like +# CodeRabbit's reply below; the marker must be its first line, because the bare phrase +# without it is not a shape Codex posts (finding 21). A CodeRabbit review that produces no comments replies # "Full review finished." (or "Review finished." for an incremental one) folded under # "Action performed". Those are verdicts, not findings: they need no disposition. Codex's # also covers the commit it names; CodeRabbit's names nothing, and its coverage comes @@ -860,11 +864,14 @@ def verdict_lines: | map(select(length > 0 and . != cr_note))) end; def codex_line_re: "^Codex Review: Didn.t find any major issues\\.?( Bravo\\.| Keep it up!| Keep them coming!| Hooray!| Swish!| You.re on a roll\\.| :\\+1:| :rocket:| :tada:| Nice work[.!]| Great job[.!]| Breezy!| ๐Ÿ‘| Delightful!| What shall we delve into next\\?| More of your lovely PRs please\\.| Can.t wait for the next one!| Chef.s kiss\\.| Already looking forward to the next diff\\.| Another round soon, please!)?$"; def reviewed_re: "^\\*\\*Reviewed commit:\\*\\* `(?[0-9a-f]{7,40})`$"; +def codex_receipt_re: "\\A\n"; def is_verdict: - verdict_lines as $l + (. // "") as $b + | verdict_lines as $l | ($l == ["
", "โœ… Action performed", "Full review finished.", "
"]) or ($l == ["
", "โœ… Action performed", "Review finished.", "
"]) - or (($l | length) == 2 and ($l[0] | test(codex_line_re)) and ($l[1] | test(reviewed_re))); + or (($l | length) == 2 and ($l[0] | test(codex_line_re)) and ($l[1] | test(reviewed_re))) + or (($l | length) == 1 and ($l[0] | test(codex_line_re)) and ($b | test(codex_receipt_re))); def verdict_sha: verdict_lines as $l | if ($l | length) == 2 and ($l[1] | test(reviewed_re)) then ($l[1] | capture(reviewed_re) | .sha) else "" end; @@ -939,15 +946,25 @@ def cr_notice_heading_re: "^## (Review limit reached|Review skipped|Reviews paus # configuration as "- **Key**: value" (each key with the values it has shown, so a list # item holding other text is not a run setting) and links the limits docs at a new path (#1199); # every "Review limit reached" notice on ejc3/fcvm PRs on 2026-10-23 matched, except #811, -# which also carries a Finishing Touches block. A line outside it makes the comment claimable, which is what it was before the -# exemption existed: the cost of an unlisted shape is one disposition, and the cost of -# accepting one is a finding nobody has to answer. +# which also carries a Finishing Touches block. From 2026-10-23 each folded summary opens +# with an icon, a of two SVGs from the CodeRabbit icon set (cr_icon_summary_re, one +# shape per icon and the name it labels). Of the 56 notices without a walkthrough on ejc3/fcvm +# PRs updated from 2026-10-01 to 2026-10-25, the 31 with icons read as findings until these +# shapes were listed. A line outside this list makes the comment claimable, which is what it +# was before the exemption existed: the cost of an unlisted shape is one disposition, and +# the cost of accepting one is a finding nobody has to answer. +def cr_icon_summary_re($icon; $name): + "^\"\" \($name)$"; def cr_notice_payload_res: [ "^$" , "^$" , "^$" , "^(โš™๏ธ Run configuration|๐Ÿ“ฅ Commits|๐Ÿ“’ Files selected for processing: [0-9]+|โ›” Files ignored due to path filters: [0-9]+|View limit details)$" , "^(โš™๏ธ Run configuration|๐Ÿ“ฅ Commits|๐Ÿ“’ Files selected for processing \\([0-9]+\\)|โ›” Files ignored due to path filters \\([0-9]+\\)|How can I continue\\?|How do review limits work\\?|Review details|View limit details)$" + , cr_icon_summary_re("cog"; "Run configuration") + , cr_icon_summary_re("git-commit-horizontal"; "Commits") + , cr_icon_summary_re("file-plus-corner"; "Files selected for processing: [0-9]+") + , cr_icon_summary_re("file-minus-corner"; "Files ignored due to path filters: [0-9]+") , "^\\* `[^`]+`( is excluded by `[^`]+`)?$" , "^- \\[ \\] ๐Ÿ” Trigger review$" , "^\\*\\*(Configuration used|Review profile|Plan)\\*\\*: [A-Za-z0-9 +]+$" diff --git a/scripts/test-check-review-threads.sh b/scripts/test-check-review-threads.sh index 85a52aa42..316ab1760 100755 --- a/scripts/test-check-review-threads.sh +++ b/scripts/test-check-review-threads.sh @@ -2126,6 +2126,152 @@ run_case "a finding inside the Autofix block is claimable" \ run_case "a finding written as a run-configuration item is claimable" \ "$(wrap9 "$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(jq -n --arg b "$CR_1199" '$b | sub("> - \\*\\*Commit\\*\\*: Unavailable"; "> - **Commit**: P1 this drops the last row")')" 2026-01-02T01:00:00Z)")" 1 "carry no disposition" +echo "== finding 51: the notice and receipt wordings the bots posted from 2026-10-23 ==" +# CodeRabbit's review-limit notice now puts an icon (a of two SVGs) in front of +# each folded summary: Run configuration, Commits, Files selected for processing, Files +# ignored due to path filters. No icon line was a listed shape, so from 2026-10-23 every +# "Review limit reached" notice on ejc3/fcvm (31 of them by 2026-10-25) read as an +# unanswered finding. CR_1285 is #1285's notice with its org, run and checkbox ids replaced +# and the range moved onto this harness's base and head. +IFS= read -r -d '' CR_1285_RAW <<'CR1285_EOF' + + + + + +> [!WARNING] +> ## Review limit reached +> +> You've used all free OSS reviews for now. Wait for the free limit to reset to keep reviewing this public repository. +> +> **Next included review available in 54 minutes.** +> +> [Check out review usage here](https://app.coderabbit.ai/dashboard/review-capacity?orgId=00000000-0000-4000-8000-000000000001). +> +>
+> View limit details +>
+>
+> +> **Limit details:** Youโ€™ve used the included review currently available. +> +> [Learn how review limits work](https://docs.coderabbit.ai/management/rate-limits). +> +> **Review configuration:** +> +>
+> Run configuration +>
+>
+> +> - **Source mode**: Auto +> - **Repository**: Unavailable +> - **Branch**: Unavailable +> - **File**: No configuration file from this repository applied +> - **Commit**: Unavailable +> - **Layers**: defaults +> - **Review profile**: CHILL +> - **Plan**: Advanced +> - **Run ID**: `00000000-0000-4000-8000-000000000002` +> +>
+>
+>
+> +>
+> Commits +>
+>
+> +> Reviewing files that changed from the base of the PR and between @BASE@ and @HEAD@. +> +>
+>
+>
+> +>
+> Files selected for processing: 2 +>
+>
+> +> * `.claude/CLAUDE.md` +> * `src/uffd/working_set.rs` +> +>
+>
+>
+> +>
+>
+>
+ + + + +- [ ] Autofix ยท Keep fixing CodeRabbit findings and required CI, and resolving merge conflicts + + + + + +Thanks for using [CodeRabbit](https://coderabbit.ai?utm_source=oss&utm_medium=github&utm_campaign=ejc3/fcvm&utm_content=1285)! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out. + +
+โค๏ธ Share +
+
+ +- [X](https://twitter.com/intent/tweet?text=I%20just%20used%20%40coderabbitai%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20the%20proprietary%20code.%20Check%20it%20out%3A&url=https%3A//coderabbit.ai) +- [Mastodon](https://mastodon.social/share?text=I%20just%20used%20%40coderabbitai%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20the%20proprietary%20code.%20Check%20it%20out%3A%20https%3A%2F%2Fcoderabbit.ai) +- [Reddit](https://www.reddit.com/submit?title=Great%20tool%20for%20code%20review%20-%20CodeRabbit&text=I%20just%20used%20CodeRabbit%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20proprietary%20code.%20Check%20it%20out%3A%20https%3A//coderabbit.ai) +- [LinkedIn](https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fcoderabbit.ai&mini=true&title=Great%20tool%20for%20code%20review%20-%20CodeRabbit&summary=I%20just%20used%20CodeRabbit%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20proprietary%20code) + +
+
+
+ + +Comment `@coderabbitai help` to get the list of available commands. + + +CR1285_EOF +CR_1285=${CR_1285_RAW//@BASE@/$BASE40}; CR_1285=${CR_1285//@HEAD@/$HEAD40} +cr1285() { jq -n --arg b "$CR_1285" --arg x "${1:-}" '$b + $x'; } +run_case "the #1285 review-limit notice with icon summaries needs no disposition" \ + "$(wrap9 "$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(cr1285)" 2026-01-02T01:00:00Z)")" 0 "CLEAR" +# Guards: a notice covers no head; the same body with anything added, or with a finding +# written where an icon summary's name goes, is a claim; and an icon from anywhere but +# CodeRabbit's icon set is not a listed shape. +run_case "the #1285 review-limit notice quoting the head's range covers no head" \ + "$(wrap7 "[$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(cr1285)" 2026-01-02T01:00:00Z)]")" 1 "UNREVIEWED HEAD" +run_case "the #1285 notice with a finding appended is claimable" \ + "$(wrap9 "$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(cr1285 $'\n\nP1: this drops the last row')" 2026-01-02T01:00:00Z)")" 1 "carry no disposition" +run_case "a finding written as an icon summary's name is claimable" \ + "$(wrap9 "$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(jq -n --arg b "$CR_1285" '$b | sub(" Commits"; " P1: this drops the last row")')" 2026-01-02T01:00:00Z)")" 1 "carry no disposition" +run_case "an icon summary whose icon is not CodeRabbit's is claimable" \ + "$(wrap9 "$(cmt coderabbitai Bot 2026-01-01T00:00:00Z "$(jq -n --arg b "$CR_1285" '$b | sub("icons-v1/cog-dark\\.svg"; "icons-v1/cog-dark.svg?P1-drops-the-last-row")')" 2026-01-02T01:00:00Z)")" 1 "carry no disposition" +# Codex's receipt can also name no commit: on #1282 (2026-10-25) the receipt was the phrase +# and its sign-off with no reviewed-commit line, and the summary comment's row for it read +# "โ€”" where the commit goes. It reports no findings, so it needs no disposition, and it names +# nothing, so it covers nothing. The receipt marker is what tells it from the bare phrase +# finding 21 rejects; receipt_nocommit is the #1282 body. +receipt_nocommit() { jq -n --argjson b "$(receipt_body "$1" 0000000)" '$b | sub("\n\n\\*\\*Reviewed commit:\\*\\* `0000000`"; "")'; } +run_case "the #1282 receipt that names no commit needs no disposition" \ + "$(wrap9 "$(cmt "$CODEX" Bot 2026-01-02T01:00:00Z "$(receipt_nocommit " Can't wait for the next one!")")")" \ + 0 "CLEAR" +run_case "the #1282 receipt that names no commit covers no head" \ + "$(wrap6 "[$(cmt "$CODEX" Bot 2026-01-02T01:00:00Z "$(receipt_nocommit " Can't wait for the next one!")")]")" \ + 1 "UNREVIEWED HEAD" +run_case "the #1282 receipt without its marker is claimable" \ + "$(wrap9 "$(cmt "$CODEX" Bot 2026-01-02T01:00:00Z "$(jq -n --argjson b "$(receipt_nocommit " Can't wait for the next one!")" '$b | sub("\n\n"; "")')")")" \ + 1 "carry no disposition" +run_case "the #1282 receipt with a finding appended is claimable" \ + "$(wrap9 "$(cmt "$CODEX" Bot 2026-01-02T01:00:00Z "$(receipt_nocommit $' Can\'t wait for the next one!\n\nP1: the lease is dropped before the rename')")")" \ + 1 "carry no disposition" +run_case "a human posting the #1282 receipt is claimable" \ + "$(wrap9 "$(cmt helpful-human User 2026-01-02T01:00:00Z "$(receipt_nocommit " Can't wait for the next one!")")")" \ + 1 "carry no disposition" + echo echo "passed=$pass failed=$fail" [ "$fail" -eq 0 ] diff --git a/tests/test_log_scan.rs b/tests/test_log_scan.rs index 3f95e0dc1..8d92e2a60 100644 --- a/tests/test_log_scan.rs +++ b/tests/test_log_scan.rs @@ -1172,6 +1172,126 @@ fn review_gate_treats_the_coderabbit_1199_notice_with_additions_as_a_claim() { } } +/// CodeRabbit's "Review limit reached" summary notice as posted on #1285 (2026-10-25), with +/// its org, run and checkbox ids replaced. JSON string literal. +const CR_1285_NOTICE: &str = r#""\n\n\n\n\n> [!WARNING]\n> ## Review limit reached\n> \n> You've used all free OSS reviews for now. Wait for the free limit to reset to keep reviewing this public repository.\n> \n> **Next included review available in 54 minutes.**\n> \n> [Check out review usage here](https://app.coderabbit.ai/dashboard/review-capacity?orgId=00000000-0000-4000-8000-000000000001).\n> \n>
\n> View limit details\n>
\n>
\n> \n> **Limit details:** Youโ€™ve used the included review currently available.\n> \n> [Learn how review limits work](https://docs.coderabbit.ai/management/rate-limits).\n> \n> **Review configuration:**\n> \n>
\n> \"\" Run configuration\n>
\n>
\n> \n> - **Source mode**: Auto\n> - **Repository**: Unavailable\n> - **Branch**: Unavailable\n> - **File**: No configuration file from this repository applied\n> - **Commit**: Unavailable\n> - **Layers**: defaults\n> - **Review profile**: CHILL\n> - **Plan**: Advanced\n> - **Run ID**: `00000000-0000-4000-8000-000000000002`\n> \n>
\n>
\n>
\n> \n>
\n> \"\" Commits\n>
\n>
\n> \n> Reviewing files that changed from the base of the PR and between c89d4fa27958c7650314e7ba9f2c212656ecc142 and 2c2089a9ffe884eed3d19ef62272121c07ada5e6.\n> \n>
\n>
\n>
\n> \n>
\n> \"\" Files selected for processing: 2\n>
\n>
\n> \n> * `.claude/CLAUDE.md`\n> * `src/uffd/working_set.rs`\n> \n>
\n>
\n>
\n> \n>
\n>
\n>
\n\n\n\n\n- [ ] Autofix ยท Keep fixing CodeRabbit findings and required CI, and resolving merge conflicts\n\n\n\n\n\nThanks for using [CodeRabbit](https://coderabbit.ai?utm_source=oss&utm_medium=github&utm_campaign=ejc3/fcvm&utm_content=1285)! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out.\n\n
\nโค๏ธ Share\n
\n
\n\n- [X](https://twitter.com/intent/tweet?text=I%20just%20used%20%40coderabbitai%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20the%20proprietary%20code.%20Check%20it%20out%3A&url=https%3A//coderabbit.ai)\n- [Mastodon](https://mastodon.social/share?text=I%20just%20used%20%40coderabbitai%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20the%20proprietary%20code.%20Check%20it%20out%3A%20https%3A%2F%2Fcoderabbit.ai)\n- [Reddit](https://www.reddit.com/submit?title=Great%20tool%20for%20code%20review%20-%20CodeRabbit&text=I%20just%20used%20CodeRabbit%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20proprietary%20code.%20Check%20it%20out%3A%20https%3A//coderabbit.ai)\n- [LinkedIn](https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fcoderabbit.ai&mini=true&title=Great%20tool%20for%20code%20review%20-%20CodeRabbit&summary=I%20just%20used%20CodeRabbit%20for%20my%20code%20review%2C%20and%20it%27s%20fantastic%21%20It%27s%20free%20for%20OSS%20and%20offers%20a%20free%20trial%20for%20proprietary%20code)\n\n
\n
\n
\n\n\nComment `@coderabbitai help` to get the list of available commands.\n\n""#; + +/// CodeRabbit's review-limit notice with icons in its folded summaries needs no disposition +/// and covers no head. +/// +/// From 2026-10-23 the notice puts a of two SVG icons in front of each folded +/// summary (Run configuration, Commits, Files selected for processing, Files ignored due to +/// path filters). No such line was a listed shape, so every "Review limit reached" notice +/// posted on ejc3/fcvm from then on read as an unanswered finding. +#[test] +fn review_gate_treats_the_coderabbit_1285_icon_notice_as_a_notice() { + require_jq(); + let (code, out) = run_gate_on_bot_comment("cr1285", "coderabbitai", CR_1285_NOTICE); + assert!( + !out.contains("carry no disposition") && !out.contains("UNANSWERED"), + "the #1285 review-limit notice is not a finding.\n{out}" + ); + assert_eq!(code, Some(1), "{out}"); + assert!( + out.contains("UNREVIEWED HEAD") && !out.contains("HEAD COVERED"), + "a notice covers no head.\n{out}" + ); +} + +/// The #1285 notice with anything added, or with a finding written where an icon summary's +/// name goes, is a claim. +/// +/// Green on the gate before and after the icon shapes were listed; the second case goes red +/// if an icon summary accepts any name. +#[test] +fn review_gate_treats_the_coderabbit_1285_notice_with_additions_as_a_claim() { + require_jq(); + let appended = format!( + "{}\\n\\nP1: this drops the last row\"", + &CR_1285_NOTICE[..CR_1285_NOTICE.len() - 1] + ); + let as_icon_name = CR_1285_NOTICE.replace( + " Commits", + " P1: this drops the last row", + ); + for (case, body) in [ + ("a finding appended", &appended), + ("a finding as an icon summary's name", &as_icon_name), + ] { + assert_ne!( + body.as_str(), + CR_1285_NOTICE, + "{case}: the fixture edit did not apply" + ); + let (code, out) = run_gate_on_bot_comment("cr1285-claim", "coderabbitai", body); + assert_eq!(code, Some(1), "{case}\n{out}"); + assert!( + out.contains("carry no disposition"), + "the #1285 notice with {case} is a claim.\n{out}" + ); + } +} + +/// Codex's receipt as posted on #1282 (2026-10-25): the marker, the phrase and its sign-off, +/// no reviewed-commit line, and the About Codex block. JSON string literal. +const CODEX_1282_RECEIPT: &str = r#""\n\nCodex Review: Didn't find any major issues. Can't wait for the next one!\n\n
โ„น๏ธ About Codex in GitHub\n
\n\n[Your team has set up Codex to review pull requests in this repo](https://chatgpt.com/codex/cloud/settings/general). Reviews are triggered when you\n- Open a pull request for review\n- Mark a draft as ready\n- Comment \"@codex review\".\n\nIf Codex has suggestions, it will comment; otherwise it will react with ๐Ÿ‘.\n\n\n\n\nCodex can also answer questions or update the PR. Try commenting \"@codex address that feedback\".\n \n
""#; + +/// A Codex receipt that names no commit reports no findings and covers nothing. +/// +/// On #1282 Codex posted its receipt without a reviewed-commit line, and its summary +/// comment's row read "โ€”" where the commit goes. The gate counted the receipt as an +/// unanswered finding. +#[test] +fn review_gate_treats_a_codex_receipt_naming_no_commit_as_a_verdict_without_coverage() { + require_jq(); + let (code, out) = run_gate_on_bot_comment( + "receipt-nocommit", + "chatgpt-codex-connector", + CODEX_1282_RECEIPT, + ); + assert!( + !out.contains("carry no disposition") && !out.contains("UNANSWERED"), + "Codex's receipt naming no commit is not a finding.\n{out}" + ); + assert_eq!(code, Some(1), "{out}"); + assert!( + out.contains("UNREVIEWED HEAD") && !out.contains("HEAD COVERED"), + "a receipt naming no commit covers no head.\n{out}" + ); +} + +/// The #1282 receipt without its marker, or with a finding after the sign-off, is a claim. +/// +/// The marker is what separates the receipt from the bare phrase, which stays claimable. +/// Green on the gate before and after; the first case goes red if the marker is not +/// required. +#[test] +fn review_gate_treats_a_codex_receipt_naming_no_commit_with_changes_as_a_claim() { + require_jq(); + let unmarked = CODEX_1282_RECEIPT.replace(r"\n\n", ""); + let appended = CODEX_1282_RECEIPT.replace( + "Can't wait for the next one!", + r"Can't wait for the next one!\n\nP1: the lease is dropped before the rename", + ); + for (case, body) in [ + ("without its marker", &unmarked), + ("with a finding appended", &appended), + ] { + assert_ne!( + body.as_str(), + CODEX_1282_RECEIPT, + "{case}: the fixture edit did not apply" + ); + let (code, out) = + run_gate_on_bot_comment("receipt-nocommit-claim", "chatgpt-codex-connector", body); + assert_eq!(code, Some(1), "{case}\n{out}"); + assert!( + out.contains("carry no disposition"), + "the #1282 receipt {case} is a claim.\n{out}" + ); + } +} + /// A bot saying that its review did NOT run is not a finding, and covers no head. /// /// Codex out of quota posts "You have reached your Codex usage limits for code reviews" as