Watcher: quote arXiv phrases and fix hint ids — 27 of 39 recent issues were noise

scripts/watch.js

@@ -36,9 +36,15 @@ const OWASP_REPOS = [
   'OWASP/www-project-top-10-for-genai-data-security',
 ];
 
+// Multi-word terms MUST be quoted (%22…%22). Unquoted, `ti:LLM+security` is
+// `ti:LLM` plus a bare `security` matched in any field, so the query admitted
+// nearly every cs.CR paper: a live check on 2026-09-14 returned 3 AI-related
+// titles in 30, and 27 of 39 watcher issues opened since 2026-08-31 were off
+// topic. Quoted and title-scoped, the same check returned 30 in 30.
 const ARXIV_URL =
   'https://export.arxiv.org/api/query?' +
-  'search_query=cat:cs.CR+AND+(ti:prompt+injection+OR+ti:jailbreak+OR+ti:LLM+security+OR+ti:agentic+AI+OR+ti:RAG+poisoning)' +
+  'search_query=cat:cs.CR+AND+(ti:%22prompt+injection%22+OR+ti:jailbreak+OR+ti:LLM+OR+ti:%22large+language+model%22' +
+  '+OR+ti:agentic+OR+ti:%22AI+agent%22+OR+ti:RAG+OR+ti:%22retrieval-augmented%22+OR+ti:MCP)' +
   '&sortBy=submittedDate&sortOrder=descending&max_results=20';
 
 const NVD_KEYWORDS = [
@@ -253,18 +259,31 @@ async function watchOwasp(state, sinceOverride) {
 /**
  * Map keywords in title/abstract to OWASP entry identifiers.
  */
+//
+// These are triage hints on a watcher issue, not mappings. Two defects fixed:
+// patterns lacked word boundaries (`rag` matched "storage" and "average"), and
+// the ids predated the 2026 renumbering (supply chain pointed at LLM10, now
+// Improper Output Handling). Each rule names the entry title it targets, and
+// scripts/watch.test.mjs checks that the id still carries that title.
+const ARXIV_HINT_RULES = [
+  { re: /\bprompt injection\b|\bjailbreak/,            ids: { LLM01: 'Prompt Injection' } },
+  { re: /\b(data|model|training|rag|knowledge) poisoning\b|\bpoisoning attack/,
+    ids: { LLM05: 'Data and Model Poisoning', DSGAI04: 'Data Model and Artifact Poisoning' } },
+  { re: /\bmemory\b.*\bagent|\bagent\b.*\bmemory\b|\bcontext poisoning\b/,
+    ids: { ASI06: 'Memory and Context Poisoning' } },
+  { re: /\btool (misuse|abuse|call)|\bagentic\b|\bllm agents?\b|\bai agents?\b/,
+    ids: { ASI01: 'Agent Goal Hijack', ASI02: 'Tool Misuse and Exploitation' } },
+  { re: /\bexfiltrat|\bdata leak|\bleakage\b/,         ids: { LLM02: 'Sensitive Information Disclosure', DSGAI01: 'Sensitive Data Leakage' } },
+  { re: /\bhallucinat|\bmisinformation\b/,             ids: { LLM07: 'Misinformation' } },
+  { re: /\bsupply chain\b/,                            ids: { LLM04: 'Supply Chain', ASI04: 'Agentic Supply Chain' } },
+];
+
 function mapArxivToOwasp(text) {
   const lower = text.toLowerCase();
   const mappings = [];
-
-  if (/prompt injection|jailbreak/.test(lower))            mappings.push('LLM01');
-  if (/data poisoning|rag/.test(lower))                    mappings.push('DSGAI04', 'LLM04');
-  if (/\bmemory\b|persistence/.test(lower))                mappings.push('ASI06');
-  if (/\btool\b|\bagent\b|agentic/.test(lower))            mappings.push('ASI01', 'ASI02');
-  if (/exfiltration|\bleak\b/.test(lower))                 mappings.push('LLM02', 'DSGAI01');
-  if (/hallucination|misinformation/.test(lower))          mappings.push('LLM07');
-  if (/supply chain/.test(lower))                          mappings.push('LLM10', 'ASI04');
-
+  for (const rule of ARXIV_HINT_RULES) {
+    if (rule.re.test(lower)) mappings.push(...Object.keys(rule.ids));
+  }
   return [...new Set(mappings)];
 }
 
@@ -783,7 +802,11 @@ async function main() {
   process.exit(0);
 }
 
-main().catch(err => {
-  console.error('Fatal error:', err);
-  process.exit(0);
-});
+if (require.main === module) {
+  main().catch(err => {
+    console.error('Fatal error:', err);
+    process.exit(0);
+  });
+}
+
+module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp };

scripts/watch.test.mjs

@@ -0,0 +1,76 @@
+/**
+ * watch.test.mjs — the weekly watcher must not flood the tracker with noise.
+ *
+ * Between 2026-08-31 and 2026-09-14 the arXiv watcher opened 39 issues, 27 of
+ * them outside GenAI security (CBDC settlement, 6G NOMA, MIMO lattices),
+ * because unquoted multi-word terms turned the query into "any cs.CR paper".
+ * Its OWASP hints also pointed at pre-2026 entry ids. These tests pin both.
+ */
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const require = createRequire(import.meta.url);
+const { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp } = require(path.join(ROOT, 'scripts', 'watch.js'));
+
+/**
+ * Field terms followed by a bare word, e.g. `ti:LLM security`. arXiv reads the
+ * bare word as a separate all-fields term, which is the defect. `+` in the URL
+ * is a space once parsed, and quoted phrases are one token.
+ */
+function unquotedPhrases(url) {
+  const query = new URL(url).searchParams.get('search_query');
+  const tokens = query.replace(/"[^"]*"/g, '"q"').replace(/[()]/g, ' $& ').split(/\s+/).filter(Boolean);
+  const bad = [];
+  tokens.forEach((tok, i) => {
+    const next = tokens[i + 1];
+    if (/^(ti|abs|all|au|cat):/.test(tok) && next && !/^(AND|OR|ANDNOT|\))$/.test(next)) bad.push(`${tok} ${next}`);
+  });
+  return bad;
+}
+
+test('every multi-word arXiv search term is quoted', () => {
+  assert.deepEqual(unquotedPhrases(ARXIV_URL), []);
+});
+
+test('the phrase check catches the query that caused the flood', () => {
+  const before = 'https://export.arxiv.org/api/query?search_query=cat:cs.CR+AND+'
+    + '(ti:prompt+injection+OR+ti:jailbreak+OR+ti:LLM+security+OR+ti:agentic+AI+OR+ti:RAG+poisoning)';
+  assert.deepEqual(unquotedPhrases(before),
+    ['ti:prompt injection', 'ti:LLM security', 'ti:agentic AI', 'ti:RAG poisoning']);
+});
+
+test('the arXiv query stays inside cs.CR', () => {
+  assert.match(decodeURIComponent(ARXIV_URL), /search_query=cat:cs\.CR\+AND\+\(/);
+});
+
+test('hints do not fire on substrings of unrelated words', () => {
+  assert.deepEqual(mapArxivToOwasp('Storage-average leverage in fragmented disks'), []);
+  assert.deepEqual(mapArxivToOwasp('Toolchain hardening for embedded bootloaders'), []);
+});
+
+test('hints use the 2026 entry ids', () => {
+  assert.deepEqual(mapArxivToOwasp('A supply chain attack on model hubs').sort(), ['ASI04', 'LLM04']);
+  assert.ok(mapArxivToOwasp('Knowledge poisoning of RAG pipelines').includes('LLM05'));
+  assert.ok(mapArxivToOwasp('Indirect prompt injection in email agents').includes('LLM01'));
+});
+
+test('every hint id exists and still carries the title its rule names', () => {
+  const titles = Object.fromEntries(
+    fs.readdirSync(path.join(ROOT, 'data', 'entries')).filter((f) => f.endsWith('.json'))
+      .map((f) => JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'entries', f), 'utf8')))
+      .map((e) => [e.id, e.name]),
+  );
+  const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+  for (const rule of ARXIV_HINT_RULES) {
+    for (const [id, title] of Object.entries(rule.ids)) {
+      assert.ok(titles[id], `${id} is not an entry`);
+      assert.equal(norm(titles[id]), norm(title), `${id} is now "${titles[id]}", rule expects "${title}"`);
+    }
+  }
+});