diff --git a/scripts/watch.js b/scripts/watch.js index 17daf60..24715e3 100644 --- a/scripts/watch.js +++ b/scripts/watch.js @@ -36,9 +36,15 @@ const OWASP_REPOS = [ 'OWASP/www-project-top-10-for-genai-data-security', ]; +// Multi-word terms MUST be quoted (%22…%22). Unquoted, `ti:LLM+security` is +// `ti:LLM` plus a bare `security` matched in any field, so the query admitted +// nearly every cs.CR paper: a live check on 2026-09-14 returned 3 AI-related +// titles in 30, and 27 of 39 watcher issues opened since 2026-08-31 were off +// topic. Quoted and title-scoped, the same check returned 30 in 30. const ARXIV_URL = 'https://export.arxiv.org/api/query?' + - 'search_query=cat:cs.CR+AND+(ti:prompt+injection+OR+ti:jailbreak+OR+ti:LLM+security+OR+ti:agentic+AI+OR+ti:RAG+poisoning)' + + 'search_query=cat:cs.CR+AND+(ti:%22prompt+injection%22+OR+ti:jailbreak+OR+ti:LLM+OR+ti:%22large+language+model%22' + + '+OR+ti:agentic+OR+ti:%22AI+agent%22+OR+ti:RAG+OR+ti:%22retrieval-augmented%22+OR+ti:MCP)' + '&sortBy=submittedDate&sortOrder=descending&max_results=20'; const NVD_KEYWORDS = [ @@ -253,18 +259,31 @@ async function watchOwasp(state, sinceOverride) { /** * Map keywords in title/abstract to OWASP entry identifiers. */ +// +// These are triage hints on a watcher issue, not mappings. Two defects fixed: +// patterns lacked word boundaries (`rag` matched "storage" and "average"), and +// the ids predated the 2026 renumbering (supply chain pointed at LLM10, now +// Improper Output Handling). Each rule names the entry title it targets, and +// scripts/watch.test.mjs checks that the id still carries that title. +const ARXIV_HINT_RULES = [ + { re: /\bprompt injection\b|\bjailbreak/, ids: { LLM01: 'Prompt Injection' } }, + { re: /\b(data|model|training|rag|knowledge) poisoning\b|\bpoisoning attack/, + ids: { LLM05: 'Data and Model Poisoning', DSGAI04: 'Data Model and Artifact Poisoning' } }, + { re: /\bmemory\b.*\bagent|\bagent\b.*\bmemory\b|\bcontext poisoning\b/, + ids: { ASI06: 'Memory and Context Poisoning' } }, + { re: /\btool (misuse|abuse|call)|\bagentic\b|\bllm agents?\b|\bai agents?\b/, + ids: { ASI01: 'Agent Goal Hijack', ASI02: 'Tool Misuse and Exploitation' } }, + { re: /\bexfiltrat|\bdata leak|\bleakage\b/, ids: { LLM02: 'Sensitive Information Disclosure', DSGAI01: 'Sensitive Data Leakage' } }, + { re: /\bhallucinat|\bmisinformation\b/, ids: { LLM07: 'Misinformation' } }, + { re: /\bsupply chain\b/, ids: { LLM04: 'Supply Chain', ASI04: 'Agentic Supply Chain' } }, +]; + function mapArxivToOwasp(text) { const lower = text.toLowerCase(); const mappings = []; - - if (/prompt injection|jailbreak/.test(lower)) mappings.push('LLM01'); - if (/data poisoning|rag/.test(lower)) mappings.push('DSGAI04', 'LLM04'); - if (/\bmemory\b|persistence/.test(lower)) mappings.push('ASI06'); - if (/\btool\b|\bagent\b|agentic/.test(lower)) mappings.push('ASI01', 'ASI02'); - if (/exfiltration|\bleak\b/.test(lower)) mappings.push('LLM02', 'DSGAI01'); - if (/hallucination|misinformation/.test(lower)) mappings.push('LLM07'); - if (/supply chain/.test(lower)) mappings.push('LLM10', 'ASI04'); - + for (const rule of ARXIV_HINT_RULES) { + if (rule.re.test(lower)) mappings.push(...Object.keys(rule.ids)); + } return [...new Set(mappings)]; } @@ -783,7 +802,11 @@ async function main() { process.exit(0); } -main().catch(err => { - console.error('Fatal error:', err); - process.exit(0); -}); +if (require.main === module) { + main().catch(err => { + console.error('Fatal error:', err); + process.exit(0); + }); +} + +module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp }; diff --git a/scripts/watch.test.mjs b/scripts/watch.test.mjs new file mode 100644 index 0000000..9984921 --- /dev/null +++ b/scripts/watch.test.mjs @@ -0,0 +1,76 @@ +/** + * watch.test.mjs — the weekly watcher must not flood the tracker with noise. + * + * Between 2026-08-31 and 2026-09-14 the arXiv watcher opened 39 issues, 27 of + * them outside GenAI security (CBDC settlement, 6G NOMA, MIMO lattices), + * because unquoted multi-word terms turned the query into "any cs.CR paper". + * Its OWASP hints also pointed at pre-2026 entry ids. These tests pin both. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp } = require(path.join(ROOT, 'scripts', 'watch.js')); + +/** + * Field terms followed by a bare word, e.g. `ti:LLM security`. arXiv reads the + * bare word as a separate all-fields term, which is the defect. `+` in the URL + * is a space once parsed, and quoted phrases are one token. + */ +function unquotedPhrases(url) { + const query = new URL(url).searchParams.get('search_query'); + const tokens = query.replace(/"[^"]*"/g, '"q"').replace(/[()]/g, ' $& ').split(/\s+/).filter(Boolean); + const bad = []; + tokens.forEach((tok, i) => { + const next = tokens[i + 1]; + if (/^(ti|abs|all|au|cat):/.test(tok) && next && !/^(AND|OR|ANDNOT|\))$/.test(next)) bad.push(`${tok} ${next}`); + }); + return bad; +} + +test('every multi-word arXiv search term is quoted', () => { + assert.deepEqual(unquotedPhrases(ARXIV_URL), []); +}); + +test('the phrase check catches the query that caused the flood', () => { + const before = 'https://export.arxiv.org/api/query?search_query=cat:cs.CR+AND+' + + '(ti:prompt+injection+OR+ti:jailbreak+OR+ti:LLM+security+OR+ti:agentic+AI+OR+ti:RAG+poisoning)'; + assert.deepEqual(unquotedPhrases(before), + ['ti:prompt injection', 'ti:LLM security', 'ti:agentic AI', 'ti:RAG poisoning']); +}); + +test('the arXiv query stays inside cs.CR', () => { + assert.match(decodeURIComponent(ARXIV_URL), /search_query=cat:cs\.CR\+AND\+\(/); +}); + +test('hints do not fire on substrings of unrelated words', () => { + assert.deepEqual(mapArxivToOwasp('Storage-average leverage in fragmented disks'), []); + assert.deepEqual(mapArxivToOwasp('Toolchain hardening for embedded bootloaders'), []); +}); + +test('hints use the 2026 entry ids', () => { + assert.deepEqual(mapArxivToOwasp('A supply chain attack on model hubs').sort(), ['ASI04', 'LLM04']); + assert.ok(mapArxivToOwasp('Knowledge poisoning of RAG pipelines').includes('LLM05')); + assert.ok(mapArxivToOwasp('Indirect prompt injection in email agents').includes('LLM01')); +}); + +test('every hint id exists and still carries the title its rule names', () => { + const titles = Object.fromEntries( + fs.readdirSync(path.join(ROOT, 'data', 'entries')).filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'entries', f), 'utf8'))) + .map((e) => [e.id, e.name]), + ); + const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); + for (const rule of ARXIV_HINT_RULES) { + for (const [id, title] of Object.entries(rule.ids)) { + assert.ok(titles[id], `${id} is not an entry`); + assert.equal(norm(titles[id]), norm(title), `${id} is now "${titles[id]}", rule expects "${title}"`); + } + } +});