test(graph-rs): add the fuzz input decoder and the Cypher and JSON fuzz targets

rust/cgr_graph_rs/fuzz/.gitignore

@@ -0,0 +1,4 @@
+/target
+/corpus
+/artifacts
+/coverage

rust/cgr_graph_rs/fuzz/fuzz_targets/cypher_parse.rs

@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: MIT
+//! Fuzz target: the Cypher lexer and parser on any query text.
+//!
+//! The text is untrusted: an LLM, an MCP client or a user writes the
+//! queries the embedded engine parses. A syntax error and a deferral to the
+//! Python parser (`Error::Defer`) are both expected outcomes; a panic, a
+//! stack overflow or a hang is a bug. The input is the query, up to 64 KiB,
+//! with each bad UTF-8 sequence replaced.
+#![no_main]
+
+use cgr_graph_rs::cypher::error::SyntaxErrorKind;
+use cgr_graph_rs::cypher::lexer::{tokenize, Token, TokenKind};
+use cgr_graph_rs::cypher::vocab::{BACKTICK, BLOCK_COMMENT_OPEN, PARAM_SIGIL, QUOTES};
+use cgr_graph_rs::cypher::{parse, DeferReason, Error};
+use cgr_graph_rs_fuzz::require;
+use libfuzzer_sys::fuzz_target;
+
+const MAX_QUERY_BYTES: usize = 64 * 1024;
+
+fuzz_target!(|data: &[u8]| {
+    let text = String::from_utf8_lossy(&data[..data.len().min(MAX_QUERY_BYTES)]);
+    let chars: Vec<char> = text.chars().collect();
+    let lexed = tokenize(&chars);
+    match &lexed {
+        Ok(tokens) => check_tokens(tokens, &chars),
+        Err(error) => check_lexer_error(error, &chars),
+    }
+    let parsed = parse(&text);
+    match (&lexed, &parsed) {
+        // `parse` lexes first, so a lexer error is the parse error,
+        // unchanged.
+        // Mutant: in parser.rs `parse`, map tokenize's error to an
+        // UnterminatedString at 0. Input: "~" aborts.
+        (Err(lexed), _) => require!(
+            parsed.as_ref().err() == Some(lexed),
+            "tokenize failed with {lexed:?} but parse returned {parsed:?}"
+        ),
+        // Once the text lexes, only the grammar can reject it: an error
+        // the lexer makes cannot come from the parser, and the position is
+        // a character offset into the query, or its end.
+        // Mutant: in parser.rs `Parser::error`, report `tok.pos + 1`.
+        // Input: the empty query aborts (position 1 > 0).
+        // Mutant: in parser.rs, defer deep nesting as `NonAscii`. Input:
+        // "RETURN " then 40 "(", "1" and 40 ")" aborts.
+        (Ok(_), Err(error)) => {
+            require!(
+                !is_lexer_error(error),
+                "parse invented a lexer error {error:?}"
+            );
+            require!(
+                position(error) <= chars.len(),
+                "{error:?} is past the end ({})",
+                chars.len()
+            );
+            if let Error::Syntax(syntax) = error {
+                // The message the Python binding raises, so it runs here.
+                let _ = syntax.to_string();
+            }
+        }
+        (Ok(_), Ok(_)) => {}
+    }
+});
+
+fn position(error: &Error) -> usize {
+    match error {
+        Error::Syntax(e) => e.pos,
+        Error::Defer(d) => d.pos,
+    }
+}
+
+fn is_lexer_error(error: &Error) -> bool {
+    match error {
+        Error::Syntax(e) => matches!(
+            e.kind,
+            SyntaxErrorKind::UnterminatedString
+                | SyntaxErrorKind::UnterminatedComment
+                | SyntaxErrorKind::UnterminatedIdent
+                | SyntaxErrorKind::UnexpectedChar(_)
+        ),
+        Error::Defer(d) => matches!(
+            d.reason,
+            DeferReason::NonAscii | DeferReason::BigInt | DeferReason::Escape
+        ),
+    }
+}
+
+/// The token list covers the text in order: tokens do not overlap, each
+/// spans at least one character, and one `Eof` closes the list at the end.
+/// Each token's span holds what its kind says it does.
+///
+/// Mutant: in lexer.rs `tokenize`, end an identifier at `j + 1`. Input:
+/// "d" aborts. Mutant: give a parameter the value `text[i..j]`, its `$`
+/// included. Input: "$" aborts. Mutant: put Eof at `n + 1`. Input: the
+/// empty query aborts.
+fn check_tokens(tokens: &[Token], chars: &[char]) {
+    let n = chars.len();
+    let (eof, body) = tokens.split_last().expect("tokenize returns at least Eof");
+    require!(
+        eof.kind == TokenKind::Eof && eof.pos == n && eof.end == n,
+        "the last token is {eof:?}, not Eof at {n}"
+    );
+    let mut previous_end = 0;
+    for token in body {
+        require!(
+            token.kind != TokenKind::Eof,
+            "Eof before the end: {token:?}"
+        );
+        require!(
+            previous_end <= token.pos && token.pos < token.end && token.end <= n,
+            "{token:?} overlaps the previous token (ending {previous_end}) or leaves the text ({n})"
+        );
+        previous_end = token.end;
+        let span = &chars[token.pos..token.end];
+        let source: String = span.iter().collect();
+        let first = span[0];
+        let last = span[span.len() - 1];
+        let ok = match token.kind {
+            // A bare word is its own name, and starts with a letter or `_`.
+            TokenKind::Ident(_) => {
+                token.value == source && (first.is_ascii_alphabetic() || first == '_')
+            }
+            TokenKind::QuotedIdent => first == BACKTICK && last == BACKTICK && span.len() >= 2,
+            TokenKind::Str => QUOTES.contains(&first) && last == first && span.len() >= 2,
+            TokenKind::Param => first == PARAM_SIGIL && token.value == source[1..],
+            // An integer token is ASCII digits naming the value it carries.
+            TokenKind::Int(v) => {
+                span.iter().all(char::is_ascii_digit) && source.parse::<i64>() == Ok(v)
+            }
+            TokenKind::Float(_) => first.is_ascii_digit() || first == '.',
+            TokenKind::Punct(_) => span.len() <= 2,
+            TokenKind::Eof => false,
+        };
+        require!(ok, "{token:?} does not match its source {source:?}");
+        // Python's `Token.text`, which the parser uses for projections.
+        let _ = token.text(chars);
+    }
+}
+
+/// A lexer error points at the character that caused it, as the Python
+/// lexer's `CypherSyntaxError` offset does, and a deferral at the
+/// character Python must judge.
+///
+/// Mutant: in lexer.rs `tokenize`, report UnexpectedChar at `i + 1`.
+/// Input: "~" aborts. Mutant: in lexer.rs `unicode_escape`, decline at
+/// `at + 1`. Input: "'\u" aborts.
+fn check_lexer_error(error: &Error, chars: &[char]) {
+    let at = |i: usize| chars.get(i).copied();
+    let ok = match error {
+        Error::Syntax(e) => match &e.kind {
+            // The lexer only reaches this for an ASCII character: every
+            // other one is whitespace or deferred before.
+            SyntaxErrorKind::UnexpectedChar(c) => at(e.pos) == Some(*c) && c.is_ascii(),
+            SyntaxErrorKind::UnterminatedString => at(e.pos).is_some_and(|c| QUOTES.contains(&c)),
+            SyntaxErrorKind::UnterminatedIdent => at(e.pos) == Some(BACKTICK),
+            SyntaxErrorKind::UnterminatedComment => {
+                chars.get(e.pos..e.pos + 2) == Some(&BLOCK_COMMENT_OPEN[..])
+            }
+            _ => false,
+        },
+        Error::Defer(d) => match d.reason {
+            // Never for ASCII text, and never for whitespace, which the
+            // lexer classifies exactly.
+            DeferReason::NonAscii => at(d.pos).is_some_and(|c| !c.is_ascii() && !c.is_whitespace()),
+            DeferReason::BigInt => at(d.pos).is_some_and(|c| c.is_ascii_digit()),
+            DeferReason::Escape => at(d.pos) == Some('\\') && at(d.pos + 1) == Some('u'),
+            DeferReason::Nesting => false,
+        },
+    };
+    require!(
+        ok,
+        "tokenize returned {error:?}, which does not point at its cause"
+    );
+}

rust/cgr_graph_rs/fuzz/fuzz_targets/json_string.rs

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: MIT
+//! Fuzz target: `json_text::push_string`, which must write a string exactly
+//! as Python's `json.dumps` writes it with its default `ensure_ascii`.
+//!
+//! The bulk writer stores this text as node and relationship properties,
+//! where names come from the repository being indexed, so any string can
+//! reach it. The input is the string, up to 64 KiB, with each bad UTF-8
+//! sequence replaced.
+#![no_main]
+
+use cgr_graph_rs::json_text::push_string;
+use cgr_graph_rs_fuzz::require;
+use libfuzzer_sys::fuzz_target;
+
+const MAX_STRING_BYTES: usize = 64 * 1024;
+
+fuzz_target!(|data: &[u8]| {
+    let s = String::from_utf8_lossy(&data[..data.len().min(MAX_STRING_BYTES)]);
+    let mut out = String::from("[");
+    push_string(&mut out, &s);
+    // It appends, and keeps what was there.
+    // Mutant: in json_text.rs, replace `out` with the opening quote in
+    // place of pushing it. Input: the empty string aborts.
+    require!(
+        out.starts_with('['),
+        "push_string changed the text before it"
+    );
+    let literal = &out[1..];
+    // `ensure_ascii`: the literal is printable ASCII between two quotes.
+    // Mutant: in json_text.rs, push each character up to U+00FF as it is.
+    // Input: b4 b4 b4 b4 7f a6 a9 07 aborts (DEL comes out raw).
+    require!(
+        literal.len() >= 2
+            && literal.starts_with('"')
+            && literal.ends_with('"')
+            && literal.bytes().all(|b| (b' '..=b'~').contains(&b)),
+        "{literal:?} is not printable ASCII between quotes"
+    );
+    // A JSON parser reads back the same string, and every escape is the
+    // one `json.dumps` writes.
+    // Mutant: in json_text.rs, push `\u{8}` as `\u0008` in place of `\b`.
+    // Input: the byte 08 aborts. Mutant: make HEX uppercase. Input: the
+    // byte c3 (read as U+FFFD) aborts. Mutant: build the low surrogate
+    // from 0xd800. Input: f1 a3 a3 9f aborts. Mutant: push `\r` as `\n`.
+    // Input: the byte 0d aborts.
+    match decode(&literal[1..literal.len() - 1]) {
+        Ok(decoded) => require!(
+            decoded == *s,
+            "{literal:?} decodes to {decoded:?}, not {s:?}"
+        ),
+        Err(why) => require!(false, "{literal:?} is not json.dumps text: {why}"),
+    }
+});
+
+/// The characters `json.dumps` writes with a two-character escape, and the
+/// escape letter of each.
+const SHORT_ESCAPES: [(char, char); 7] = [
+    ('"', '"'),
+    ('\\', '\\'),
+    ('\n', 'n'),
+    ('\r', 'r'),
+    ('\t', 't'),
+    ('\u{8}', 'b'),
+    ('\u{c}', 'f'),
+];
+
+/// The body of a JSON string literal, decoded; an error if any character
+/// is not the form `json.dumps(ensure_ascii=True)` writes for it: printable
+/// ASCII as it is, the seven short escapes, and `\u` with four lowercase
+/// hex digits for every other character, as a surrogate pair above the
+/// BMP.
+fn decode(body: &str) -> Result<String, String> {
+    let mut out = String::new();
+    let mut chars = body.chars();
+    while let Some(c) = chars.next() {
+        if c != '\\' {
+            if c == '"' {
+                return Err("an unescaped quote".into());
+            }
+            out.push(c);
+            continue;
+        }
+        let letter = chars.next().ok_or("a backslash at the end")?;
+        if letter != 'u' {
+            let &(c, _) = SHORT_ESCAPES
+                .iter()
+                .find(|&&(_, l)| l == letter)
+                .ok_or_else(|| format!("the escape \\{letter}"))?;
+            out.push(c);
+            continue;
+        }
+        let unit = hex4(&mut chars)?;
+        let c = match unit {
+            0xd800..=0xdbff => {
+                if chars.next() != Some('\\') || chars.next() != Some('u') {
+                    return Err(format!("the high surrogate {unit:04x} is alone"));
+                }
+                let low = hex4(&mut chars)?;
+                if !(0xdc00..=0xdfff).contains(&low) {
+                    return Err(format!("{unit:04x} is followed by {low:04x}"));
+                }
+                let code = 0x10000 + ((unit - 0xd800) << 10) + (low - 0xdc00);
+                char::from_u32(code).ok_or("a pair outside Unicode")?
+            }
+            0xdc00..=0xdfff => return Err(format!("the low surrogate {unit:04x} is alone")),
+            _ => char::from_u32(unit).ok_or("not a scalar value")?,
+        };
+        if (' '..='~').contains(&c) || SHORT_ESCAPES.iter().any(|&(e, _)| e == c) {
+            return Err(format!(
+                "{c:?} has a \\u escape that json.dumps does not write"
+            ));
+        }
+        out.push(c);
+    }
+    Ok(out)
+}
+
+fn hex4(chars: &mut std::str::Chars) -> Result<u32, String> {
+    let mut unit = 0;
+    for _ in 0..4 {
+        let c = chars.next().ok_or("a short \\u escape")?;
+        if !matches!(c, '0'..='9' | 'a'..='f') {
+            return Err(format!("{c:?} in a \\u escape"));
+        }
+        unit = unit * 16 + c.to_digit(16).expect("a hex digit");
+    }
+    Ok(unit)
+}