fix: 89 non-reserved keywords now work as identifiers (#696)

.openspec/grammar/sqlite.ebnf

@@ -411,7 +411,38 @@ rollback-stmt   ::= "ROLLBACK" [ "TRANSACTION" ] ;
 
 table-name      ::= identifier ;                                          (* V2 *)
 column-name     ::= identifier ;                                          (* V2 *)
-identifier      ::= ID ;   (* bare, "double-quoted", [bracketed], `backticked` -- V2 tokenizer (#60) *)
+identifier      ::= ID | fallback-keyword ;
+                                                           (* V2 tokenizer (#60): ID is bare,
+                                                              "double-quoted", [bracketed], `backticked`.
+                                                              fallback-keyword added by #696. *)
+
+fallback-keyword ::= "ABORT" | "ACTION" | "AFTER" | "ALWAYS" | "ANALYZE"
+                  | "ASC" | "ATTACH" | "BEFORE" | "BEGIN" | "BY" | "CASCADE"
+                  | "CAST" | "COLUMN" | "CONFLICT" | "CROSS" | "CURRENT"
+                  | "CURRENT_DATE" | "CURRENT_TIME" | "CURRENT_TIMESTAMP"
+                  | "DATABASE" | "DEFERRED" | "DESC" | "DETACH" | "DO"
+                  | "EACH" | "END" | "EXCLUDE" | "EXCLUSIVE" | "EXPLAIN"
+                  | "FAIL" | "FILTER" | "FIRST" | "FOLLOWING" | "FOR"
+                  | "FULL" | "GENERATED" | "GLOB" | "GROUPS" | "IF"
+                  | "IGNORE" | "IMMEDIATE" | "INDEXED" | "INITIALLY"
+                  | "INNER" | "INSTEAD" | "KEY" | "LAST" | "LEFT" | "LIKE"
+                  | "MATCH" | "MATERIALIZED" | "NATURAL" | "NO" | "NULLS"
+                  | "OF" | "OFFSET" | "OTHERS" | "OUTER" | "OVER"
+                  | "PARTITION" | "PLAN" | "PRAGMA" | "PRECEDING" | "QUERY"
+                  | "RAISE" | "RANGE" | "RECURSIVE" | "REGEXP" | "REINDEX"
+                  | "RELEASE" | "RENAME" | "REPLACE" | "RESTRICT" | "RIGHT"
+                  | "ROLLBACK" | "ROW" | "ROWS" | "SAVEPOINT" | "TEMP"
+                  | "TEMPORARY" | "TIES" | "TRIGGER" | "UNBOUNDED"
+                  | "VACUUM" | "VIEW" | "VIRTUAL" | "WINDOW" | "WITH"
+                  | "WITHOUT" ;
+                                                           (* V2 [parse.y:272 fallback] -- the 89
+                                                              non-reserved keywords Lemon's
+                                                              `%fallback ID` retries as a plain ID
+                                                              wherever the grammar can't shift them
+                                                              as the keyword itself (#696). The other
+                                                              57 keywords in the tokenizer's table
+                                                              stay fully reserved, matching the
+                                                              oracle. *)
 
 (* ===================== Future blocks (stubs -- the denominator) =====================
  * V4:  subqueries in FROM (table-valued)

.openspec/specs/002-parser/spec.md

@@ -464,6 +464,14 @@ The parser MUST accept all SQL that SQLite accepts, and reject all SQL that SQLi
 
 **Tests:** `tests/unit/parser.rs::test_error_on_missing_columns`, `tests/corpus/parser_oracle_test.rs::parser_matches_oracle_three_way_outcome`
 
+#### Scenario: Non-reserved (fallback) keywords work as identifiers
+
+- GIVEN the 89 keywords `parse.y:272`'s `%fallback ID` declares non-reserved (e.g. `KEY`, `VALUE`-adjacent words, `MATCH`, `FIRST`, `ROW`) used as a column/table name or `AS`-alias, such as `CREATE TABLE p(namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(namespace, key))` followed by `SELECT key, value FROM p`
+- WHEN parsed
+- THEN parse succeeds, with the word still working as its keyword in keyword position in the same statement (`PRIMARY KEY(key)`, `ORDER BY key`); the other 57 keywords stay reserved and are rejected as bare identifiers, matching the oracle
+
+**Tests:** `tests/corpus/fallback_keyword_test.rs::every_fallback_word_is_accepted_as_a_column_name_matching_the_oracle`, `tests/corpus/fallback_keyword_test.rs::every_reserved_word_is_rejected_as_a_bare_column_name_matching_the_oracle`, `tests/corpus/fallback_keyword_test.rs::sqe_namespace_properties_table_matches_the_oracle`, `tests/corpus/fallback_keyword_test.rs::a_fallback_word_works_as_both_keyword_and_identifier_in_one_statement`, `tests/corpus/fallback_keyword_test.rs::as_alias_using_a_fallback_word_parses`
+
 #### Scenario: SQL text corpus labels match real SQLite
 
 - GIVEN the three-way labeled corpus at `tests/corpus/sql/{valid_in_subset,valid_out_of_subset,invalid}/*.sql` (#2), covering the V2 SELECT-core subset plus representative V3/V4+ statements and malformed SQL

CHANGELOG.md

@@ -8,6 +8,21 @@ All notable changes to sqlite-rs. Format follows [Keep a Changelog](https://keep
 
 ### Fixed
 
+- 89 of our 146 keywords were reserved unconditionally, unlike real
+  SQLite, which treats them as ordinary identifiers outside keyword
+  position (`parse.y:272`'s `%fallback ID`) — this blocked schemas as
+  ordinary as `CREATE TABLE p(namespace TEXT, key TEXT, value TEXT,
+  PRIMARY KEY(namespace, key))`. `parser::grammar`'s `identifier()` (the
+  single choke point nearly every identifier-accepting production already
+  funneled through) and `primary_expr`'s column-reference arm now accept
+  the 89-word fallback set; `KEY`/`FIRST`/`MATCH`/etc. still work as
+  keywords in keyword position in the same statement. The other 57
+  keywords stay fully reserved. Deliberately not extended to a bare
+  (no-`AS`) alias, since that regressed `a NATURAL JOIN b`/`t LEFT JOIN
+  (...)` — those keywords are the join operator there, a shift/reduce
+  call this recursive-descent parser can't make the way SQLite's LALR
+  table does; `AS <fallback-keyword>` is the safe form (#696, spec 002
+  Req 2).
 - A SQL comment before or after a statement (`-- c\nCREATE TABLE t(a);`,
   `CREATE TABLE t(a); -- c`, or comment-only input) was a parse error,
   even though `split_statements` already grouped a leading comment with

src/parser/grammar.rs

@@ -27,7 +27,7 @@
 
 use super::ast::*;
 use super::error::{PResult, ParseFail};
-use super::tokenizer::{Keyword, Param, Span, Token, TokenKind};
+use super::tokenizer::{is_fallback_keyword, keyword_text, Keyword, Param, Span, Token, TokenKind};
 
 /// Recursive-descent parser state: the token stream, a cursor into it, and
 /// the current expression-nesting depth (see [`MAX_EXPR_DEPTH`]).
@@ -211,6 +211,13 @@ impl Parser {
                 let span = self.advance_span();
                 Ok((name, span))
             }
+            // `parse.y:272`'s `%fallback ID` list (#696): 89 keywords are
+            // non-reserved in real SQLite and double as ordinary
+            // identifiers wherever a keyword isn't expected.
+            TokenKind::Keyword(kw) if is_fallback_keyword(kw) => {
+                let span = self.advance_span();
+                Ok((keyword_text(kw).to_string(), span))
+            }
             _ => {
                 let tok = self.peek().clone();
                 Err(ParseFail::Invalid {
@@ -1381,6 +1388,13 @@ impl Parser {
             self.advance();
             return Ok(Some(name));
         }
+        // Deliberately not extended to bare (no `AS`) fallback keywords:
+        // real SQLite's LALR table resolves e.g. `a NATURAL JOIN b`'s
+        // `NATURAL` as the join operator, not a bare alias for `a`, via
+        // shift/reduce precedence this recursive-descent parser has no
+        // equivalent for. Accepting it here regressed exactly that case
+        // (#696) — `AS <fallback-keyword>` (`identifier()`, above) is
+        // the safe, unambiguous form and is what the ticket requires.
         Ok(None)
     }
 
@@ -2235,7 +2249,33 @@ impl Parser {
                 self.advance();
                 self.exists_tail(start, false)
             }
-            TokenKind::Identifier(name) => {
+            // SQLite treats most keywords as usable function names when
+            // followed by `(` (e.g. `replace(...)`, `glob(...)`) — only
+            // the handful matched above (CASE/CAST/EXISTS/CURRENT_*)
+            // are true reserved words in expression position. Checked
+            // before the #696 fallback-identifier arm below so that a
+            // fallback keyword followed by `(` (e.g. `key(x)`) is still
+            // a function call, not a column reference.
+            TokenKind::Keyword(kw) if matches!(self.peek_at(1).kind, TokenKind::LParen) => {
+                self.advance();
+                self.function_call(format!("{kw:?}"), tok.span)
+            }
+            // `parse.y:272`'s `%fallback ID` list (#696): a non-reserved
+            // keyword not followed by `(` is an ordinary column
+            // reference, same as `TokenKind::Identifier` below.
+            TokenKind::Identifier(_) | TokenKind::Keyword(_) => {
+                let name = match tok.kind {
+                    TokenKind::Identifier(name) => name,
+                    TokenKind::Keyword(kw) if is_fallback_keyword(kw) => {
+                        keyword_text(kw).to_string()
+                    }
+                    other => {
+                        return Err(ParseFail::Invalid {
+                            message: format!("expected column or expression, found {other:?}"),
+                            span: tok.span,
+                        })
+                    }
+                };
                 self.advance();
                 if matches!(self.peek().kind, TokenKind::LParen) {
                     return self.function_call(name, tok.span);
@@ -2271,14 +2311,6 @@ impl Parser {
                 };
                 Ok(Expr { kind, span })
             }
-            // SQLite treats most keywords as usable function names when
-            // followed by `(` (e.g. `replace(...)`, `glob(...)`) — only
-            // the handful matched above (CASE/CAST/EXISTS/CURRENT_*)
-            // are true reserved words in expression position.
-            TokenKind::Keyword(kw) if matches!(self.peek_at(1).kind, TokenKind::LParen) => {
-                self.advance();
-                self.function_call(format!("{kw:?}"), tok.span)
-            }
             TokenKind::LParen => {
                 self.advance();
                 if self.at_kw(Keyword::SELECT) {

src/parser/tokenizer.rs

@@ -589,6 +589,123 @@ const KEYWORDS: &[(&str, Keyword)] = &[
     ("WITHOUT", Keyword::WITHOUT),
 ];
 
+/// `parse.y:272`'s `%fallback ID` list (3.53.4, pinned oracle): these 89
+/// keywords are non-reserved in real SQLite — Lemon retries them as a
+/// plain `ID` wherever the grammar can't shift them as the keyword
+/// itself, so they double as ordinary identifiers (column/table/alias
+/// names) everywhere a keyword isn't expected. The other 57 keywords in
+/// `KEYWORDS` above stay fully reserved. See #696.
+const FALLBACK_KEYWORDS: &[Keyword] = &[
+    Keyword::ABORT,
+    Keyword::ACTION,
+    Keyword::AFTER,
+    Keyword::ALWAYS,
+    Keyword::ANALYZE,
+    Keyword::ASC,
+    Keyword::ATTACH,
+    Keyword::BEFORE,
+    Keyword::BEGIN,
+    Keyword::BY,
+    Keyword::CASCADE,
+    Keyword::CAST,
+    Keyword::COLUMN,
+    Keyword::CONFLICT,
+    Keyword::CROSS,
+    Keyword::CURRENT,
+    Keyword::CURRENT_DATE,
+    Keyword::CURRENT_TIME,
+    Keyword::CURRENT_TIMESTAMP,
+    Keyword::DATABASE,
+    Keyword::DEFERRED,
+    Keyword::DESC,
+    Keyword::DETACH,
+    Keyword::DO,
+    Keyword::EACH,
+    Keyword::END,
+    Keyword::EXCLUDE,
+    Keyword::EXCLUSIVE,
+    Keyword::EXPLAIN,
+    Keyword::FAIL,
+    Keyword::FILTER,
+    Keyword::FIRST,
+    Keyword::FOLLOWING,
+    Keyword::FOR,
+    Keyword::FULL,
+    Keyword::GENERATED,
+    Keyword::GLOB,
+    Keyword::GROUPS,
+    Keyword::IF,
+    Keyword::IGNORE,
+    Keyword::IMMEDIATE,
+    Keyword::INDEXED,
+    Keyword::INITIALLY,
+    Keyword::INNER,
+    Keyword::INSTEAD,
+    Keyword::KEY,
+    Keyword::LAST,
+    Keyword::LEFT,
+    Keyword::LIKE,
+    Keyword::MATCH,
+    Keyword::MATERIALIZED,
+    Keyword::NATURAL,
+    Keyword::NO,
+    Keyword::NULLS,
+    Keyword::OF,
+    Keyword::OFFSET,
+    Keyword::OTHERS,
+    Keyword::OUTER,
+    Keyword::OVER,
+    Keyword::PARTITION,
+    Keyword::PLAN,
+    Keyword::PRAGMA,
+    Keyword::PRECEDING,
+    Keyword::QUERY,
+    Keyword::RAISE,
+    Keyword::RANGE,
+    Keyword::RECURSIVE,
+    Keyword::REGEXP,
+    Keyword::REINDEX,
+    Keyword::RELEASE,
+    Keyword::RENAME,
+    Keyword::REPLACE,
+    Keyword::RESTRICT,
+    Keyword::RIGHT,
+    Keyword::ROLLBACK,
+    Keyword::ROW,
+    Keyword::ROWS,
+    Keyword::SAVEPOINT,
+    Keyword::TEMP,
+    Keyword::TEMPORARY,
+    Keyword::TIES,
+    Keyword::TRIGGER,
+    Keyword::UNBOUNDED,
+    Keyword::VACUUM,
+    Keyword::VIEW,
+    Keyword::VIRTUAL,
+    Keyword::WINDOW,
+    Keyword::WITH,
+    Keyword::WITHOUT,
+];
+
+/// True for the 89 keywords `parse.y:272`'s `%fallback ID` declares
+/// non-reserved (see [`FALLBACK_KEYWORDS`]) — these double as ordinary
+/// identifiers outside keyword position. The other 57 keywords in
+/// `KEYWORDS` stay fully reserved, matching the oracle.
+pub fn is_fallback_keyword(kw: Keyword) -> bool {
+    FALLBACK_KEYWORDS.contains(&kw)
+}
+
+/// The literal source text a keyword token was recognized from
+/// (uppercase, per [`KEYWORDS`]) — used to recover an identifier's text
+/// when a [`is_fallback_keyword`] keyword is accepted in an identifier
+/// position (#696).
+pub fn keyword_text(kw: Keyword) -> &'static str {
+    KEYWORDS
+        .iter()
+        .find(|(_, k)| *k == kw)
+        .map_or("", |(text, _)| text)
+}
+
 /// Case-insensitive ASCII ordering of `a` against `b`, without
 /// allocating an uppercased copy of either — used by [`lookup_word`]'s
 /// binary search so per-identifier lookup costs no heap allocation.

tests/corpus/extracted_sql_test.rs

@@ -278,7 +278,7 @@ fn deeply_nested_expressions_hit_the_depth_guard_instead_of_the_stack() {
 /// never raise it without a documented cause like the #240/#257/#403 bumps
 /// above. A raise means a regression that reclassified valid SQL as
 /// malformed.
-const SELECT_INVALID_BASELINE: usize = 2;
+const SELECT_INVALID_BASELINE: usize = 1;
 
 /// Invariant 2: the parser must not call real, SQLite-accepted SELECT invalid.
 /// `Unsupported` is expected and fine — the V2 grammar is a deliberate slice.