feat(sql,core): non-recursive CTEs — WITH name AS (SELECT ...) SELECT ...

core/src/main/resources/help/commands/dql/select.json

@@ -3,6 +3,7 @@
   "category": "DQL",
   "shortDescription": "Retrieve data from one or more tables",
   "syntax": [
+    "WITH name AS (SELECT ...) [, name2 AS (SELECT ...)]",
     "SELECT [DISTINCT] columns",
     "FROM table [AS alias]",
     "FROM (SELECT ...) [AS] alias",
@@ -31,6 +32,11 @@
   ],
   "description": "The SELECT statement retrieves rows from Elasticsearch indices. It supports most standard SQL features including joins, aggregations, and subqueries.",
   "clauses": [
+    {
+      "name": "WITH",
+      "description": "Common table expressions: name a subquery once and reference it in FROM/JOIN like a table. Non-recursive; each CTE may reference the CTEs defined before it.",
+      "optional": true
+    },
     {
       "name": "DISTINCT",
       "description": "Remove duplicate rows from results",
@@ -131,6 +137,12 @@
       "description": "Filter by a set computed by another query: the subquery runs first and its values are pushed into the outer query",
       "sql": "SELECT id, amount FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')",
       "output": null
+    },
+    {
+      "title": "Common table expression",
+      "description": "Aggregate in a named subquery, then select from it by name",
+      "sql": "WITH monthly AS (SELECT category, SUM(amount) AS total FROM bi_events GROUP BY category) SELECT * FROM monthly",
+      "output": null
     }
   ],
   "notes": [
@@ -140,12 +152,16 @@
     "A FROM-less SELECT is evaluated against a hidden softclient4es_handshake index created on first use, so a read-only account needs it pre-created",
     "A derived table (a SELECT in FROM or JOIN) must carry an alias; a SELECT * body exposes an unchecked projection",
     "A WHERE subquery that reads an outer alias (a correlated subquery) executes through the relational engine shipped in softclient4es-arrow-extensions, and is refused with HTTP 400 at every venue without it. An outer reference must be QUALIFIED with the outer table's alias - a bare column name inside the subquery is read as the subquery's own column - and an object path inside the body should be qualified with the body's own alias",
-    "NOT IN follows SQL: when the subquery's values contain NULL, no row matches. ANY/SOME over an empty subquery is false and ALL over an empty subquery is true"
+    "NOT IN follows SQL: when the subquery's values contain NULL, no row matches. ANY/SOME over an empty subquery is false and ALL over an empty subquery is true",
+    "A CTE name shadows an index of the same name for the duration of the statement, so a CTE cannot be named after the index it reads: write WITH orders_f AS (SELECT ... FROM orders ...) rather than WITH orders AS (SELECT ... FROM orders ...). A CTE referenced twice is executed twice (inline semantics)"
   ],
   "limitations": [
     "Derived tables parse and are validated but execute only through the relational engine (softclient4es-arrow-extensions); without it the statement is refused with HTTP 400",
     "A WHERE subquery may return at most 65536 distinct values (Elasticsearch index.max_terms_count); a larger set must be written as a JOIN",
-    "Subqueries are accepted in WHERE only (SELECT, DELETE, UPDATE): not in HAVING, CASE, JOIN ON, MATERIALIZED VIEW or WATCHER definitions; UNION ALL and FROM-less subquery bodies are rejected, and NOT IN over a GROUP BY subquery cannot see a NULL group"
+    "Subqueries are accepted in WHERE only (SELECT, DELETE, UPDATE): not in HAVING, CASE, JOIN ON, MATERIALIZED VIEW or WATCHER definitions; UNION ALL and FROM-less subquery bodies are rejected, and NOT IN over a GROUP BY subquery cannot see a NULL group",
+    "WITH RECURSIVE and CTE column lists (WITH a (x, y) AS ...) are not supported; a WITH clause is accepted only at the top of a SELECT, not inside a subquery body, CTAS, INSERT ... SELECT or MATERIALIZED VIEW",
+    "A statement with a WITH clause executes only through the relational engine (softclient4es-arrow-extensions); without it the statement is refused with HTTP 400",
+    "A CTE body may not name the CTE itself: unlike PostgreSQL and other engines, which bind such a name to the base table, this engine rejects it. Rename the CTE (WITH orders_f AS (SELECT ... FROM orders ...)). WITH RECURSIVE is a separate unsupported feature."
   ],
   "seeAlso": [
     "INSERT",

core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala

@@ -17,7 +17,12 @@
 package app.softnetwork.elastic.client
 
 import app.softnetwork.elastic.client.result.ElasticError
-import app.softnetwork.elastic.sql.query.{closureSearches, derivedTablesPresent, Statement}
+import app.softnetwork.elastic.sql.query.{
+  closureSearches,
+  ctesPresent,
+  derivedTablesPresent,
+  Statement
+}
 
 /** The ONE rejection every venue WITHOUT the relational engine emits for a closure-shaped statement
   * — a cross-index JOIN or a derived table (epic 22 AD-5; #157's discipline, widened).
@@ -38,10 +43,21 @@ object RelationalClosureGuard {
     * did not choose.
     */
   def shapeOf(statement: Statement): String =
-    if (closureSearches(statement).exists(_.hasCorrelatedSubqueries)) CorrelatedShape
+    // Story 22.5 — FIRST: a CTE reference IS a derived table, so a WITH statement would otherwise
+    // be reported as "a derived table (subquery in FROM/JOIN)" — a shape the user did not write.
+    // Naming what was actually typed is the whole point of this method.
+    if (ctesPresent(statement)) CteShape
+    else if (closureSearches(statement).exists(_.hasCorrelatedSubqueries)) CorrelatedShape
     else if (derivedTablesPresent(statement)) "A derived table (subquery in FROM/JOIN)"
     else "A cross-index JOIN"
 
+  /** Story 22.5. 🔴 The words `WITH clause` sit in the first 25 characters on purpose:
+    * `GatewayApi.excerpt` caps a rejection at 200 characters and elides the MIDDLE (head 120 + tail
+    * 77), so a term that must reach the user has to fit in the FIRST 120 (story 22.3's measured
+    * `LATERAL` incident).
+    */
+  private val CteShape = "A WITH clause (common table expression)"
+
   /** Story 22.3 — reported FIRST, for the same reason the derived table outranks the JOIN: it is
     * the construct with the narrowest remedy (qualify differently, or rewrite as a JOIN), and a
     * statement that carries both is refused for the one the analyst is least likely to have chosen.

core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala

@@ -18,7 +18,12 @@ package app.softnetwork.elastic.client
 
 import app.softnetwork.elastic.client.result._
 import app.softnetwork.elastic.sql.parser.Parser
-import app.softnetwork.elastic.sql.query.{relationalClosureRequired, SearchStatement, SingleSearch}
+import app.softnetwork.elastic.sql.query.{
+  ctesPresent,
+  relationalClosureRequired,
+  SearchStatement,
+  SingleSearch
+}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import org.slf4j.{Logger, LoggerFactory}
@@ -65,6 +70,11 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers {
   private val CorrelatedSelect =
     "SELECT c.id FROM customers c WHERE EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.id)"
 
+  /** Story 22.5's corpus witness, `superset.flightsql.w6.006`, verbatim. */
+  private val CteSelect =
+    "WITH monthly AS (SELECT category, SUM(amount) AS total FROM bi_events GROUP BY category) " +
+    "SELECT * FROM monthly"
+
   // ---- the ONE seam ------------------------------------------------------------------------
 
   behavior of "SearchApi.resolveWithSchema (the direct-API seam)"
@@ -233,6 +243,53 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers {
     err.message should include("A correlated subquery")
   }
 
+  /** Story 22.5 — the CTE shape is reported FIRST, and the test that matters is the one below it: a
+    * CTE reference IS a derived table, so WITHOUT the arm this statement is refused as "a derived
+    * table (subquery in FROM/JOIN)" — a construct the analyst never wrote.
+    */
+  it should "name the CTE shape, and prefer it over the derived table it is made of" in {
+    val cte = searchStatement(CteSelect)
+    relationalClosureRequired(cte) shouldBe true
+    ctesPresent(cte) shouldBe true
+    RelationalClosureGuard.shapeOf(cte) should include("WITH clause")
+    // The falsifiable half: it must NOT fall through to the derived-table wording.
+    RelationalClosureGuard.shapeOf(cte) should not include "derived table"
+    // ... and the plain derived table must still get its own name (the arm did not swallow it).
+    RelationalClosureGuard.shapeOf(searchStatement(DerivedSelect)) should include("derived table")
+  }
+
+  it should "refuse a CTE statement on the direct API at the seam" in {
+    val err = refusalOf(client().search(searchStatement(CteSelect)))
+    err.statusCode shouldBe Some(400)
+    err.operation shouldBe Some("search")
+    err.message should include("WITH clause")
+    err.message should include(RelationalClosureGuard.ExtensionJar)
+  }
+
+  it should "refuse a CTE statement even when NO CTE is referenced" in {
+    // The classifier cannot count references, and the AST predicate must agree with it.
+    val err =
+      refusalOf(client().search(searchStatement("WITH u AS (SELECT 1 AS x) SELECT a FROM t")))
+    err.statusCode shouldBe Some(400)
+    err.message should include("WITH clause")
+  }
+
+  /** Story 22.5, Task 0 row 13. `IndicesApi.parseQueryForDeletion` ALREADY sniffed a leading `WITH`
+    * as SQL before this story, so a CTE statement handed to `deleteByQuery` used to die on a lexer
+    * error. It now PARSES, and both routes must still refuse it — never a delete-by-query over an
+    * index named after the CTE's alias.
+    */
+  it should "refuse a CTE statement handed to deleteByQuery, on both routes" in {
+    val sql = "WITH c AS (SELECT id FROM t) SELECT * FROM c"
+    // alias != index: the SingleSearch arm's index check fires first (loud, 400).
+    val mismatch = refusalOf(client().asInstanceOf[IndicesApi].deleteByQuery("t", sql))
+    mismatch.statusCode shouldBe Some(400)
+    // alias == index: the arm reaches `resolveDmlWithSchema` and the seam's closure term refuses.
+    val seam = refusalOf(client().asInstanceOf[IndicesApi].deleteByQuery("c", sql))
+    seam.statusCode shouldBe Some(400)
+    seam.message should include("WITH clause")
+  }
+
   it should "say the statement was refused rather than executed against the first index (PD-2)" in {
     val msg = RelationalClosureGuard.rejection(searchStatement(JoinSelect)).message
     msg should include("refused rather than executed against the first index it names")

core/src/test/scala/app/softnetwork/elastic/client/SplitStatementsSpec.scala

@@ -31,6 +31,15 @@ class SplitStatementsSpec extends AnyFlatSpec with Matchers {
     GatewayApi.splitStatements("SELECT 1; SELECT 2") shouldBe List("SELECT 1", "SELECT 2")
   }
 
+  /** Story 22.5 — `splitStatements` is a quote-aware `;` scanner keyed on `'`, `"`, backtick and
+    * `--`. A `WITH` token is none of those, and the parentheses of a CTE body are not either, so
+    * the splitter needs no change: this pin is what says so rather than assuming it.
+    */
+  it should "split a CTE statement from a following statement on the `;`" in {
+    GatewayApi.splitStatements("WITH a AS (SELECT 1 AS x) SELECT * FROM a; SELECT 1") shouldBe
+    List("WITH a AS (SELECT 1 AS x) SELECT * FROM a", "SELECT 1")
+  }
+
   it should "return a single statement unchanged" in {
     GatewayApi.splitStatements("SELECT * FROM t WHERE a = 1") shouldBe
     List("SELECT * FROM t WHERE a = 1")

core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala

@@ -477,6 +477,42 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers {
     client.searchedStatement.get() shouldBe null
   }
 
+  /** Story 22.5 — a CTE statement reaches the SAME guard through the SAME predicate, with zero edit
+    * in this extension: `relationalClosureRequired` gained one disjunct and every venue inherited
+    * it.
+    *
+    * The falsifiable half is the pair of `null` seam assertions. A CTE statement reports
+    * `returnsRows == true` (no aggregate in the OUTER select), so had it slipped past the closure
+    * arm it would have been SCROLLED against an index named after the CTE's alias — `monthly` is
+    * not an index, and the alias is what `sources` yields.
+    */
+  it should "reject a CTE statement, naming the WITH clause, and never reach scroll or search" in {
+    val (client, res) = run(
+      "WITH monthly AS (SELECT category, SUM(amount) AS total FROM bi_events GROUP BY category) " +
+      "SELECT * FROM monthly",
+      Quota.Community
+    )
+    res shouldBe a[ElasticFailure]
+    val err = res.asInstanceOf[ElasticFailure].elasticError
+    err.statusCode shouldBe Some(400)
+    err.message should include("WITH clause")
+    err.message should include("softclient4es-arrow-extensions")
+    // It must be named as the construct the analyst WROTE, not as the derived table it becomes.
+    err.message should not include "derived table"
+    client.scrolledStatement.get() shouldBe null
+    client.searchedStatement.get() shouldBe null
+  }
+
+  it should "reject a CTE statement even when no CTE is referenced" in {
+    // The arrow regex classifier keys on the leading token and cannot count references; the AST
+    // predicate must agree with it or the two venues disagree about who owns the statement.
+    val (client, res) = run("WITH u AS (SELECT 1 AS x) SELECT a FROM t", Quota.Community)
+    res shouldBe a[ElasticFailure]
+    res.asInstanceOf[ElasticFailure].elasticError.message should include("WITH clause")
+    client.scrolledStatement.get() shouldBe null
+    client.searchedStatement.get() shouldBe null
+  }
+
   it should "reject INSERT ... SELECT and CTAS carrying a derived table, and claim them" in {
     Seq(
       "INSERT INTO target SELECT COL FROM (SELECT 1 AS COL) AS d",

core/src/test/scala/app/softnetwork/elastic/client/help/HelpCorpusSpec.scala

@@ -810,6 +810,17 @@ class HelpCorpusSpec extends AnyFlatSpec with Matchers {
     // `start ~> derivedTableBodyInner <~ end`, i.e. the already-enumerated `derivedTableBodyInner`
     // in parentheses, so the package walk's coverage is unchanged (the superset assertion below
     // stays green and no new help document is required).
+    // Story 22.5 adds `cteBody` and `withQuery`, and NEITHER names a new statement leaf, which is
+    // the check this comment block exists to record:
+    //   - `cteBody` is `start ~> (derivedTableBodyInner | err(...)) <~ end`, i.e. the
+    //     already-enumerated `derivedTableBodyInner` in parentheses with a message of its own. Its
+    //     leaves are `SingleSearch` / `MultiSearch` / `FromlessSelect`, all already walked.
+    //   - `withQuery` is `withClause ~ searchStatement`, typed `SearchStatement` because the WITH
+    //     list is a FIELD on `SingleSearch` (`ctes`) rather than a new `Statement` kind — which is
+    //     precisely why story 22.5 needs no new help document. A `WITH …` statement IS a
+    //     `SingleSearch` (or a `MultiSearch` for a `UNION ALL` outer), both already enumerated.
+    // Both facts are re-checked mechanically by the superset assertion below and by this file's
+    // parser -> doc gate, which stays green with no new document.
     val expectedAbstract =
       Set(
         "statement",
@@ -818,6 +829,8 @@ class HelpCorpusSpec extends AnyFlatSpec with Matchers {
         "dmlStatement",
         "searchStatement",
         "derivedTableBodyInner",
+        "cteBody",
+        "withQuery",
         "app$softnetwork$elastic$sql$parser$WhereParser$$subqueryBody"
       )
     withClue(

macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala

@@ -40,6 +40,68 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers {
       )""")
   }
 
+  // Story 22.5 — a CTE statement is also a `SingleSearch`, so it reaches the SAME guarded arm with
+  // no new macro branch.
+  //
+  // 🔴 The SQL is an UNREFERENCED CTE, and that choice is the whole test. A statement that
+  // REFERENCES its CTE carries a derived table, so story 22.1's `from.relationalClosureRequired`
+  // arm already aborts it — MEASURED: with such a statement, deleting `|| ctes.nonEmpty` from
+  // `SingleSearch.relationalClosureRequired` left this suite 25/25 GREEN, i.e. the row could not
+  // fail for the reason its title named. `WITH u AS (...) SELECT a FROM t` has NO derived table,
+  // so only the new disjunct can reject it.
+  it should "REJECT a CTE at compile time, even when no CTE is referenced" in {
+    assertDoesNotCompile("""
+      import app.softnetwork.elastic.client.macros.TestElasticClientApi
+      import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats
+      import app.softnetwork.elastic.sql.query.SelectStatement
+
+      case class Row(a: Int)
+
+      TestElasticClientApi.searchAs[Row](
+        "WITH u AS (SELECT a FROM t) SELECT a FROM t"
+      )""")
+  }
+
+  // …and the REFERENCED form too, which 22.1's arm would also catch — kept as the neighbour, not
+  // as the guard.
+  it should "REJECT a referenced CTE at compile time" in {
+    assertDoesNotCompile("""
+      import app.softnetwork.elastic.client.macros.TestElasticClientApi
+      import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats
+      import app.softnetwork.elastic.sql.query.SelectStatement
+
+      case class Row(a: Int)
+
+      TestElasticClientApi.searchAs[Row](
+        "WITH m AS (SELECT a FROM t) SELECT a FROM m"
+      )""")
+  }
+
+  /** 🔴 `assertDoesNotCompile` reports only THAT a snippet failed, never WHY — so the two rows
+    * above cannot see the shape naming at all: disabling the `ctesPresent` branch in
+    * `closureAbortMessage` left them GREEN. The message is asserted DIRECTLY, the way
+    * `RelationalClosureGuardSpec` asserts `RelationalClosureGuard.shapeOf` for the runtime guard.
+    */
+  it should "name the WITH clause in the abort message, not the derived table it becomes" in {
+    def parsed(sql: String): app.softnetwork.elastic.sql.query.Statement =
+      app.softnetwork.elastic.sql.parser.Parser(sql) match {
+        case Right(s) => s
+        case Left(e)  => fail(s"[$sql] rejected: ${e.msg}")
+      }
+    val cte = "WITH m AS (SELECT a FROM t) SELECT a FROM m"
+    val msg = SQLQueryValidator.closureAbortMessage(parsed(cte), cte)
+    msg should include("WITH clauses (common table expressions)")
+    // The falsifiable half: it must NOT fall through to the construct the author never wrote.
+    msg should not include "Derived tables"
+    // …and the neighbours must keep their own names (the arm did not swallow them).
+    val derived = "SELECT COL FROM (SELECT 1 AS COL) AS d"
+    SQLQueryValidator.closureAbortMessage(parsed(derived), derived) should include(
+      "Derived tables (subqueries in FROM/JOIN)"
+    )
+    val join = "SELECT o.id, c.name FROM orders o JOIN customers c ON o.cid = c.id"
+    SQLQueryValidator.closureAbortMessage(parsed(join), join) should include("Cross-index JOINs")
+  }
+
   // ============================================================
   // Positive Tests (Should Compile)
   // ============================================================