feat(sql,core): derived tables in FROM and JOIN — grammar, AST, column scope, loud guard (story 22.1)

core/src/main/resources/help/commands/dql/select.json

@@ -5,7 +5,9 @@
   "syntax": [
     "SELECT [DISTINCT] columns",
     "FROM table [AS alias]",
+    "FROM (SELECT ...) [AS] alias",
     "[JOIN table ON condition]",
+    "[JOIN (SELECT ...) [AS] alias ON condition]",
     "[WHERE condition]",
     "[GROUP BY columns]",
     "[HAVING condition]",
@@ -96,15 +98,24 @@
       "description": "UNION ALL keeps every row of both sides; bare UNION is not accepted",
       "sql": "SELECT id, amount FROM orders UNION ALL SELECT id, amount FROM archived_orders",
       "output": null
+    },
+    {
+      "title": "Derived table",
+      "description": "Aggregate in a subquery, filter the aggregate outside",
+      "sql": "SELECT d.category, d.total FROM (SELECT category, SUM(amount) AS total FROM orders GROUP BY category) AS d WHERE d.total > 100",
+      "output": null
     }
   ],
   "notes": [
     "An alias must not be a reserved word: count, day, hour, now, today, offset, distance and every function name are rejected as aliases",
     "Result sets larger than the index max_result_window are paged automatically; the query itself needs no scroll or search_after handling",
     "Nested fields are addressed with JOIN UNNEST(<path>) AS <alias>; see the JOIN clause above",
-    "A FROM-less SELECT is evaluated against a hidden softclient4es_handshake index created on first use, so a read-only account needs it pre-created"
+    "A FROM-less SELECT is evaluated against a hidden softclient4es_handshake index created on first use, so a read-only account needs it pre-created",
+    "A derived table (a SELECT in FROM or JOIN) must carry an alias; a SELECT * body exposes an unchecked projection"
+  ],
+  "limitations": [
+    "Derived tables parse and are validated but execute only through the relational engine (softclient4es-arrow-extensions); without it the statement is refused with HTTP 400"
   ],
-  "limitations": [],
   "seeAlso": ["INSERT", "UPDATE", "DELETE"],
   "minVersion": null,
   "aliases": []

core/src/main/scala/app/softnetwork/elastic/client/RelationalClosureGuard.scala

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2025 SOFTNETWORK
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package app.softnetwork.elastic.client
+
+import app.softnetwork.elastic.client.result.ElasticError
+import app.softnetwork.elastic.sql.query.{derivedTablesPresent, Statement}
+
+/** The ONE rejection every venue WITHOUT the relational engine emits for a closure-shaped statement
+  * — a cross-index JOIN or a derived table (epic 22 AD-5; #157's discipline, widened).
+  *
+  * Two readers, one message: `CoreDqlExtension.execute` (the gateway path, where a join-capable
+  * extension would have claimed the statement first) and `SearchApi.resolveWithSchema` (the direct
+  * client API, which no extension sees). A second message would drift; a second PREDICATE would
+  * drift faster, which is why both read
+  * `app.softnetwork.elastic.sql.query.relationalClosureRequired`.
+  */
+object RelationalClosureGuard {
+
+  val ExtensionJar = "softclient4es-arrow-extensions"
+
+  /** Which construct triggered the refusal — NAMED because the remedy differs: a cross-index JOIN
+    * can be rewritten by hand, a derived table is usually emitted by a BI tool the user does not
+    * control. When BOTH are present the derived table is reported, because it is the one the user
+    * did not choose.
+    */
+  def shapeOf(statement: Statement): String =
+    if (derivedTablesPresent(statement)) "A derived table (subquery in FROM/JOIN)"
+    else "A cross-index JOIN"
+
+  /** `operation` stays `"join"` on the gateway path for BOTH shapes: nothing downstream
+    * distinguishes them (the JDBC driver relays `message` verbatim), so changing it would be a
+    * second behaviour change with no consumer.
+    *
+    * 🔴 The wording is deliberately VENUE-NEUTRAL, which #157's original text was not. Since this
+    * story widened `SearchApi.resolveWithSchema` (lead ruling on OQ-1) the same message is returned
+    * to a Scala embedder calling `client.search(...)` and — through `gateway.run` — to the JDBC
+    * driver's `handleFailure` and the Flight sidecar, verbatim. "Re-run the installer" and a
+    * `repl.md` anchor are not actionable there, so the remedy names the JAR and the REPL flag is
+    * given as the REPL's spelling of it rather than as the only one.
+    */
+  def rejection(statement: Statement, operation: String = "join"): ElasticError =
+    ElasticError(
+      message =
+        s"${shapeOf(statement)} requires the relational engine shipped in the $ExtensionJar jar " +
+        "(Java 11+); this venue has none, so the statement is refused rather than executed " +
+        s"against the first index it names. Put $ExtensionJar on the classpath (at the REPL: " +
+        "re-run the installer, or drop --no-extensions). See " +
+        "documentation/client/repl.md#extensions-cross-index-joins-materialized-views.",
+      statusCode = Some(400),
+      operation = Some(operation)
+    )
+}

core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala

@@ -137,6 +137,24 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC
     * i.e. `shouldBeScripted` across the clauses.
     */
   private[client] def resolveWithSchema(single: SingleSearch): ElasticResult[SingleSearch] = {
+    // Story 22.1 (epic 22 AD-5) — THE seam. Every direct-API path crosses it BEFORE routing
+    // (`search`, `searchAsync`, `searchWithInnerHits`, `ScrollApi.scroll`, and `IndicesApi`'s
+    // by-query DELETE/UPDATE bodies through `resolveDmlWithSchema`), and none of them passes
+    // through `CoreDqlExtension` — so a guard placed here cannot be bypassed by a routing
+    // decision, and two guards would be two places for it to go missing.
+    //
+    // A derived table has no index to resolve a schema against and `single.sources` yields its
+    // ALIAS; a cross-index JOIN resolves only its FIRST table. MEASURED on `origin/main` before
+    // this story: `client.search("SELECT o.id, c.name FROM orders o JOIN customers c ON …")`
+    // returned `ElasticSuccess` over index list `[orders]` with a `match_all` body — the JOIN leg
+    // silently dropped, HTTP 200, wrong answer. That is the #157 residual on the raw client API
+    // (no sibling production code calls this API — every one goes through `gateway.run` — which is
+    // why it survived), and the lead's ruling for this story is to close it here rather than file
+    // it.
+    if (single.relationalClosureRequired)
+      return ElasticResult.failure(
+        RelationalClosureGuard.rejection(single, operation = "search")
+      )
     // #306 -- this used to return early for a statement whose WHERE carried no temporal literal.
     // That was correct while the only job was rewriting those literals, and is WRONG now that the
     // schema is also attached to the AST: almost no statement carries a temporal WHERE literal,

core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala

@@ -94,7 +94,12 @@ class CoreDqlExtension extends ExtensionSpi {
     // SELECT with a cross-index JOIN, mirroring the arrow JoinExtension's canHandle) so they
     // fail loudly in execute instead of falling through to the core DML/DDL executors, which
     // would silently run the inner SELECT as a single-index search and write wrong data.
-    case other => joinRequired(other)
+    // Issue #157 / story 22.1 — also claim write-with-CLOSURE statements (INSERT … SELECT /
+    // CREATE TABLE … AS SELECT carrying a cross-index JOIN or a derived table, mirroring the arrow
+    // JoinExtension's canHandle) so they fail loudly in execute instead of falling through to the
+    // core DML/DDL executors, which would silently run the inner SELECT as a single-index search
+    // and write wrong data.
+    case other => relationalClosureRequired(other)
   }
 
   override def execute(
@@ -110,19 +115,8 @@ class CoreDqlExtension extends ExtensionSpi {
       // INSERT … SELECT / CTAS, write that wrong data. With a join-capable extension
       // registered ahead (priority < 100) join statements never get here; without one, fail
       // loudly instead of returning wrong data.
-      case s if joinRequired(s) =>
-        Future.successful(
-          ElasticFailure(
-            ElasticError(
-              message =
-                "Cross-index JOIN requires the softclient4es-arrow-extensions jar (Java 11+). " +
-                "Re-run the installer, or run with --no-extensions removed. " +
-                "See documentation/client/repl.md#extensions.",
-              statusCode = Some(400),
-              operation = Some("join")
-            )
-          )
-        )
+      case s if relationalClosureRequired(s) =>
+        Future.successful(ElasticFailure(RelationalClosureGuard.rejection(s)))
 
       case dql: DqlStatement =>
         licenseManager match {
@@ -150,18 +144,11 @@ class CoreDqlExtension extends ExtensionSpi {
     "SELECT ... FROM ... WHERE ... GROUP BY ... HAVING ... LIMIT ..."
   )
 
-  /** True when the statement carries a cross-index JOIN (`from.enrichmentRequired`), including
-    * embedded in INSERT … SELECT / CREATE TABLE … AS SELECT. UNNEST joins are excluded by
-    * construction — `joinedTables` collects `StandardJoin` sources only.
-    */
-  private[this] def joinRequired(statement: Statement): Boolean = statement match {
-    case single: SingleSearch    => single.from.enrichmentRequired
-    case multi: MultiSearch      => multi.requests.exists(_.from.enrichmentRequired)
-    case select: SelectStatement => select.statement.exists(joinRequired)
-    case insert: Insert          => insert.values.left.exists(joinRequired)
-    case create: CreateTable     => create.ddl.left.exists(joinRequired)
-    case _                       => false
-  }
+  // Story 22.1 — `joinRequired` DELETED. The predicate now lives in the `sql` module
+  // (`app.softnetwork.elastic.sql.query.relationalClosureRequired`, folded over `closureSearches`)
+  // so core, the `searchAs` macro and the arrow venue read ONE definition with ONE list of
+  // statement arms. It covers the same shapes it always did — a cross-index JOIN, embedded in
+  // INSERT … SELECT / CTAS, with UNNEST excluded by construction — plus derived tables.
 
   // ════════════════════════════════════════════════════════════════════
   // ✅ QUOTA CHECK LOGIC (in extension, not in core)