diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 3f57345f9..37f64eefd 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -31,10 +31,13 @@ import app.softnetwork.elastic.sql.query.{ IsNotNullExpr, IsNullCriteria, IsNullExpr, + MatchAllCriteria, MatchCriteria, + MatchNoneCriteria, NestedElement, NestedElements, - Predicate + Predicate, + SubqueryCriteria } import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.requests.searches.queries.{InnerHit, Query} @@ -186,6 +189,20 @@ case class ElasticBridge(filter: ElasticFilter) { case matchExpression: MatchCriteria => matchExpression case isNull: IsNullCriteria => isNull case isNotNull: IsNotNullCriteria => isNotNull + // Story 22.2 — the two RESOLVED sentinels a WHERE subquery collapses to. + case _: MatchAllCriteria => matchAllQuery() + case _: MatchNoneCriteria => matchNoneQuery() + // 🔴 Story 22.2 — an UNRESOLVED subquery node must never reach a bridge. It is replaced by a + // literal criteria at `SearchApi.resolveWithSchema` (the ONE seam), so arriving here means a + // statement was handed straight to `singleSearch` / `singleSearchToJsonQuery`, which bypass + // it. Named rather than swallowed by the `Unsupported filter type` default below, because + // the alternative — emitting nothing for the predicate — is a silent wrong answer. + case s: SubqueryCriteria => + throw new IllegalArgumentException( + s"Unresolved WHERE subquery reached the query builder: ${s.sql}. Subqueries are " + + "executed at SearchApi.resolveWithSchema before translation; a statement handed " + + "straight to singleSearch / singleSearchToJsonQuery bypasses it." + ) case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 58f784a48..5be05bab0 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -1,7 +1,8 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.Criteria +import app.softnetwork.elastic.sql.operator.NOT +import app.softnetwork.elastic.sql.query.{Criteria, InExpr, MatchAllCriteria, MatchNoneCriteria} import com.fasterxml.jackson.databind.JsonNode import com.sksamuel.elastic4s.ElasticApi.matchAllQuery import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} @@ -1340,4 +1341,45 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { json should not include "o.status" } + /** Story 22.2 — the two RESOLVED sentinels a WHERE subquery collapses to, and the loud refusal of + * an UNRESOLVED node. + * + * 🔴 The `terms` row pins the EXACT AST `SubqueryResolver` produces, so the resolver spec and + * this one share one shape: if the resolver ever built a differently typed `Values`, the emitted + * query would move and this row would say so. + */ + private def asQueryOf(criteria: Criteria): String = { + import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + SearchBodyBuilderFn(SearchRequest("*") query criteria.asQuery()).string + } + + it should "emit match_all / match_none for the resolved subquery sentinels (story 22.2)" in { + asQueryOf(MatchAllCriteria()).replaceAll("\\s", "") should include("\"match_all\":{}") + asQueryOf(MatchNoneCriteria()).replaceAll("\\s", "") should include("\"match_none\":{}") + } + + it should "emit the terms clause for the resolver's IN shape (story 22.2)" in { + val in: Criteria = + InExpr(GenericIdentifier("customer_id"), LongValues(Seq(LongValue(3), LongValue(1))), None) + asQueryOf(in).replaceAll("\\s", "") should include("\"terms\":{\"customer_id\":[3,1]}") + asQueryOf( + InExpr(GenericIdentifier("customer_id"), LongValues(Seq(LongValue(3))), Some(NOT)) + ).replaceAll("\\s", "") should include("\"must_not\"") + } + + it should "refuse an UNRESOLVED subquery node BY NAME, never silently (story 22.2)" in { + implicit def timestamp: Long = 0L + val node = parser + .Parser("SELECT id FROM t WHERE a IN (SELECT a FROM u)") + .toOption + .collect { case s: query.SingleSearch => s } + .flatMap(_.where.flatMap(_.criteria)) + .getOrElse(fail("expected a WHERE subquery")) + val ex = the[IllegalArgumentException] thrownBy node.asQuery() + ex.getMessage should include("Unresolved WHERE subquery") + ex.getMessage should include("resolveWithSchema") + } + } diff --git a/build.sbt b/build.sbt index e9ed399b1..546ed3193 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.23.0" +ThisBuild / version := "0.24.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/resources/help/commands/dql/select.json b/core/src/main/resources/help/commands/dql/select.json index 3df98b93a..99b4f0c5c 100644 --- a/core/src/main/resources/help/commands/dql/select.json +++ b/core/src/main/resources/help/commands/dql/select.json @@ -9,6 +9,17 @@ "[JOIN table ON condition]", "[JOIN (SELECT ...) [AS] alias ON condition]", "[WHERE condition]", + "", + "-- A WHERE condition may carry an UNCORRELATED subquery. The inner query runs FIRST and its", + "-- values are pushed into the outer query as a terms query or a literal comparison:", + "SELECT ... WHERE col IN (SELECT col FROM ...)", + "SELECT ... WHERE col NOT IN (SELECT col FROM ...)", + "SELECT ... WHERE EXISTS (SELECT 1 FROM ...)", + "SELECT ... WHERE NOT EXISTS (SELECT 1 FROM ...)", + "SELECT ... WHERE col > (SELECT MAX(col) FROM ...)", + "SELECT ... WHERE col = ANY (SELECT col FROM ...)", + "SELECT ... WHERE col > ALL (SELECT col FROM ...)", + "", "[GROUP BY columns]", "[HAVING condition]", "[ORDER BY columns [ASC|DESC]]", @@ -34,7 +45,12 @@ "name": "JOIN", "description": "Combine rows from multiple tables", "optional": true, - "variants": ["LEFT JOIN", "RIGHT JOIN", "INNER JOIN", "OUTER JOIN"] + "variants": [ + "LEFT JOIN", + "RIGHT JOIN", + "INNER JOIN", + "OUTER JOIN" + ] }, { "name": "WHERE", @@ -55,7 +71,12 @@ "name": "ORDER BY", "description": "Sort result rows", "optional": true, - "modifiers": ["ASC", "DESC", "NULLS FIRST", "NULLS LAST"] + "modifiers": [ + "ASC", + "DESC", + "NULLS FIRST", + "NULLS LAST" + ] }, { "name": "UNION ALL", @@ -104,6 +125,12 @@ "description": "Aggregate in a subquery, filter the aggregate outside", "sql": "SELECT d.category, d.total FROM (SELECT category, SUM(amount) AS total FROM orders GROUP BY category) AS d WHERE d.total > 100", "output": null + }, + { + "title": "Subquery in WHERE", + "description": "Filter by a set computed by another query: the subquery runs first and its values are pushed into the outer query", + "sql": "SELECT id, amount FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')", + "output": null } ], "notes": [ @@ -111,12 +138,20 @@ "Result sets larger than the index max_result_window are paged automatically; the query itself needs no scroll or search_after handling", "Nested fields are addressed with JOIN UNNEST() AS ; see the JOIN clause above", "A FROM-less SELECT is evaluated against a hidden softclient4es_handshake index created on first use, so a read-only account needs it pre-created", - "A derived table (a SELECT in FROM or JOIN) must carry an alias; a SELECT * body exposes an unchecked projection" + "A derived table (a SELECT in FROM or JOIN) must carry an alias; a SELECT * body exposes an unchecked projection", + "A WHERE subquery must be self-contained: a reference to an outer alias (a correlated subquery) is refused until the relational engine executes it, and a bare column name inside the subquery is read as the subquery's own column", + "NOT IN follows SQL: when the subquery's values contain NULL, no row matches. ANY/SOME over an empty subquery is false and ALL over an empty subquery is true" ], "limitations": [ - "Derived tables parse and are validated but execute only through the relational engine (softclient4es-arrow-extensions); without it the statement is refused with HTTP 400" + "Derived tables parse and are validated but execute only through the relational engine (softclient4es-arrow-extensions); without it the statement is refused with HTTP 400", + "A WHERE subquery may return at most 65536 distinct values (Elasticsearch index.max_terms_count); a larger set must be written as a JOIN", + "Subqueries are accepted in WHERE only (SELECT, DELETE, UPDATE): not in HAVING, CASE, JOIN ON, MATERIALIZED VIEW or WATCHER definitions; UNION ALL and FROM-less subquery bodies are rejected, and NOT IN over a GROUP BY subquery cannot see a NULL group" + ], + "seeAlso": [ + "INSERT", + "UPDATE", + "DELETE" ], - "seeAlso": ["INSERT", "UPDATE", "DELETE"], "minVersion": null, "aliases": [] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index e5c577802..b6361ade4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -319,6 +319,11 @@ trait ElasticConversion { val aggsNode = Option(json.path("aggregations")) .filter(!_.isMissingNode) + // Story 22.2 — how many documents a top-level aggregation actually aggregated over. `size: 0` + // still reports `hits.total`, so this is present on every aggregation response and on every + // major (a bare number on ES 6, `{value, relation}` on ES 7+). + val rootDocCount: Option[Long] = docCountOf(json.path("hits"), "total") + val rows = (hitsNode, aggsNode) match { case (Some(hits), None) if hits.nonEmpty => // Case 1 : only hits @@ -328,12 +333,12 @@ trait ElasticConversion { case (None, Some(aggs)) => // Case 2 : only aggregations - val ret = parseAggregations(aggs, rowInvariants, fieldAliases, aggregations) + val ret = parseAggregations(aggs, rowInvariants, fieldAliases, aggregations, rootDocCount) combineAggregationRows(ret) case (Some(hits), Some(aggs)) if hits.isEmpty => // Case 3 : aggregations with no hits - val ret = parseAggregations(aggs, rowInvariants, fieldAliases, aggregations) + val ret = parseAggregations(aggs, rowInvariants, fieldAliases, aggregations, rootDocCount) combineAggregationRows(ret) case (Some(hits), Some(aggs)) if hits.nonEmpty => @@ -635,11 +640,17 @@ trait ElasticConversion { /** Parse recursively aggregations from Elasticsearch response with parent context */ + /** @param docCount + * how many documents the enclosing scope aggregated over, threaded so an aggregate over ZERO + * documents can answer ANSI NULL (story 22.2). `hits.total` at the root, a bucket's + * `doc_count` inside one, a wrapper aggregation's own `doc_count` under that. + */ def parseAggregations( aggsNode: JsonNode, parentContext: ListMap[String, Any], fieldAliases: ListMap[String, String], - aggregations: ListMap[String, ClientAggregation] + aggregations: ListMap[String, ClientAggregation], + docCount: Option[Long] = None ): Seq[ListMap[String, Any]] = { if (aggsNode.isMissingNode || !aggsNode.isObject) { @@ -721,11 +732,17 @@ trait ElasticConversion { } // Recursively parse subaggregations - parseAggregations(subAggsNode, currentContext, fieldAliases, aggregations) + parseAggregations( + subAggsNode, + currentContext, + fieldAliases, + aggregations, + docCountOf(aggValue, "doc_count").orElse(docCount) + ) } } else if (bucketAggs.isEmpty) { // No buckets : it is a leaf aggregation (metrics or top_hits) - val metrics = extractMetrics(aggsNode, aggregations) + val metrics = extractMetrics(aggsNode, aggregations, docCount) val allTopHits = extractAllTopHits(aggsNode, fieldAliases, aggregations) if (allTopHits.nonEmpty) { @@ -739,7 +756,8 @@ trait ElasticConversion { // Handle each aggregation with buckets bucketAggs.flatMap { case (aggName, buckets, _) => buckets.flatMap { bucket => - val metrics = extractMetrics(bucket, aggregations) + val bucketDocCount = docCountOf(bucket, "doc_count") + val metrics = extractMetrics(bucket, aggregations, bucketDocCount) val allTopHits = extractAllTopHits(bucket, fieldAliases, aggregations) val bucketKey = extractBucketKey(bucket) @@ -775,7 +793,13 @@ trait ElasticConversion { /*subAggFields.foreach { entry => subAggsNode.set(entry.getKey, entry.getValue) // FIXME }*/ - parseAggregations(subAggsNode, currentContext, fieldAliases, aggregations) + parseAggregations( + subAggsNode, + currentContext, + fieldAliases, + aggregations, + bucketDocCount + ) } else { Seq(currentContext) } @@ -784,6 +808,21 @@ trait ElasticConversion { } } + /** The document count a scope aggregated over, when the response states it. + * + * Handles both shapes Elasticsearch uses for `hits.total`: a bare number (ES 6) and `{"value": + * n, "relation": "eq"}` (ES 7+). A `doc_count` is always a bare number. + */ + private[client] def docCountOf(node: JsonNode, field: String): Option[Long] = { + val n = node.path(field) + if (n.isMissingNode) None + else if (n.isNumber) Some(n.asLong()) + else if (n.isObject) { + val v = n.path("value") + if (v.isNumber) Some(v.asLong()) else None + } else None + } + /** Extract the bucket key with proper typing (String, Long, Double, DateTime, etc.) */ def extractBucketKey(bucket: JsonNode): Any = { @@ -902,9 +941,15 @@ trait ElasticConversion { /** Extract metrics from an aggregation node */ + /** @param docCount + * how many documents the enclosing scope aggregated over, when the response states it + * (`hits.total` at the root, a bucket's `doc_count` inside one). `Some(0)` is what makes an + * aggregate ANSI-NULL — see [[ClientAggregation.nullOverEmptyInput]]. + */ def extractMetrics( aggsNode: JsonNode, - aggregations: ListMap[String, ClientAggregation] + aggregations: ListMap[String, ClientAggregation], + docCount: Option[Long] = None ): ListMap[String, Any] = { aggsNode match { case n: ObjectNode => @@ -917,8 +962,25 @@ trait ElasticConversion { bucketRoot = Some(agg.bucketRoot) case _ => } - // Detect simple metric values + // 🔴 Story 22.2 — ANSI: an aggregate computed over ZERO documents is NULL (COUNT and SUM + // excepted — `ClientAggregation.nullOverEmptyInput` is the ONE place that rule lives). + // + // This arm comes FIRST, before the value is read, because on ES 8 the value CANNOT be + // trusted here: that module reads the TYPED response, `SingleMetricAggregateBase` holds a + // primitive `double`, and Elasticsearch's `null` has already become `0.0` inside the + // vendor's model before any of our code runs. MEASURED: identical wire responses on 6.8 / + // 7.17 / 8.18 / 9.0 (`{"value":null}`), but ES 8 alone converted it to `0.0` — so + // `WHERE x > (SELECT MAX(y) FROM t WHERE )` reduced to `x > 0` and returned + // EVERY row there while returning none elsewhere. + // + // The document count is the recoverable signal and it is EXACT, not a heuristic: a + // genuine `MAX` of `0.0` needs at least one document, where this rule cannot fire. On the + // majors that were already correct the value is `null` anyway, so this is a no-op for + // them beyond making the NULL explicit rather than an absent key. + val emptyInput = + docCount.contains(0L) && aggregations.get(name).exists(_.nullOverEmptyInput) Option(value.get("value")) + .filter(_ => !emptyInput) .filter(!_.isNull) .map { metricValue => val numericValue = if (metricValue.isIntegralNumber) { @@ -989,7 +1051,12 @@ trait ElasticConversion { } else { None } - } match { + } + // An empty input yields an EXPLICIT null column rather than an absent key, so every + // consumer sees the same thing on every major (an absent key was the pre-existing + // behaviour on 6.8 / 7.17 / 9.0 and reads as NULL only because `rowNormalizer` + // null-fills under NativeContext). + .orElse(if (emptyInput) Some(name -> (null: Any)) else None) match { case Some(m) => // Skip auxiliary aggregations (from HAVING/WHERE/ORDER BY only, not in SELECT) val isAuxiliary = aggregations.get(m._1).exists(_.auxiliary) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 3a7de7467..5ecf72dfa 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -321,7 +321,11 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { ElasticQuery( single, collection.immutable.Seq(single.sources: _*), - sql = Some(single.sql), + // Story 22.2 — the statement AS WRITTEN, not the resolved one, exactly as `search` + // has always done (`sql = Some(query)` there). A resolved WHERE subquery carries up to + // 65,536 literals, and this render reaches the `Row query …` INFO line and + // `ElasticResponse.sql`. + sql = Some(parsed.sql), explodeNested = single.explodeNested ) scrollWithMetrics( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 2c4af57cf..4ddc2e88e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -34,8 +34,11 @@ import app.softnetwork.elastic.sql.query.{ SQLAggregation, SearchStatement, SelectStatement, - SingleSearch + SingleSearch, + SubqueryScope } +import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.schema.Schema import app.softnetwork.elastic.sql.query.TemporalLiterals import com.fasterxml.jackson.databind.JsonNode import com.typesafe.config.ConfigFactory @@ -155,18 +158,33 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC return ElasticResult.failure( RelationalClosureGuard.rejection(single, operation = "search") ) + // Story 22.2 — PHASE ONE. Every UNCORRELATED WHERE subquery is executed and its predicate is + // rewritten into the literal form Elasticsearch already runs (`terms` / a literal comparison / + // `match_all` / `match_none`) BEFORE the schema attach and the temporal resolution below, so + // the values it injects get EXACTLY the treatment a hand-written literal list gets (an ISO-8601 + // instant string is normalised against the OUTER column's mapped format, custom formats + // included). It runs for EVERY source shape — the single-concrete-source guard below is about + // the OUTER schema, not about this, and `FROM a, b WHERE x IN (SELECT …)` must be rewritten too. + // `hasWhereSubqueries` is ONE boolean per statement: zero cost for every statement without one. + val phaseOne: SingleSearch = + if (!single.hasWhereSubqueries) single + else + SubqueryResolver.resolve(single, this) match { + case ElasticSuccess(rewritten) => rewritten + case ElasticFailure(error) => return ElasticResult.failure(error) + } // #306 -- this used to return early for a statement whose WHERE carried no temporal literal. // That was correct while the only job was rewriting those literals, and is WRONG now that the // schema is also attached to the AST: almost no statement carries a temporal WHERE literal, // so the lookup would be skipped for almost every query and `baseType` would stay `Any`. - single.sources.distinct match { + phaseOne.sources.distinct match { case Seq(source) if !source.contains("*") && !source.contains(",") => this match { case indices: IndicesApi if !schemaMissed(source) => Try(indices.loadSchema(source)) match { case Success(ElasticSuccess(schema)) => schemaMisses.remove(source) - TemporalLiterals(single, schema) match { + TemporalLiterals(phaseOne, schema) match { case Right(literalsResolved) => // #306 -- ATTACH the schema to the AST. `GenericIdentifier.baseType` is // `col.map(_.dataType)` and `col` is populated only here, inside `update`, from @@ -187,7 +205,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // routes a row query through `scrollRows` -> `ScrollApi.scroll`, which resolves // AGAIN, so this runs twice on that path (pinned in SchemaAttachSpec). val resolved = literalsResolved.update(Some(schema)) - if (resolved ne single) + if (resolved ne phaseOne) logger.debug( s"Temporal literals resolved against the mapping of '$source':${resolved.where .map(_.sql) @@ -210,19 +228,84 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC logger.debug( s"Schema of '$source' unavailable (${error.message}) - temporal literals forwarded verbatim" ) - ElasticResult.success(single) + ElasticResult.success(phaseOne) case Failure(e) => logger.debug( s"Schema lookup for '$source' failed with ${e.getClass.getName} - temporal literals forwarded verbatim" ) - ElasticResult.success(single) + ElasticResult.success(phaseOne) } - case _ => ElasticResult.success(single) + case _ => ElasticResult.success(phaseOne) } - case _ => ElasticResult.success(single) + case _ => ElasticResult.success(phaseOne) } } + /** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free + * (story 22.2). + * + * `None` under every #306 skip condition — several sources, a wildcard, a client that is not an + * `IndicesApi`, a remembered miss, a failed or absent schema. The caller (`SubqueryResolver`) + * reads it for ONE decision: a `text` column cannot carry a terms aggregation, so mode P is + * declined for it. An unknown mapping therefore keeps the cheap default, and an Elasticsearch + * refusal propagates loudly with the inner statement's own message. + */ + private[client] def innerColumnType(inner: SingleSearch, name: String): Option[SQLType] = + resolvedSchema(inner).flatMap(_.find(name).map(_.dataType)) + + /** Story 22.2 (PD-2) — the SCHEMA-aware half of the correlation rule. + * + * A QUALIFIED reference to an outer alias is caught structurally at parse time + * (`SubqueryScope`). A BARE one cannot be: SQL resolves a bare name innermost-first, so `cid` + * inside the subquery is the INNER column whenever the inner index has it — and that is the + * right reading in the overwhelming majority of statements. When BOTH mappings are in hand, + * though, a bare name the inner index does NOT map while the outer one DOES is a correlated + * reference beyond reasonable doubt, and executing it as uncorrelated would send an unknown + * field to Elasticsearch and answer zero rows with HTTP 200 — the silent-wrong-answer mode epic + * 22 exists to close. + * + * Any schema-absent condition answers `None` (assume inner — PD-2's documented boundary), so + * this never turns a schema outage into a rejection. + */ + private[client] def bareNameCorrelation( + outer: SingleSearch, + inner: SingleSearch + ): Option[String] = + for { + innerMapping <- resolvedSchema(inner) + outerMapping <- resolvedSchema(outer) + // 🔴 A name the INNER SELECT list defines as an alias is an inner name, whatever the mappings + // say: `IN (SELECT id AS cid FROM customers ORDER BY cid)` references `cid`, which no mapping + // carries — and rejecting it because the OUTER index happens to have a `cid` column would + // refuse a perfectly self-contained subquery. `referencedIdentifiers` covers ORDER BY and + // GROUP BY, which is exactly where such an alias is referenced. + innerAliases = inner.select.fieldAliases.values.toSet + offender <- inner.referencedIdentifiers.find { id => + id.tableAlias.isEmpty && !id.nested && !id.name.contains(".") && + id.functions.isEmpty && id.name.nonEmpty && id.name != "*" && + !innerAliases.contains(id.name) && + innerMapping.find(id.name).isEmpty && outerMapping.find(id.name).isDefined + } + } yield SubqueryScope.bareCorrelatedMessage( + offender.name, + innerMapping.name, + outerMapping.name + ) + + private def resolvedSchema(s: SingleSearch): Option[Schema] = + s.sources.distinct match { + case Seq(source) if !source.contains("*") && !source.contains(",") => + this match { + case indices: IndicesApi if !schemaMissed(source) => + Try(indices.loadSchema(source)) match { + case Success(ElasticSuccess(schema)) => Some(schema) + case _ => None + } + case _ => None + } + case _ => None + } + /** Sources whose schema answered 404, with the time of the miss (see [[resolveWithSchema]]). Per * client instance, like the schema cache it shadows. Keys are caller-supplied FROM names, so the * map is bounded the way #238's `shardCountCache` is: above [[schemaMissPurgeThreshold]] entries @@ -557,7 +640,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC .getOrElse(query)}\nin indices '$indices' -> ${error.message}" ) ElasticResult.failure( - enrichMaxResultWindowError(error).copy( + enrichBoundError(error).copy( operation = Some("search"), index = Some(elasticQuery.indices.mkString(",")) ) @@ -664,7 +747,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC s"❌ Failed to execute multi-search for query \n$elasticQueries\n -> ${error.message}" ) ElasticResult.failure( - enrichMaxResultWindowError(error).copy( + enrichBoundError(error).copy( operation = Some("multiSearch") ) ) @@ -866,7 +949,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC ) Future.successful( ElasticResult.failure( - enrichMaxResultWindowError(error).copy( + enrichBoundError(error).copy( operation = Some("searchAsync"), index = Some(elasticQuery.indices.mkString(",")) ) @@ -884,7 +967,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC .getOrElse(query)}\nin indices '$indices' -> ${t.getMessage}" ) ElasticResult.failure( - enrichMaxResultWindowError( + enrichBoundError( ElasticError( message = s"Failed to execute search: ${t.getMessage}", cause = Some(t), @@ -982,7 +1065,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC ) Future.successful( ElasticResult.failure( - enrichMaxResultWindowError(error).copy( + enrichBoundError(error).copy( operation = Some("multiSearchAsync") ) ) @@ -996,7 +1079,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC s"❌ Failed to execute asynchronous multi-search for query \n$elasticQueries\n -> ${t.getMessage}" ) ElasticResult.failure( - enrichMaxResultWindowError( + enrichBoundError( ElasticError( message = s"Failed to execute multi-search: ${t.getMessage}", cause = Some(t), @@ -2076,6 +2159,46 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC * can still surface the rejection despite the scroll routing, so translate it into an actionable * message here; every other error passes through unchanged. */ + private[client] def enrichBoundError(error: ElasticError): ElasticError = + enrichMaxTermsCountError(enrichMaxResultWindowError(error)) + + /** Story 22.2 (AD-9) — the residual Elasticsearch-side bound, translated exactly as + * `index.max_result_window` was by #224. + * + * An index tuned BELOW the default (`index.max_terms_count = 100`) rejects a larger `terms` + * query with a shard-level `illegal_argument_exception` that names neither the subquery that + * produced the list nor the remedy; and mode P's own overflow on ES >= 7.10 is a + * `too_many_buckets_exception` against `search.max_buckets`. Both are the SAME bound from the + * analyst's point of view, so both become the SAME message `SubqueryResolver.tooMany` emits — + * one text whichever side hit the limit first. Every other error passes through unchanged. + */ + private def enrichMaxTermsCountError(error: ElasticError): ElasticError = { + def mentionsTerms(message: String): Boolean = + message != null && + (message.contains("max_terms_count") || message.contains("too_many_buckets")) + // The REST high-level clients (ES 6/7) surface the per-shard root cause as SUPPRESSED + // exceptions on an "all shards failed" wrapper, so the scan walks both chains (bounded). + def throwableMentionsTerms(t: Throwable, depth: Int = 10): Boolean = + t != null && depth > 0 && + (mentionsTerms(t.getMessage) || + t.getSuppressed.exists(x => throwableMentionsTerms(x, depth - 1)) || + throwableMentionsTerms(t.getCause, depth - 1)) + if (mentionsTerms(error.message) || error.cause.exists(t => throwableMentionsTerms(t))) + error.copy( + // 🔴 The scan is TEXTUAL and this helper now sits on EVERY search path, so the wording has + // to fit BOTH causes: a plain high-cardinality `GROUP BY` is the commonest way to exceed + // `search.max_buckets`, and it must not be told it wrote a subquery it did not write. + message = "This query exceeded an Elasticsearch cardinality bound: a `terms` query may " + + "carry at most `index.max_terms_count` values (default 65536) and an aggregation at " + + "most `search.max_buckets` buckets. Narrow the grouping, or the subquery that produced " + + "the values, raise the setting on the index, or - for a subquery - rewrite the " + + "statement as a JOIN (executed by the relational engine, " + + s"softclient4es-arrow-extensions). Elasticsearch said: ${error.message}", + statusCode = error.statusCode.orElse(Some(400)) + ) + else error + } + private def enrichMaxResultWindowError(error: ElasticError): ElasticError = { def mentionsWindow(message: String): Boolean = message != null && diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala b/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala new file mode 100644 index 000000000..2291986c9 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala @@ -0,0 +1,740 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql._ +// 🔴 RENAMED, deliberately: `app.softnetwork.elastic.client` (this package) declares its OWN +// `BooleanValue` / `StringValue` for aggregate results, and on the 2.12 leg a member of the +// enclosing package outranks BOTH a wildcard and an explicit import — so the two names silently +// resolve to the wrong classes there while the 2.13 leg compiles. Aliasing is the only spelling +// that means the same thing on both legs. Caught only by `+ core/compile`. +import app.softnetwork.elastic.sql.{BooleanValue => SqlBoolean, StringValue => SqlString} +import app.softnetwork.elastic.sql.`type`.{SQLTemporal, SQLType, SQLTypes} +import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.query._ + +import java.time.format.DateTimeFormatter +import java.time.{Instant, ZoneOffset} +import scala.collection.immutable.ListMap + +/** Story 22.2 / epic 22 AD-2 — ES-native TWO-PHASE execution of UNCORRELATED WHERE subqueries. + * + * Phase one runs each inner statement through the SAME client (`api.search`, so it crosses the + * seam itself: schema attach, temporal literals, scroll routing, #224 window translation and — + * recursively — its own subqueries) and rewrites the node into a literal criteria the bridges + * already emit. Phase two is the outer statement, unchanged. + * + * Executed SYNCHRONOUSLY, on the async path too (lead ruling OQ-3, 2026-09-14): a second, async + * seam would be two places for the subquery phase to go missing (#306's argument, verbatim). + * THREAD MODEL, read from source rather than assumed: phase one runs on the thread that CONSTRUCTS + * the outer request — `searchAsync` resolves BEFORE any `Future`, `ScrollApi.scroll` resolves at + * `Source` construction, i.e. the gateway `ActorSystem`'s dispatcher thread that called + * `SearchExecutor.execute`, or the caller's own thread on the direct API. An inner ROW-shaped body + * executes through `search(inner)`, which `Await`s its scroll on `SearchApi.scrollRoutingSystem` — + * a DIFFERENT system, so the blocked thread is never one the inner stream needs: no self-deadlock + * at any nesting depth. COST BOUND per subquery: mode P / scalar / EXISTS = one round trip; mode W + * row-shaped = at most `ceil(65,537 / scrollSize)` pages (66 at the default 1,000), sliced on ES + * >= 7.15. + * + * 🔴 Inner statements execute under `EntityContext`, NOT `NativeContext`: under `NativeContext` + * `ElasticConversion.rowNormalizer` NULL-FILLS a requested output name the response does not + * carry, so a mismatch between `Field.outputName` and the key the converter produced (an + * un-aliased `AVG(amount)`, a qualified `c.id`) would read as a genuine NULL — a + * `MatchNoneCriteria`, i.e. a SILENT WRONG ANSWER. Under `EntityContext` the key is ABSENT and + * [[cell]] / [[collect]] answer a 400 naming the column and the columns that ARE there. + */ +object SubqueryResolver { + + /** Elasticsearch's default `index.max_terms_count` — the ceiling on the values a `terms` query + * may carry (6.8 -> 9.x, per index). ALSO this resolver's own hard bound: more distinct values + * than this is a LOUD 400 naming the limit and the JOIN rewrite, never a truncated `terms` (the + * #205 family). Equal to `Bucket.DefaultSize` by coincidence of Elasticsearch's defaults, not by + * construction — the two are kept separate deliberately. + */ + val MaxTerms: Int = 65536 + + /** 🔴 `MaxTerms + 1`, never `MaxTerms`. With a bucket size of exactly 65,536 a 70,000-value + * column would come back as EXACTLY 65,536 buckets — under the limit, silently truncated (#205 + * one story over). One bucket more is what makes the overflow VISIBLE to [[tooMany]]. + */ + private val Bound: Limit = Limit(MaxTerms + 1, None) + + private type Execute = SingleSearch => ElasticResult[ElasticResponse] + private type ColumnType = (SingleSearch, String) => Option[SQLType] + + /** The rewritten statement (the SAME instance when nothing changed), or the FIRST failure: the + * inner statement's own error with ITS status (never flattened, #184), or a 400 of this + * object's. + */ + def resolve(single: SingleSearch, api: SearchApi): ElasticResult[SingleSearch] = + resolve( + single, + s => api.search(s)(EntityContext), // an absent key stays ABSENT — never null-filled + api.bareNameCorrelation(single, _), + api.innerColumnType + ) + + /** Test seam: the inner executor, the bare-name correlation check and the inner column's mapped + * type are injected, so the whole two-phase contract is provable Docker-free. `columnType(inner, + * name)` answers `None` when no schema is loadable (the #306 skip list) — mode P stays the + * default then. + */ + private[client] def resolve( + single: SingleSearch, + execute: Execute, + correlation: SingleSearch => Option[String] = _ => None, + columnType: ColumnType = (_, _) => None + ): ElasticResult[SingleSearch] = + single.where.flatMap(_.criteria) match { + case None => ElasticResult.success(single) + case Some(criteria) => + rewrite(criteria, execute, correlation, columnType) match { + case Right(c) if c eq criteria => ElasticResult.success(single) + case Right(c) => ElasticResult.success(single.copy(where = Some(Where(Some(c))))) + case Left(error) => ElasticResult.failure(error) + } + } + + /** The SAME walk shape as `TemporalLiterals.rewrite` — predicate / relations / leaf, preserving + * the instance (`eq`) wherever nothing changed, so a statement with no subquery is returned + * untouched and the seam's second crossing (`search -> scrollRows -> ScrollApi.scroll`, #306) is + * a genuine no-op. + */ + private def rewrite( + criteria: Criteria, + execute: Execute, + correlation: SingleSearch => Option[String], + columnType: ColumnType + ): Either[ElasticError, Criteria] = criteria match { + case p: Predicate => + for { + l <- rewrite(p.leftCriteria, execute, correlation, columnType) + r <- rewrite(p.rightCriteria, execute, correlation, columnType) + } yield + if ((l eq p.leftCriteria) && (r eq p.rightCriteria)) p + else p.copy(leftCriteria = l, rightCriteria = r) + case n: ElasticNested => + rewrite(n.criteria, execute, correlation, columnType).map(c => + if (c eq n.criteria) n else n.copy(criteria = c) + ) + case n: ElasticChild => + rewrite(n.criteria, execute, correlation, columnType).map(c => + if (c eq n.criteria) n else n.copy(criteria = c) + ) + case n: ElasticParent => + rewrite(n.criteria, execute, correlation, columnType).map(c => + if (c eq n.criteria) n else n.copy(criteria = c) + ) + case s: SubqueryCriteria => resolveOne(s, execute, correlation, columnType) + case other => Right(other) + } + + private def resolveOne( + node: SubqueryCriteria, + execute: Execute, + correlation: SingleSearch => Option[String], + columnType: ColumnType + ): Either[ElasticError, Criteria] = + node.inner match { + // Unreachable after `validate()`; `GatewayApi.run(statement)` does not validate a + // programmatically built statement, so the arm is a named 400 rather than a MatchError. + case None => Left(bad(s"Unsupported subquery body in ${node.sql}")) + case Some(inner) => + correlation(inner) match { + case Some(reason) => Left(bad(reason)) + case None => + node match { + case in: InSubquery => resolveIn(in, inner, execute, columnType) + case ex: ExistsSubquery => resolveExists(ex, inner, execute) + case sc: ScalarSubquery => resolveScalar(sc, inner, execute) + case qs: QuantifiedSubquery => + resolveQuantified(qs, inner, execute, columnType) + } + } + } + + // ── IN ───────────────────────────────────────────────────────────────────────────────────── + + /** MODE P (projection): the body is `SELECT FROM … [WHERE …] [ORDER BY …]` with no + * GROUP BY / HAVING / LIMIT / window / EXCEPT / JOIN. Its DISTINCT values ARE a terms + * aggregation over the column — one bounded request whatever the row count — so it is executed + * as `… GROUP BY LIMIT MaxTerms+1` (a LIMIT on a grouped statement IS the bucket size, + * `Bucket.update`). MODE W (as written): everything else runs unchanged, except that a + * row-shaped body with no LIMIT (or a LIMIT window above the bound) is bounded at `MaxTerms + 1` + * rows so the scroll it routes to cannot run away. + * + * 🔴 Mode P is a terms AGGREGATION, and a terms aggregation over a `text` column is an + * Elasticsearch 400 (no fielddata) for a statement the analyst wrote WITHOUT a GROUP BY. When + * the inner mapping is loadable and says `text`, the body runs AS WRITTEN (mode W); an unknown + * mapping keeps mode P (the cheap default) and an ES refusal then propagates loudly with the + * inner statement's own message. + */ + private def valueSet( + inner: SingleSearch, + node: SubqueryCriteria, + execute: Execute, + columnType: ColumnType + ): Either[ElasticError, (Seq[Any], Boolean, Boolean)] = { + val field = inner.select.fields.head + val projection = if (declinesModeP(inner, field, columnType)) None else modeP(inner, field) + // 🔴 The column the CONVERTER produces for the statement ACTUALLY EXECUTED, derived the SAME way + // `SearchApi.extractOutputFieldNames` derives it — `fieldsWithComputedAliases`, not `fields`. + // MEASURED on real ES 8.18: an UN-ALIASED aggregate (`SELECT AVG(amount)`, the headline scalar + // shape) comes back under the synthetic alias `__c1` while `select.fields.head.outputName` is + // `amount`, so reading the plain field list answered "no column 'amount'" for a statement that + // had run perfectly. One derivation, two consumers (the 21.3 lesson). + for { + statement <- projection + .map(Right(_): Either[ElasticError, SingleSearch]) + .getOrElse(bounded(inner, node)) + column = columnNameOf(statement) + rows <- run(statement, execute, node) + collected <- collect(rows, column, node, strict = projection.isDefined) + values = collected._1 + hadNull = collected._2 + // A terms aggregation NEVER yields a null key (documents without the field are skipped), so + // a null cell on mode P can only mean the column was not produced UNDER `column` — a naming + // defect between `Field.outputName` and the converter's bucket key. Loud, never MatchNone. + _ <- + if (projection.isDefined && hadNull) + Left( + bad( + s"Subquery ${node.sql}: the projected column '$column' was not produced by the " + + "aggregation (naming defect, report it)" + ) + ) + else Right(()) + _ <- if (values.size > MaxTerms) Left(tooMany(values.size, node)) else Right(()) + } yield (values, hadNull, projection.isDefined) + } + + private def resolveIn( + node: InSubquery, + inner: SingleSearch, + execute: Execute, + columnType: ColumnType + ): Either[ElasticError, Criteria] = + for { + resolved <- valueSet(inner, node, execute, columnType) + values = resolved._1 + hadNull = resolved._2 + modePUsed = resolved._3 + // ANSI NULL rule (PD-5): a NULL among the inner values is invisible to `IN` (no row can equal + // it) and makes `NOT IN` UNKNOWN for EVERY row — zero rows. Mode P cannot see NULLs (terms + // aggregations skip missing values), so `NOT IN` probes for one explicitly. + nullSeen <- + if (node.maybeNot.isEmpty || hadNull) Right(hadNull) + else if (modePUsed) nullProbe(inner, execute, node) + else Right(false) + } yield + if (node.maybeNot.isDefined && nullSeen) MatchNoneCriteria() + else if (values.isEmpty) + if (node.maybeNot.isDefined) MatchAllCriteria() else MatchNoneCriteria() + else TermValues.in(node.identifier, values, node.maybeNot) + + private[client] def columnNameOf(s: SingleSearch): String = + s.select.fieldsWithComputedAliases.head.outputName + + /** Mode P is a terms AGGREGATION over the projected column, and TWO mapped types cannot carry + * one: + * + * - `text` has no fielddata, so Elasticsearch answers 400; + * - a `date` column buckets as a DATE HISTOGRAM, which needs an interval nobody wrote — + * MEASURED on real ES 8.18 as `all shards failed; Invalid interval specified, must be + * non-null and non-empty` for `IN (SELECT since FROM customers WHERE region = 'EU')`. + * + * Both therefore run the body AS WRITTEN (mode W). An UNKNOWN mapping keeps mode P, the cheap + * default, and an Elasticsearch refusal then propagates loudly with the inner statement's own + * message rather than being guessed at here. + */ + private def declinesModeP(inner: SingleSearch, field: Field, columnType: ColumnType): Boolean = + columnType(inner, field.identifier.name).exists { + case SQLTypes.Text => true + case _: SQLTemporal => true + case _ => false + } + + private def modeP(inner: SingleSearch, field: Field): Option[SingleSearch] = { + val id = field.identifier + val bare = id.functions.isEmpty && id.name.nonEmpty && id.name != "*" && !id.nested + if ( + bare && inner.groupBy.isEmpty && inner.having.isEmpty && inner.limit.isEmpty && + inner.windowFunctions.isEmpty && inner.select.except.isEmpty && inner.from.joins.isEmpty + ) + Some( + inner + .copy( + select = Select(Seq(field), None), + groupBy = Some(GroupBy(Seq(Bucket(id)))), + orderBy = None, // order is irrelevant to a SET + limit = Some(Bound) // = the bucket size, MaxTerms + 1 + ) + .update() + ) + else None + } + + /** Mode W with the row bound: a row-shaped body KEEPS its own LIMIT when it has one within the + * bound (`IN (SELECT id FROM t ORDER BY score DESC LIMIT 100)` means those 100); otherwise it is + * bounded at `MaxTerms + 1` rows, which `SearchApi.search` routes through scroll with + * `maxDocuments`. An aggregation-shaped body is bounded by its own bucket sizes. + */ + private def bounded( + inner: SingleSearch, + node: SubqueryCriteria + ): Either[ElasticError, SingleSearch] = + if (!inner.returnsRows) Right(inner) + else + inner.limit match { + case Some(l) + if l.limit.toLong + l.offset.map(_.offset.toLong).getOrElse(0L) <= MaxTerms + 1 => + Right(inner) + // 🔴 REJECTED, never rewritten. Replacing a written `LIMIT 10 OFFSET 100000` with + // `LIMIT 65537` executes a COMPLETELY DIFFERENT window and, if the rows it reads hold at + // most MaxTerms distinct values, the count check never fires — a silent wrong answer in the + // one arm whose whole point is "the body means what it says". + case Some(l) => + Left( + bad( + s"Subquery ${node.sql} asks for a window beyond $MaxTerms rows (${l.sql.trim}). " + + "Narrow the subquery's LIMIT/OFFSET, or rewrite the statement as a JOIN (executed by " + + "the relational engine, softclient4es-arrow-extensions)." + ) + ) + case None => Right(inner.copy(limit = Some(Bound))) + } + + /** `SELECT FROM … WHERE () AND IS NULL LIMIT 1` — one row means the + * inner set contains a NULL. Built on the AST (`Predicate`, `IsNullExpr`, `Limit`), never by + * re-parsing a render; its `.sql` is pinned as a parser fixed point in `SubqueryResolverSpec`. + */ + private def nullProbe( + inner: SingleSearch, + execute: Execute, + node: SubqueryCriteria + ): Either[ElasticError, Boolean] = { + val field = inner.select.fields.head + val isNull: Criteria = IsNullExpr(field.identifier) + val where = inner.where.flatMap(_.criteria) match { + // NOT-FOLD: this CONSTRUCTS a predicate rather than reading one — the fourth argument is the + // literal `None`, i.e. the probe carries no NOT at all, so there is no negation to fold onto + // either operand. (`PainlessNullSurvivalSpec`'s scan cannot type a receiver and counts the + // positional `Predicate(_, _, _, x, _)` shape whichever direction it runs in.) + case Some(c) => Predicate(c, AND, isNull, None, group = true) + case None => isNull + } + val probe = inner + .copy( + select = Select(Seq(field), None), + where = Some(Where(Some(where))), + orderBy = None, + limit = Some(Limit(1, None)) + ) + .update() + run(probe, execute, node).map(_.nonEmpty) + } + + // ── quantified (lead ruling OQ-4) ─────────────────────────────────────────────────────────── + + /** ` ANY|SOME|ALL ()` for the ten combinations `IN` / `NOT IN` do not already + * cover. The body resolves EXACTLY as [[resolveIn]] 's does; the quantifier is reduced HERE, + * from the resolved value list. + */ + private def resolveQuantified( + node: QuantifiedSubquery, + inner: SingleSearch, + execute: Execute, + columnType: ColumnType + ): Either[ElasticError, Criteria] = + for { + resolved <- valueSet(inner, node, execute, columnType) + values = resolved._1 + hadNull = resolved._2 + modePUsed = resolved._3 + // Which forms need to KNOW whether the inner set carries a NULL: + // - every `ALL` form — a NULL makes the universal UNKNOWN for every row; + // - 🔴 every NEGATED form, `ANY` included. `NOT (x > ANY S)` is NOT `NOT (x > min S)` once S + // holds a NULL: with `S = {1, NULL}` and `x = 0` the inner is `FALSE OR UNKNOWN` = + // UNKNOWN, so its negation is UNKNOWN and no row matches — while `must_not(range gt 1)` + // would RETURN that row. A plain (un-negated) `ANY` genuinely ignores inner NULLs. + // Mode P cannot see a NULL (terms aggregations skip missing values), so it probes — exactly + // as `NOT IN` does, through the SAME helper. + nullSeen <- + if (!(node.universal || node.maybeNot.isDefined) || hadNull) Right(hadNull) + else if (modePUsed) nullProbe(inner, execute, node) + else Right(false) + reduced <- reduceQuantified(node, values, nullSeen) + } yield reduced + + /** 🔴 The ANSI reduction table. Order matters and the two empty-set rows are OPPOSITE — the + * single most likely thing to ship backwards: + * + * 1. `ALL` over a set containing NULL is UNKNOWN for every row (never TRUE, never FALSE), so + * it answers no rows and an outer `NOT` does NOT flip it — UNKNOWN negated is still + * UNKNOWN. `ANY` simply ignores inner NULLs. + * 1. Over an EMPTY set an existential (`ANY`/`SOME`) is FALSE and a universal (`ALL`) is TRUE. + * Both are genuine truth values, so an outer `NOT` DOES flip them. + * 1. Otherwise: `> ANY` is `> min`, `> ALL` is `> max`, and the two irreducible forms use both + * ends — `= ALL` is `min == max && x = min`, `<> ANY` is its negation. + * + * Story 22.3 runs the CORRELATED twins of these forms on DuckDB, which is ANSI; the two venues + * must agree on the same statement, which is the argument that decided PD-5. + */ + private[client] def reduceQuantified( + node: QuantifiedSubquery, + values: Seq[Any], + nullSeen: Boolean + ): Either[ElasticError, Criteria] = { + val negate = node.maybeNot.isDefined + def truth(b: Boolean): Criteria = + if (b != negate) MatchAllCriteria() else MatchNoneCriteria() + // UNKNOWN, and UNKNOWN negated is still UNKNOWN — so this answer is NOT run through `truth`. + if ((node.universal || negate) && nullSeen) Right(MatchNoneCriteria()) + else if (values.isEmpty) Right(truth(node.universal)) + else { + val ordered = TermValues.sorted(values) + val lo = ordered.head + val hi = ordered.last + def cmp(op: ComparisonOperator, v: Any): Criteria = + GenericExpression(node.identifier, op, TermValues.literal(v), node.maybeNot) + (node.operator, node.universal) match { + case (GT, false) => Right(cmp(GT, lo)) + case (GE, false) => Right(cmp(GE, lo)) + case (LT, false) => Right(cmp(LT, hi)) + case (LE, false) => Right(cmp(LE, hi)) + case (GT, true) => Right(cmp(GT, hi)) + case (GE, true) => Right(cmp(GE, hi)) + case (LT, true) => Right(cmp(LT, lo)) + case (LE, true) => Right(cmp(LE, lo)) + // `x = ALL (S)` — true only when S is a singleton value and x equals it. + case (EQ, true) => Right(if (lo == hi) cmp(EQ, lo) else truth(false)) + // `x <> ANY (S)` — the negation of the row above: true unless S is a singleton x equals. + case (NE | DIFF, false) => Right(if (lo == hi) cmp(NE, lo) else truth(true)) + // `= ANY` and `<> ALL` collapse to `InSubquery` at PARSE time, so no PARSED statement can + // reach this arm — but `GatewayApi.run(statement)` accepts a programmatically built AST, and + // answering `match_none` there would be a silent wrong answer. Loud, like the sibling arm in + // `resolveOne`. + case (op, universal) => + Left( + bad( + s"Unsupported quantified comparison '$op ${if (universal) "ALL" else "ANY"}' in " + + s"${node.sql}: this combination is expressed as IN / NOT IN." + ) + ) + } + } + } + + // ── EXISTS ───────────────────────────────────────────────────────────────────────────────── + + /** True iff the body returns >= 1 row. A row-shaped body is capped at `LIMIT 1` (its own `LIMIT + * 0` is KEPT: zero rows, EXISTS false — DuckDB agrees); an aggregation-shaped body runs as + * written (a metric-only SELECT always yields one row => EXISTS true, as in every ANSI engine). + * A GROUP BY body with NO HAVING is capped at one BUCKET; with a HAVING it must run as written, + * because the `bucket_selector` filters AFTER the size cut. + */ + private def resolveExists( + node: ExistsSubquery, + inner: SingleSearch, + execute: Execute + ): Either[ElasticError, Criteria] = { + val statement = + if (inner.returnsRows && !inner.limit.exists(_.limit == 0)) + inner.copy(limit = Some(Limit(1, None))) + else if (inner.groupBy.isDefined && inner.having.isEmpty && inner.limit.isEmpty) + inner.copy(limit = Some(Limit(1, None))) + else inner + run(statement, execute, node).map { rows => + if (rows.nonEmpty != node.maybeNot.isDefined) MatchAllCriteria() else MatchNoneCriteria() + } + } + + // ── scalar ───────────────────────────────────────────────────────────────────────────────── + + /** 0 rows -> the subquery is NULL -> the comparison is UNKNOWN -> no row (`MatchNoneCriteria`, + * and NOT negated by an outer `NOT`: `NOT (x > NULL)` is UNKNOWN too — ANSI, and DuckDB); 1 row + * -> a literal comparison the bridge already emits; > 1 rows -> 400 (ANSI cardinality + * violation). + */ + private def resolveScalar( + node: ScalarSubquery, + inner: SingleSearch, + execute: Execute + ): Either[ElasticError, Criteria] = { + // 🔴 BOUNDED, like every other inner execution. `ScalarSubquery.validate` keeps a scalar body + // to "metric-only, or LIMIT 1", but `GatewayApi.run(statement)` does not validate a + // programmatically built AST — and two rows are all this method ever needs in order to report + // the ANSI cardinality violation below, so an unbounded row-shaped body can never run away here. + val statement = + if (inner.returnsRows && !inner.limit.exists(_.limit <= 2)) + inner.copy(limit = Some(Limit(2, None))) + else inner + val column = columnNameOf(statement) + for { + rows <- run(statement, execute, node) + value <- rows match { + case Seq() => Right(None) + case Seq(row) => cell(row, column, node).flatMap(scalarCell(_, node)) + case many => + Left( + bad(s"Scalar subquery returned ${many.size} rows, expected at most one: ${node.sql}") + ) + } + } yield value match { + case None => MatchNoneCriteria() + case Some(v) => + GenericExpression(node.identifier, node.operator, TermValues.literal(v), node.maybeNot) + } + } + + /** A scalar cell may arrive as a ONE-element sequence (the row path's array wrap of a script + * field — `UPPER(a) AS up` -> `List("X")`), so it is unwrapped; a genuinely multi-valued cell is + * not a scalar (400, the ANSI cardinality violation on the VALUE axis). + */ + private def scalarCell(v: Any, node: SubqueryCriteria): Either[ElasticError, Option[Any]] = + v match { + case null => Right(None) + case s: scala.collection.Seq[_] if s.size <= 1 => Right(s.headOption.flatMap(Option(_))) + case s: scala.collection.Seq[_] => + Left(bad(s"Scalar subquery ${node.sql} returned a multi-valued cell (${s.size} values)")) + case other => Right(Some(other)) + } + + // ── plumbing ─────────────────────────────────────────────────────────────────────────────── + + private def run( + s: SingleSearch, + execute: Execute, + node: SubqueryCriteria + ): Either[ElasticError, Seq[ListMap[String, Any]]] = + execute(s) match { + case ElasticSuccess(response) => Right(response.results) + // #184 — the inner failure keeps ITS status and cause; only the message says where it + // happened. Mode P's ES-side overflow on >= 7.10 is a `too_many_buckets_exception`, which is + // the SAME bound the resolver's own count check enforces, so the analyst sees ONE message + // whichever side hit it first. + case ElasticFailure(error) => + if (mentionsTooManyBuckets(error)) Left(tooManyFrom(error, node)) + else Left(error.copy(message = s"Subquery ${node.sql} failed: ${error.message}")) + } + + private def mentionsTooManyBuckets(error: ElasticError): Boolean = { + def mentions(message: String): Boolean = + message != null && message.contains("too_many_buckets") + def walk(t: Throwable, depth: Int): Boolean = + t != null && depth > 0 && + (mentions(t.getMessage) || + t.getSuppressed.exists(s => walk(s, depth - 1)) || + walk(t.getCause, depth - 1)) + mentions(error.message) || error.cause.exists(t => walk(t, 10)) + } + + /** The projected column's values, DISTINCT and non-null, plus whether a null was seen. + * + * A cell that is a sequence (the row path's array wrap of a script field, or a genuinely + * multi-valued field) is FLATTENED — SQL-on-arrays semantics: `x IN (SELECT tags FROM t)` + * matches any tag. A MISSING column is a 400, never guessed. + */ + private def collect( + rows: Seq[ListMap[String, Any]], + column: String, + node: SubqueryCriteria, + strict: Boolean + ): Either[ElasticError, (Seq[Any], Boolean)] = { + val builder = scala.collection.mutable.LinkedHashSet.empty[Any] + var hadNull = false + var failure: Option[ElasticError] = None + // 🔴 Under `EntityContext` a document carrying NO VALUE for the projected column yields a row + // WITHOUT the key — which on the row path (mode W) is an ordinary SQL NULL, not a defect. + // Treating it as one turned `x NOT IN (SELECT FROM t)`, the exact ANSI shape + // PD-5 exists for, into an HTTP 400 instead of zero rows. `strict` is true only for mode P, + // where the aggregation contract really does guarantee the key; on the row path the + // naming-defect 400 is kept for the case that can only BE one — rows came back and NOT ONE of + // them carries the column. + if (!strict && rows.nonEmpty && !rows.exists(_.contains(column))) + return Left(missingColumn(rows.head, column, node)) + rows.foreach { row => + if (failure.isEmpty) + (if (strict) cell(row, column, node) + else Right(row.getOrElse(column, null))) match { + case Left(e) => failure = Some(e) + case Right(v) => + val flat: Seq[Any] = v match { + case null => Seq(null) + case s: scala.collection.Seq[_] => if (s.isEmpty) Seq(null) else s.toList + case other => Seq(other) + } + flat.foreach { + case null => hadNull = true + // 🔴 CANONICALISED before the set, not after: a `ZonedDateTime` and the `Instant` it + // converts to are distinct objects that render to the SAME ISO-8601 literal, and the + // converters produce both shapes for one `date` column. Deduplicating on the raw cell + // would emit the term twice and — worse — count it twice against MaxTerms. + case x => builder += TermValues.canonical(x) + } + } + } + failure.map(Left(_)).getOrElse(Right((builder.toList, hadNull))) + } + + private def cell( + row: ListMap[String, Any], + column: String, + node: SubqueryCriteria + ): Either[ElasticError, Any] = + row.get(column).toRight(missingColumn(row, column, node)) + + private def missingColumn( + row: ListMap[String, Any], + column: String, + node: SubqueryCriteria + ): ElasticError = + bad( + s"Subquery ${node.sql} returned no column '$column' " + + s"(columns: ${row.keys.mkString(", ")})" + ) + + private def tooMany(n: Int, node: SubqueryCriteria): ElasticError = + bad( + s"Subquery ${node.sql} returned more than $MaxTerms distinct values ($n or more), the " + + "maximum a terms query accepts (index.max_terms_count / search.max_buckets). Narrow the " + + "subquery, or rewrite the statement as a JOIN (executed by the relational engine, " + + "softclient4es-arrow-extensions)." + ) + + private def tooManyFrom(error: ElasticError, node: SubqueryCriteria): ElasticError = + tooMany(MaxTerms + 1, node).copy(cause = error.cause) + + private def bad(message: String): ElasticError = + ElasticError(message = message, statusCode = Some(400), operation = Some("subquery")) +} + +/** Typed literal values for the rewritten predicate, built from the cells the CONVERTER returned + * (Jackson's smallest type per value — `ElasticConversion.jsonNodeToAny`; `extractBucketKey` for a + * terms key). + * + * - every integral -> `LongValues`; + * - ANY floating value -> the WHOLE list becomes `DoubleValues` (a column mixing `1` and `2.5` + * must not be typed on its FIRST value, which is how `inToQuery` types the terms query); + * - strings -> `StringValues`; booleans -> `BooleanValues` (rendered `true` / `false`, which an + * Elasticsearch boolean field accepts). + * + * 🔴 TEMPORAL cells are the arm the converters make NECESSARY: `extractBucketKey` prefers + * `key_as_string` and parses it into a `ZonedDateTime`, turns an integral key in the epoch-millis + * range into one too, and `jsonNodeToAny` parses date-shaped strings on the row path. So every + * `java.time.temporal.Temporal` / `java.util.Date` cell is rendered as an ISO-8601 INSTANT string + * (UTC) inside a `StringValue` — exactly what a hand-written `IN ('2024-01-01T00:00:00Z')` is — so + * `TemporalLiterals`' `InExpr(StringValues)` / `GenericExpression(StringValue)` arms then + * normalise it against the OUTER column's mapped format, custom formats included. NEVER `toString` + * (a `ZonedDateTime` renders `…Z[UTC]`, which no ES date format accepts) and never epoch millis (a + * custom format without `epoch_millis` rejects them). + * + * Documented residual: the converter's millis-range heuristic also turns a `long` id between 1e12 + * and 1e13 into a date — the same value already DISPLAYS as a date at the REPL, so such a column + * is unusable in mode P until that heuristic is revisited (out of scope here). + * + * [[in]] returns the CRITERIA (an `InExpr` built INSIDE, where `R` / `T` are concrete) rather than + * an existential `Values[_, _ <: Value[_]]`: `InExpr[R, +T <: Value[R]]` will not infer through an + * existential on the 2.12 leg. + */ +private[client] object TermValues { + + /** 🔴 FIXED-WIDTH, deliberately — NOT `DateTimeFormatter.ISO_INSTANT`. + * + * `ISO_INSTANT` emits 0, 3, 6 or 9 fractional digits depending on the value, so + * `"2024-01-01T00:00:00.500Z"` and `"2024-01-01T00:00:00Z"` differ in LENGTH and compare + * `'.'(0x2E) < 'Z'(0x5A)` — the LATER instant sorts FIRST. [[sorted]] reads that order to pick + * the `min`/`max` a quantified comparison reduces to, so a millisecond-bearing `date` column + * would have produced `> ALL` bounds that are too low: rows Elasticsearch must not return. With + * three digits always present, lexicographic order IS chronological order, and + * `strict_date_optional_time` accepts the spelling on every supported major. + */ + private val Iso: DateTimeFormatter = + DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC) + + def in(identifier: Identifier, values: Seq[Any], maybeNot: Option[NOT.type]): Criteria = + if (values.forall(isIntegral)) + InExpr(identifier, LongValues(values.map(v => LongValue(asLong(v)))), maybeNot) + else if (values.forall(isNumeric)) + InExpr(identifier, DoubleValues(values.map(v => DoubleValue(asDouble(v)))), maybeNot) + else if (values.forall(_.isInstanceOf[java.lang.Boolean])) + InExpr( + identifier, + BooleanValues(values.map(v => SqlBoolean(v.asInstanceOf[java.lang.Boolean]))), + maybeNot + ) + else InExpr(identifier, StringValues(values.map(v => SqlString(asString(v)))), maybeNot) + + /** The form a cell is COMPARED and DEDUPLICATED in: the exact literal the predicate will carry. + * Only temporals move (to their UTC ISO-8601 instant); everything else is its own canonical + * form. + */ + def canonical(v: Any): Any = v match { + case _: java.util.Date | _: java.time.temporal.Temporal => asString(v) + case other => other + } + + def literal(v: Any): Value[_] = v match { + case b: java.lang.Boolean => SqlBoolean(b) + case x if isIntegral(x) => LongValue(asLong(x)) + case x if isNumeric(x) => DoubleValue(asDouble(x)) + case other => SqlString(asString(other)) + } + + /** The ordering the quantified reduction's `min` / `max` are read from. + * + * Numeric values compare numerically; everything else compares as the STRING the predicate will + * carry — which is what Elasticsearch's own `range` query over a keyword field does, and which + * is chronological for the ISO-8601 instants temporal cells render as (same length, same `Z` + * suffix, so lexicographic order IS time order). + */ + def sorted(values: Seq[Any]): Seq[Any] = + // Integral values are ordered as LONGS: `asDouble` loses precision above 2^53, which would pick + // the wrong neighbour as the max of a `long` id column. + if (values.forall(isIntegral)) values.sortBy(asLong) + else if (values.forall(isNumeric)) values.sortBy(asDouble) + else values.sortBy(asString) + + private def isIntegral(v: Any): Boolean = v match { + case _: java.lang.Byte | _: java.lang.Short | _: java.lang.Integer | _: java.lang.Long => true + case _: java.math.BigInteger => true + case _ => false + } + + private def isNumeric(v: Any): Boolean = isIntegral(v) || (v match { + case _: java.lang.Float | _: java.lang.Double => true + case _: java.math.BigDecimal => true + case _ => false + }) + + private def asLong(v: Any): Long = v.asInstanceOf[Number].longValue() + + private def asDouble(v: Any): Double = v.asInstanceOf[Number].doubleValue() + + private def asString(v: Any): String = v match { + case s: String => s + case i: Instant => Iso.format(i) + case d: java.util.Date => Iso.format(d.toInstant) + case t: java.time.ZonedDateTime => Iso.format(t.toInstant) + case t: java.time.OffsetDateTime => Iso.format(t.toInstant) + case t: java.time.LocalDateTime => Iso.format(t.toInstant(ZoneOffset.UTC)) + case t: java.time.LocalDate => Iso.format(t.atStartOfDay(ZoneOffset.UTC).toInstant) + case t: java.time.temporal.Temporal => Iso.format(Instant.from(t)) + case other => String.valueOf(other) + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 17884fe9a..da4b1c0a5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -440,6 +440,27 @@ package object client extends SerializationApi { aggType == AggregationType.DenseRank def singleValued: Boolean = !multivalued + /** ANSI: an aggregate computed over ZERO input rows is NULL — with two deliberate exceptions. + * + * - `COUNT` is 0, never NULL. That is ANSI's own rule and it is the single most damaging + * thing to get backwards here, so it is stated first. + * - `SUM` keeps Elasticsearch's own answer, `0.0`. RECORDED DECISION (story 22.2, Winston): + * ANSI says NULL, but the `sum` aggregation returns `0.0` over no documents on EVERY + * supported major (measured on 6.8, 7.17, 8.18 and 9.0), so passing it through reports + * what the engine computed instead of inventing a different value; synthesising NULL would + * mean overriding the engine on every major and silently flipping `SUM` from `0` to NULL + * for every existing consumer — a far larger blast radius than the defect being fixed. The + * divergence from ANSI is documented rather than papered over. + * + * Everything else — `MIN`, `MAX`, `AVG`, the STDDEV/VARIANCE family, the percentile family and + * `bucket_script` arithmetic over them — is unambiguous: Elasticsearch answers `null` and so + * must we. + */ + def nullOverEmptyInput: Boolean = aggType match { + case AggregationType.Count | AggregationType.Sum => false + case _ => singleValued + } + def ranking: Boolean = aggType == AggregationType.RowNumber || aggType == AggregationType.Rank || diff --git a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala index ee16233b8..a7d88cc0b 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala @@ -18,7 +18,7 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{SearchStatement, SingleSearch} +import app.softnetwork.elastic.sql.query.{relationalClosureRequired, SearchStatement, SingleSearch} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.slf4j.{Logger, LoggerFactory} @@ -99,6 +99,22 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers { err.message should not include RelationalClosureGuard.ExtensionJar } + /** Story 22.2 (AD-8) — an UNCORRELATED WHERE subquery is PASSTHROUGH: it executes ES-natively in + * core at every venue, so it must NOT trip this guard. + * + * The assertion is falsifiable in BOTH directions here. The refusal must not be the closure one; + * and the message it IS must name the SUBQUERY, which proves phase one actually ran — the + * resolver executed the inner statement through this same (cluster-less) client and reported its + * failure. A seam that skipped the rewrite would report the OUTER statement's failure instead. + */ + it should "let an uncorrelated WHERE subquery through the guard, and RUN phase one" in { + val sql = "SELECT id FROM orders WHERE cid IN (SELECT id FROM customers)" + relationalClosureRequired(searchStatement(sql)) shouldBe false + val err = refusalOf(client().search(searchStatement(sql))) + err.message should not include RelationalClosureGuard.ExtensionJar + err.message should include("Subquery") + } + it should "leave a JOIN UNNEST alone — it is not a cross-index JOIN" in { val err = refusalOf( client().search( diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala new file mode 100644 index 000000000..d7612b2bc --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala @@ -0,0 +1,676 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql._ +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.operator.{DIFF, GT, NOT} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.collection.immutable.ListMap + +/** Story 22.2 — the two-phase rewrite, Docker-free, through the resolver's injected executor. + * + * Every statement the resolver DERIVES (mode P's GROUP BY, mode W's bound, the NULL probe) is + * asserted on its SHAPE and pinned as a parser fixed point, so a derived statement that renders + * into something the engine could not re-read fails here rather than at a customer's cluster. + */ +class SubqueryResolverSpec extends AnyFlatSpec with Matchers { + + private def single(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s + case other => fail(s"[$sql] $other") + } + + private def rows(column: String, values: Any*): ElasticResult[ElasticResponse] = + ElasticSuccess( + ElasticResponse( + None, + "{}", + values.map(v => ListMap[String, Any](column -> v)), + ListMap.empty, + ListMap.empty + ) + ) + + /** Records every inner statement the resolver executes and answers from the table. */ + private class Recording(answers: PartialFunction[SingleSearch, ElasticResult[ElasticResponse]]) { + val executed = scala.collection.mutable.ArrayBuffer.empty[SingleSearch] + val execute: SingleSearch => ElasticResult[ElasticResponse] = { s => + executed += s + answers.applyOrElse(s, (x: SingleSearch) => fail(s"unexpected inner statement: ${x.sql}")) + } + } + + private def whereOf(r: ElasticResult[SingleSearch]): Criteria = + r match { + case ElasticSuccess(s) => + s.where.flatMap(_.criteria).getOrElse(fail(s"no WHERE in ${s.sql}")) + case other => fail(s"expected a success, got $other") + } + + private def errorOf(r: ElasticResult[_]): ElasticError = r match { + case ElasticFailure(e) => e + case other => fail(s"expected a failure, got $other") + } + + private def valuesOf(c: Criteria): Values[_, _] = c match { + case in: InExpr[_, _] => in.values + case other => fail(s"expected an InExpr, got $other") + } + + // ── IN ─────────────────────────────────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - IN" + + it should "run a bare-column body as a GROUP BY with bucket size MaxTerms+1 (mode P)" in { + val outer = + single("SELECT id FROM orders WHERE customer_id IN (SELECT id FROM c WHERE r = 'EU')") + val rec = new Recording({ case s if s.groupBy.isDefined => rows("id", 3L, 1L, 2L, 3L) }) + val res = SubqueryResolver.resolve(outer, rec.execute) + val inner = rec.executed.head + inner.groupBy.map(_.buckets.map(_.identifier.name)) shouldBe Some(Seq("id")) + inner.limit shouldBe Some(Limit(SubqueryResolver.MaxTerms + 1, None)) + inner.where.map(_.sql.trim) shouldBe Some("WHERE r = 'EU'") + inner.returnsRows shouldBe false + Parser(inner.sql) shouldBe Right(inner) // the DERIVED statement is a parser fixed point + // distinct, order-preserving, typed on the WHOLE list + valuesOf(whereOf(res)) shouldBe LongValues(Seq(LongValue(3), LongValue(1), LongValue(2))) + } + + it should "widen a mixed integral/floating column to DoubleValues (never typed by the first value)" in { + val outer = single("SELECT id FROM t WHERE amount IN (SELECT amount FROM u)") + val res = SubqueryResolver.resolve( + outer, + new Recording({ case _ => rows("amount", 1L, 2.5d) }).execute + ) + valuesOf(whereOf(res)) shouldBe DoubleValues(Seq(DoubleValue(1.0), DoubleValue(2.5))) + } + + it should "emit StringValues and BooleanValues for string and boolean columns" in { + val s = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE name IN (SELECT name FROM u)"), + new Recording({ case _ => rows("name", "a", "b") }).execute + ) + valuesOf(whereOf(s)) shouldBe StringValues(Seq(StringValue("a"), StringValue("b"))) + val b = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE flag IN (SELECT flag FROM u)"), + new Recording({ case _ => + rows("flag", java.lang.Boolean.TRUE, java.lang.Boolean.FALSE) + }).execute + ) + valuesOf(whereOf(b)) shouldBe BooleanValues(Seq(BooleanValue(true), BooleanValue(false))) + } + + /** 🔴 FIXED-WIDTH milliseconds, not `ISO_INSTANT`. `ISO_INSTANT` varies its fraction width, so + * `"…T00:00:00.500Z"` sorts BEFORE `"…T00:00:00Z"` (`'.' < 'Z'`) and `TermValues.sorted` — which + * reads that order to pick the min/max a quantified comparison reduces to — would have inverted + * the bound for any millisecond-bearing date column. Three digits always present makes + * lexicographic order chronological, and `strict_date_optional_time` accepts the spelling. + */ + it should "render TEMPORAL cells as fixed-width ISO-8601 instants, chronologically ordered" in { + val zdt = java.time.ZonedDateTime.parse("2024-01-01T00:00:00Z") + val res = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE created IN (SELECT created FROM u)"), + new Recording({ case _ => rows("created", zdt, zdt.toInstant) }).execute + ) + // both cells are the SAME instant, so the set collapses to one value + valuesOf(whereOf(res)) shouldBe StringValues(Seq(StringValue("2024-01-01T00:00:00.000Z"))) + // and the ordering the quantified reduction reads IS chronological across fraction widths + val half = zdt.plusNanos(500000000L) + TermValues.sorted(Seq(half, zdt).map(TermValues.canonical)) shouldBe + Seq("2024-01-01T00:00:00.000Z", "2024-01-01T00:00:00.500Z") + val sc = single("SELECT id FROM t WHERE created > (SELECT MAX(created) AS m FROM u)") + whereOf( + SubqueryResolver.resolve(sc, new Recording({ case _ => rows("m", zdt) }).execute) + ) shouldBe GenericExpression( + sc.whereSubqueries.head.asInstanceOf[ScalarSubquery].identifier, + GT, + StringValue("2024-01-01T00:00:00.000Z"), + None + ) + } + + it should "run a `text` inner column in mode W, and an unknown mapping in mode P" in { + val outer = single("SELECT id FROM t WHERE name IN (SELECT name FROM u)") + val r1 = new Recording({ case _ => rows("name", "x") }) + SubqueryResolver.resolve(outer, r1.execute, columnType = (_, _) => Some(SQLTypes.Text)) + r1.executed.head.groupBy shouldBe None // a terms aggregation on `text` is an ES 400 + val r2 = new Recording({ case _ => rows("name", "x") }) + SubqueryResolver.resolve(outer, r2.execute) + r2.executed.head.groupBy shouldBe defined // unknown mapping: the cheap default + } + + it should "fail LOUDLY when the projected column is absent from the rows (EntityContext keeps it absent)" in { + // the headline UN-ALIASED aggregate: under NativeContext this would null-fill and answer + // MatchNone, a silent wrong answer + val outer = single("SELECT id FROM t WHERE amount > (SELECT AVG(amount) FROM u)") + // 🔴 The column the CONVERTER produces, which for an UN-ALIASED aggregate is the SYNTHETIC + // alias `__c1`, NOT the source field `amount`. Measured on real ES 8.18: reading + // `select.fields.head.outputName` here asked for `amount` and the resolver answered "no column" + // for a statement that had run perfectly. + val column = SubqueryResolver.columnNameOf(outer.whereSubqueries.head.inner.get) + column shouldBe "__c1" + val err = errorOf( + SubqueryResolver.resolve(outer, new Recording({ case _ => rows("other_key", 1.5d) }).execute) + ) + err.statusCode shouldBe Some(400) + err.message should include(column) + err.message should include("other_key") + // the same shape with the RIGHT key resolves — the resolver reads the ONE name it requests + whereOf( + SubqueryResolver.resolve(outer, new Recording({ case _ => rows(column, 100.5d) }).execute) + ) shouldBe a[GenericExpression] + } + + it should "fail LOUDLY on a null produced by mode P (a terms bucket never yields one)" in { + val err = errorOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u)"), + new Recording({ case _ => rows("a", null) }).execute + ) + ) + err.message should include("naming defect") + } + + it should "flatten the row-path array wrap and multi-valued cells" in { + val res = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE tag IN (SELECT UPPER(tag) AS up FROM u)"), + new Recording({ case _ => rows("up", List("A"), List("B", "C")) }).execute + ) + valuesOf(whereOf(res)) shouldBe StringValues(Seq("A", "B", "C").map(StringValue)) + } + + it should "keep a body's own LIMIT and bound a row-shaped no-LIMIT body at MaxTerms+1" in { + val r1 = new Recording({ case _ => rows("a", "x") }) + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u ORDER BY score DESC LIMIT 100)"), + r1.execute + ) + r1.executed.head.limit shouldBe Some(Limit(100, None)) + val r2 = new Recording({ case _ => rows("up", "X") }) + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT UPPER(a) AS up FROM u)"), + r2.execute + ) + r2.executed.head.groupBy shouldBe None // a scripted projection is not a bare column: mode W + r2.executed.head.limit shouldBe Some(Limit(SubqueryResolver.MaxTerms + 1, None)) + } + + it should "resolve an EMPTY set to match_none for IN and match_all for NOT IN" in { + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u)"), + new Recording({ case _ => rows("a") }).execute + ) + ) shouldBe MatchNoneCriteria() + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a NOT IN (SELECT a FROM u)"), + // mode P cannot see a NULL, so `NOT IN` probes for one: the probe answers no row here + new Recording({ + case s if s.groupBy.isDefined => rows("a") + case s if s.limit.contains(Limit(1, None)) => rows("a") + }).execute + ) + ) shouldBe MatchAllCriteria() + } + + // ── NULL semantics (PD-5) ──────────────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - ANSI NULL semantics" + + it should "ignore inner NULLs for IN" in { + val res = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u LIMIT 10)"), + new Recording({ case _ => rows("a", 1L, null, 2L) }).execute + ) + valuesOf(whereOf(res)) shouldBe LongValues(Seq(LongValue(1), LongValue(2))) + } + + it should "resolve NOT IN to match_none when a mode-W set contains NULL" in { + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a NOT IN (SELECT a FROM u LIMIT 10)"), + new Recording({ case _ => rows("a", 1L, null) }).execute + ) + ) shouldBe MatchNoneCriteria() + } + + it should "probe for NULL with `... AND a IS NULL LIMIT 1` for NOT IN in mode P, and honour it" in { + val outer = single("SELECT id FROM t WHERE a NOT IN (SELECT a FROM u WHERE b = 1)") + val rec = new Recording({ + case s if s.groupBy.isDefined => rows("a", 1L, 2L) + case s if s.limit.contains(Limit(1, None)) => rows("a", null) // one row = a NULL exists + }) + whereOf(SubqueryResolver.resolve(outer, rec.execute)) shouldBe MatchNoneCriteria() + val probe = rec.executed(1) + probe.where.map(_.sql.trim) shouldBe Some("WHERE (b = 1 AND a IS NULL)") + Parser(probe.sql) shouldBe Right(probe) + // and with NO null the predicate is the ordinary NOT IN + val rec2 = new Recording({ + case s if s.groupBy.isDefined => rows("a", 1L, 2L) + case _ => rows("a") + }) + whereOf(SubqueryResolver.resolve(outer, rec2.execute)) + .asInstanceOf[InExpr[_, _]] + .maybeNot shouldBe Some(NOT) + } + + // ── EXISTS ─────────────────────────────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - EXISTS" + + it should "cap a row-shaped body at LIMIT 1, keep LIMIT 0, and run an aggregate body as written" in { + val r1 = new Recording({ case _ => rows("id", "x") }) + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE EXISTS (SELECT 1 FROM u)"), + r1.execute + ) + ) shouldBe MatchAllCriteria() + r1.executed.head.limit shouldBe Some(Limit(1, None)) + val r2 = new Recording({ case _ => rows("id") }) + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE EXISTS (SELECT id FROM u LIMIT 0)"), + r2.execute + ) + ) shouldBe MatchNoneCriteria() + r2.executed.head.limit shouldBe Some(Limit(0, None)) + val r3 = new Recording({ case _ => rows("n", 0L) }) + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE EXISTS (SELECT COUNT(*) AS n FROM u)"), + r3.execute + ) + ) shouldBe MatchAllCriteria() // a metric-only SELECT always yields one row + } + + it should "cap a GROUP BY body without HAVING at one bucket and run a HAVING body as written" in { + val r1 = new Recording({ case _ => rows("a", "x") }) + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE EXISTS (SELECT a FROM u GROUP BY a)"), + r1.execute + ) + r1.executed.head.limit shouldBe Some(Limit(1, None)) + val r2 = new Recording({ case _ => rows("a", "x") }) + SubqueryResolver.resolve( + single( + "SELECT id FROM t WHERE EXISTS (SELECT a, COUNT(*) AS n FROM u GROUP BY a HAVING n > 1)" + ), + r2.execute + ) + r2.executed.head.limit shouldBe None + } + + it should "negate under NOT" in { + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE NOT EXISTS (SELECT 1 FROM u)"), + new Recording({ case _ => rows("id", "x") }).execute + ) + ) shouldBe MatchNoneCriteria() + whereOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE NOT EXISTS (SELECT 1 FROM u)"), + new Recording({ case _ => rows("id") }).execute + ) + ) shouldBe MatchAllCriteria() + } + + // ── scalar ─────────────────────────────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - scalar" + + it should "rewrite one row into a literal comparison, zero rows into match_none, two rows into a 400" in { + val outer = single("SELECT id FROM t WHERE amount > (SELECT AVG(amount) AS a FROM u)") + whereOf( + SubqueryResolver.resolve(outer, new Recording({ case _ => rows("a", 12.5d) }).execute) + ) shouldBe GenericExpression( + outer.whereSubqueries.head.asInstanceOf[ScalarSubquery].identifier, + GT, + DoubleValue(12.5), + None + ) + // 0 rows -> the subquery is NULL -> UNKNOWN -> no row, NOT negated by an outer NOT + whereOf( + SubqueryResolver.resolve(outer, new Recording({ case _ => rows("a") }).execute) + ) shouldBe MatchNoneCriteria() + val many = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE amount > (SELECT amount AS a FROM u LIMIT 1)"), + new Recording({ case _ => rows("a", 1L, 2L) }).execute + ) + errorOf(many).statusCode shouldBe Some(400) + errorOf(many).message should include("returned 2 rows") + } + + it should "unwrap a one-element scalar cell and reject a multi-valued one" in { + val outer = single("SELECT id FROM t WHERE tag = (SELECT UPPER(tag) AS up FROM u LIMIT 1)") + whereOf( + SubqueryResolver.resolve(outer, new Recording({ case _ => rows("up", List("X")) }).execute) + ) shouldBe a[GenericExpression] + errorOf( + SubqueryResolver + .resolve(outer, new Recording({ case _ => rows("up", List("X", "Y")) }).execute) + ).message should include("multi-valued") + } + + // ── quantified (lead ruling OQ-4) ──────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - quantified comparison" + + /** An `ALL` form in mode P probes for an inner NULL exactly as `NOT IN` does, so the NULL probe + * (the only inner statement with `LIMIT 1`) is answered with NO row unless a row explicitly + * passes `null` among its values. + */ + private def quantified(sql: String, values: Any*): Criteria = + whereOf( + SubqueryResolver.resolve( + single(sql), + new Recording({ + case s if s.limit.contains(Limit(1, None)) => + if (values.contains(null)) rows("a", null) else rows("a") + case _ => rows("a", values: _*) + }).execute + ) + ) + + private def idOf(sql: String): Identifier = + single(sql).whereSubqueries.head.asInstanceOf[QuantifiedSubquery].identifier + + /** 🔴 The full reduction table, every row, values `{1, 5, 9}` so `min != max`. */ + it should "reduce every ordering combination to a MIN/MAX comparison" in { + val table = Seq( + "> ANY" -> (GT -> 1L), + ">= ANY" -> (app.softnetwork.elastic.sql.operator.GE -> 1L), + "< ANY" -> (app.softnetwork.elastic.sql.operator.LT -> 9L), + "<= ANY" -> (app.softnetwork.elastic.sql.operator.LE -> 9L), + "> ALL" -> (GT -> 9L), + ">= ALL" -> (app.softnetwork.elastic.sql.operator.GE -> 9L), + "< ALL" -> (app.softnetwork.elastic.sql.operator.LT -> 1L), + "<= ALL" -> (app.softnetwork.elastic.sql.operator.LE -> 1L) + ) + table.foreach { case (spelling, (op, bound)) => + val sql = s"SELECT id FROM t WHERE amount $spelling (SELECT a FROM u)" + withClue(s"$spelling: ") { + quantified(sql, 5L, 1L, 9L) shouldBe + GenericExpression(idOf(sql), op, LongValue(bound), None) + } + } + } + + it should "reduce = ALL and <> ANY, which need BOTH ends" in { + val eqAll = "SELECT id FROM t WHERE amount = ALL (SELECT a FROM u)" + quantified(eqAll, 7L, 7L) shouldBe + GenericExpression(idOf(eqAll), app.softnetwork.elastic.sql.operator.EQ, LongValue(7), None) + quantified(eqAll, 7L, 8L) shouldBe MatchNoneCriteria() // not a singleton: no row can equal all + val neAny = "SELECT id FROM t WHERE amount <> ANY (SELECT a FROM u)" + quantified(neAny, 7L, 7L) shouldBe + GenericExpression(idOf(neAny), app.softnetwork.elastic.sql.operator.NE, LongValue(7), None) + quantified(neAny, 7L, 8L) shouldBe MatchAllCriteria() // some value differs from every x + quantified("SELECT id FROM t WHERE amount != ANY (SELECT a FROM u)", 7L, 8L) shouldBe + MatchAllCriteria() + } + + /** 🔴 The two empty-set rows are OPPOSITE — ANSI: an existential over an empty set is FALSE, a + * universal over an empty set is TRUE. The single most likely thing to ship backwards. + */ + it should "answer FALSE for ANY and TRUE for ALL over an EMPTY subquery" in { + quantified("SELECT id FROM t WHERE amount > ANY (SELECT a FROM u)") shouldBe MatchNoneCriteria() + quantified("SELECT id FROM t WHERE amount < ANY (SELECT a FROM u)") shouldBe MatchNoneCriteria() + quantified("SELECT id FROM t WHERE amount = ALL (SELECT a FROM u)") shouldBe MatchAllCriteria() + quantified("SELECT id FROM t WHERE amount > ALL (SELECT a FROM u)") shouldBe MatchAllCriteria() + quantified("SELECT id FROM t WHERE amount <> ANY (SELECT a FROM u)") shouldBe + MatchNoneCriteria() + } + + /** 🔴 An inner NULL makes every `ALL` form UNKNOWN for every row — and UNKNOWN negated is still + * UNKNOWN, so an outer NOT does NOT flip it. `ANY` ignores inner NULLs. + */ + it should "answer no rows for ALL when the inner set carries a NULL, and ignore it for ANY" in { + quantified("SELECT id FROM t WHERE amount > ALL (SELECT a FROM u LIMIT 9)", 1L, null) shouldBe + MatchNoneCriteria() + quantified( + "SELECT id FROM t WHERE NOT amount > ALL (SELECT a FROM u LIMIT 9)", + 1L, + null + ) shouldBe MatchNoneCriteria() + val anySql = "SELECT id FROM t WHERE amount > ANY (SELECT a FROM u LIMIT 9)" + quantified(anySql, 5L, null, 1L) shouldBe + GenericExpression(idOf(anySql), GT, LongValue(1), None) + } + + it should "probe for a NULL in mode P for an ALL form, exactly as NOT IN does" in { + val outer = single("SELECT id FROM t WHERE amount > ALL (SELECT a FROM u WHERE b = 1)") + val rec = new Recording({ + case s if s.groupBy.isDefined => rows("a", 1L, 9L) + case s if s.limit.contains(Limit(1, None)) => rows("a", null) + }) + whereOf(SubqueryResolver.resolve(outer, rec.execute)) shouldBe MatchNoneCriteria() + rec.executed should have size 2 + rec.executed(1).where.map(_.sql.trim) shouldBe Some("WHERE (b = 1 AND a IS NULL)") + } + + it should "flip a genuine truth value under NOT, and a comparison's own maybeNot" in { + quantified("SELECT id FROM t WHERE NOT amount > ANY (SELECT a FROM u)") shouldBe + MatchAllCriteria() // NOT FALSE + quantified("SELECT id FROM t WHERE NOT amount > ALL (SELECT a FROM u)") shouldBe + MatchNoneCriteria() // NOT TRUE + val sql = "SELECT id FROM t WHERE NOT amount > ANY (SELECT a FROM u)" + quantified(sql, 1L, 9L) shouldBe GenericExpression(idOf(sql), GT, LongValue(1), Some(NOT)) + } + + // ── bounds, status, recursion ──────────────────────────────────────────────────────────────── + + behavior of "SubqueryResolver - bounds, status, purity" + + it should "fail loudly above MaxTerms distinct values, naming the setting and the JOIN rewrite" in { + val outer = single("SELECT id FROM t WHERE a IN (SELECT a FROM u)") + val err = errorOf( + SubqueryResolver.resolve( + outer, + new Recording({ case _ => + rows("a", (1L to (SubqueryResolver.MaxTerms + 1).toLong).map(Long.box): _*) + }).execute + ) + ) + err.statusCode shouldBe Some(400) + err.message should include("index.max_terms_count") + err.message should include("JOIN") + // exactly MaxTerms values is fine + SubqueryResolver.resolve( + outer, + new Recording({ case _ => + rows("a", (1L to SubqueryResolver.MaxTerms.toLong).map(Long.box): _*) + }).execute + ) shouldBe a[ElasticSuccess[_]] + } + + it should "translate an inner too_many_buckets_exception into the same bound message" in { + val err = errorOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u)"), + new Recording({ case _ => + ElasticFailure( + ElasticError( + "all shards failed: too_many_buckets_exception: Trying to create too many buckets", + None, + Some(400) + ) + ) + }).execute + ) + ) + err.message should include("index.max_terms_count") + err.message should include("JOIN") + } + + it should "propagate an inner failure with ITS status and cause (#184)" in { + val boom = new RuntimeException("shard down") + val err = errorOf( + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u)"), + new Recording({ case _ => + ElasticFailure(ElasticError("unavailable", Some(boom), Some(503))) + }).execute + ) + ) + err.statusCode shouldBe Some(503) + err.cause shouldBe Some(boom) + err.message should include("Subquery") + } + + it should "return the SAME instance when the statement carries no subquery" in { + val s = single("SELECT id FROM t WHERE a = 1") + SubqueryResolver + .resolve(s, new Recording({ case _ => fail("must not execute") }).execute) + .asInstanceOf[ElasticSuccess[SingleSearch]] + .value should be theSameInstanceAs s + } + + it should "reject a correlated bare name through the injected mapping check" in { + val outer = single("SELECT id FROM orders WHERE cid IN (SELECT id FROM customers WHERE r = 1)") + val res = SubqueryResolver.resolve( + outer, + new Recording({ case _ => rows("id", 1L) }).execute, + _ => Some("Correlated subquery: 'r' is not a column of 'customers'") + ) + errorOf(res).statusCode shouldBe Some(400) + errorOf(res).message should include("Correlated subquery") + } + + it should "rewrite BOTH sides of a predicate and keep every untouched criteria instance" in { + val outer = single( + "SELECT id FROM t WHERE a = 1 AND b IN (SELECT b FROM u) OR EXISTS (SELECT 1 FROM v)" + ) + val rec = new Recording({ + case s if s.sources.contains("u") => rows("b", 7L) + case s if s.sources.contains("v") => rows("id", "x") + }) + val rewritten = SubqueryResolver.resolve(outer, rec.execute) match { + case ElasticSuccess(s) => s + case other => fail(s"$other") + } + rec.executed should have size 2 + rewritten.whereSubqueries shouldBe empty + rewritten.where.map(_.sql.trim) shouldBe Some("WHERE a = 1 AND b IN (7) OR 1 = 1") + } + + it should "refuse LOUDLY a quantified combination the parser cannot build" in { + // `= ANY` and `<> ALL` collapse to InSubquery at PARSE time, so no parsed statement reaches this + // arm — but `GatewayApi.run(statement)` takes a programmatically built AST, where answering + // `match_none` would be a silent wrong answer. + val node = single("SELECT id FROM t WHERE amount > ANY (SELECT a FROM u)").whereSubqueries.head + .asInstanceOf[QuantifiedSubquery] + val res = SubqueryResolver.reduceQuantified( + node.copy(operator = DIFF, quantifier = app.softnetwork.elastic.sql.operator.ALL), + Seq(1L, 2L), + nullSeen = false + ) + res.isLeft shouldBe true + res.swap.toOption.get.message should include("Unsupported quantified comparison") + } + + /** 🔴 Independent-review H4: a NEGATED `ANY` over a NULL-bearing set is UNKNOWN, not + * `must_not(min)`. With `S = {1, NULL}` and `x = 0`, `x > ANY S` is `FALSE OR UNKNOWN` = + * UNKNOWN, so its negation is UNKNOWN and NO row matches — while `must_not(range gt 1)` would + * return it. + */ + it should "answer no rows for a NEGATED ANY when the inner set carries a NULL" in { + quantified( + "SELECT id FROM t WHERE NOT amount > ANY (SELECT a FROM u LIMIT 9)", + 1L, + null + ) shouldBe + MatchNoneCriteria() + // and it PROBES for that NULL in mode P, exactly as NOT IN and the ALL family do + val outer = single("SELECT id FROM t WHERE NOT amount > ANY (SELECT a FROM u WHERE b = 1)") + val rec = new Recording({ + case s if s.groupBy.isDefined => rows("a", 1L, 9L) + case s if s.limit.contains(Limit(1, None)) => rows("a", null) + }) + whereOf(SubqueryResolver.resolve(outer, rec.execute)) shouldBe MatchNoneCriteria() + rec.executed should have size 2 + } + + /** 🔴 Independent-review M1: a written window BEYOND the bound is REFUSED, never silently + * replaced — executing `LIMIT 65537 OFFSET 0` in place of `LIMIT 10 OFFSET 100000` reads a + * completely different window and the count check would never notice. + */ + it should "refuse a body whose own LIMIT window exceeds the bound" in { + val res = SubqueryResolver.resolve( + single("SELECT id FROM t WHERE a IN (SELECT a FROM u ORDER BY a LIMIT 10 OFFSET 100000)"), + new Recording({ case _ => fail("must not execute") }).execute + ) + errorOf(res).statusCode shouldBe Some(400) + errorOf(res).message should include("beyond 65536 rows") + } + + /** 🔴 Independent-review H3: on the ROW path a document that carries no value for the projected + * column yields a row WITHOUT the key, and that is an ordinary SQL NULL — not the naming defect + * mode P's contract makes it. Turning it into a 400 broke the exact ANSI shape PD-5 exists for. + */ + it should "read an ABSENT key on the row path as a NULL, and stay loud when NO row carries it" in { + val notIn = single("SELECT id FROM t WHERE a NOT IN (SELECT a FROM u LIMIT 10)") + val mixed = ElasticSuccess( + ElasticResponse( + None, + "{}", + Seq(ListMap[String, Any]("a" -> 1L), ListMap.empty[String, Any]), + ListMap.empty, + ListMap.empty + ) + ) + whereOf(SubqueryResolver.resolve(notIn, new Recording({ case _ => mixed }).execute)) shouldBe + MatchNoneCriteria() // the absent key IS the NULL that makes NOT IN empty + val none = ElasticSuccess( + ElasticResponse( + None, + "{}", + Seq(ListMap[String, Any]("other" -> 1L)), + ListMap.empty, + ListMap.empty + ) + ) + errorOf( + SubqueryResolver.resolve(notIn, new Recording({ case _ => none }).execute) + ).message should include("returned no column 'a'") + } + + /** 🔴 Independent-review M4: a scalar body is bounded at two rows — enough to report the ANSI + * cardinality violation, impossible to run away even when `validate()` was bypassed. + */ + it should "cap a row-shaped scalar body at LIMIT 2" in { + val rec = new Recording({ case _ => rows("a", 1L) }) + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE amount > (SELECT a FROM u LIMIT 1)"), + rec.execute + ) + rec.executed.head.limit shouldBe Some(Limit(1, None)) // its own smaller LIMIT is kept + val rec2 = new Recording({ case _ => rows("a", 1L) }) + SubqueryResolver.resolve( + single("SELECT id FROM t WHERE amount > (SELECT MAX(a) AS a FROM u)"), + rec2.execute + ) + rec2.executed.head.limit shouldBe None // an aggregate body is bounded by its own shape + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala index 98dc5b124..6ec57e785 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala @@ -182,6 +182,26 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { trunc.flatMap(t => Some(t.warning)).getOrElse("") should not be empty } + /** Story 22.2 (AD-8 / PD-4) — ROUTING and the licensed cap do not move for a WHERE subquery. + * + * ⚠️ Scope of this row, stated so it is not read as more than it is: `RecordingClient` overrides + * `scroll` wholesale and therefore never crosses `resolveWithSchema`, so nothing here exercises + * the subquery PHASE — it asserts only that the statement is claimed by `canHandle` as an + * ordinary `DqlStatement`, takes the scroll arm rather than `searchAsync`, and is capped at the + * OUTER quota (the inner statement is not quota-checked, PD-4). The phase itself is proved + * Docker-free in `SubqueryResolverSpec` and `RelationalClosureGuardSpec`, and on a real cluster + * in `WhereSubqueryCompletenessSpec`. + */ + it should "route an uncorrelated WHERE subquery exactly like a plain SELECT (story 22.2)" in { + val (client, res) = + run("SELECT a, b FROM idx WHERE a IN (SELECT a FROM other)", Quota.Community) + res shouldBe a[ElasticSuccess[_]] + client.scrolledStatement.get() shouldBe a[SingleSearch] + client.scrolledConfig.get().maxDocuments shouldBe Some(10000L) // the OUTER cap, unchanged + client.searchedStatement.get() shouldBe null + truncationOf(res).map(_.limit) shouldBe Some(10000L) + } + it should "cap a no-LIMIT query at the Pro quota (1,000,000) via scroll, never searchAsync" in { val (client, res) = run("SELECT a, b FROM idx", Quota.Pro, LicenseType.Pro) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index c39e9bc52..6eca023d4 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -31,10 +31,13 @@ import app.softnetwork.elastic.sql.query.{ IsNotNullExpr, IsNullCriteria, IsNullExpr, + MatchAllCriteria, MatchCriteria, + MatchNoneCriteria, NestedElement, NestedElements, - Predicate + Predicate, + SubqueryCriteria } import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.FetchSourceContext @@ -184,6 +187,20 @@ case class ElasticBridge(filter: ElasticFilter) { case matchExpression: MatchCriteria => matchExpression case isNull: IsNullCriteria => isNull case isNotNull: IsNotNullCriteria => isNotNull + // Story 22.2 — the two RESOLVED sentinels a WHERE subquery collapses to. + case _: MatchAllCriteria => matchAllQuery() + case _: MatchNoneCriteria => matchNoneQuery() + // 🔴 Story 22.2 — an UNRESOLVED subquery node must never reach a bridge. It is replaced by a + // literal criteria at `SearchApi.resolveWithSchema` (the ONE seam), so arriving here means a + // statement was handed straight to `singleSearch` / `singleSearchToJsonQuery`, which bypass + // it. Named rather than swallowed by the `Unsupported filter type` default below, because + // the alternative — emitting nothing for the predicate — is a silent wrong answer. + case s: SubqueryCriteria => + throw new IllegalArgumentException( + s"Unresolved WHERE subquery reached the query builder: ${s.sql}. Subqueries are " + + "executed at SearchApi.resolveWithSchema before translation; a statement handed " + + "straight to singleSearch / singleSearchToJsonQuery bypasses it." + ) case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index d66281469..f76d43367 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -1,7 +1,8 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.Criteria +import app.softnetwork.elastic.sql.operator.NOT +import app.softnetwork.elastic.sql.query.{Criteria, InExpr, MatchAllCriteria, MatchNoneCriteria} import com.fasterxml.jackson.databind.JsonNode import com.sksamuel.elastic4s.ElasticApi.matchAllQuery import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn @@ -1341,4 +1342,45 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { json should not include "o.status" } + /** Story 22.2 — the two RESOLVED sentinels a WHERE subquery collapses to, and the loud refusal of + * an UNRESOLVED node. + * + * 🔴 The `terms` row pins the EXACT AST `SubqueryResolver` produces, so the resolver spec and + * this one share one shape: if the resolver ever built a differently typed `Values`, the emitted + * query would move and this row would say so. + */ + private def asQueryOf(criteria: Criteria): String = { + import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + SearchBodyBuilderFn(SearchRequest("*") query criteria.asQuery()).string + } + + it should "emit match_all / match_none for the resolved subquery sentinels (story 22.2)" in { + asQueryOf(MatchAllCriteria()).replaceAll("\\s", "") should include("\"match_all\":{}") + asQueryOf(MatchNoneCriteria()).replaceAll("\\s", "") should include("\"match_none\":{}") + } + + it should "emit the terms clause for the resolver's IN shape (story 22.2)" in { + val in: Criteria = + InExpr(GenericIdentifier("customer_id"), LongValues(Seq(LongValue(3), LongValue(1))), None) + asQueryOf(in).replaceAll("\\s", "") should include("\"terms\":{\"customer_id\":[3,1]}") + asQueryOf( + InExpr(GenericIdentifier("customer_id"), LongValues(Seq(LongValue(3))), Some(NOT)) + ).replaceAll("\\s", "") should include("\"must_not\"") + } + + it should "refuse an UNRESOLVED subquery node BY NAME, never silently (story 22.2)" in { + implicit def timestamp: Long = 0L + val node = parser + .Parser("SELECT id FROM t WHERE a IN (SELECT a FROM u)") + .toOption + .collect { case s: query.SingleSearch => s } + .flatMap(_.where.flatMap(_.criteria)) + .getOrElse(fail("expected a WHERE subquery")) + val ex = the[IllegalArgumentException] thrownBy node.asQuery() + ex.getMessage should include("Unresolved WHERE subquery") + ex.getMessage should include("resolveWithSchema") + } + } diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWhereSubqueryCompletenessSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..2728909ef --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWhereSubqueryCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientWhereSubqueryCompletenessSpec extends WhereSubqueryCompletenessSpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..b9c162500 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientWhereSubqueryCompletenessSpec extends WhereSubqueryCompletenessSpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..b9c162500 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWhereSubqueryCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientWhereSubqueryCompletenessSpec extends WhereSubqueryCompletenessSpec diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..a39af6e04 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientWhereSubqueryCompletenessSpec extends WhereSubqueryCompletenessSpec diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..a39af6e04 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWhereSubqueryCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientWhereSubqueryCompletenessSpec extends WhereSubqueryCompletenessSpec diff --git a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala index b51f85ccc..516cbe20b 100644 --- a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala +++ b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala @@ -44,6 +44,22 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { // Positive Tests (Should Compile) // ============================================================ + // Story 22.2 — an uncorrelated WHERE subquery IS typeable at compile time: the statement is a + // `SingleSearch` over the OUTER index, which is exactly what the row type binds against, and the + // inner query is an execution step the macro never has to type. No new macro arm was needed; this + // row is the guard that none is silently added. + it should "ACCEPT an uncorrelated WHERE subquery at compile time (story 22.2)" in { + assertCompiles(""" + import app.softnetwork.elastic.client.macros.TestElasticClientApi + import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats + import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Strings + import app.softnetwork.elastic.sql.query.SelectStatement + + TestElasticClientApi.searchAs[Strings]( + "SELECT vchar::VARCHAR, c::CHAR, text FROM strings WHERE c IN (SELECT c FROM others)" + )""") + } + "SQLQueryValidator" should "VALIDATE all numeric types" in { assertCompiles(""" import app.softnetwork.elastic.client.macros.TestElasticClientApi diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala index b58aa3998..157e99d6f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLKeywords.scala @@ -113,9 +113,12 @@ import app.softnetwork.elastic.sql.function.time.{ } import app.softnetwork.elastic.sql.operator.{ AGAINST, + ALL, AND, + ANY, BETWEEN, Child, + EXISTS, IN, IS_NOT_NULL, IS_NULL, @@ -126,6 +129,7 @@ import app.softnetwork.elastic.sql.operator.{ OR, Parent, RLIKE, + SOME, UNION } import app.softnetwork.elastic.sql.query.{ @@ -218,6 +222,13 @@ object SQLKeywords { Nested, Child, Parent, + // Story 22.2 — WHERE subqueries. `EXISTS` and `ALL` were already words (`statementWords`, + // `wordsOf("UNION ALL")`); `ANY` and `SOME` are new ones the `Expr` scan of `SQLKeywordsSpec` + // requires. Registering all four keeps REPL highlighting uniform. + EXISTS, + ANY, + SOME, + ALL, Case, WHEN, THEN, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index bd8fc4f84..fc36179b7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -36,7 +36,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes } import app.softnetwork.elastic.sql.parser.Validator -import app.softnetwork.elastic.sql.query.{CriteriaWithConditionalFunction, Expression} +import app.softnetwork.elastic.sql.query.{Criteria, CriteriaWithConditionalFunction, Expression} package object cond { @@ -50,7 +50,6 @@ package object cond { case object NullIf extends Expr("NULLIF") with ConditionalOp case object Greatest extends Expr("GREATEST") with ConditionalOp case object Least extends Expr("LEAST") with ConditionalOp - // case object Exists extends Expr("EXISTS") with ConditionalOp case object Case extends Expr("CASE") with ConditionalOp @@ -259,7 +258,22 @@ package object cond { override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(argTypes) override def validate(): Either[String, Unit] = { - if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") + // Story 22.2 (AD-6) — a CASE-WHEN condition is parsed by `case_condition`, which shares + // `whereCriteria` with WHERE, so a subquery node is grammatically reachable here. It is + // reached at PARSE time (`Field.validate` -> `FunctionChain.validate` -> + // `Validator.validateChain` -> `functions.map(_.validate())`), and the check below passes a + // criteria (its `out` IS BOOLEAN), so without this arm the statement parses and dies later in + // the node's `painless` throw — a rendering-time internal error where the analyst wants a + // named rejection. + val subquery = conditions.collectFirst { + case (c: Criteria, _) if c.subqueries.nonEmpty => c + } + if (subquery.isDefined) + Left( + s"A subquery is not supported in a CASE WHEN condition: ${subquery.get.sql}. " + + "Filter in WHERE, or compute the flag in a separate query." + ) + else if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") else if ( expression.isEmpty && conditions.exists { case (cond, _) => cond.out != SQLTypes.Boolean } ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala index 43a6ac652..3865614b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -93,6 +93,28 @@ package object operator { case object UNION extends Expr("UNION ALL") with Operator with TokenRegex + /** Story 22.2 — the WHERE-subquery operators. + * + * `EXISTS` and the three quantifiers are plain `Operator`s, NOT `ComparisonOperator`s: + * `ComparisonOperator` is sealed and its `maybeNegated` match is exhaustive over that set, so + * adding a member there would force an arm for a token that never reaches a negation. `UNION` + * above is the precedent for an `Operator with TokenRegex` that carries no comparison semantics. + */ + case object EXISTS extends Expr("EXISTS") with Operator with TokenRegex + + /** `ANY` / `SOME` / `ALL` in ` ()`. + * + * 🔴 None of the three is ADDED to `Parser.reservedKeywords` by this story: `all` and `exists` + * were already reserved, and `any` / `some` are NOT — `WHERE any = 1` and `SELECT some FROM t` + * parse today (measured) and must keep parsing. What makes the quantified form win is the ORDER + * of the alternation in `WhereParser.criteria`, never a new reserved word. + */ + sealed trait Quantifier extends Operator with TokenRegex + + case object ANY extends Expr("ANY") with Quantifier + case object SOME extends Expr("SOME") with Quantifier + case object ALL extends Expr("ALL") with Quantifier + sealed trait ElasticOperator extends Operator with TokenRegex case object Nested extends Expr("NESTED") with ElasticOperator diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 71583d895..7d87441c9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -1042,6 +1042,18 @@ object Parser f.joins match { case Nil => val criteria = resolveWhere(f, w).flatMap(_.criteria) + // Story 22.2 — a watcher input renders its WHERE into a watcher search body WITHOUT + // crossing `SearchApi.resolveWithSchema`, so the two-phase subquery rewrite never runs + // for it and an unresolved node would reach the query builder at deployment time. Same + // shape, same reason, as the derived-table refusal above. + if (criteria.exists(_.subqueries.nonEmpty)) + err( + "A watcher input cannot carry a WHERE subquery (IN (SELECT ...), EXISTS " + + "(SELECT ...), a scalar or quantified subquery): a watcher input is a single " + + "Elasticsearch search and cannot run the inner query. Pre-compute the subquery's " + + "values as a MATERIALIZED VIEW and watch the view." + ) + else // `FROM a, b` stays a legitimate multi-index search; only a qualifier over it is // unserviceable — see `qualifiedOverManyIndices`. if (qualifiedOverManyIndices(f, criteria)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index 16a7d117c..c807cfd61 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -33,12 +33,15 @@ import app.softnetwork.elastic.sql.{ } import app.softnetwork.elastic.sql.operator.{ AGAINST, + ALL, AND, + ANY, BETWEEN, Child, ComparisonOperator, DIFF, EQ, + EXISTS, ExpressionOperator, GE, GT, @@ -55,23 +58,30 @@ import app.softnetwork.elastic.sql.operator.{ OR, Parent, PredicateOperator, - RLIKE + Quantifier, + RLIKE, + SOME } import app.softnetwork.elastic.sql.query.{ BetweenExpr, ConditionalFunctionAsCriteria, Criteria, DistanceCriteria, + DqlStatement, ElasticChild, ElasticNested, ElasticParent, ElasticRelation, + ExistsSubquery, GenericExpression, InExpr, + InSubquery, IsNotNullExpr, IsNullExpr, MultiMatchCriteria, Predicate, + QuantifiedSubquery, + ScalarSubquery, Where } @@ -94,7 +104,26 @@ trait WhereParser { def diff: PackratParser[ComparisonOperator] = DIFF.sql ^^ (_ => DIFF) - private def any_identifier: PackratParser[Identifier] = + /** 🔴 `lazy val`, NOT `def` — and it is a PERFORMANCE contract, not a style choice. + * + * `PackratParsers` memoises on *(parser INSTANCE, position)*: `recall(p, in)` and + * `updateCacheAndGet(p, …)` key the cache on `p` itself (scala-parser-combinators 1.1.2, + * `PackratParsers.scala:237-289`), and the library's own scaladoc states the rule outright — + * *"each grammar production previously declared as a `def` without formal parameters becomes a + * `lazy val`"* (`:34-38`). A `def` builds a FRESH instance on every reference, so each + * alternative of [[criteria]] that begins with `any_identifier` got its OWN memo entry at the + * same position and re-did the identical 9-way alternation from scratch. + * + * MEASURED (`ParserSpec`, 10 timed runs, median): **3.490 s -> 1.228 s (-64 %) on `main` with + * this one word changed**, and 4.927 s -> 1.240 s on this branch. The cost was therefore + * GRAMMAR-WIDE and pre-existing — every statement the engine has ever parsed paid it — not + * something story 22.2's four new alternatives created; what those alternatives did was make it + * visible, because they multiplied the number of times the same `any_identifier` was re-parsed. + * + * With memoisation actually firing, the four extra alternatives cost +0.95 % (1.228 -> 1.240, + * ranges fully overlapping): below this suite's measurement resolution. + */ + private lazy val any_identifier: PackratParser[Identifier] = // #284 - see quotedIdentifierUnlessArithmetic. quotedIdentifierUnlessArithmetic | identifierWithArithmeticExpression | @@ -233,8 +262,84 @@ trait WhereParser { c } - def criteria: PackratParser[Criteria] = - (equality | + /** Story 22.2 — the parenthesised body of a WHERE subquery. + * + * It is 22.1's `derivedTableBodyInner` (`searchStatement | fromlessSelect`, in that order, for + * the same `|`-commit reason) with its OWN parentheses. 🔴 NOT `derivedTable`, whose `err("A + * derived table body must be a SELECT …")` alternative would fire on `WHERE a = (b + 1)` — an + * `Error` does not backtrack, so the alternation below could never fall back to `equality` and a + * statement that parses today (MEASURED at Task 0: `WHERE a = (b + 1)` is `Right`) would be + * rejected. With `derivedTableBodyInner` the failure inside the parentheses is a plain `Failure` + * and the fall-through keeps the parenthesised-expression reading. Pinned by the neighbour test. + */ + private def subqueryBody: PackratParser[DqlStatement] = start ~> derivedTableBodyInner <~ end + + private def comparisonOp: PackratParser[ComparisonOperator] = eq | ne | diff | ge | gt | le | lt + + /** `SOME` is canonicalised to `ANY` here (ANSI synonyms), so the AST carries one spelling and `x + * > SOME (S)` renders — and re-parses — as `x > ANY (S)`. + */ + private def quantifier: PackratParser[Quantifier] = + ANY.regex ^^ (_ => ANY) | SOME.regex ^^ (_ => ANY) | ALL.regex ^^ (_ => ALL) + + private def existsSubquery: PackratParser[Criteria] = + not.? ~ (EXISTS.regex ~> subqueryBody) ^^ { case n ~ q => ExistsSubquery(q, n) } + + private def inSubquery: PackratParser[Criteria] = + any_identifier ~ not.? ~ in ~ subqueryBody ^^ { case i ~ n ~ _ ~ q => InSubquery(i, q, n) } + + private def scalarSubquery: PackratParser[Criteria] = + not.? ~ any_identifier ~ comparisonOp ~ subqueryBody ^^ { case n ~ i ~ o ~ q => + ScalarSubquery(i, o, q, n) + } + + /** ` ANY|SOME|ALL ()`. + * + * The two combinations that ARE the `IN` machinery collapse onto [[InSubquery]] here (PD-3): `= + * ANY` / `= SOME` is `IN`, `<> ALL` / `!= ALL` is `NOT IN`. Every OTHER combination becomes a + * [[QuantifiedSubquery]], reduced from the resolved value list at execution time (lead ruling + * OQ-4, 2026-09-14 — this REPLACES the spec's `err` naming a MIN/MAX rewrite). + * + * 🔴 MUST precede `equality` / `comparison` in [[criteria]]. `ANY` and `SOME` are NOT reserved + * words and this story reserves nothing (`WHERE any = 1` and `SELECT some FROM t` parse today — + * measured — and must keep parsing), so `x = ANY (…)` otherwise SUCCEEDS as `x = ` + * and the `(SELECT …)` is left to `whereCriteria`'s bare delimiters. What makes the quantified + * form win is the ORDER of the alternation, never a new reserved word: on `x = any` this + * production fails at `subqueryBody` and the alternation falls through to `equality` with the + * column reading intact. + */ + private def quantifiedSubquery: PackratParser[Criteria] = + not.? ~ any_identifier ~ comparisonOp ~ quantifier ~ subqueryBody ^^ { + case n ~ i ~ EQ ~ ANY ~ q => InSubquery(i, q, n) + case n ~ i ~ (NE | DIFF) ~ ALL ~ q => + // `NOT x <> ALL (S)` is `x IN (S)`: the two negations cancel, and the canonical render is + // the positive `IN`. + InSubquery(i, q, if (n.isDefined) None else Some(NOT)) + case n ~ i ~ o ~ qf ~ q => QuantifiedSubquery(i, o, qf, q, n) + } + + /** `lazy val` for the same reason as [[any_identifier]]: this is the hottest production in the + * grammar (every WHERE, HAVING, JOIN-ON and CASE-WHEN condition reaches it) and it is referenced + * from several places, each of which would otherwise rebuild the whole alternation and defeat + * the cache at the same position. + */ + lazy val criteria: PackratParser[Criteria] = + // Story 22.2 — the four subquery productions come FIRST, and the order is load-bearing: + // - `existsSubquery`: `EXISTS` is a reserved word, so no identifier-headed production can + // match its prefix and nothing else can start with it; + // - `quantifiedSubquery`: MUST precede `equality` / `comparison` — see its scaladoc; + // - `inSubquery` vs `inLiteral`/`inLongs`/`inDoubles`: disjoint at the character after `(` + // (`SELECT` is neither a quote nor a digit), so either order is correct; the subquery form + // is first only because it fails fastest; + // - `scalarSubquery` before `equality` / `comparison`: those FAIL (they do not partially + // succeed) on `(SELECT`, and on `(b + 1)` it is `scalarSubquery` that FAILS — at + // `derivedTableBodyInner` — so `WHERE a = (b + 1)` keeps its parenthesised-expression + // reading. Every neighbour is pinned byte-identical in `WhereSubquerySpec`. + (existsSubquery | + quantifiedSubquery | + inSubquery | + scalarSubquery | + equality | like | rlike | comparison | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 5e7dd1e21..44fad1a36 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -85,7 +85,22 @@ case class Having(criteria: Option[Criteria]) extends Updateable { criteria.map(c => Having.resolveAggregateAliases(c, request).update(request)) ) - override def validate(): Either[String, Unit] = criteria.map(_.validate()).getOrElse(Right(())) + /** Story 22.2 (AD-6) — HAVING reaches `criteria` through the SHARED `whereCriteria` production, + * so a subquery node is grammatically reachable here. It is DECLINED in `validate()` rather than + * by duplicating the alternation: one grammar, one reduction + * (`project_parser_rejection_semantics`). The subquery phase runs on the WHERE only, so a + * subquery left in a HAVING would reach the bucket-selector script and die in the node's + * `painless` throw — this arm is the named rejection that comes first. + */ + override def validate(): Either[String, Unit] = + criteria.map(_.subqueries).getOrElse(Nil).headOption match { + case Some(s) => + Left( + s"A subquery is not supported in HAVING: ${s.sql}. " + + "Compute the value in a separate query, or move the condition to WHERE." + ) + case None => criteria.map(_.validate()).getOrElse(Right(())) + } def nestedElements: Seq[NestedElement] = criteria.map(_.nestedElements).getOrElse(Seq.empty).groupBy(_.path).map(_._2.head).toList diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SubqueryScope.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SubqueryScope.scala index 73c3a3b1a..969fc4242 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SubqueryScope.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SubqueryScope.scala @@ -70,10 +70,13 @@ object SubqueryScope { id.tableAlias.isEmpty && !id.nested && id.name.contains(".") && outerOnly.contains(id.name.split("\\.", 2)(0)) } - // A derived table NESTED in this body is walked with this statement's names added. - direct ++ inner.from.derivedTables.values.toSeq.flatMap(d => - correlatedReferences(d.query, outerScopes ++ innerNames) - ) + // A derived table NESTED in this body, and (story 22.2) a WHERE SUBQUERY nested in it, + // are walked with this statement's names added — so a reference two levels in to the + // OUTERMOST alias is caught at the outermost `update()` too. + val deeper = outerScopes ++ innerNames + direct ++ + inner.from.derivedTables.values.toSeq.flatMap(d => correlatedReferences(d.query, deeper)) ++ + inner.whereSubqueries.flatMap(sq => correlatedReferences(sq.query, deeper)) case multi: MultiSearch => multi.requests.flatMap(r => correlatedReferences(r, outerScopes)) case _ => Nil // a FROM-less body names no source and can reference nothing } @@ -118,6 +121,30 @@ object SubqueryScope { } } + /** Story 22.2 — the message a CORRELATED WHERE subquery is refused with. It names the offending + * reference, the outer alias it reads, the story that will execute it, and the two rewrites that + * work today — plus the object-field disambiguation, because a dotted path into an object field + * whose head happens to equal an outer alias lands here too (loud beats zero rows with HTTP + * 200). + */ + def correlatedMessage(id: Identifier, node: Criteria): String = { + val alias = id.name.split("\\.", 2)(0) + s"Correlated subquery: '${id.name}' reads the outer alias '$alias' inside ${node.sql}. " + + "Correlated subqueries require the relational engine (story 22.3, softclient4es-arrow-extensions) " + + "and are not executed yet. Rewrite as a JOIN, make the subquery self-contained, or " + + s"if '$alias' is an object field of the inner table, qualify it with the inner table's alias." + } + + /** Story 22.2 (PD-2) — the BARE-name half of the correlation rule, decided by the two MAPPINGS at + * the seam rather than structurally. A bare name has no alias to quote, so the message names the + * two indices instead. + */ + def bareCorrelatedMessage(name: String, innerIndex: String, outerIndex: String): String = + s"Correlated subquery: '$name' is not a column of '$innerIndex' but is a column of " + + s"'$outerIndex', so the subquery reads the outer row. Correlated subqueries require the " + + "relational engine (story 22.3, softclient4es-arrow-extensions) and are not executed yet. " + + "Rewrite as a JOIN, or make the subquery self-contained." + def lateralMessage(id: Identifier, derivedAlias: String): String = s"A derived table cannot reference an outer alias: '${id.name}' inside derived table " + s"'$derivedAlias' reads the enclosing FROM (SQL-92 §7.6; this is LATERAL, which is not " + diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/TemporalLiterals.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/TemporalLiterals.scala index 12f702da9..60b578355 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/TemporalLiterals.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/TemporalLiterals.scala @@ -22,7 +22,8 @@ import app.softnetwork.elastic.sql.schema.{Column, Schema} import app.softnetwork.elastic.sql.`type`.{SQLTemporal, SQLTime, SQLType} import java.time.format.DateTimeFormatter -import java.time.{LocalDate, LocalTime} +import java.time.temporal.TemporalAccessor +import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, OffsetDateTime, ZoneOffset} import scala.util.Try import scala.util.matching.Regex @@ -169,6 +170,17 @@ object TemporalLiterals { /** True when one of the custom patterns parses the literal -- Elasticsearch will too. */ def acceptsAsCustom(literal: String): Boolean = customFormatters.exists(formatter => Try(formatter.parse(literal)).isSuccess) + + /** Render an instant in the FIRST custom pattern, in UTC (story 22.2). + * + * `withZone` is not optional: a pattern such as `yyyy-MM-dd` cannot format a zone-less + * `Instant` without one, and UTC is the zone every literal this class handles is normalised + * to. + */ + def renderCustom(temporal: TemporalAccessor): Option[String] = + customFormatters.headOption.flatMap(f => + Try(f.withZone(ZoneOffset.UTC).format(temporal)).toOption + ) } object FieldFormat { @@ -211,7 +223,7 @@ object TemporalLiterals { case _ => Right(None) } } else if (format.acceptsAsCustom(literal)) Right(None) - else if (!format.acceptsIsoOptionalTime) Right(None) + else if (!format.acceptsIsoOptionalTime) downConvert(literal, format) else literal match { case CalendarLiteral(date, separator, time, zone) => @@ -225,6 +237,32 @@ object TemporalLiterals { } } + /** The column's format accepts NO ISO alternative and none of its custom patterns parses this + * literal, so forwarding it verbatim is a GUARANTEED Elasticsearch rejection. When the literal + * is itself a readable ISO date or instant, re-render it in the column's own pattern instead. + * + * 🔴 Found by story 22.2 on real ES 8.18, and it is a PRE-EXISTING gap of #276, not a new one: + * `WHERE placed IN ('2024-01-01T00:00:00Z')` against a `"format": "yyyy-MM-dd"` column failed + * the same way when written by hand. Story 22.2 makes it reachable without anyone typing an ISO + * literal — a subquery over a DATE column yields `java.time` cells, which are rendered as ISO + * instants before they reach this class. + * + * Conservative by construction: it only fires where the alternative is a certain failure, it + * never rejects (an unreadable literal is still forwarded for Elasticsearch to judge), and a + * literal that already round-trips through a custom pattern was returned two lines above. + */ + private def downConvert(literal: String, format: FieldFormat): Either[String, Option[String]] = + readIso(literal).flatMap(format.renderCustom) match { + case Some(rendered) if rendered != literal => Right(Some(rendered)) + case _ => Right(None) + } + + private def readIso(literal: String): Option[TemporalAccessor] = + Try(Instant.parse(literal)).toOption + .orElse(Try(OffsetDateTime.parse(literal).toInstant).toOption) + .orElse(Try(LocalDateTime.parse(literal).toInstant(ZoneOffset.UTC)).toOption) + .orElse(Try(LocalDate.parse(literal).atStartOfDay(ZoneOffset.UTC).toInstant).toOption) + private def calendar( literal: String, date: String, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 6fe06906e..43a7e2577 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -58,7 +58,10 @@ sealed trait Criteria extends Updateable with PainlessScript { case c: Expression => c.dependencies case relation: ElasticRelation => relation.criteria.dependencies case m: MultiMatchCriteria => m.dependencies - case _ => Nil + // Story 22.2 — a subquery node's LEFT operand is an ordinary outer identifier and must not be + // hidden by the `case _` default below. + case s: SubqueryCriteria => s.outerIdentifiers.flatMap(_.dependencies) + case _ => Nil } /** Every identifier this criteria names directly — both operands of an equality included. @@ -75,7 +78,24 @@ sealed trait Criteria extends Updateable with PainlessScript { }) case relation: ElasticRelation => relation.criteria.referencedIdentifiers case m: MultiMatchCriteria => m.identifiers - case _ => Nil + // Story 22.2 — the OUTER operand only. A subquery's own body is a separate scope; walking into + // it here would make `derivedScopeCheck` and `SubqueryScope` read inner columns as outer ones. + case s: SubqueryCriteria => s.outerIdentifiers + case _ => Nil + } + + /** Every WHERE-subquery node of this criteria tree, in statement order (story 22.2). + * + * Relation wrappers and predicates recurse; a subquery's OWN body is NOT walked here — an inner + * subquery is resolved when the inner statement crosses the seam itself (`api.search(inner)` -> + * `resolveWithSchema(inner)`), which is what makes the recursion depth-unbounded without this + * walker having to know about it. + */ + def subqueries: Seq[SubqueryCriteria] = this match { + case Predicate(left, _, right, _, _) => left.subqueries ++ right.subqueries + case relation: ElasticRelation => relation.criteria.subqueries + case s: SubqueryCriteria => Seq(s) + case _ => Nil } def nested: Boolean = false @@ -130,6 +150,12 @@ sealed trait Criteria extends Updateable with PainlessScript { .flatMap { id => id.metricName.map(name => Field(id, Some(Alias(name)))) } + // Story 22.2 — so `COUNT(x) IN (SELECT …)` is rejected by `Where.validate` exactly like any + // other aggregate written in a WHERE clause. + case s: SubqueryCriteria => + s.outerIdentifiers + .filter(_.aggregations.nonEmpty) + .flatMap(id => id.metricName.map(name => Field(id, Some(Alias(name))))) case _ => Seq.empty } @@ -1250,6 +1276,323 @@ case class InExpr[R, +T <: Value[R]]( } +/** A WHERE predicate whose right-hand side is a SUBQUERY (story 22.2 — ANSI-92 `` / + * `` / `` with a `` / ``). + * + * It has NO Elasticsearch form of its own. The inner statement is executed FIRST, at the ONE + * `SingleSearch -> ElasticQuery` seam (`SearchApi.resolveWithSchema`, epic 22 AD-2), and the node + * is REPLACED by a literal criteria the bridges already translate — an `InExpr` over the inner + * column's distinct values, a `GenericExpression` over a literal, or [[MatchAllCriteria]] / + * [[MatchNoneCriteria]]. A node that reaches a bridge unresolved is a programming error and fails + * LOUDLY there (`ElasticBridge.query`), never silently. + * + * Extends `Criteria` directly rather than `Expression`: `Expression` is keyed on an `identifier` + * and a `maybeValue: Option[Token]` and its `validate()` compares `identifier.out` with the + * value's type — a subquery has no value type until it has run, and `EXISTS` has no identifier at + * all. + * + * `correlatedRefs` is RECORDED in `update(outer)` (the house pattern: record in update, format in + * validate — `FieldSort.bareTableAlias`), because `update` runs inside `Parser.single`'s action + * and cannot report, and because the OUTER scope is only known one level up. See + * [[SubqueryScope]]. + */ +sealed trait SubqueryCriteria extends Criteria with ElasticFilter { + def query: DqlStatement + def maybeNot: Option[NOT.type] + def correlatedRefs: Seq[Identifier] + + /** The identifiers of the OUTER statement this node names directly (its left operand) — what + * `referencedIdentifiers` / `derivedScopeCheck` / the aggregate-in-WHERE rejection must see. + * `EXISTS` names none. + */ + def outerIdentifiers: Seq[Identifier] + + /** The body as a `SingleSearch` when it is one — the only body kind this story EXECUTES (PD-7). + */ + final def inner: Option[SingleSearch] = query match { + case s: SingleSearch => Some(s) + case _ => None + } + + override def group: Boolean = false + + override def nestedElement: Option[NestedElement] = None + + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this + + /** A subquery has no Painless: it is never a script predicate (WHERE only — AD-6). Reachable only + * through a CASE-WHEN condition that `Case.validate()` did not catch, so it is the LAST line of + * defence rather than the rejection — loud and named either way. + */ + override def painless(context: Option[PainlessContext]): String = + throw new IllegalArgumentException( + s"A WHERE subquery has no Painless form and cannot appear in a CASE / HAVING / script " + + s"context: $sql" + ) + + protected def notAsString: String = maybeNot.map(_ => "NOT ").getOrElse("") + + /** Shared validation, in order: a body kind this story executes; the body's OWN rules + * (`Parser.apply` validates the TOP level only — story 22.1's trap, one clause over); NOT + * correlated (PD-2). + */ + protected def commonChecks: Either[String, Unit] = + for { + _ <- query match { + case s: SingleSearch => s.validate() + case _: MultiSearch => + Left( + s"UNION ALL inside a WHERE subquery is not supported yet: $sql. " + + "Write one subquery per branch, or wait for set-operator support (story 22.6)." + ) + case _: FromlessSelect => + Left( + s"A WHERE subquery must read a table: $sql. " + + "For constant values write the literal list (IN ('a', 'b')) or the literal itself." + ) + case other => + Left(s"A WHERE subquery body must be a SELECT, got ${other.getClass.getSimpleName}") + } + _ <- correlatedRefs.headOption match { + case Some(id) => Left(SubqueryScope.correlatedMessage(id, this)) + case None => Right(()) + } + } yield () + + /** The projected column of the inner statement — exactly ONE, never `*` — for IN / quantified / + * scalar. `identifierName` is the SAME test `SearchApi.extractOutputFieldNames` makes: ONE + * spelling of "is this a SELECT *" (the story 21.3 two-derivations lesson). + */ + protected def singleColumnCheck(position: String): Either[String, Unit] = + inner.map(_.select.fields) match { + case Some(Seq(f)) if f.identifier.identifierName == "*" => + Left(s"A subquery in $position position must project exactly one column, not *: $sql") + case Some(Seq(_)) => Right(()) + case Some(fs) => + Left( + s"A subquery in $position position must project exactly one column, got ${fs.size}: $sql" + ) + case None => Right(()) // the body KIND was already rejected by commonChecks + } +} + +/** ` [NOT] IN ()` — also what `= ANY|SOME (…)` and `<> ALL (…)` / `!= ALL + * (…)` reduce to AT PARSE TIME (PD-3): the SAME node, so the rewrite, the correlation rule and the + * render have ONE implementation. The render is the canonical `IN` spelling — `x = ANY (SELECT …)` + * re-parses as `x IN (SELECT …)`, an equal AST by construction (the fixed point holds on the + * CANONICAL text, as `LEFT OUTER JOIN` -> `LEFT JOIN` already does). + */ +case class InSubquery( + identifier: Identifier, + query: DqlStatement, + maybeNot: Option[NOT.type] = None, + correlatedRefs: Seq[Identifier] = Nil +) extends SubqueryCriteria { + override def operator: Operator = IN + override def sql: String = s"$identifier $notAsString$operator (${query.sql})" + + /** 🔴 Required by `PainlessOperandFormSpec`, and it is not bookkeeping: without it a `NOT` + * written AFTER a predicate operator (`a = 1 AND NOT `) falls back to wrapping the + * UN-negated criterion in an Elasticsearch `must_not`, which MATCHES a document lacking the + * field - the story BIDC-8 defect. Folding the NOT into `maybeNot` hands the negation to the + * RESOLVER, which is the only place that knows the ANSI three-valued answer (an `ALL` form over + * a NULL-bearing set stays `MatchNone` under a NOT, an empty `ANY` flips to `MatchAll`). + */ + override def negated: Option[Criteria] = + Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) + override def outerIdentifiers: Seq[Identifier] = Seq(identifier) + override def nested: Boolean = identifier.nested + override def nestedElement: Option[NestedElement] = identifier.nestedElement + override lazy val limit: Option[Limit] = identifier.limit + + override def update(request: SingleSearch): Criteria = { + val updated = this.copy( + identifier = identifier.update(request), + correlatedRefs = SubqueryScope.correlatedReferences(query, request) + ) + // the same shape as InExpr.update: a nested (UNNEST) left operand is wrapped like any other + if (updated.nested) ElasticNested(updated, limit) else updated + } + + override def validate(): Either[String, Unit] = + for { + _ <- identifier.validate() + _ <- commonChecks + _ <- singleColumnCheck("IN") + } yield () +} + +/** `[NOT] EXISTS ()` — true iff the inner statement returns at least one row. */ +case class ExistsSubquery( + query: DqlStatement, + maybeNot: Option[NOT.type] = None, + correlatedRefs: Seq[Identifier] = Nil +) extends SubqueryCriteria { + override def operator: Operator = EXISTS + override def sql: String = s"$notAsString$operator (${query.sql})" + + /** 🔴 Required by `PainlessOperandFormSpec`, and it is not bookkeeping: without it a `NOT` + * written AFTER a predicate operator (`a = 1 AND NOT `) falls back to wrapping the + * UN-negated criterion in an Elasticsearch `must_not`, which MATCHES a document lacking the + * field - the story BIDC-8 defect. Folding the NOT into `maybeNot` hands the negation to the + * RESOLVER, which is the only place that knows the ANSI three-valued answer (an `ALL` form over + * a NULL-bearing set stays `MatchNone` under a NOT, an empty `ANY` flips to `MatchAll`). + */ + override def negated: Option[Criteria] = + Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) + override def outerIdentifiers: Seq[Identifier] = Nil + override def update(request: SingleSearch): Criteria = + this.copy(correlatedRefs = SubqueryScope.correlatedReferences(query, request)) + override def validate(): Either[String, Unit] = commonChecks +} + +/** ` ()` where the subquery yields ONE row, ONE column (ANSI ``). Statically enforced as "an aggregate with no GROUP BY, or LIMIT 1"; the RUN-TIME + * row count is re-checked by the resolver (0 rows = NULL = no match; > 1 = 400, ANSI cardinality + * violation). + */ +case class ScalarSubquery( + identifier: Identifier, + operator: ComparisonOperator, + query: DqlStatement, + maybeNot: Option[NOT.type] = None, + correlatedRefs: Seq[Identifier] = Nil +) extends SubqueryCriteria { + override def sql: String = s"$notAsString$identifier $operator (${query.sql})" + + /** 🔴 Required by `PainlessOperandFormSpec`, and it is not bookkeeping: without it a `NOT` + * written AFTER a predicate operator (`a = 1 AND NOT `) falls back to wrapping the + * UN-negated criterion in an Elasticsearch `must_not`, which MATCHES a document lacking the + * field - the story BIDC-8 defect. Folding the NOT into `maybeNot` hands the negation to the + * RESOLVER, which is the only place that knows the ANSI three-valued answer (an `ALL` form over + * a NULL-bearing set stays `MatchNone` under a NOT, an empty `ANY` flips to `MatchAll`). + */ + override def negated: Option[Criteria] = + Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) + override def outerIdentifiers: Seq[Identifier] = Seq(identifier) + override def nested: Boolean = identifier.nested + override def nestedElement: Option[NestedElement] = identifier.nestedElement + override lazy val limit: Option[Limit] = identifier.limit + + override def update(request: SingleSearch): Criteria = { + val updated = this.copy( + identifier = identifier.update(request), + correlatedRefs = SubqueryScope.correlatedReferences(query, request) + ) + if (updated.nested) ElasticNested(updated, limit) else updated + } + + override def validate(): Either[String, Unit] = + for { + _ <- identifier.validate() + _ <- commonChecks + _ <- singleColumnCheck("scalar") + _ <- inner match { + case Some(s) if (!s.returnsRows && s.groupBy.isEmpty) || s.limit.exists(_.limit == 1) => + Right(()) + case Some(_) => + Left( + s"A scalar subquery must return a single row: $sql. " + + "Use an aggregate with no GROUP BY (MAX, MIN, AVG, SUM, COUNT ...) or add LIMIT 1." + ) + case None => Right(()) + } + } yield () +} + +/** ` ANY|SOME|ALL ()` for the TEN combinations that are not already the + * `IN` machinery (lead ruling OQ-4, 2026-09-14 — this REVERSES the spec's PD-1, which rejected + * them with a MIN/MAX rewrite message). + * + * 🔴 The body resolves EXACTLY as [[InSubquery]] 's does — a bounded value LIST (mode P / mode W, + * the same typing, the same 65,536 bound) — and the quantifier is reduced IN THE RESOLVER from + * that list (`SubqueryResolver.reduceQuantified`). The rejected alternative, rewriting the body to + * `MIN(…)` / `MAX(…)` at parse time, cannot wrap a body that already carries `GROUP BY` / `HAVING` + * / `LIMIT` (that needs 22.4's derived tables), needs TWO aggregates for `= ALL` and `<> ANY`, and + * would have to recover the ANSI empty-set rule from a NULL aggregate result — the `rowNormalizer` + * null-vs-absent trap this story's own review flagged. + * + * It is a [[SubqueryCriteria]] sibling and NOT a [[ScalarSubquery]] variant precisely because its + * body yields a LIST where a scalar subquery's yields one cell. + * + * `SOME` is CANONICALISED to `ANY` (they are synonyms in ANSI SQL), the same canonical-render rule + * PD-3 applies to `= ANY` -> `IN`: `x > SOME (S)` renders `x > ANY (S)` and re-parses to an EQUAL + * AST. + */ +case class QuantifiedSubquery( + identifier: Identifier, + operator: ComparisonOperator, + quantifier: Quantifier, + query: DqlStatement, + maybeNot: Option[NOT.type] = None, + correlatedRefs: Seq[Identifier] = Nil +) extends SubqueryCriteria { + override def sql: String = + s"$notAsString$identifier $operator $quantifier (${query.sql})" + + /** 🔴 Required by `PainlessOperandFormSpec`, and it is not bookkeeping: without it a `NOT` + * written AFTER a predicate operator (`a = 1 AND NOT `) falls back to wrapping the + * UN-negated criterion in an Elasticsearch `must_not`, which MATCHES a document lacking the + * field - the story BIDC-8 defect. Folding the NOT into `maybeNot` hands the negation to the + * RESOLVER, which is the only place that knows the ANSI three-valued answer (an `ALL` form over + * a NULL-bearing set stays `MatchNone` under a NOT, an empty `ANY` flips to `MatchAll`). + */ + override def negated: Option[Criteria] = + Some(this.copy(maybeNot = if (maybeNot.isDefined) None else Some(NOT))) + override def outerIdentifiers: Seq[Identifier] = Seq(identifier) + override def nested: Boolean = identifier.nested + override def nestedElement: Option[NestedElement] = identifier.nestedElement + override lazy val limit: Option[Limit] = identifier.limit + + /** `true` for `ALL`, `false` for `ANY` / `SOME` — the ONE place the two families are told apart, + * so the resolver's opposite empty-set rules cannot be keyed off two different tests. + */ + def universal: Boolean = quantifier == ALL + + override def update(request: SingleSearch): Criteria = { + val updated = this.copy( + identifier = identifier.update(request), + correlatedRefs = SubqueryScope.correlatedReferences(query, request) + ) + if (updated.nested) ElasticNested(updated, limit) else updated + } + + override def validate(): Either[String, Unit] = + for { + _ <- identifier.validate() + _ <- commonChecks + _ <- singleColumnCheck(s"$operator $quantifier") + } yield () +} + +/** The two RESOLVED sentinels (story 22.2). Produced ONLY by `SubqueryResolver`, never by the + * parser; their `sql` renders (`1 = 1` / `1 = 0`) exist so a resolved statement still logs and + * re-parses as a `SingleSearch` — they are not a fixed point of THESE classes (the re-parse yields + * a `GenericExpression`), which is fine: nothing persists a RESOLVED statement + * (`MaterializedViewExtension` persists the PARSED one, and a MV carrying a WHERE subquery is + * refused at `validate()`). + */ +case class MatchAllCriteria() extends Criteria with ElasticFilter { + override def operator: Operator = EQ + override def sql: String = "1 = 1" + override def group: Boolean = false + override def nestedElement: Option[NestedElement] = None + override def update(request: SingleSearch): Criteria = this + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this + override def painless(context: Option[PainlessContext]): String = "true" +} + +case class MatchNoneCriteria() extends Criteria with ElasticFilter { + override def operator: Operator = EQ + override def sql: String = "1 = 0" + override def group: Boolean = false + override def nestedElement: Option[NestedElement] = None + override def update(request: SingleSearch): Criteria = this + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this + override def painless(context: Option[PainlessContext]): String = "false" +} + case class BetweenExpr( identifier: Identifier, fromTo: FromTo, @@ -1448,6 +1791,21 @@ sealed abstract class ElasticRelation(val criteria: Criteria, val operator: Elas override def group: Boolean = criteria.group + /** 🔴 Story 22.2 (independent review, H2). Without this override a relation inherits + * `Validator`'s no-op `validate()` and NEVER recurses into the criteria it wraps — so EVERY rule + * that lives in a leaf's `validate()` is skipped inside `NESTED(…)` / `CHILD(…)` / `PARENT(…)`, + * and inside the `ElasticNested` wrapper the subquery nodes put THEMSELVES in when their left + * operand is nested (`InSubquery.update` and friends). + * + * Measured consequence for this story: `WHERE items.sku IN (SELECT a, b FROM u)` skipped the + * single-column check and silently resolved against the FIRST of two projected columns, and a + * CORRELATED body inside a relation ran as if it were uncorrelated. `Criteria.subqueries` DOES + * walk relations, so the resolver executed the node either way — validation was the only thing + * missing. The fix is not subquery-specific: `InExpr`'s and `Expression`'s own type checks were + * being skipped in the same position. + */ + override def validate(): Either[String, Unit] = criteria.validate() + } case class ElasticNested( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index ee1fa4956..9ab48d37d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -86,6 +86,14 @@ package object query { def derivedTablesPresent(statement: Statement): Boolean = closureSearches(statement).exists(_.hasDerivedTables) + /** Story 22.2 — the statement carries a WHERE subquery somewhere. Read by the MATERIALIZED VIEW + * and WATCHER guards, which must refuse one: both render the SELECT into an Elasticsearch + * artefact (a transform, a watcher input) WITHOUT crossing `SearchApi.resolveWithSchema`, so the + * two-phase rewrite never runs for them and an unresolved node would reach the query builder. + */ + def whereSubqueriesPresent(statement: Statement): Boolean = + closureSearches(statement).exists(_.hasWhereSubqueries) + sealed trait Statement extends Token sealed trait DqlStatement extends Statement @@ -165,6 +173,22 @@ package object query { */ lazy val relationalClosureRequired: Boolean = from.relationalClosureRequired + /** Every WHERE-subquery node this statement carries, in statement order (story 22.2). + * + * 🔴 NOT part of [[relationalClosureRequired]], and that is the epic's routing rule, not an + * oversight: an UNCORRELATED WHERE subquery executes ES-natively in core at EVERY venue (lead + * ruling OQ-1), so such a statement is PASSTHROUGH — `relationalClosureRequired == false` and + * `hasWhereSubqueries == true`. Story 22.3 widens the closure predicate with the CORRELATED + * ones only. + */ + lazy val whereSubqueries: Seq[SubqueryCriteria] = + where.flatMap(_.criteria).map(_.subqueries).getOrElse(Nil) + + /** The ONE boolean `SearchApi.resolveWithSchema` tests per statement: decided once, on the AST, + * so a statement without a subquery pays nothing (`feedback_no_per_row_hot_path_work`). + */ + lazy val hasWhereSubqueries: Boolean = whereSubqueries.nonEmpty + /** Every identifier this statement NAMES, across every clause that can carry one — the SELECT * list (through each item's function chain), WHERE, HAVING, GROUP BY, ORDER BY and each * standard JOIN's ON. @@ -1506,6 +1530,12 @@ package object query { "Elasticsearch transform reads indices. Materialize the subquery as its own view and " + "reference it." ) + else if (whereSubqueriesPresent(dql)) + Left( + "MATERIALIZED VIEW over a WHERE subquery (IN (SELECT ...), EXISTS (SELECT ...), a " + + "scalar or quantified subquery) is not supported: an Elasticsearch transform cannot run " + + "the inner query. Materialize the subquery's values first and reference them." + ) else dql.validate() override def sql: String = { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/census/DialectCensus.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/census/DialectCensus.scala index 02638308c..b441d302c 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/census/DialectCensus.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/census/DialectCensus.scala @@ -2487,7 +2487,55 @@ object DialectCensus { Ansi, "SQL:2016 Part 2 (Foundation) Feature E061-03 IN predicate with list of values", QueryClause, - "terms query; literal, long and double lists each have their own production" + "terms query; literal, long and double lists each have their own production, and since " + + "story 22.2 the operand may also be an uncorrelated subquery (SQL:2016 E061-11), executed " + + "first and pushed into the outer query as the same terms clause" + ), + e( + "op.predicate.exists", + Op, + "EXISTS", + "EXISTS", + OP, + """case object EXISTS extends Expr("EXISTS") with Operator with TokenRegex""", + "SELECT id FROM emp WHERE EXISTS (SELECT 1 FROM dept)", + "1", + Ansi, + "SQL:2016 Part 2 (Foundation) Feature E061-08 EXISTS predicate", + QueryClause, + "story 22.2 - the inner statement runs FIRST at SearchApi.resolveWithSchema and the " + + "predicate is rewritten to match_all / match_none; correlated bodies are refused" + ), + e( + "op.predicate.quantified.any", + Op, + "ANY", + "ANY", + OP, + """case object ANY extends Expr("ANY") with Quantifier""", + "SELECT id FROM emp WHERE salary > ANY (SELECT salary FROM dept)", + "2", + Ansi, + "SQL:2016 Part 2 (Foundation) Feature E061-07 Quantified comparison predicate", + QueryClause, + "story 22.2 - SOME is the ANSI synonym and is canonicalised to ANY; = ANY reduces to IN " + + "at parse time, the ordering forms reduce to a MIN/MAX comparison in the resolver", + Some(List("SOME")) + ), + e( + "op.predicate.quantified.all", + Op, + "ALL", + "ALL", + OP, + """case object ALL extends Expr("ALL") with Quantifier""", + "SELECT id FROM emp WHERE salary > ALL (SELECT salary FROM dept)", + "2", + Ansi, + "SQL:2016 Part 2 (Foundation) Feature E061-07 Quantified comparison predicate", + QueryClause, + "story 22.2 - <> ALL reduces to NOT IN at parse time; the ordering forms reduce to a " + + "MIN/MAX comparison in the resolver, and ALL over an EMPTY set is TRUE (ANSI)" ), e( "op.predicate.like", diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/WhereSubquerySpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/WhereSubquerySpec.scala new file mode 100644 index 000000000..f20603744 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/WhereSubquerySpec.scala @@ -0,0 +1,383 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.operator.{ALL, ANY, DIFF, GT, NE, NOT} +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 22.2 — WHERE subqueries: grammar, AST, validation, correlation, the render fixed point + * WITH its rendered text, and the neighbour productions the new alternation order could move. + */ +class WhereSubquerySpec extends AnyFlatSpec with Matchers { + + private def parse(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s + case Right(other) => fail(s"[$sql] expected SingleSearch, got ${other.getClass.getSimpleName}") + case Left(e) => fail(s"[$sql] rejected: ${e.msg}") + } + + private def reasonOf(sql: String): String = + Parser(sql).swap.toOption.map(_.msg).getOrElse("") + + private def whereOf(sql: String): Criteria = + parse(sql).where.flatMap(_.criteria).getOrElse(fail(s"[$sql] no WHERE")) + + /** `ParserTotalitySpec.rejects` (private there): no throw, THEN `Left`, THEN NOT the boundary + * catch, THEN the reasons. + * + * 🔴 The third assertion is what keeps every case FALSIFIABLE. Since `Parser.apply` carries a + * `NonFatal` boundary catch (#250), a restored `throw` still yields a `Left` whose message + * CONTAINS the same words — `noException` + `isLeft` + `include` all stay green. Only the + * `InternalParseFailure` prefix tells the two apart. + */ + private def rejects(sql: String, reasons: String*): Unit = { + withClue(s"[$sql] ") { noException should be thrownBy Parser(sql) } + withClue(s"[$sql] ") { Parser(sql).isLeft shouldBe true } + val msg = reasonOf(sql) + withClue( + s"[$sql] msg=[$msg] - this rejection must come from an `err(...)` in the grammar or from a " + + "`validate()`, never from `Parser.apply`'s NonFatal boundary catch. " + ) { + msg should not startWith Parser.InternalParseFailure + } + reasons.foreach(r => withClue(s"[$sql] msg=[$msg] ") { msg should include(r) }) + () + } + + // ── grammar + AST ──────────────────────────────────────────────────────────────────────────── + + "IN (SELECT ...)" should "build an InSubquery whose body is the inner SingleSearch" in { + val s = parse( + "SELECT id FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')" + ) + val in = s.where.flatMap(_.criteria).get.asInstanceOf[InSubquery] + in.identifier.name shouldBe "customer_id" + in.maybeNot shouldBe None + in.inner.map(_.from.tables.head.name) shouldBe Some("customers") + in.correlatedRefs shouldBe Nil + s.hasWhereSubqueries shouldBe true + s.whereSubqueries should have size 1 + // AD-8 — an uncorrelated WHERE subquery executes in core at every venue: PASSTHROUGH. + s.relationalClosureRequired shouldBe false + relationalClosureRequired(s) shouldBe false + s.returnsRows shouldBe true // the outer statement's SHAPE is unchanged + } + + it should "parse a body whose LAST clause is WHERE or HAVING (story 22.1's AD-2b scanner)" in { + // Without the depth-aware `whereCriteria` these are `Left("Unbalanced parentheses")`: the inner + // clause swallows the subquery's own `)`. + parse( + "SELECT id FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')" + ) + parse("SELECT id FROM t WHERE EXISTS (SELECT 1 FROM u WHERE x = 1)") + parse("SELECT id FROM t WHERE a IN (SELECT a FROM u GROUP BY a HAVING COUNT(*) > 1)") + parse("SELECT id FROM t WHERE a IN (SELECT a FROM u WHERE (b = 1 OR c = 2) AND d = 3)") + // and the top level keeps its contract: an unmatched OPENING paren is still loud + rejects("SELECT a FROM t WHERE (b = 1", "Unbalanced parentheses") + // a stray `)` stays a rejection; its wording is `phrase`'s and is never pinned + rejects("SELECT a FROM t WHERE a = 1)") + } + + it should "carry NOT in the spelling the grammar accepts" in { + whereOf("SELECT id FROM t WHERE a NOT IN (SELECT a FROM u)") + .asInstanceOf[InSubquery] + .maybeNot shouldBe Some(NOT) + // MEASURED at Task 0 on `origin/main`: the prefix form `WHERE NOT a IN (…)` does NOT parse for + // a literal list either (`end of input expected`), so this story neither gains nor loses it. + rejects("SELECT a FROM t WHERE NOT a IN (1, 2)") + rejects("SELECT a FROM t WHERE NOT a IN (SELECT a FROM u)") + } + + "EXISTS (SELECT ...)" should "build an ExistsSubquery, negated by NOT" in { + whereOf("SELECT id FROM t WHERE EXISTS (SELECT 1 FROM u WHERE x = 1)") shouldBe + a[ExistsSubquery] + whereOf("SELECT id FROM t WHERE NOT EXISTS (SELECT 1 FROM u)") + .asInstanceOf[ExistsSubquery] + .maybeNot shouldBe Some(NOT) + } + + " (SELECT ...)" should "build a ScalarSubquery for every comparison operator" in { + Seq("=", "<>", "!=", ">=", ">", "<=", "<").foreach { op => + withClue(op) { + whereOf(s"SELECT id FROM t WHERE amount $op (SELECT AVG(amount) FROM t)") shouldBe + a[ScalarSubquery] + } + } + whereOf("SELECT id FROM t WHERE amount > (SELECT AVG(amount) FROM t)") + .asInstanceOf[ScalarSubquery] + .operator shouldBe GT + } + + "Quantified forms" should "reduce = ANY / = SOME to IN and <> ALL / != ALL to NOT IN (PD-3)" in { + val in = whereOf("SELECT id FROM t WHERE a IN (SELECT a FROM u)").asInstanceOf[InSubquery] + whereOf("SELECT id FROM t WHERE a = ANY (SELECT a FROM u)") shouldBe in + whereOf("SELECT id FROM t WHERE a = SOME (SELECT a FROM u)") shouldBe in + whereOf("SELECT id FROM t WHERE a <> ALL (SELECT a FROM u)") + .asInstanceOf[InSubquery] + .maybeNot shouldBe Some(NOT) + whereOf("SELECT id FROM t WHERE a != ALL (SELECT a FROM u)") + .asInstanceOf[InSubquery] + .maybeNot shouldBe Some(NOT) + } + + /** 🔴 Lead ruling OQ-4 (2026-09-14) REVERSES PD-1: these SHIP. The `err` naming a MIN/MAX rewrite + * is deleted, not retargeted. + */ + it should "build a QuantifiedSubquery for the ten remaining combinations" in { + val cases = Seq( + ">" -> ANY, + ">=" -> ANY, + "<" -> ANY, + "<=" -> ANY, + ">" -> ALL, + ">=" -> ALL, + "<" -> ALL, + "<=" -> ALL + ) + cases.foreach { case (op, q) => + val c = whereOf(s"SELECT id FROM t WHERE a $op $q (SELECT a FROM u)") + withClue(s"$op $q: ") { + c shouldBe a[QuantifiedSubquery] + c.asInstanceOf[QuantifiedSubquery].quantifier shouldBe q + c.asInstanceOf[QuantifiedSubquery].universal shouldBe (q == ALL) + } + } + whereOf("SELECT id FROM t WHERE a = ALL (SELECT a FROM u)") + .asInstanceOf[QuantifiedSubquery] + .universal shouldBe true + whereOf("SELECT id FROM t WHERE a <> ANY (SELECT a FROM u)") + .asInstanceOf[QuantifiedSubquery] + .operator shouldBe NE + whereOf("SELECT id FROM t WHERE a != ANY (SELECT a FROM u)") + .asInstanceOf[QuantifiedSubquery] + .operator shouldBe DIFF + } + + it should "canonicalise SOME to ANY so the two spellings share one node" in { + whereOf("SELECT id FROM t WHERE a > SOME (SELECT a FROM u)") shouldBe + whereOf("SELECT id FROM t WHERE a > ANY (SELECT a FROM u)") + } + + it should "not reserve ANY / SOME: a column named any or some still parses" in { + parse("SELECT id FROM t WHERE any = 1") + parse("SELECT some FROM t WHERE some > 1") + parse("SELECT a FROM t WHERE a = any") + } + + // ── validation (AC 3) ──────────────────────────────────────────────────────────────────────── + + "Validation" should "require exactly one projected column in IN, quantified and scalar position" in { + rejects("SELECT id FROM t WHERE a IN (SELECT a, b FROM u)", "exactly one column", "got 2") + rejects("SELECT id FROM t WHERE a IN (SELECT * FROM u)", "exactly one column, not *") + rejects("SELECT id FROM t WHERE a > (SELECT a, b FROM u LIMIT 1)", "exactly one column") + rejects("SELECT id FROM t WHERE a > ANY (SELECT a, b FROM u)", "exactly one column") + parse("SELECT id FROM t WHERE EXISTS (SELECT a, b FROM u)") // EXISTS projects anything + parse("SELECT id FROM t WHERE EXISTS (SELECT * FROM u)") + } + + it should "require a single-row scalar body: metric-only or LIMIT 1" in { + parse("SELECT id FROM t WHERE a > (SELECT MAX(a) FROM u)") + parse("SELECT id FROM t WHERE a > (SELECT a FROM u ORDER BY a DESC LIMIT 1)") + rejects("SELECT id FROM t WHERE a > (SELECT a FROM u)", "must return a single row") + rejects("SELECT id FROM t WHERE a > (SELECT MAX(a) FROM u GROUP BY b)", "single row") + // a QUANTIFIED body is a LIST, so the single-row rule must NOT apply to it + parse("SELECT id FROM t WHERE a > ANY (SELECT a FROM u)") + } + + it should "decline UNION ALL and FROM-less bodies (PD-7)" in { + rejects( + "SELECT id FROM t WHERE a IN (SELECT a FROM u UNION ALL SELECT a FROM v)", + "UNION ALL inside a WHERE subquery" + ) + rejects("SELECT id FROM t WHERE a IN (SELECT 1)", "must read a table") + rejects("SELECT id FROM t WHERE EXISTS (SELECT 1)", "must read a table") + } + + it should "validate the body one level down" in { + // `Parser.apply` validates the TOP level only, so the node's own validate() must run the body's + rejects("SELECT id FROM t WHERE a IN (SELECT a, b FROM u GROUP BY a)", "Non-aggregated") + } + + it should "reject an aggregate as the left operand (aggregates are not allowed in WHERE)" in { + rejects( + "SELECT id FROM t WHERE COUNT(a) IN (SELECT a FROM u)", + "Aggregate functions are not allowed in WHERE" + ) + } + + it should "decline a subquery in HAVING, CASE and JOIN ON (AD-6)" in { + rejects( + "SELECT a, COUNT(*) AS n FROM t GROUP BY a HAVING n IN (SELECT n FROM u)", + "not supported in HAVING" + ) + rejects( + "SELECT CASE WHEN a IN (SELECT a FROM u) THEN 1 ELSE 0 END AS f FROM t", + "CASE WHEN condition" + ) + // the EXISTING `On` rule already refuses it — no new arm was added, and the substring below is + // what tells a later change that a DIFFERENT rule started firing + rejects( + "SELECT o.id FROM orders o JOIN customers c ON o.cid IN (SELECT id FROM x)", + "ON clause", + "equality" + ) + } + + it should "decline a subquery in a materialized view and in a watcher input" in { + rejects( + "CREATE MATERIALIZED VIEW v AS SELECT a FROM t WHERE a IN (SELECT a FROM u)", + "cannot run the inner query" + ) + rejects( + """CREATE WATCHER my_watcher AS + | EVERY 5 MINUTES + | FROM t WHERE a IN (SELECT a FROM u) WITHIN 2 MINUTES + | ALWAYS DO + | log_action AS LOG "Watcher triggered" AT INFO + | END""".stripMargin, + "cannot carry a WHERE subquery" + ) + } + + // ── correlation (AC 4) ─────────────────────────────────────────────────────────────────────── + + "A correlated subquery" should "be rejected with the 22.3 message when it names an outer alias" in { + rejects( + "SELECT o.id FROM orders o WHERE o.cid IN " + + "(SELECT c.id FROM customers c WHERE c.region = o.region)", + "Correlated subquery", + "o.region", + "outer alias 'o'", + "story 22.3" + ) + rejects( + "SELECT o.id FROM orders o WHERE EXISTS (SELECT 1 FROM customers c WHERE c.id = o.cid)", + "Correlated subquery", + "o.cid" + ) + rejects( // the outer INDEX name is a correlation name too + "SELECT id FROM orders WHERE cid IN (SELECT id FROM customers WHERE region = orders.region)", + "Correlated subquery", + "orders.region" + ) + } + + it should "honour shadowing: an alias the inner statement declares is the INNER one" in { + parse( + "SELECT o.id FROM orders o WHERE o.cid IN (SELECT o.id FROM customers o WHERE o.r = 'EU')" + ) + } + + it should "see a nested body's reference to the OUTERMOST alias" in { + rejects( + "SELECT o.id FROM orders o WHERE o.cid IN (SELECT c.id FROM customers c WHERE c.k IN " + + "(SELECT k FROM z WHERE z.r = o.region))", + "Correlated subquery", + "o.region" + ) + } + + it should "assume a BARE name is the inner column at parse time (PD-2)" in { + val s = parse("SELECT id FROM orders WHERE cid IN (SELECT id FROM customers WHERE r = 'EU')") + s.whereSubqueries.head.correlatedRefs shouldBe Nil + } + + // ── render: the fixed point AND the text (constraint 2) ────────────────────────────────────── + + private val renders = Seq( + "SELECT id FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')" -> + "SELECT id FROM orders WHERE customer_id IN (SELECT id FROM customers WHERE region = 'EU')", + "SELECT id FROM t WHERE a NOT IN (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a NOT IN (SELECT a FROM u)", + "SELECT id FROM t WHERE a = ANY (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a IN (SELECT a FROM u)", + "SELECT id FROM t WHERE a = SOME (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a IN (SELECT a FROM u)", + "SELECT id FROM t WHERE a <> ALL (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a NOT IN (SELECT a FROM u)", + "SELECT id FROM t WHERE a > ANY (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a > ANY (SELECT a FROM u)", + "SELECT id FROM t WHERE a > SOME (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a > ANY (SELECT a FROM u)", + "SELECT id FROM t WHERE a <= ALL (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a <= ALL (SELECT a FROM u)", + "SELECT id FROM t WHERE a = ALL (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a = ALL (SELECT a FROM u)", + "SELECT id FROM t WHERE a <> ANY (SELECT a FROM u)" -> + "SELECT id FROM t WHERE a <> ANY (SELECT a FROM u)", + "SELECT id FROM t WHERE NOT EXISTS (SELECT 1 FROM u WHERE x = 1)" -> + "SELECT id FROM t WHERE NOT EXISTS (SELECT 1 FROM u WHERE x = 1)", + "SELECT id FROM t WHERE amount > (SELECT AVG(amount) FROM t)" -> + "SELECT id FROM t WHERE amount > (SELECT AVG(amount) FROM t)", + "SELECT id FROM t WHERE a = 1 AND b IN (SELECT b FROM u) OR EXISTS (SELECT 1 FROM v)" -> + "SELECT id FROM t WHERE a = 1 AND b IN (SELECT b FROM u) OR EXISTS (SELECT 1 FROM v)", + "SELECT id FROM t WHERE a IN (SELECT a FROM u WHERE b IN (SELECT b FROM v))" -> + "SELECT id FROM t WHERE a IN (SELECT a FROM u WHERE b IN (SELECT b FROM v))", + "DELETE FROM t WHERE id IN (SELECT id FROM u WHERE flag = true)" -> + "DELETE FROM t WHERE id IN (SELECT id FROM u WHERE flag = true)" + ) + + renders.foreach { case (in, text) => + it should s"render [$in] as [$text] and re-parse to an EQUAL ast" in { + val stmt = Parser(in).toOption.getOrElse(fail(s"[$in] rejected: ${reasonOf(in)}")) + stmt.sql shouldBe text + // 🔴 `== Right(stmt)`, never `isRight`: a LOSSY render satisfies the fixed point on both + // sides. The TEXT assertion above and this one only bite together. + Parser(stmt.sql) shouldBe Right(stmt) + } + } + + // ── neighbour pins (Task 0 row 6; the alternation order is what could move them) ───────────── + + /** Every render here was MEASURED on the branch base before the grammar changed and is asserted + * byte-for-byte, so a subquery production that started winning one of these inputs fails here + * rather than in a customer's query. + */ + private val neighbours = Seq( + "SELECT a FROM t WHERE a = (b + 1)" -> "SELECT a FROM t WHERE a = (b + 1)", + "SELECT a FROM t WHERE (a = 1 AND b = 2)" -> "SELECT a FROM t WHERE (a = 1 AND b = 2)", + "SELECT a FROM t WHERE a IN ('x', 'y')" -> "SELECT a FROM t WHERE a IN ('x','y')", + "SELECT a FROM t WHERE a IN (1, 2)" -> "SELECT a FROM t WHERE a IN (1,2)", + "SELECT a FROM t WHERE a IN (1.5, 2.5)" -> "SELECT a FROM t WHERE a IN (1.5,2.5)", + "SELECT a FROM t WHERE a NOT IN (1, 2)" -> "SELECT a FROM t WHERE a NOT IN (1,2)", + "SELECT a FROM t WHERE a = 1 AND b IN (2, 3)" -> "SELECT a FROM t WHERE a = 1 AND b IN (2,3)", + "SELECT a FROM t WHERE a = b" -> "SELECT a FROM t WHERE a = b", + "SELECT a FROM t WHERE a > 1" -> "SELECT a FROM t WHERE a > 1", + "SELECT a FROM t WHERE exists_flag = true" -> "SELECT a FROM t WHERE exists_flag = true", + "SELECT a FROM t WHERE a = any" -> "SELECT a FROM t WHERE a = any", + "SELECT a FROM t WHERE any = 1" -> "SELECT a FROM t WHERE any = 1", + "SELECT some FROM t WHERE some > 1" -> "SELECT some FROM t WHERE some > 1", + "SELECT a FROM t WHERE ABS(a) > 1" -> "SELECT a FROM t WHERE ABS(a) > 1", + "SELECT a FROM t WHERE MATCH(title) AGAINST('x')" -> + "SELECT a FROM t WHERE MATCH (title) AGAINST ('x')", + "SELECT a FROM t WHERE a BETWEEN 1 AND 2" -> "SELECT a FROM t WHERE a BETWEEN 1 AND 2", + "SELECT a FROM t WHERE id = 1 AND CHILD(x = 2 AND y = 3 AND z = 4)" -> + "SELECT a FROM t WHERE id = 1 AND CHILD(x = 2 AND y = 3 AND z = 4)", + "SELECT a FROM t WHERE CASE WHEN a = 1 THEN 1 ELSE 0 END = 1" -> + "SELECT a FROM t WHERE CASE WHEN a = 1 THEN 1 ELSE 0 END = 1" + ) + + "Neighbouring productions" should "not move" in { + neighbours.foreach { case (sql, expected) => + val stmt = Parser(sql).toOption.getOrElse(fail(s"[$sql] rejected: ${reasonOf(sql)}")) + withClue(s"[$sql] ") { stmt.sql shouldBe expected } + withClue(s"[$sql] ") { Parser(stmt.sql) shouldBe Right(stmt) } + } + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/SubqueryScopeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SubqueryScopeSpec.scala new file mode 100644 index 000000000..93f5fad11 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SubqueryScopeSpec.scala @@ -0,0 +1,97 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Story 22.2 — the correlation detector itself, on parsed scopes. + * + * `WhereSubquerySpec` pins what the PARSER does with a correlated statement; this pins the walk + * story 22.3 consumes as THE detector, so a change that keeps the rejections but loses the + * shadowing rule or the nested walk fails here. + */ +class SubqueryScopeSpec extends AnyFlatSpec with Matchers { + + private def single(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s + case other => fail(s"[$sql] $other") + } + + /** The correlated statements cannot be obtained through `Parser` (they are rejected at + * `validate()`), so the body and the outer scope are taken apart the way `update` sees them: the + * body is parsed on its own — where its qualifiers resolve against NOTHING — and measured + * against the outer statement's names. + */ + private def refs(body: String, outer: String): Seq[String] = + SubqueryScope.correlatedReferences(single(body), single(outer)).map(_.name) + + "correlationNames" should "carry every alias, index name and UNNEST alias of the FROM" in { + val s = single("SELECT o.id FROM orders o") + SubqueryScope.correlationNames(s) should contain allOf ("orders", "o") + } + + "A qualified reference to an outer name" should "be reported" in { + refs( + "SELECT c.id FROM customers c WHERE c.region = o.region", + "SELECT id FROM orders o" + ) shouldBe + Seq("o.region") + // the outer INDEX name is a correlation name too + refs( + "SELECT id FROM customers WHERE region = orders.region", + "SELECT id FROM orders" + ) shouldBe Seq("orders.region") + } + + it should "be silent when the inner statement declares the same name (innermost wins)" in { + refs("SELECT o.id FROM customers o WHERE o.region = 'EU'", "SELECT id FROM orders o") shouldBe + Nil + } + + it should "be silent for a bare name (SQL resolves it innermost-first — PD-2)" in { + refs("SELECT id FROM customers WHERE region = 'EU'", "SELECT id FROM orders o") shouldBe Nil + } + + it should "be silent for a dotted path whose head is in NEITHER scope (an object field)" in { + refs("SELECT id FROM customers WHERE address.city = 'X'", "SELECT id FROM orders o") shouldBe + Nil + } + + "A body nested one level deeper" should "still see the OUTERMOST scope" in { + refs( + "SELECT c.id FROM customers c WHERE c.k IN (SELECT k FROM z WHERE z.r = o.region)", + "SELECT id FROM orders o" + ) shouldBe Seq("o.region") + } + + "The messages" should "name the offender, the scope and the story that will execute it" in { + val node = single("SELECT id FROM t WHERE a IN (SELECT a FROM u)").whereSubqueries.head + val id = single("SELECT id FROM customers WHERE region = orders.region").referencedIdentifiers + .find(_.name.contains(".")) + .getOrElse(fail("no qualified identifier")) + val qualified = SubqueryScope.correlatedMessage(id, node) + qualified should include("Correlated subquery") + qualified should include("orders.region") + qualified should include("story 22.3") + val bare = SubqueryScope.bareCorrelatedMessage("vip", "orders", "customers") + bare should include("'vip' is not a column of 'orders'") + bare should include("is a column of 'customers'") + bare should include("story 22.3") + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala index aa24fa76e..e1a4ad85d 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala @@ -245,18 +245,46 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { ) } - it should "never reject under a custom or opaque format" in { + /** ⚠️ RETARGETED by story 22.2, not deleted (the 21.4 rule for a contract pin whose behaviour + * deliberately moves). + * + * The `"2026-06-04T00:00:00" -> Verbatim` row read *"not ours to fix: no ISO alternative"*. It + * WAS ours: forwarding an ISO literal verbatim to a column whose format accepts no ISO + * alternative is a GUARANTEED Elasticsearch rejection — measured on real ES 8.18 through a + * story-22.2 subquery over a `date` column, and reproducible by hand with `WHERE ts = + * '2026-06-04T00:00:00'`. It is now re-rendered in the column's own pattern. Nothing that worked + * before changes: the alternative to the rewrite was a certain failure, an unreadable literal is + * still forwarded, and a literal a custom pattern already parses is returned before this rule is + * reached (the row above). + */ + it should "re-render an ISO literal under a custom format, and still never reject" in { check( custom, Seq( - "not-a-date" -> Verbatim, - "2026-06-04 00:00:00" -> Verbatim, // parity: the custom pattern parses it - "2026-06-04T00:00:00" -> Verbatim // not ours to fix: no ISO alternative + "not-a-date" -> Verbatim, + "2026-06-04 00:00:00" -> Verbatim, // parity: the custom pattern parses it + "2026-06-04T00:00:00" -> Rewrite("2026-06-04 00:00:00"), + "2026-06-04T00:00:00Z" -> Rewrite("2026-06-04 00:00:00"), + "2026-06-04" -> Rewrite("2026-06-04 00:00:00") ) ) check(opaque, Seq("not-a-date" -> Verbatim, "2026-06-04 00:00:00" -> Verbatim)) } + /** The date-only companion of the row above: the shape story 22.2's subquery over a `date` column + * actually produces, against the `yyyy-MM-dd` mapping the completeness fixture uses. + */ + it should "re-render an ISO instant into a date-only custom format" in { + check( + FieldFormat("yyyy-MM-dd"), + Seq( + "2024-01-01T00:00:00Z" -> Rewrite("2024-01-01"), + "2024-01-01" -> Verbatim, // the custom pattern parses it + "not-a-date" -> Verbatim + ) + ) + } + it should "prefer parity with a custom alternative, and still fix the space form where ISO is accepted" in { check( mixed, diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/WhereSubqueryCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/WhereSubqueryCompletenessSpec.scala new file mode 100644 index 000000000..0b918c681 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/WhereSubqueryCompletenessSpec.scala @@ -0,0 +1,482 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.language.implicitConversions + +case class WsqId(id: String) +case class WsqCount(n: Long) + +/** Story 22.2 — uncorrelated WHERE subqueries, executed ES-natively against a REAL cluster. + * + * Two-index, MULTI-SHARD fixture with CLOSED-FORM oracles: every expectation below is computed + * from the fixture rule, never from a document total, so a wrong answer cannot coincide with the + * right count (the story 21.3 lesson). The failure modes this guards are silent ones — an + * unresolved predicate answering `match_all`, a truncated `terms` list, an ANSI NULL rule inverted + * — so the assertions are exact row sets or exact sizes, never "non-empty". + */ +trait WhereSubqueryCompletenessSpec + extends AnyFlatSpecLike + with ElasticDockerTestKit + with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val customers = "wsq_customers" + private val orders = "wsq_orders" + private val narrow = "wsq_narrow" + private val wide = "wsq_wide" + + /** 40 customers, 200 orders: customer `cNN` owns exactly 5 orders, so a set of `k` customers + * selects exactly `5 * k` orders. + */ + private val customerCount = 40 + private val orderCount = 200 + private val euCustomers = 15 // c01..c15 + private val vipCustomers = 5 // c01..c05 + + private val threeShards = """{"number_of_shards": 3, "number_of_replicas": 0}""" + + private def customerId(n: Int): String = f"c${(n - 1) % customerCount + 1}%02d" + + override def beforeAll(): Unit = { + super.beforeAll() + + client.createIndex(customers, settings = threeShards).get shouldBe true + client + .setMapping( + customers, + """{ + | "properties": { + | "id": { "type": "keyword" }, + | "region": { "type": "keyword" }, + | "vip": { "type": "boolean" }, + | "since": { "type": "date" }, + | "tier": { "type": "text" } + | } + |}""".stripMargin + ) + .get shouldBe true + + client.createIndex(orders, settings = threeShards).get shouldBe true + client + .setMapping( + orders, + """{ + | "properties": { + | "id": { "type": "keyword" }, + | "customer_id": { "type": "keyword" }, + | "amount": { "type": "integer" }, + | "placed": { "type": "date", "format": "yyyy-MM-dd" } + | } + |}""".stripMargin + ) + .get shouldBe true + + // `max_terms_count` tuned BELOW the default so a 150-value terms query is refused (AD-9). + client + .createIndex( + narrow, + settings = """{"number_of_shards": 1, "number_of_replicas": 0, "max_terms_count": 100}""" + ) + .get shouldBe true + client + .setMapping( + narrow, + """{"properties": {"id": {"type": "keyword"}, "k": {"type": "integer"}}}""" + ) + .get shouldBe true + client.createIndex(wide, settings = threeShards).get shouldBe true + client + .setMapping(wide, """{"properties": {"id": {"type": "keyword"}, "k": {"type": "integer"}}}""") + .get shouldBe true + + // c01..c15 = EU, c16..c39 = US, c40 has NO region at all (the ANSI NULL of the NOT IN rule). + val customerDocs = (1 to customerCount).map { c => + val id = f"c$c%02d" + val region = + if (c <= euCustomers) """"region":"EU",""" + else if (c < customerCount) """"region":"US",""" + else "" + val since = if (c <= euCustomers) "2024-01-01" else "2024-06-01" + val tier = if (c <= vipCustomers) "gold" else "silver" + s"""{"id":"$id",$region"vip":${c <= vipCustomers},"since":"$since","tier":"$tier"}""" + }.toList + + val orderDocs = (1 to orderCount).map { n => + val cid = customerId(n) + val placed = if (cid.drop(1).toInt <= euCustomers) "2024-01-01" else "2024-06-01" + f"""{"id":"o$n%03d","customer_id":"$cid","amount":$n,"placed":"$placed"}""" + }.toList + + val narrowDocs = (1 to 10).map(k => s"""{"id":"n$k","k":$k}""").toList + val wideDocs = (1 to 150).map(k => s"""{"id":"w$k","k":$k}""").toList + + index(customers, customerDocs) + index(orders, orderDocs) + index(narrow, narrowDocs) + index(wide, wideDocs) + } + + private def index(name: String, docs: List[String]): Unit = { + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = name, logEvery = 1000) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + client.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => + case ElasticFailure(error) => + error.cause.foreach(_.printStackTrace()) + fail(s"Bulk indexing $name failed: ${error.message}") + } + client.refresh(name) + () + } + + override def afterAll(): Unit = { + Seq(customers, orders, narrow, wide).foreach(client.deleteIndex) + super.afterAll() + } + + /** 🔴 `searchAs` is a MACRO that validates the statement at COMPILE time, so every query below is + * an inline string LITERAL — a `val` holding the same text is rejected by the macro. That is + * also why the correlated-subquery rejection (a parse-time `Left`) lives in the REPL integration + * spec, which dispatches at run time, and not here. + */ + private def idsOf(result: ElasticResult[Seq[WsqId]]): Seq[String] = result match { + case ElasticSuccess(rows) => rows.map(_.id) + case ElasticFailure(error) => fail(s"failed: ${error.message}") + } + + // ── IN / NOT IN ────────────────────────────────────────────────────────────────────────────── + + "IN (SELECT ...)" should "return exactly the orders whose customer is in the inner set" in { + val ids = idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT id FROM wsq_customers WHERE region = 'EU') ORDER BY id" + ) + ) + // the exact SET, not merely its size: a wrong answer of the right cardinality fails here + ids shouldBe (1 to orderCount) + .filter(n => customerId(n).drop(1).toInt <= euCustomers) + .map(n => f"o$n%03d") + ids should have size (5L * euCustomers) // 75 + } + + it should "read the column the converter actually produces for a QUALIFIED or ALIASED projection" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT c.id FROM wsq_customers c WHERE c.region = 'EU')" + ) + ) should have size 75L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT id AS cid FROM wsq_customers WHERE region = 'EU')" + ) + ) should have size 75L + } + + it should "normalise a DATE inner projection against the OUTER column's own format" in { + // `since` is a default-format `date` on wsq_customers while `placed` is `yyyy-MM-dd` on + // wsq_orders: the inner terms key arrives as a java.time value, is rendered as an ISO-8601 + // instant, and TemporalLiterals then normalises it against `placed`'s own format. + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE placed IN (SELECT since FROM wsq_customers WHERE region = 'EU')" + ) + ) should have size 75L + } + + it should "run a `text` inner column in mode W (a terms aggregation on text is an ES 400)" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_customers WHERE tier IN (SELECT tier FROM wsq_customers WHERE vip = true)" + ) + ) should have size vipCustomers.toLong + } + + "NOT IN" should "be the exact complement over a NULL-free inner set" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id NOT IN (SELECT id FROM wsq_customers WHERE region = 'EU')" + ) + ) should have size (orderCount - 5L * euCustomers) // 125 + } + + /** 🔴 ANSI (PD-5), and the two rows are DIFFERENT rules. `c40` carries no `region`, so the inner + * set contains a NULL: `IN` ignores it (and matches nothing, since no `customer_id` is a region + * name) while `NOT IN` is UNKNOWN for EVERY row and matches NOTHING — not "everything except". + */ + it should "return NO row when the inner set contains a NULL" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id NOT IN (SELECT region FROM wsq_customers)" + ) + ) shouldBe empty + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT region FROM wsq_customers)" + ) + ) shouldBe empty + } + + "The quantified spellings" should "agree with IN / NOT IN" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id = ANY (SELECT id FROM wsq_customers WHERE region = 'EU')" + ) + ) should have size 75L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id <> ALL (SELECT id FROM wsq_customers WHERE region = 'EU')" + ) + ) should have size 125L + } + + /** 🔴 Lead ruling OQ-4 — the ordering quantifiers EXECUTE, reduced from the resolved value list. + * `amount` runs 1..200 and the inner set is `{1..5}`, so `> ANY` is `> 1` (199 rows) while `> + * ALL` is `> 5` (195 rows): a test that confused the two ends would be off by four. + */ + it should "reduce an ordering quantifier to the right end of the inner set" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > ANY (SELECT amount FROM wsq_orders WHERE amount <= 5)" + ) + ) should have size 199L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > ALL (SELECT amount FROM wsq_orders WHERE amount <= 5)" + ) + ) should have size 195L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount <= ALL (SELECT amount FROM wsq_orders WHERE amount <= 5)" + ) + ) should have size 1L // <= 1 + } + + /** 🔴 The two empty-set rules are OPPOSITE: an existential over nothing is FALSE, a universal + * over nothing is TRUE. The single most likely thing to ship backwards. + */ + it should "answer no rows for ANY and every row for ALL over an EMPTY subquery" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > ANY (SELECT amount FROM wsq_orders WHERE amount > 100000)" + ) + ) shouldBe empty + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > ALL (SELECT amount FROM wsq_orders WHERE amount > 100000)" + ) + ) should have size orderCount.toLong + } + + // ── EXISTS ─────────────────────────────────────────────────────────────────────────────────── + + "EXISTS" should "be true or false on a row-shaped body and true on an aggregate body" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE EXISTS (SELECT 1 FROM wsq_customers WHERE vip = true)" + ) + ) should have size orderCount.toLong + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE EXISTS (SELECT 1 FROM wsq_customers WHERE region = 'MARS')" + ) + ) shouldBe empty + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE NOT EXISTS (SELECT 1 FROM wsq_customers WHERE region = 'MARS')" + ) + ) should have size orderCount.toLong + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE EXISTS (SELECT id FROM wsq_customers LIMIT 0)" + ) + ) shouldBe empty + // a metric-only SELECT always yields exactly one row, so EXISTS is true even over no documents + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE EXISTS (SELECT COUNT(*) AS n FROM wsq_customers WHERE region = 'MARS')" + ) + ) should have size orderCount.toLong + } + + // ── scalar ─────────────────────────────────────────────────────────────────────────────────── + + "A scalar subquery" should "compare against the aggregate and against a LIMIT 1 row" in { + // amounts are 1..200, so AVG = 100.5 and exactly 100 orders are above it. + // 🔴 The UN-ALIASED shape is the headline one: under NativeContext its requested output name + // would be NULL-FILLED and the predicate would silently collapse to match_none. + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > (SELECT AVG(amount) FROM wsq_orders)" + ) + ) should have size 100L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > (SELECT AVG(amount) AS a FROM wsq_orders)" + ) + ) should have size 100L + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount = (SELECT MAX(amount) AS m FROM wsq_orders)" + ) + ) shouldBe Seq("o200") + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount >= (SELECT amount FROM wsq_orders ORDER BY amount DESC LIMIT 1)" + ) + ) shouldBe Seq("o200") + // 🔴 ANSI: an aggregate over ZERO matching documents is NULL, so the comparison is UNKNOWN and + // NO row matches. This is the statement the empty-aggregate defect broke: before the fix it + // reduced to `amount > 0` and returned EVERY one of the 200 orders on ES 8.18 (which reported + // `0.0`), while ES 7.17 answered a 400. An exact-count assertion, on a 3-shard index, is the + // guard — an execution-success assertion would have passed throughout. + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > (SELECT MAX(amount) AS m FROM wsq_orders WHERE amount > 100000)" + ) + ) shouldBe empty + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount < (SELECT MIN(amount) AS m FROM wsq_orders WHERE amount > 100000)" + ) + ) shouldBe empty + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE amount > (SELECT AVG(amount) AS a FROM wsq_orders WHERE amount > 100000)" + ) + ) shouldBe empty + } + + /** 🔴 The empty-aggregate rule itself, asserted on the VALUES rather than through a subquery, so + * a regression is attributed to the conversion layer and not to the subquery resolver. + * + * MEASURED before the fix, same request on every major: Elasticsearch answers identically on the + * wire (`{"value":null}` for max/min/avg, `0.0` for sum, `0` for value_count), but ES 8.18 alone + * converted the nulls to `0.0` — its module reads the TYPED response, where a primitive `double` + * has already swallowed the null before any of our code runs. + */ + it should "answer NULL for MAX / MIN / AVG over zero documents, 0 for COUNT, and keep SUM at 0" in { + def cell(sql: String, column: String): Option[Any] = + client.search( + app.softnetwork.elastic.sql.parser + .Parser(sql) + .toOption + .collect { case s: app.softnetwork.elastic.sql.query.SearchStatement => s } + .getOrElse(fail(s"[$sql] did not parse")) + )(NativeContext) match { + case ElasticSuccess(r) => + r.results.headOption.getOrElse(fail(s"[$sql] returned no row")).get(column) + case ElasticFailure(e) => fail(s"[$sql] ${e.message}") + } + val empty = "FROM wsq_orders WHERE amount > 100000" + // ANSI NULL — the aggregate had no input + cell(s"SELECT MAX(amount) AS m $empty", "m") shouldBe Some(null) + cell(s"SELECT MIN(amount) AS m $empty", "m") shouldBe Some(null) + cell(s"SELECT AVG(amount) AS m $empty", "m") shouldBe Some(null) + // 🔴 COUNT is 0, NEVER null — ANSI's own exception, and the single most damaging thing to get + // backwards here. + cell(s"SELECT COUNT(*) AS c $empty", "c").map(_.toString.toDouble) shouldBe Some(0.0d) + cell(s"SELECT COUNT(amount) AS c $empty", "c").map(_.toString.toDouble) shouldBe Some(0.0d) + // SUM keeps Elasticsearch's own answer on every major (recorded decision, NOT ANSI's NULL) + cell(s"SELECT SUM(amount) AS s $empty", "s").map(_.toString.toDouble) shouldBe Some(0.0d) + // and a NON-empty aggregate is untouched: amounts are 1..200 + cell("SELECT MAX(amount) AS m FROM wsq_orders", "m").map(_.toString.toDouble) shouldBe + Some(200.0d) + cell("SELECT COUNT(*) AS c FROM wsq_orders", "c").map(_.toString.toDouble) shouldBe Some(200.0d) + } + + // ── recursion, correlation, bounds ─────────────────────────────────────────────────────────── + + "Nested subqueries" should "resolve recursively" in { + // orders o001..o005 belong to c01..c05, which own 25 orders in total + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT id FROM wsq_customers WHERE id IN (SELECT customer_id FROM wsq_orders WHERE amount <= 5))" + ) + ) should have size 25L + } + + "A bare outer column inside a subquery" should "be refused by the MAPPINGS at the seam (PD-2)" in { + // `vip` is a wsq_customers column, NOT a wsq_orders one, so the inner statement is reading the + // OUTER row. Executed as uncorrelated it would send an unknown field to Elasticsearch and + // answer zero rows with HTTP 200 — the silent mode epic 22 exists to close. + client.searchAs[WsqId]( + "SELECT id FROM wsq_customers WHERE id IN (SELECT customer_id FROM wsq_orders WHERE vip = true)" + ) match { + case ElasticFailure(error) => error.message should include("Correlated subquery") + case ElasticSuccess(rows) => fail(s"a bare outer column was read as an inner one: $rows") + } + } + + /** The false-positive guard for the rule above: `cid` is an alias the INNER SELECT defines and no + * mapping carries, while the outer index HAS a `customer_id`. Without the alias filter in + * `bareNameCorrelation` a self-contained subquery would be rejected. + */ + it should "NOT refuse an inner SELECT alias that no mapping carries" in { + idsOf( + client.searchAs[WsqId]( + "SELECT id FROM wsq_orders WHERE customer_id IN (SELECT id AS cid FROM wsq_customers WHERE region = 'EU' ORDER BY cid)" + ) + ) should have size 75L + } + + /** 🔴 ES 6.8 does NOT enforce a per-index `max_terms_count` set at create time — MEASURED: the + * 150-value terms query is accepted there, on both the REST and the Jest client, while 7.17 / + * 8.18 / 9.0 refuse it. The bound itself is enforced on EVERY major by the resolver's own count + * check (`SubqueryResolver.MaxTerms`, pinned Docker-free); what is version-dependent is only + * whether ELASTICSEARCH ALSO refuses a smaller, index-tuned limit — which is what the error + * translation exists for. Gated on the measured version rather than skipped, so the translation + * stays asserted wherever it can fire. + */ + private lazy val enforcesMaxTermsCount: Boolean = + client.version match { + case ElasticSuccess(v) => !v.trim.startsWith("6.") + case ElasticFailure(_) => true + } + + "The terms bound" should "surface the index's own max_terms_count loudly, with the remedy" in { + assume(enforcesMaxTermsCount) + client.searchAs[WsqCount]( + "SELECT COUNT(*) AS n FROM wsq_narrow WHERE k IN (SELECT k FROM wsq_wide)" + ) match { + case ElasticFailure(error) => + error.message should include("max_terms_count") + error.message should include("JOIN") + case ElasticSuccess(_) => + fail("a 150-value terms query was accepted by an index capped at 100") + } + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala index d55f603b9..7c505704c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala @@ -1317,6 +1317,114 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { rows shouldBe Seq(Map("1" -> 1)) } + // ========================================================================= + // 6d. WHERE subqueries — story 22.2: the UNCORRELATED forms EXECUTE at the plain REPL + // ========================================================================= + + behavior of "REPL - WHERE subqueries without the relational engine" + + /** 🔴 Every SELECT below carries an explicit `LIMIT`, deliberately. Without one the licensed + * gateway routes a row query through the capped scroll and answers a `StreamResult`, which + * `assertSelectResult` only LOGS — an assertion on it could not fail. The LIMIT keeps the result + * a materialised `QueryRows` so these rows are falsifiable; the no-LIMIT scroll route is covered + * against a real cluster by `WhereSubqueryCompletenessSpec`. + */ + it should "execute IN (SELECT ...) ES-natively through the gateway path" in { + // `dql_orders` (section 5) holds ids 1 and 2 with customer_id 1 and 2. The inner query selects + // customer_id 2, so the outer must return exactly order 2 — never both rows (which is what an + // unresolved predicate answering `match_all` would give) and never none. + assertSelectResult( + System.nanoTime(), + executeSync( + "SELECT id FROM dql_orders WHERE customer_id IN " + + "(SELECT customer_id FROM dql_orders WHERE id = 2) ORDER BY id LIMIT 10" + ), + rows = Seq(Map("id" -> 2)) + ) + } + + it should "execute EXISTS and a scalar subquery the same way" in { + assertSelectResult( + System.nanoTime(), + executeSync( + "SELECT id FROM dql_orders WHERE EXISTS (SELECT 1 FROM dql_orders WHERE id = 2) " + + "ORDER BY id LIMIT 10" + ), + rows = Seq(Map("id" -> 1), Map("id" -> 2)) + ) + assertSelectResult( + System.nanoTime(), + executeSync( + "SELECT id FROM dql_orders WHERE NOT EXISTS (SELECT 1 FROM dql_orders WHERE id = 99) " + + "ORDER BY id LIMIT 10" + ), + rows = Seq(Map("id" -> 1), Map("id" -> 2)) + ) + assertSelectResult( + System.nanoTime(), + executeSync( + "SELECT id FROM dql_orders WHERE id = (SELECT MAX(id) AS m FROM dql_orders) LIMIT 10" + ), + rows = Seq(Map("id" -> 2)) + ) + } + + it should "execute an ordering quantifier (lead ruling OQ-4)" in { + // ids are 1 and 2, so `> ANY {1, 2}` is `> 1` (one row) and `> ALL {1, 2}` is `> 2` (none): + // a reduction that took the wrong end of the set would answer the other way round. + assertSelectResult( + System.nanoTime(), + executeSync("SELECT id FROM dql_orders WHERE id > ANY (SELECT id FROM dql_orders) LIMIT 10"), + rows = Seq(Map("id" -> 2)) + ) + assertSelectResult( + System.nanoTime(), + executeSync("SELECT id FROM dql_orders WHERE id > ALL (SELECT id FROM dql_orders) LIMIT 10"), + nbResults = Some(0) + ) + } + + it should "delete by query through a WHERE subquery" in { + // on a throw-away table, so section 5's fixture is untouched + assertDdl( + System.nanoTime(), + executeSync("CREATE TABLE IF NOT EXISTS dql_sub_del (id INT NOT NULL, tag VARCHAR)") + ) + assertDml( + System.nanoTime(), + executeSync("INSERT INTO dql_sub_del (id, tag) VALUES (1, 'keep'), (2, 'drop'), (3, 'drop')"), + Some(DmlResult(inserted = 3)) + ) + executeSync("REFRESH TABLE dql_sub_del") + executeSync( + "DELETE FROM dql_sub_del WHERE id IN (SELECT id FROM dql_sub_del WHERE tag = 'drop')" + ) + executeSync("REFRESH TABLE dql_sub_del") + assertSelectResult( + System.nanoTime(), + executeSync("SELECT id FROM dql_sub_del ORDER BY id LIMIT 10"), + rows = Seq(Map("id" -> 1)) + ) + executeSync("DROP TABLE dql_sub_del") + () + } + + it should "refuse a CORRELATED subquery with the story-22.3 message, not the JOIN message" in { + val res = executeSync( + "SELECT o.id FROM dql_orders o WHERE EXISTS " + + "(SELECT 1 FROM dql_orders x WHERE x.customer_id = o.customer_id)" + ) + res shouldBe a[ExecutionFailure] + val error = res.asInstanceOf[ExecutionFailure].error + error.statusCode shouldBe Some(400) + error.message should include("Correlated subquery") + } + + it should "still answer the handshake — the subquery phase did not widen" in { + val rows = assertQueryRows(System.nanoTime(), executeSync("SELECT 1")) + rows shouldBe Seq(Map("1" -> 1)) + } + // ========================================================================= // 7. Error handling // =========================================================================