diff --git a/AGENTS.md b/AGENTS.md index 00b17796..33796920 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,39 +99,47 @@ Mandatory linting: `cargo clippy --all-targets --all-features -- -D warnings` ### When working on css or js Frontend formatting: `npm run format` +Mandatory frontend validation: `npm test` More about testing: see [github actions](./.github/workflows/ci.yml). -Project structure: see [contribution guide](./CONTRIBUTING.md) +Contributor setup and validation: see [CONTRIBUTING.md](./CONTRIBUTING.md). Module overview: see [src/lib.rs](./src/lib.rs); architecture diagram: [docs/architecture-detailed.png](./docs/architecture-detailed.png). NEVER reformat/lint/touch files unrelated to your task. Always run tests/lints/format before stopping when you changed code. ### Testing ``` -cargo test # tests with inmemory sqlite by default +cargo test # tests with in-memory SQLite by default ``` For other databases, see [docker testing setup](./docker-compose.yml) ``` -docker compose up -d mssql # or postgres or mysql -DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' cargo test # all dbms use the same user:pass and db name +docker compose up --wait mssql # or postgres, mysql, mariadb, oracle +DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' cargo test ``` +ODBC tests require the database-specific ODBC driver on the host; starting the container is not sufficient. See the PostgreSQL ODBC and Oracle matrix entries in [CI](./.github/workflows/ci.yml) for driver setup and connection strings. On Linux and macOS, `cargo test --features odbc-static` matches CI's static unixODBC linking. + +For dynamic frontend changes, run the Playwright tests under `tests/end-to-end/` as described in [CONTRIBUTING.md](./CONTRIBUTING.md). For examples containing `test.hurl`, run `scripts/test-examples-hurl.sh `. + ### Documentation -Components and functions are documented in [official website](./examples/official-site/sqlpage/migrations/); one migration per component and per function. You CAN update existing migrations, the official site database is recreated from scratch on each deployment. +Components and functions are documented in [official-site migrations](./examples/official-site/sqlpage/migrations/). Edit the existing migration for an existing entity; add an appropriately ordered migration for a new entity. The official-site database is recreated from migrations on each deployment. official documentation website sql tables: + - `parameter_type(type)` - `component(name,description,icon,introduced_in_version)` -- icon name from tabler icon - `parameter(top_level BOOLEAN, name, component REFERENCES component(name), description, description_md, type, optional BOOLEAN)` parameter types: BOOLEAN, COLOR, HTML, ICON, INTEGER, JSON, REAL, TEXT, TIMESTAMP, URL - `example(component REFERENCES component(name), description, properties JSON)` + - `sqlpage_functions(name,icon,description_md,return_type,introduced_in_version)` + - `sqlpage_function_parameters(function,index,name,description_md,type)` #### Project Conventions -- Components: defined in `./sqlpage/templates/*.handlebars` -- Functions: `src/webserver/database/sqlpage_functions/functions.rs` registered with `make_function!`. +- Built-in UI component templates: `sqlpage/templates/*.handlebars`; header/control components: `src/render.rs`. +- SQLPage functions: one `async fn` module under `src/webserver/database/sqlpage_functions/functions/`, registered with `sqlpage_functions!` in `functions.rs`. - [Configuration](./configuration.md): see [AppConfig](./src/app_config.rs) -- Routing: file-based in `src/webserver/routing.rs`; not found handled via `src/default_404.sql`. +- Routing: file-based in `src/webserver/routing.rs`. Missing paths use the nearest ancestor `404.sql`; without one, HTML uses `src/default_404.sql` and other formats receive a plain-text 404. - Follow patterns from similar modules before introducing new abstractions. - frontend: see [css](./sqlpage/sqlpage.css) and [js](./sqlpage/sqlpage.js) diff --git a/CHANGELOG.md b/CHANGELOG.md index d25915e2..beed7d45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ ## unreleased +- **SQLPage functions can now be composed with database results.** Direct calls such as `SELECT sqlpage.url_encode(url) FROM links` already ran once per row. Per-row evaluation now also works through parentheses, concatenation, `COALESCE`, JSON constructors, and nested SQLPage functions. The database first decides which rows exist, then SQLPage evaluates the selected expression for each row. This enables patterns that were not previously possible, such as fetching only missing cached values or rendering a reusable SQL file with parameters from each row: + + ```sql + SELECT + id, + COALESCE(cached_response, sqlpage.fetch(api_url)) AS response + FROM integrations; + + SELECT + 'dynamic' AS component, + sqlpage.run_sql('item.sql', json_object('id', id)) AS properties + FROM items; + ``` + + `COALESCE` is evaluated from left to right, so the first query only calls the API for rows where `cached_response` is `NULL`. If a query returns no rows, none of its selected SQLPage functions run. This also makes database-controlled conditional work possible with scalar `SET` queries: + + ```sql + SET refresh_result = ( + SELECT sqlpage.fetch(refresh_url) + FROM cache_entries + WHERE cache_key = $key AND expires_at < CURRENT_TIMESTAMP + ); + ``` + + **Upgrade notes:** + - A `sqlpage.*` call with only constants or request variables that is the whole value of a selected column now also runs once per returned row. For example, `SELECT id, sqlpage.fetch($url) AS body FROM jobs` previously made one HTTP request and reused its result; it now makes one request per job. The same applies to side-effecting or expensive functions such as `sqlpage.exec`, `sqlpage.run_sql`, and `sqlpage.persist_uploaded_file`. To run a function once per page, store its result first: `SET body = sqlpage.fetch($url);`, then select `$body`. + - `SELECT id, sqlpage.random_string(8) AS token FROM invitations` now creates a different token for every invitation instead of repeating one token. If one shared batch token is intended, use `SET token = sqlpage.random_string(8);` first. + - A selected function is no longer called when its query returns no rows. Move the call to a separate `SET` statement if it must run unconditionally. + - SQLPage-computed values do not exist yet when the database performs `DISTINCT`, filtering, grouping, or sorting. Queries that use `SELECT DISTINCT` with a computed projection, reference a computed alias from `WHERE`, `GROUP BY`, or `ORDER BY`, or use ordinal `GROUP BY`/`ORDER BY` with a computed projection now return a clear error instead of producing database-dependent results. Apply those operations to the source database columns, or compute the value once with `SET` when it does not depend on a row. + - `SET x = (SELECT ...)` now has explicit scalar-query behavior: zero rows set `x` to `NULL`, exactly one output column is required, and multiple columns return a clear SQLPage error. Multiple rows keep the database's scalar-subquery behavior: SQLite uses the first row, while other supported databases return an error. For portable results, make the query intrinsically single-row with a unique predicate or aggregate, or use the database's one-row limiter. - **Access logs now go to stdout.** SQLPage now writes the single per-request completion log line to stdout with the target `sqlpage::access`, matching common application-server and container logging conventions. Diagnostic logs, warnings, and internal errors still go to stderr. If your `LOG_LEVEL` or `RUST_LOG` filter is scoped to a specific old target such as `sqlpage::webserver::http=info`, add `sqlpage::access=info` so request-completion logs are still emitted. If your log pipeline only collects stderr, update it to collect stdout too. - **OIDC redirects are no longer cacheable.** Authorization redirects contain one-time state and post-login redirects set session cookies. SQLPage now sends `Cache-Control: no-store` for these responses, preventing a browser or intermediary from replaying an expired authorization redirect. diff --git a/docker-compose.yml b/docker-compose.yml index 98b42cf4..6bf53790 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,8 +66,14 @@ services: oracle: profiles: ["oracle"] ports: ["1521:1521"] - image: gvenzl/oracle-free:slim + image: gvenzl/oracle-free:23.26.2-slim environment: ORACLE_PASSWORD: Password123! APP_USER: root APP_USER_PASSWORD: Password123! + healthcheck: + test: ["CMD", "healthcheck.sh"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 5s diff --git a/examples/official-site/functions.sql b/examples/official-site/functions.sql index ad91cfa7..b8064343 100644 --- a/examples/official-site/functions.sql +++ b/examples/official-site/functions.sql @@ -18,12 +18,20 @@ select ' In addition to normal SQL functions supported by your database, SQLPage provides a few special functions to help you extract data from user requests. -These functions are special, because they are not executed inside your database, -but by SQLPage itself before sending the query to your database. -Thus, they require all the parameters to be known at the time the query is sent to your database. -Function parameters cannot reference columns from the rest of your query. -The only case when you can call a SQLPage function with a parameter that is not a constant is when it appears at the top level of a `SELECT` statement. -For example, `SELECT sqlpage.url_encode(url) FROM t` is allowed because SQLPage can execute `SELECT url FROM t` and then apply the `url_encode` function to each value. +These functions are special because they are not database functions. +SQLPage evaluates them itself, either before or after the database query. + +When a SQLPage function call is the whole value of a selected column, SQLPage first lets the database decide which rows exist. +It selects the function arguments as hidden columns, then calls the SQLPage function once for each returned row. +For example, `SELECT sqlpage.url_encode(url) AS encoded FROM t` runs the database query first, then applies `url_encode` to every returned `url` value. +If the query returns no row, the function is not called. +This also applies inside scalar `SET` subqueries, for instance `SET body = (SELECT sqlpage.fetch(url) FROM cache_misses WHERE enabled)`. + +In other positions, SQLPage functions run before the query, but only when their arguments can be evaluated without reading database columns. +For instance, `WHERE sqlpage.cookie(''x'') = ''1''` can run before the query, but `WHERE sqlpage.fetch(url) IS NOT NULL` cannot because `url` is a database column. + +If a SQLPage function is expensive or has side effects and should run only once, store its result with `SET` first and reuse the variable. +If a function should run only when a database row exists, put it as a standalone selected column in that row-producing query. For more information about how SQLPage functions are evaluated, and data types in SQLPage, read [the SQLPage data model documentation](/extensions-to-sql). ' as contents_md where $function IS NULL; diff --git a/examples/official-site/sqlpage/migrations/08_functions.sql b/examples/official-site/sqlpage/migrations/08_functions.sql index 3a5fd4a4..30a8a783 100644 --- a/examples/official-site/sqlpage/migrations/08_functions.sql +++ b/examples/official-site/sqlpage/migrations/08_functions.sql @@ -204,6 +204,9 @@ VALUES ( 'arrows-shuffle', 'Returns a cryptographically secure random string of the given length. +When used as a standalone selected column in a query that returns several rows, `random_string` runs once per returned row. +Use `SET token = sqlpage.random_string(32)` first if you want one token reused later in the page. + ### Example Generate a random string of 32 characters and use it as a session ID stored in a cookie: @@ -414,9 +417,10 @@ from json_each(sqlpage.exec(''curl'', ''https://jsonplaceholder.typicode.com/use This means that the SQLPage server will not be blocked while the command is running, it will be able to serve other requests, but it will not be able to serve the current request until the command has finished. You should generally avoid long running commands. - If the program name is NULL, the result will be NULL. - - If any argument is NULL, it will be passed to the command as an empty string. - - If the command exits with a non-zero exit code, the function will raise an error. - - Arbitrary SQL operations are not allowed as sqlpage function arguments. Use `SET` to assign the result of a SQL query to a variable, and then use that variable as an argument to `sqlpage.exec`. + - If any argument is NULL, it will be passed to the command as an empty string. + - If the command exits with a non-zero exit code, the function will raise an error. + - Arbitrary SQL operations are not allowed as sqlpage function arguments. Use `SET` to assign the result of a SQL query to a variable, and then use that variable as an argument to `sqlpage.exec`. + - When `sqlpage.exec(...)` is a standalone selected column, the command runs once per returned row. Use `SET command_result = sqlpage.exec(...)` first if the command should run only once for the page. ' ); INSERT INTO sqlpage_function_parameters ( diff --git a/examples/official-site/sqlpage/migrations/38_run_sql.sql b/examples/official-site/sqlpage/migrations/38_run_sql.sql index 27a98983..314755f2 100644 --- a/examples/official-site/sqlpage/migrations/38_run_sql.sql +++ b/examples/official-site/sqlpage/migrations/38_run_sql.sql @@ -56,6 +56,8 @@ for help with manipulating the json array returned by `run_sql`. - **variables**: the included file will have access to the same variables (URL parameters, POST variables, etc.) as the calling file. If the included file changes the value of a variable or creates a new variable, the change will not be visible in the calling file. + - **per-row execution**: when `sqlpage.run_sql(...)` is a standalone selected column in a query that returns several rows, the included file runs once per returned row. + Use `SET included = sqlpage.run_sql(...)` first if the included file should run only once for the page. ### Parameters diff --git a/examples/official-site/sqlpage/migrations/40_fetch.sql b/examples/official-site/sqlpage/migrations/40_fetch.sql index f93d6f50..f56020b3 100644 --- a/examples/official-site/sqlpage/migrations/40_fetch.sql +++ b/examples/official-site/sqlpage/migrations/40_fetch.sql @@ -106,6 +106,20 @@ set api_value = sqlpage.fetch($target_url); -- no http request made if the field update my_table set field = $api_value where id = 1 and $api_value is not null; -- update the field only if it was not present before ``` +You can also make the HTTP request depend on whether a database query returns a row. +When `fetch` is a standalone selected column, SQLPage runs it only for rows returned by the database: + +```sql +set api_value = ( + select sqlpage.fetch(url) + from cache_misses + where key = $key +); +``` + +If `cache_misses` has no matching row, no HTTP request is made and `$api_value` is set to `NULL`. +If the query returns more than one row or more than one column, SQLPage returns a clear scalar `SET` error. + ## Advanced usage If you need to handle errors or inspect the response headers or the status code, diff --git a/examples/official-site/sqlpage/migrations/72_set_variable.sql b/examples/official-site/sqlpage/migrations/72_set_variable.sql index 1a213656..94ae7aad 100644 --- a/examples/official-site/sqlpage/migrations/72_set_variable.sql +++ b/examples/official-site/sqlpage/migrations/72_set_variable.sql @@ -29,6 +29,10 @@ select from categories; ``` +When `sqlpage.set_variable(...)` is used as a standalone selected column in a query that returns several rows, it runs once per returned row. +This is useful for generating one link per row, as in the example above. +If you need a single link reused later in the page, store it with `SET` first. + ### Parameters - `name` (TEXT): The name of the variable to set. - `value` (TEXT): The value to set the variable to. If `NULL` is passed, the variable is removed from the URL. diff --git a/src/filesystem.rs b/src/filesystem.rs index ce508b09..4530cc98 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -445,9 +445,17 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> { let create_table_sql = DbFsQueries::get_create_table_sql(state.db.info.database_type); let db = &state.db; let conn = &db.connection; - conn.execute("DROP TABLE IF EXISTS sqlpage_files").await?; - log::debug!("Creating table sqlpage_files: {create_table_sql}"); - conn.execute(create_table_sql).await?; + // Other tests share this database, so never drop a table their initialized state may use. + if conn + .execute("SELECT 1 FROM sqlpage_files WHERE 1 = 0") + .await + .is_err() + { + log::debug!("Creating table sqlpage_files: {create_table_sql}"); + conn.execute(create_table_sql).await?; + } + conn.execute("DELETE FROM sqlpage_files WHERE path = 'unit test file.txt'") + .await?; let dbms = db.info.kind; let insert_sql = format!( diff --git a/src/lib.rs b/src/lib.rs index d0c13b3f..3b4d869f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ pub mod webserver; use crate::app_config::AppConfig; use crate::filesystem::FileSystem; -use crate::webserver::database::ParsedSqlFile; +use crate::webserver::database::SqlFile; use crate::webserver::oidc::OidcState; use file_cache::FileCache; use std::path::{Path, PathBuf}; @@ -106,7 +106,7 @@ pub const DEFAULT_404_FILE: &str = "default_404.sql"; pub struct AppState { pub db: Database, all_templates: AllTemplates, - sql_file_cache: FileCache, + sql_file_cache: FileCache, file_system: FileSystem, config: AppConfig, pub oidc_state: Option>, @@ -124,11 +124,11 @@ impl AppState { let file_system = FileSystem::init(&config.web_root, &db).await; sql_file_cache.add_static( PathBuf::from("index.sql"), - ParsedSqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")), + SqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")), ); sql_file_cache.add_static( PathBuf::from(DEFAULT_404_FILE), - ParsedSqlFile::new( + SqlFile::new( &db, include_str!("default_404.sql"), Path::new(DEFAULT_404_FILE), diff --git a/src/webserver/database/error_highlighting.rs b/src/webserver/database/error_highlighting.rs index c2b7595f..57d1356f 100644 --- a/src/webserver/database/error_highlighting.rs +++ b/src/webserver/database/error_highlighting.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use super::sql::{SourceSpan, StmtWithParams}; +use super::sql::SourceSpan; #[derive(Debug)] struct NiceDatabaseError { @@ -84,7 +84,7 @@ struct NicePositionedError { impl std::fmt::Display for NicePositionedError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "In \"{}\": {}", self.source_file.display(), self.error)?; + write!(f, "In \"{}\": {:#}", self.source_file.display(), self.error)?; write_source_position_info(f, &self.source_file, Some(self.query_position)) } } @@ -114,14 +114,15 @@ pub fn display_db_error( #[must_use] pub fn display_stmt_db_error( source_file: &Path, - stmt: &StmtWithParams, + query: &str, + query_position: SourceSpan, db_err: sqlx::error::Error, ) -> anyhow::Error { anyhow::Error::new(NiceDatabaseError { source_file: source_file.to_path_buf(), db_err, - query: stmt.query.clone(), - query_position: Some(stmt.query_position), + query: query.to_owned(), + query_position: Some(query_position), }) } @@ -184,6 +185,23 @@ fn test_display_stmt_error_includes_file_and_line() { assert!(message.contains("example.sql: line 12")); } +#[test] +fn test_display_stmt_error_includes_error_chain() { + let err = display_stmt_error( + Path::new("example.sql"), + SourceSpan { + start: super::sql::SourceLocation { line: 4, column: 1 }, + end: super::sql::SourceLocation { + line: 4, + column: 20, + }, + }, + anyhow::anyhow!("native database error").context("Failed to set variable x"), + ); + let message = err.to_string(); + assert!(message.contains("Failed to set variable x: native database error")); +} + #[test] fn test_quote_source_with_highlight() { let source = "SELECT *\nFROM table\nWHERE "; diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs index c06d60de..8ba29e3d 100644 --- a/src/webserver/database/execute_queries.rs +++ b/src/webserver/database/execute_queries.rs @@ -10,18 +10,17 @@ use tracing::Instrument; use super::csv_import::run_csv_import; use super::error_highlighting::{display_stmt_db_error, display_stmt_error}; use super::sql::{ - DelayedFunctionCall, ParsedSqlFile, ParsedStatement, SimpleSelectValue, StmtWithParams, + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, + SqlFile, }; +use super::sqlpage_expr::{NoInputs, RowExpr, RowInputs}; use crate::dynamic_component::parse_dynamic_rows; use crate::utils::add_value_to_map; use crate::webserver::ErrorWithStatus; -use crate::webserver::database::sql_to_json::row_to_string; use crate::webserver::http_request_info::ExecutionContext; -use crate::webserver::request_variables::SetVariablesMap; use crate::webserver::single_or_vec::SingleOrVec; -use super::syntax_tree::{StmtParam, extract_req_param}; -use super::{Database, DbItem, error_highlighting::display_db_error}; +use super::{Database, DbItem, ScalarSubqueryBehavior, error_highlighting::display_db_error}; use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo}; use sqlx::pool::PoolConnection; use sqlx::{ @@ -30,6 +29,14 @@ use sqlx::{ pub type DbConn = Option>; +/// One database result together with private values reserved for computed +/// columns and therefore omitted from the user-visible row. +struct QueryResult { + item: DbItem, + inputs: RowInputs, + output_column_count: usize, +} + fn source_line_number(line: usize) -> i64 { i64::try_from(line).unwrap_or(i64::MAX) } @@ -135,6 +142,29 @@ fn create_db_query_span( (span, operation_name) } +fn create_query_metrics<'a>( + request: &'a ExecutionContext, + source_file: &Path, + source_span: SourceSpan, + query: &BoundQuery<'_>, +) -> (tracing::Span, DbQueryMetricsContext<'a>) { + let db_system_name = request.app_state.db.info.database_type.otel_name(); + let (query_span, operation_name) = create_db_query_span( + query.sql, + source_file, + source_span.start.line, + db_system_name, + ); + let query_metrics = DbQueryMetricsContext::new( + query_span.clone(), + operation_name, + db_system_name, + &request.app_state.telemetry_metrics, + ); + record_query_params(&query_metrics.span, &query.param_values); + (query_span, query_metrics) +} + impl Database { pub(crate) async fn prepare_with( &self, @@ -149,8 +179,9 @@ impl Database { } } +#[allow(clippy::too_many_lines)] // Keeps the single-connection statement dispatcher together. pub fn stream_query_results_with_conn<'a>( - sql_file: &'a ParsedSqlFile, + sql_file: &'a SqlFile, request: &'a ExecutionContext, db_connection: &'a mut DbConn, ) -> impl Stream + 'a { @@ -158,91 +189,101 @@ pub fn stream_query_results_with_conn<'a>( async_stream::try_stream! { for res in &sql_file.statements { match res { - ParsedStatement::CsvImport(csv_import) => { + FileStatement::CsvImport(csv_import) => { let connection = take_connection(&request.app_state.db, db_connection, request).await?; log::debug!("Executing CSV import: {csv_import:?}"); run_csv_import(connection, csv_import, request).await.with_context(|| format!("Failed to import the CSV file {:?} into the table {:?}", csv_import.uploaded_file, csv_import.table_name))?; }, - ParsedStatement::StmtWithParams(stmt) => { - let query = bind_parameters(stmt, request, db_connection) + FileStatement::Query(statement) => match &statement.body { + QueryBody::SingleRow(query) => { + let row = execute_single_row(query, request, db_connection) + .await + .map_err(|error| with_stmt_position(source_file, statement.source_span, error))?; + for item in parse_dynamic_rows(DbItem::Row(row)) { + yield item; + } + } + QueryBody::Database(stmt) => { + let query = bind_query(stmt, request, db_connection) .await - .map_err(|e| with_stmt_position(source_file, stmt.query_position, e))?; + .map_err(|error| with_stmt_position(source_file, statement.source_span, error))?; request.server_timing.record("bind_params"); - let connection = take_connection(&request.app_state.db, db_connection, request).await?; log::trace!("Executing query {:?}", query.sql); - let db_system_name = request.app_state.db.info.database_type.otel_name(); - let (query_span, operation_name) = create_db_query_span( - query.sql, - source_file, - stmt.query_position.start.line, - db_system_name, - ); - let mut query_metrics = DbQueryMetricsContext::new( - query_span.clone(), - operation_name, - db_system_name, - &request.app_state.telemetry_metrics, - ); - record_query_params(&query_metrics.span, &query.param_values); - let mut stream = connection.fetch_many(query); + let (query_span, mut query_metrics) = create_query_metrics(request, source_file, statement.source_span, &query); let mut error = None; let mut returned_rows: i64 = 0; - loop { + let buffer_rows = stmt.must_buffer_rows(); + let mut deferred_query_results = Vec::new(); + { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + let mut stream = connection.fetch_many(query); + loop { let start_next = std::time::Instant::now(); let next_elem = stream.next().instrument(query_span.clone()).await; query_metrics.add_duration(start_next.elapsed()); let Some(elem) = next_elem else { break; }; - let mut query_result = parse_single_sql_result(source_file, stmt, elem); - if let DbItem::Error(e) = query_result { + let mut query_result = parse_single_sql_result(source_file, stmt, statement.source_span, elem); + if let DbItem::Error(e) = query_result.item { error = Some(e); break; } - if matches!(query_result, DbItem::Row(_)) { + if matches!(query_result.item, DbItem::Row(_)) { returned_rows += 1; } - apply_json_columns(&mut query_result, &stmt.json_columns); - if let Err(err) = apply_delayed_functions(request, &stmt.delayed_functions, &mut query_result) + apply_json_columns(&mut query_result.item, &stmt.json_columns); + if buffer_rows { + deferred_query_results.push(query_result); + } else { + let mut computed_connection = None; + if let Err(err) = evaluate_computed_columns(request, &stmt.computed_columns, &mut query_result, &mut computed_connection) + .instrument(query_span.clone()) + .await + { + error = Some(err); + break; + } + for db_item in parse_dynamic_rows(query_result.item) { + yield db_item; + } + } + } + drop(stream); + } + if error.is_none() && buffer_rows { + for mut query_result in deferred_query_results { + if let Err(err) = evaluate_computed_columns( + request, + &stmt.computed_columns, + &mut query_result, + db_connection, + ) .instrument(query_span.clone()) .await - { - error = Some(err); - break; - } - for db_item in parse_dynamic_rows(query_result) { - yield db_item; + { + error = Some(err); + break; + } + for db_item in parse_dynamic_rows(query_result.item) { + yield db_item; + } } } - drop(stream); if let Some(error) = error { - query_metrics.record_error(returned_rows, &error); - try_rollback_transaction(connection).await; + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + let error = record_error_and_rollback(connection, &query_metrics, returned_rows, error).await; yield DbItem::Error(error); } else { query_metrics.record_success(returned_rows); } + } }, - ParsedStatement::SetVariable { variable, value} => { - execute_set_variable_query(db_connection, request, variable, value, source_file).await - .with_context(|| - format!("Failed to set the {variable} variable to {value:?}") - )?; - }, - ParsedStatement::StaticSimpleSet { variable, value} => { - execute_set_simple_static(db_connection, request, variable, value, source_file).await - .with_context(|| - format!("Failed to set the {variable} variable to {value:?}") - )?; + FileStatement::SetVariable { target, value} => { + execute_set_variable_query(db_connection, request, target, value, source_file).await + .with_context(|| format!("Failed to set variable {}", target.0)) + .map_err(|error| with_stmt_position(source_file, value.source_span, error))?; }, - ParsedStatement::StaticSimpleSelect { values, query_position } => { - let row = exec_static_simple_select(values, request, db_connection) - .await - .map_err(|e| with_stmt_position(source_file, *query_position, e))?; - for i in parse_dynamic_rows(DbItem::Row(row)) { - yield i; - } - } - ParsedStatement::Error(e) => yield DbItem::Error(clone_anyhow_err(source_file, e)), + FileStatement::Error(e) => yield DbItem::Error(clone_anyhow_err(source_file, e)), } } } @@ -284,21 +325,20 @@ pub fn stop_at_first_error( .take_until(error_rx) } -/// Executes the sqlpage pseudo-functions contained in a static simple select -async fn exec_static_simple_select( - columns: &[(String, SimpleSelectValue)], +async fn execute_single_row( + query: &SingleRowQuery, req: &ExecutionContext, db_connection: &mut DbConn, ) -> anyhow::Result { - let mut map = serde_json::Map::with_capacity(columns.len()); - for (name, value) in columns { - let value = match value { - SimpleSelectValue::Static(s) => s.clone(), - SimpleSelectValue::Dynamic(p) => { - extract_req_param_as_json(p, req, db_connection).await? - } - }; - map = add_value_to_map(map, (name.clone(), value)); + let mut map = serde_json::Map::with_capacity(query.columns.len()); + let mut inputs = NoInputs; + for column in &query.columns { + let value = column + .value + .evaluate(req, db_connection, &mut inputs) + .await? + .into_json(); + map = add_value_to_map(map, (column.name.clone(), value)); } Ok(serde_json::Value::Object(map)) } @@ -313,24 +353,10 @@ async fn try_rollback_transaction(db_connection: &mut AnyConnection) { } } -/// Extracts the value of a parameter from the request. -/// Returns `Ok(None)` when NULL should be used as the parameter value. -async fn extract_req_param_as_json( - param: &StmtParam, - request: &ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result { - if let Some(val) = extract_req_param(param, request, db_connection).await? { - Ok(serde_json::Value::String(val.into_owned())) - } else { - Ok(serde_json::Value::Null) - } -} - /// This function is used to create a pinned boxed stream of query results. /// This allows recursive calls. pub fn stream_query_results_boxed<'a>( - sql_file: &'a ParsedSqlFile, + sql_file: &'a SqlFile, request: &'a ExecutionContext, db_connection: &'a mut DbConn, ) -> Pin + 'a>> { @@ -344,108 +370,167 @@ pub fn stream_query_results_boxed<'a>( async fn execute_set_variable_query<'a>( db_connection: &'a mut DbConn, request: &'a ExecutionContext, - variable: &StmtParam, - statement: &StmtWithParams, + variable: &super::sql::VariableName, + statement: &Query, source_file: &Path, ) -> anyhow::Result<()> { - let query = bind_parameters(statement, request, db_connection).await?; - let connection = take_connection(&request.app_state.db, db_connection, request).await?; - log::debug!( - "Executing query to set the {variable:?} variable: {:?}", - query.sql - ); - - let db_system_name = request.app_state.db.info.database_type.otel_name(); - let (query_span, operation_name) = create_db_query_span( - query.sql, - source_file, - statement.query_position.start.line, - db_system_name, - ); - let mut query_metrics = DbQueryMetricsContext::new( - query_span.clone(), - operation_name, - db_system_name, - &request.app_state.telemetry_metrics, - ); - record_query_params(&query_metrics.span, &query.param_values); - let start_time = std::time::Instant::now(); - let value = match connection - .fetch_optional(query) - .instrument(query_span.clone()) - .await - { - Ok(Some(row)) => { - query_metrics.add_duration(start_time.elapsed()); - query_metrics.record_success(1_i64); - row_to_string(&row) - } - Ok(None) => { - query_metrics.add_duration(start_time.elapsed()); - query_metrics.record_success(0_i64); - None - } - Err(e) => { - query_metrics.add_duration(start_time.elapsed()); - try_rollback_transaction(connection).await; - let err = display_stmt_db_error(source_file, statement, e); - query_metrics.record_error(0_i64, &err); - return Err(err); - } - }; - - let (mut vars, name) = vars_and_name(request, variable)?; + let value = execute_scalar_query(db_connection, request, statement, source_file).await?; - log::debug!("Setting variable {name} to {value:?}"); - vars.insert(name.to_owned(), value.map(SingleOrVec::Single)); + log::debug!("Setting variable {} to {value:?}", variable.0); + request + .set_variables + .borrow_mut() + .insert(variable.0.clone(), value.map(SingleOrVec::Single)); Ok(()) } -async fn execute_set_simple_static<'a>( +async fn execute_scalar_query<'a>( db_connection: &'a mut DbConn, request: &'a ExecutionContext, - variable: &StmtParam, - value: &SimpleSelectValue, - _source_file: &Path, -) -> anyhow::Result<()> { - let value_str = match value { - SimpleSelectValue::Static(json_value) => match json_value { - serde_json::Value::Null => None, - serde_json::Value::String(s) => Some(s.clone()), - other => Some(other.to_string()), - }, - SimpleSelectValue::Dynamic(stmt_param) => { - extract_req_param(stmt_param, request, db_connection) - .await? - .map(std::borrow::Cow::into_owned) + statement: &Query, + source_file: &Path, +) -> anyhow::Result> { + let QueryBody::Database(database_query) = &statement.body else { + let QueryBody::SingleRow(single_row) = &statement.body else { + unreachable!() + }; + ensure_scalar_column_count(single_row.columns.len())?; + let row = execute_single_row(single_row, request, db_connection).await?; + return scalar_value_from_row(DbItem::Row(row)); + }; + let query = bind_query(database_query, request, db_connection).await?; + log::debug!("Executing scalar query: {:?}", query.sql); + let (query_span, mut query_metrics) = + create_query_metrics(request, source_file, statement.source_span, &query); + + let mut scalar_row = None; + let mut returned_rows: i64 = 0; + let mut error = None; + let scalar_subquery_behavior = request + .app_state + .db + .info + .database_type + .scalar_subquery_behavior(); + { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + let mut stream = connection.fetch_many(query); + loop { + let start_next = std::time::Instant::now(); + let next_elem = stream.next().instrument(query_span.clone()).await; + query_metrics.add_duration(start_next.elapsed()); + let Some(elem) = next_elem else { break }; + + let result = + parse_single_sql_result(source_file, database_query, statement.source_span, elem); + let output_column_count = result.output_column_count; + match result.item { + row @ DbItem::Row(_) => { + returned_rows += 1; + if scalar_row.is_some() { + debug_assert_eq!( + scalar_subquery_behavior, + ScalarSubqueryBehavior::ErrorOnMultipleRows + ); + error = Some(anyhow!( + "SET scalar query returned more than one row. A SET subquery must return zero or one row." + )); + break; + } + scalar_row = Some(QueryResult { + item: row, + inputs: result.inputs, + output_column_count, + }); + if scalar_subquery_behavior == ScalarSubqueryBehavior::FirstRow { + break; + } + } + DbItem::FinishedQuery => {} + DbItem::Error(err) => { + error = Some(err); + break; + } + } + } + drop(stream); + } + + let value = if let Some(error) = error { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + return Err( + record_error_and_rollback(connection, &query_metrics, returned_rows, error).await, + ); + } else if let Some(mut row) = scalar_row { + ensure_scalar_column_count(row.output_column_count)?; + apply_json_columns(&mut row.item, &database_query.json_columns); + if let Err(error) = evaluate_computed_columns( + request, + &database_query.computed_columns, + &mut row, + db_connection, + ) + .instrument(query_span.clone()) + .await + { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + return Err(record_error_and_rollback( + connection, + &query_metrics, + returned_rows, + error, + ) + .await); } + scalar_value_from_row(row.item)? + } else { + None }; - let (mut vars, name) = vars_and_name(request, variable)?; + query_metrics.record_success(returned_rows); + Ok(value) +} - log::debug!("Setting variable {name} to static value {value_str:?}"); - vars.insert(name.to_owned(), value_str.map(SingleOrVec::Single)); - Ok(()) +async fn record_error_and_rollback( + connection: &mut AnyConnection, + query_metrics: &DbQueryMetricsContext<'_>, + returned_rows: i64, + error: anyhow::Error, +) -> anyhow::Error { + query_metrics.record_error(returned_rows, &error); + try_rollback_transaction(connection).await; + error } -fn vars_and_name<'a, 'b>( - request: &'a ExecutionContext, - variable: &'b StmtParam, -) -> anyhow::Result<(std::cell::RefMut<'a, SetVariablesMap>, &'b str)> { - match variable { - StmtParam::PostOrGet(name) | StmtParam::Get(name) => { - if request.post_variables.contains_key(name) { - log::warn!( - "Deprecation warning! Setting the value of ${name}, but there is already a form field named :{name}. This will stop working soon. Please rename the variable, or use :{name} directly if you intended to overwrite the posted form field value." - ); - } - Ok((request.set_variables.borrow_mut(), name)) - } - StmtParam::Post(name) => Ok((request.set_variables.borrow_mut(), name)), - _ => Err(anyhow!( - "Only GET and POST variables can be set, not {variable:?}" - )), +fn scalar_value_from_row(item: DbItem) -> anyhow::Result> { + let DbItem::Row(Value::Object(row)) = item else { + anyhow::bail!("SET scalar query did not return a row object"); + }; + ensure_scalar_column_count(row.len())?; + Ok(row + .into_iter() + .next() + .and_then(|(_, value)| json_to_scalar_string(value))) +} + +fn ensure_scalar_column_count(column_count: usize) -> anyhow::Result<()> { + match column_count { + 0 => anyhow::bail!( + "SET scalar query returned no columns. A SET subquery must select exactly one column." + ), + 1 => Ok(()), + _ => anyhow::bail!( + "SET scalar query returned more than one column. A SET subquery must select exactly one column." + ), + } +} + +fn json_to_scalar_string(value: Value) -> Option { + match value { + Value::Null => None, + Value::String(s) => Some(s), + other => Some(other.to_string()), } } @@ -522,23 +607,47 @@ async fn set_trace_context(connection: &mut AnyConnection, db: &Database) { #[inline] fn parse_single_sql_result( source_file: &Path, - stmt: &StmtWithParams, + query: &DatabaseQuery, + source_span: SourceSpan, res: sqlx::Result>, -) -> DbItem { +) -> QueryResult { match res { Ok(Either::Right(r)) => { if log::log_enabled!(log::Level::Trace) { debug_row(&r); } - DbItem::Row(super::sql_to_json::row_to_json(&r)) + match super::sql_to_json::row_to_json_with_inputs(&r, query.row_input_json.len()) { + Ok((row, mut inputs)) => { + decode_json_values(&mut inputs, &query.row_input_json); + QueryResult { + item: DbItem::Row(row), + inputs: RowInputs::new(inputs), + output_column_count: r.columns().len() - query.row_input_json.len() + + query.computed_columns.len(), + } + } + Err(error) => QueryResult { + item: DbItem::Error(error), + inputs: RowInputs::new(Vec::new()), + output_column_count: 0, + }, + } } Ok(Either::Left(res)) => { log::debug!("Finished query with result: {res:?}"); - DbItem::FinishedQuery + QueryResult { + item: DbItem::FinishedQuery, + inputs: RowInputs::new(Vec::new()), + output_column_count: 0, + } } Err(err) => { - let nice_err = display_stmt_db_error(source_file, stmt, err); - DbItem::Error(nice_err) + let nice_err = display_stmt_db_error(source_file, &query.sql, source_span, err); + QueryResult { + item: DbItem::Error(nice_err), + inputs: RowInputs::new(Vec::new()), + output_column_count: 0, + } } } } @@ -564,16 +673,6 @@ fn debug_row(r: &AnyRow) { } fn clone_anyhow_err(source_file: &Path, err: &anyhow::Error) -> anyhow::Error { - if let Some(func_err) = err.downcast_ref::() { - let line = func_err.line; - let loc = if line > 0 { - format!(":{line}") - } else { - String::new() - }; - return anyhow::anyhow!("{}{loc} {}", source_file.display(), func_err); - } - let mut e = anyhow!( "{} contains a syntax error preventing SQLPage from parsing and preparing its SQL statements.", source_file.display() @@ -584,18 +683,22 @@ fn clone_anyhow_err(source_file: &Path, err: &anyhow::Error) -> anyhow::Error { e } -async fn bind_parameters<'a>( - stmt: &'a StmtWithParams, +async fn bind_query<'a>( + query: &'a DatabaseQuery, request: &'a ExecutionContext, db_connection: &mut DbConn, -) -> anyhow::Result> { - let sql = stmt.query.as_str(); +) -> anyhow::Result> { + let sql = query.sql.as_str(); log::debug!("Preparing statement: {sql}"); let mut arguments = AnyArguments::default(); - let mut param_values = Vec::with_capacity(stmt.params.len()); - for (param_idx, param) in stmt.params.iter().enumerate() { - log::trace!("\tevaluating parameter {}: {}", param_idx + 1, param); - let argument = extract_req_param(param, request, db_connection).await?; + let mut param_values = Vec::with_capacity(query.bindings.len()); + let mut inputs = NoInputs; + for (param_idx, binding) in query.bindings.iter().enumerate() { + log::trace!("\tevaluating binding {}: {:?}", param_idx + 1, binding); + let argument = binding + .evaluate(request, db_connection, &mut inputs) + .await? + .into_function_argument(); log::debug!( "\tparameter {}: {}", param_idx + 1, @@ -608,8 +711,8 @@ async fn bind_parameters<'a>( Some(Cow::Borrowed(v)) => arguments.add(v), } } - let has_arguments = !stmt.params.is_empty(); - Ok(StatementWithParams { + let has_arguments = !query.bindings.is_empty(); + Ok(BoundQuery { sql, arguments, has_arguments, @@ -617,59 +720,26 @@ async fn bind_parameters<'a>( }) } -async fn apply_delayed_functions( - request: &ExecutionContext, - delayed_functions: &[DelayedFunctionCall], - item: &mut DbItem, -) -> anyhow::Result<()> { - // We need to open new connections for each delayed function call, because we are still fetching the results of the current query in the main connection. - let mut db_conn = None; - if let DbItem::Row(serde_json::Value::Object(results)) = item { - for f in delayed_functions { - log::trace!("Applying delayed function {} to {:?}", f.function, results); - apply_single_delayed_function(request, &mut db_conn, f, results).await?; - log::trace!( - "Delayed function applied {}. Result: {:?}", - f.function, - results - ); - } - } - Ok(()) -} - -async fn apply_single_delayed_function( +async fn evaluate_computed_columns( request: &ExecutionContext, + columns: &[OutputColumn], + result: &mut QueryResult, db_connection: &mut DbConn, - f: &DelayedFunctionCall, - row: &mut serde_json::Map, ) -> anyhow::Result<()> { - let mut params = Vec::new(); - for arg in &f.argument_col_names { - let Some(arg_value) = row.remove(arg) else { - anyhow::bail!( - "The column {arg} is missing in the result set, but it is required by the {} function.", - f.function - ); - }; - params.push(json_to_fn_param(arg_value)); + if let DbItem::Row(serde_json::Value::Object(results)) = &mut result.item { + for column in columns { + let value = column + .value + .evaluate(request, db_connection, &mut result.inputs) + .await? + .into_json(); + let old_results = std::mem::take(results); + *results = add_value_to_map(old_results, (column.name.clone(), value)); + } } - let result_str = f.function.evaluate(request, db_connection, params).await?; - let result_json = result_str - .map(Cow::into_owned) - .map_or(serde_json::Value::Null, serde_json::Value::String); - row.insert(f.target_col_name.clone(), result_json); Ok(()) } -fn json_to_fn_param(json: serde_json::Value) -> Option> { - match json { - serde_json::Value::String(s) => Some(Cow::Owned(s)), - serde_json::Value::Null => None, - _ => Some(Cow::Owned(json.to_string())), - } -} - fn apply_json_columns(item: &mut DbItem, json_columns: &[String]) { if let DbItem::Row(Value::Object(row)) = item { for column in json_columns { @@ -700,14 +770,27 @@ fn apply_json_columns(item: &mut DbItem, json_columns: &[String]) { } } -pub struct StatementWithParams<'a> { +fn decode_json_values(values: &mut [Value], json_flags: &[bool]) { + debug_assert_eq!(values.len(), json_flags.len()); + for (value, decode_as_json) in values.iter_mut().zip(json_flags) { + if *decode_as_json + && let Value::String(json) = value + && let Ok(parsed) = serde_json::from_str(json) + { + *value = parsed; + } + } +} + +/// Rewritten SQL and evaluated arguments in the form consumed by `sqlx`. +pub struct BoundQuery<'a> { sql: &'a str, arguments: AnyArguments<'a>, has_arguments: bool, param_values: Vec>, } -impl<'q> sqlx::Execute<'q, Any> for StatementWithParams<'q> { +impl<'q> sqlx::Execute<'q, Any> for BoundQuery<'q> { fn sql(&self) -> &'q str { self.sql } diff --git a/src/webserver/database/mod.rs b/src/webserver/database/mod.rs index b73e0cb1..000e1647 100644 --- a/src/webserver/database/mod.rs +++ b/src/webserver/database/mod.rs @@ -4,14 +4,13 @@ mod csv_import; pub mod execute_queries; pub mod migrations; mod sql; +mod sqlpage_expr; mod sqlpage_functions; -mod syntax_tree; mod error_highlighting; mod sql_to_json; -pub use sql::ParsedSqlFile; -use sql::{DB_PLACEHOLDERS, DbPlaceHolder}; +pub use sql::SqlFile; use sqlx::any::AnyKind; // SupportedDatabase is defined in this module @@ -28,6 +27,12 @@ pub enum SupportedDatabase { Generic, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScalarSubqueryBehavior { + FirstRow, + ErrorOnMultipleRows, +} + impl SupportedDatabase { /// Detect the database type from a connection's `dbms_name` #[must_use] @@ -79,6 +84,32 @@ impl SupportedDatabase { Self::Generic => "other_sql", } } + + /// Mirrors how the backend handles a scalar subquery that returns multiple rows. + fn scalar_subquery_behavior(self) -> ScalarSubqueryBehavior { + match self { + Self::Sqlite => ScalarSubqueryBehavior::FirstRow, + _ => ScalarSubqueryBehavior::ErrorOnMultipleRows, + } + } + + fn concat_function_null_behavior(self) -> sqlpage_expr::ConcatNullBehavior { + use sqlpage_expr::ConcatNullBehavior::{IgnoreNull, PropagateNull}; + + match self { + Self::Sqlite | Self::Duckdb | Self::Oracle | Self::Postgres | Self::Mssql => IgnoreNull, + Self::MySql | Self::Snowflake | Self::Generic => PropagateNull, + } + } + + fn concat_operator_null_behavior(self) -> sqlpage_expr::ConcatNullBehavior { + use sqlpage_expr::ConcatNullBehavior::{IgnoreNull, PropagateNull}; + + match self { + Self::Oracle | Self::Mssql => IgnoreNull, + _ => PropagateNull, + } + } } impl From for SupportedDatabase { @@ -131,12 +162,75 @@ impl std::fmt::Display for Database { #[inline] #[must_use] pub fn make_placeholder(dbms: AnyKind, arg_number: usize) -> String { - if let Some((_, placeholder)) = DB_PLACEHOLDERS.iter().find(|(kind, _)| *kind == dbms) { - match *placeholder { - DbPlaceHolder::PrefixedNumber { prefix } => format!("{prefix}{arg_number}"), - DbPlaceHolder::Positional { placeholder } => placeholder.to_string(), + match dbms { + AnyKind::Sqlite => format!("?{arg_number}"), + AnyKind::Postgres => format!("${arg_number}"), + AnyKind::Mssql => format!("@p{arg_number}"), + AnyKind::MySql | AnyKind::Odbc => "?".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::sqlpage_expr::ConcatNullBehavior::{IgnoreNull, PropagateNull}; + use super::{ScalarSubqueryBehavior, SupportedDatabase}; + + #[test] + fn scalar_subquery_behavior_matches_backends() { + assert_eq!( + SupportedDatabase::Sqlite.scalar_subquery_behavior(), + ScalarSubqueryBehavior::FirstRow + ); + for database in [ + SupportedDatabase::Duckdb, + SupportedDatabase::Oracle, + SupportedDatabase::Postgres, + SupportedDatabase::MySql, + SupportedDatabase::Mssql, + SupportedDatabase::Snowflake, + SupportedDatabase::Generic, + ] { + assert_eq!( + database.scalar_subquery_behavior(), + ScalarSubqueryBehavior::ErrorOnMultipleRows + ); + } + } + + #[test] + fn concat_null_behavior_matches_backends() { + for database in [ + SupportedDatabase::Sqlite, + SupportedDatabase::Duckdb, + SupportedDatabase::Oracle, + SupportedDatabase::Postgres, + SupportedDatabase::Mssql, + ] { + assert_eq!(database.concat_function_null_behavior(), IgnoreNull); + } + for database in [ + SupportedDatabase::MySql, + SupportedDatabase::Snowflake, + SupportedDatabase::Generic, + ] { + assert_eq!(database.concat_function_null_behavior(), PropagateNull); + } + } + + #[test] + fn concat_operator_null_behavior_matches_backends() { + for database in [SupportedDatabase::Oracle, SupportedDatabase::Mssql] { + assert_eq!(database.concat_operator_null_behavior(), IgnoreNull); + } + for database in [ + SupportedDatabase::Sqlite, + SupportedDatabase::Duckdb, + SupportedDatabase::Postgres, + SupportedDatabase::MySql, + SupportedDatabase::Snowflake, + SupportedDatabase::Generic, + ] { + assert_eq!(database.concat_operator_null_behavior(), PropagateNull); } - } else { - unreachable!("missing dbms: {dbms:?} in DB_PLACEHOLDERS ({DB_PLACEHOLDERS:?})") } } diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index 0d4bdfef..40f40d3c 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -1,270 +1,192 @@ -use super::SupportedDatabase; -use super::csv_import::{CsvImport, extract_csv_copy_statement}; -use super::sqlpage_functions::functions::SqlPageFunctionName; -use super::syntax_tree::StmtParam; -use crate::file_cache::AsyncFromStrWithState; -use crate::webserver::database::DbInfo; -use crate::webserver::database::error_highlighting::quote_source_with_highlight; -use crate::{AppState, Database}; +//! SQL-file front end that turns source text into a cacheable execution plan. +//! +//! A routed `.sql` file reaches this module before request-specific values are +//! available. It selects the connected DBMS's parser dialect, tokenizes and +//! parses statements with source locations, and classifies each statement as a +//! query, `SQLPage` `SET` assignment, CSV import, or error. Parse and rewrite +//! errors are retained as statements so they can flow through `SQLPage`'s normal +//! execution and rendering error path, while the resulting [`SqlFile`] remains +//! request-independent and safe to reuse from the file cache. +//! +//! Ordinary statements and `SET` value queries are delegated to [`rewrite`], +//! which lowers the parsed AST into database SQL plus SQLPage-owned expressions. +//! The immutable statement types live in [`statement`]; `execute_queries` later +//! supplies request variables, runs the database work, and evaluates the +//! `SQLPage` expressions. This module therefore owns parsing and statement +//! classification, but not query execution or database-row decoding. + +use std::fmt::Write as _; +use std::path::Path; + use async_trait::async_trait; use sqlparser::ast::helpers::attached_token::AttachedToken; use sqlparser::ast::{ - CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, - FunctionArguments, Ident, ObjectName, ObjectNamePart, SelectFlavor, SelectItem, Set, SetExpr, - Spanned, Statement, Value, ValueWithSpan, -}; -use sqlparser::dialect::{ - Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, OracleDialect, - PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, + Expr, Ident, ObjectName, ObjectNamePart, SelectFlavor, SelectItem, Set, SetExpr, Spanned, + Statement, Value, }; +use sqlparser::dialect::Dialect; use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::Token::{self, EOF, SemiColon}; use sqlparser::tokenizer::{Location, Span, TokenWithSpan, Tokenizer}; -use sqlx::any::AnyKind; -use std::fmt::Write; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -mod parameter_extraction; -pub(super) use self::parameter_extraction::{ - DB_PLACEHOLDERS, DbPlaceHolder, ParamExtractContext, SqlPageFunctionError, - function_args_to_stmt_params, -}; -use self::parameter_extraction::{ - ParameterExtractor, TEMP_PLACEHOLDER_PREFIX, extract_ident_param, validate_function_calls, -}; +use super::csv_import::extract_csv_copy_statement; +use super::{Database, DbInfo, SupportedDatabase}; +use crate::AppState; +use crate::file_cache::AsyncFromStrWithState; +use crate::webserver::database::error_highlighting::quote_source_with_highlight; -#[derive(Default)] -pub struct ParsedSqlFile { - pub(super) statements: Vec, - pub source_path: PathBuf, -} +mod dialect; +mod rewrite; +mod statement; + +#[cfg(test)] +pub(super) use statement::SourceLocation; +pub use statement::SqlFile; +pub(super) use statement::{ + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, + VariableName, +}; -impl ParsedSqlFile { +impl SqlFile { #[must_use] - pub fn new(db: &Database, sql: &str, source_path: &Path) -> ParsedSqlFile { - let dialect = dialect_for_db(db.info.database_type); + pub fn new(db: &Database, sql: &str, source_path: &Path) -> Self { + let dialect = dialect::parser_dialect(db.info.database_type); log::debug!( "Parsing SQL file {} using dialect {:?}", source_path.display(), dialect ); - let parsed_statements = match parse_sql(&db.info, dialect.as_ref(), sql) { - Ok(parsed) => parsed, - Err(err) => return Self::from_err(err, source_path), + let statements = match parse_sql(&db.info, dialect.as_ref(), sql) { + Ok(statements) => statements.collect::>().into_boxed_slice(), + Err(error) => { + return Self::from_error(error, source_path); + } }; - let statements = parsed_statements.collect(); - ParsedSqlFile { + Self { statements, source_path: source_path.to_path_buf(), } } - fn from_err(e: impl Into, source_path: &Path) -> Self { + fn from_error(error: impl Into, source_path: &Path) -> Self { Self { - statements: vec![ParsedStatement::Error( - e.into() + statements: vec![FileStatement::Error( + error + .into() .context(format!("Error parsing file {}", source_path.display())), - )], + )] + .into_boxed_slice(), source_path: source_path.to_path_buf(), } } } -#[async_trait(? Send)] -impl AsyncFromStrWithState for ParsedSqlFile { +#[async_trait(?Send)] +impl AsyncFromStrWithState for SqlFile { async fn from_str_with_state( app_state: &AppState, source: &str, source_path: &Path, ) -> anyhow::Result { - Ok(ParsedSqlFile::new(&app_state.db, source, source_path)) + Ok(Self::new(&app_state.db, source, source_path)) } } -/// A single SQL statement that has been parsed from a SQL file. -#[derive(Debug, PartialEq)] -pub(super) struct StmtWithParams { - /// The SQL query with placeholders for parameters. - pub query: String, - /// The line and column of the first token in the query. - pub query_position: SourceSpan, - /// Parameters that should be bound to the query. - /// They can contain functions that will be called before the query is executed, - /// the result of which will be bound to the query. - pub params: Vec, - /// Functions that are called on the result set after the query has been executed, - /// and which can be passed the result of the query as an argument. - pub delayed_functions: Vec, - /// Columns that are JSON columns, and which should be converted to JSON objects after the query is executed. - /// Only relevant for databases that do not have a native JSON type, and which return JSON values as text. - pub json_columns: Vec, -} - -/// A location in the source code. -#[derive(Debug, PartialEq, Clone, Copy)] -pub(super) struct SourceSpan { - pub start: SourceLocation, - pub end: SourceLocation, -} - -/// A location in the source code. -#[derive(Debug, PartialEq, Clone, Copy)] -pub(super) struct SourceLocation { - pub line: usize, - pub column: usize, -} - -#[derive(Debug)] -pub(super) enum ParsedStatement { - StmtWithParams(StmtWithParams), - StaticSimpleSelect { - values: Vec<(String, SimpleSelectValue)>, - query_position: SourceSpan, - }, - SetVariable { - variable: StmtParam, - value: StmtWithParams, - }, - StaticSimpleSet { - variable: StmtParam, - value: SimpleSelectValue, - }, - CsvImport(CsvImport), - Error(anyhow::Error), -} - -#[derive(Debug, PartialEq)] -pub(super) enum SimpleSelectValue { - Static(serde_json::Value), - Dynamic(StmtParam), -} - fn parse_sql<'a>( - db_info: &'a DbInfo, + database: &'a DbInfo, dialect: &'a dyn Dialect, sql: &'a str, -) -> anyhow::Result + 'a> { - log::trace!("Parsing {} SQL: {sql}", db_info.dbms_name); - +) -> anyhow::Result + 'a> { + log::trace!("Parsing {} SQL: {sql}", database.dbms_name); let tokens = Tokenizer::new(dialect, sql) .tokenize_with_location() - .map_err(|err| { - let location = err.location; - anyhow::Error::new(err).context(format!("The SQLPage parser could not understand the SQL file. Tokenization failed. Please check for syntax errors:\n{}", quote_source_with_highlight(sql, location.line, location.column))) + .map_err(|error| { + let location = error.location; + anyhow::Error::new(error).context(format!( + "The SQLPage parser could not understand the SQL file. Tokenization failed. Please check for syntax errors:\n{}", + quote_source_with_highlight(sql, location.line, location.column) + )) })?; let mut parser = Parser::new(dialect).with_tokens_with_locations(tokens); let mut has_error = false; Ok(std::iter::from_fn(move || { if has_error { - // Return the first error and ignore the rest return None; } - let statement = parse_single_statement(&mut parser, db_info, sql); - log::debug!("Parsed statement: {statement:?}"); - if let Some(ParsedStatement::Error(_)) = &statement { + let statement = parse_single_statement(&mut parser, database, sql); + if matches!(statement, Some(FileStatement::Error(_))) { has_error = true; } statement })) } -fn transform_to_positional_placeholders(stmt: &mut StmtWithParams, kind: AnyKind) { - if let Some((_, DbPlaceHolder::Positional { placeholder })) = DB_PLACEHOLDERS - .iter() - .find(|(placeholder_kind, _)| *placeholder_kind == kind) - { - let mut new_params = Vec::new(); - let mut query = stmt.query.clone(); - while let Some(pos) = query.find(TEMP_PLACEHOLDER_PREFIX) { - let start_of_number = pos + TEMP_PLACEHOLDER_PREFIX.len(); - let end = query[start_of_number..] - .find(|c: char| !c.is_ascii_digit()) - .map_or(query.len(), |i| start_of_number + i); - let param_idx = query[start_of_number..end].parse::().unwrap_or(1) - 1; - query.replace_range(pos..end, placeholder); - new_params.push(stmt.params[param_idx].clone()); - } - stmt.query = query; - stmt.params = new_params; - } -} - fn parse_single_statement( parser: &mut Parser<'_>, - db_info: &DbInfo, + database: &DbInfo, source_sql: &str, -) -> Option { +) -> Option { if parser.peek_token() == EOF { return None; } - let mut stmt = match parser.parse_statement() { - Ok(stmt) => stmt, - Err(err) => return Some(syntax_error(err, parser, source_sql)), + let mut statement = match parser.parse_statement() { + Ok(statement) => statement, + Err(error) => return Some(syntax_error(error, parser, source_sql)), }; let mut semicolon = false; while parser.consume_token(&SemiColon) { semicolon = true; } - let mut params = match ParameterExtractor::extract_parameters(&mut stmt, db_info.clone()) { - Ok(p) => p, - Err(err) => return Some(ParsedStatement::Error(err)), - }; - let dbms = db_info.database_type; - if let Some(parsed) = extract_set_variable(&mut stmt, &mut params, db_info) { - return Some(parsed); + if let Some(statement) = extract_set_variable(&mut statement, database) { + return Some(statement); } - if let Some(csv_import) = extract_csv_copy_statement(&mut stmt) { - return Some(ParsedStatement::CsvImport(csv_import)); - } - if let Some(static_statement) = extract_static_simple_select(&stmt, ¶ms) { - log::debug!( - "Optimised a static simple select to avoid a trivial database query: {stmt} optimized to {static_statement:?}" - ); - return Some(ParsedStatement::StaticSimpleSelect { - values: static_statement, - query_position: extract_query_start(&stmt), - }); + if let Some(csv_import) = extract_csv_copy_statement(&mut statement) { + return Some(FileStatement::CsvImport(csv_import)); } - let delayed_functions = extract_toplevel_functions(&mut stmt); + Some( + match rewrite::rewrite_query(statement, database, semicolon) { + Ok(query) => FileStatement::Query(query), + Err(error) => FileStatement::Error(error), + }, + ) +} - if let Err(err) = validate_function_calls(&stmt) { - return Some(ParsedStatement::Error(err)); - } - let json_columns = extract_json_columns(&stmt, dbms); - let query = format!( - "{stmt}{semicolon}", - semicolon = if semicolon { ";" } else { "" } - ); - let mut stmt_with_params = StmtWithParams { - query, - query_position: extract_query_start(&stmt), - params, - delayed_functions, - json_columns, +fn extract_set_variable(statement: &mut Statement, database: &DbInfo) -> Option { + let Statement::Set(Set::SingleAssignment { + variable: ObjectName(name), + values, + scope: None, + hivevar: false, + }) = statement + else { + return None; + }; + let ([ObjectNamePart::Identifier(identifier)], [value]) = + (name.as_mut_slice(), values.as_mut_slice()) + else { + return None; }; - transform_to_positional_placeholders(&mut stmt_with_params, db_info.kind); - log::debug!("Final transformed statement: {}", stmt_with_params.query); - Some(ParsedStatement::StmtWithParams(stmt_with_params)) -} -fn extract_query_start(stmt: &impl Spanned) -> SourceSpan { - let location = stmt.span(); - SourceSpan { - start: SourceLocation { - line: usize::try_from(location.start.line).unwrap_or(0), - column: usize::try_from(location.start.column).unwrap_or(0), - }, - end: SourceLocation { - line: usize::try_from(location.end.line).unwrap_or(0), - column: usize::try_from(location.end.column).unwrap_or(0), + let mut target = std::mem::take(&mut identifier.value); + if target.starts_with(['$', ':', '?']) { + target.remove(0); + } + let expression = std::mem::replace(value, Expr::value(Value::Null)); + let value_statement = expression_to_query(expression); + Some( + match rewrite::rewrite_query(value_statement, database, false) { + Ok(value) => FileStatement::SetVariable { + target: VariableName(target), + value, + }, + Err(error) => FileStatement::Error(error), }, - } + ) } -fn syntax_error(err: ParserError, parser: &Parser, sql: &str) -> ParsedStatement { +fn syntax_error(error: ParserError, parser: &Parser, sql: &str) -> FileStatement { let Span { start: Location { line: start_line, @@ -272,437 +194,138 @@ fn syntax_error(err: ParserError, parser: &Parser, sql: &str) -> ParsedStatement }, end: Location { line: end_line, .. }, } = parser.peek_token_no_skip().span; - - let mut msg = String::from( + let mut message = String::from( "Parsing failed: SQLPage couldn't understand the SQL file. Please check for syntax errors on ", ); if start_line == end_line { - write!(&mut msg, "line {start_line}:").unwrap(); + write!(&mut message, "line {start_line}:").unwrap(); } else { - write!(&mut msg, "lines {start_line} to {end_line}:").unwrap(); + write!(&mut message, "lines {start_line} to {end_line}:").unwrap(); } write!( - &mut msg, + &mut message, "\n{}", quote_source_with_highlight(sql, start_line, start_column) ) .unwrap(); - ParsedStatement::Error(anyhow::Error::from(err).context(msg)) + FileStatement::Error(anyhow::Error::from(error).context(message)) } -fn dialect_for_db(dbms: SupportedDatabase) -> Box { - match dbms { - SupportedDatabase::Duckdb => Box::new(DuckDbDialect {}), - SupportedDatabase::Oracle => Box::new(OracleDialect {}), - SupportedDatabase::Postgres => Box::new(PostgreSqlDialect {}), - SupportedDatabase::Generic => Box::new(GenericDialect {}), - SupportedDatabase::Mssql => Box::new(MsSqlDialect {}), - SupportedDatabase::MySql => Box::new(MySqlDialect {}), - SupportedDatabase::Sqlite => Box::new(SQLiteDialect {}), - SupportedDatabase::Snowflake => Box::new(SnowflakeDialect {}), - } -} +const SQLPAGE_FUNCTION_NAMESPACE: &str = "sqlpage"; -#[derive(Debug, PartialEq)] -pub struct DelayedFunctionCall { - pub function: SqlPageFunctionName, - pub argument_col_names: Vec, - pub target_col_name: String, +pub(super) fn is_sqlpage_func(parts: &[ObjectNamePart]) -> bool { + matches!( + parts, + [ + ObjectNamePart::Identifier(Ident { + value, + quote_style: None, + .. + }), + ObjectNamePart::Identifier(Ident { quote_style: None, .. }) + ] if value.eq_ignore_ascii_case(SQLPAGE_FUNCTION_NAMESPACE) + ) } -/// The execution of top-level functions is delayed until after the query has been executed. -/// For instance, `SELECT sqlpage.fetch(x) FROM t` will be executed as `SELECT x as _sqlpage_f0_a0 FROM t` -/// and the `sqlpage.fetch` function will be called with the value of `_sqlpage_f0_a0` after the query has been executed, -/// on each row of the result set. -fn extract_toplevel_functions(stmt: &mut Statement) -> Vec { - struct SelectItemToAdd { - expr_to_insert: SelectItem, - position: usize, +pub(super) fn extract_json_columns( + statement: &Statement, + database: SupportedDatabase, +) -> Vec { + if matches!( + database, + SupportedDatabase::Postgres | SupportedDatabase::Mssql + ) { + return Vec::new(); } - let mut delayed_function_calls: Vec = Vec::new(); - let set_expr = match stmt { - Statement::Query(q) => q.body.as_mut(), - _ => return delayed_function_calls, + let Statement::Query(query) = statement else { + return Vec::new(); }; - let select_items = match set_expr { - sqlparser::ast::SetExpr::Select(s) => &mut s.projection, - _ => return delayed_function_calls, + let SetExpr::Select(select) = query.body.as_ref() else { + return Vec::new(); }; - let mut select_items_to_add: Vec = Vec::new(); - - for (position, select_item) in select_items.iter_mut().enumerate() { - let SelectItem::ExprWithAlias { - expr: - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }), - alias, - } = select_item - else { - continue; - }; - let Some(func_name) = extract_sqlpage_function_name(func_name_parts) else { - continue; - }; - func_name_parts.clear(); // mark the function for deletion - let mut argument_col_names = Vec::with_capacity(args.len()); - for (arg_idx, arg) in args.iter_mut().enumerate() { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) - | FunctionArg::Named { - arg: FunctionArgExpr::Expr(expr), - .. - } => { - let func_idx = delayed_function_calls.len(); - let argument_col_name = format!("_sqlpage_f{func_idx}_a{arg_idx}"); - argument_col_names.push(argument_col_name.clone()); - let expr_to_insert = SelectItem::ExprWithAlias { - expr: std::mem::replace(expr, Expr::value(Value::Null)), - alias: Ident::with_quote('"', argument_col_name), - }; - select_items_to_add.push(SelectItemToAdd { - expr_to_insert, - position, - }); - } - other => { - log::error!("Unsupported argument to {func_name}: {other}"); - } - } - } - delayed_function_calls.push(DelayedFunctionCall { - function: func_name, - argument_col_names, - target_col_name: alias.value.clone(), - }); - } - // Insert the new select items (the function arguments) at the positions where the function calls were - let mut it = select_items_to_add.into_iter().peekable(); - *select_items = std::mem::take(select_items) - .into_iter() - .enumerate() - .flat_map(|(position, item)| { - let mut items = Vec::with_capacity(1); - while it.peek().is_some_and(|x| x.position == position) { - items.push(it.next().unwrap().expr_to_insert); - } - if items.is_empty() { - items.push(item); + select + .projection + .iter() + .filter_map(|item| match item { + SelectItem::ExprWithAlias { expr, alias } if is_json_expression(expr) => { + Some(alias.value.clone()) } - items + _ => None, }) - .collect(); - delayed_function_calls -} - -fn extract_static_simple_select( - stmt: &Statement, - params: &[StmtParam], -) -> Option> { - let set_expr = match stmt { - Statement::Query(q) - if q.limit_clause.is_none() - && q.fetch.is_none() - && q.order_by.is_none() - && q.with.is_none() - && q.locks.is_empty() => - { - q.body.as_ref() - } - _ => return None, - }; - let select_items = match set_expr { - sqlparser::ast::SetExpr::Select(s) - if s.cluster_by.is_empty() - && s.distinct.is_none() - && s.distribute_by.is_empty() - && s.from.is_empty() - && s.group_by == sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]) - && s.having.is_none() - && s.into.is_none() - && s.lateral_views.is_empty() - && s.named_window.is_empty() - && s.qualify.is_none() - && s.selection.is_none() - && s.sort_by.is_empty() - && s.top.is_none() => - { - &s.projection - } - _ => return None, - }; - let mut items = Vec::with_capacity(select_items.len()); - let mut params_iter = params.iter().cloned(); - for select_item in select_items { - let sqlparser::ast::SelectItem::ExprWithAlias { expr, alias } = select_item else { - return None; - }; - let value = expr_to_simple_select_val(&mut params_iter, expr)?; - let key = alias.value.clone(); - items.push((key, value)); - } - if let Some(p) = params_iter.next() { - log::error!("static select extraction failed because of extraneous parameter: {p:?}"); - return None; - } - Some(items) -} - -fn expr_to_simple_select_val( - params_iter: &mut impl Iterator, - expr: &Expr, -) -> Option { - use SimpleSelectValue::{Dynamic, Static}; - use serde_json::Value::{Bool, Null, Number, String}; - Some(match expr { - Expr::Value(ValueWithSpan { - value: Value::Boolean(b), - .. - }) => Static(Bool(*b)), - Expr::Value(ValueWithSpan { - value: Value::Number(n, _), - .. - }) => Static(Number(n.parse().ok()?)), - Expr::Value(ValueWithSpan { - value: Value::SingleQuotedString(s), - .. - }) => Static(String(s.clone())), - Expr::Value(ValueWithSpan { - value: Value::Null, .. - }) => Static(Null), - e if is_simple_select_placeholder(e) => { - if let Some(p) = params_iter.next() { - Dynamic(p) - } else { - log::error!("Parameter not extracted for placehorder: {expr:?}"); - return None; - } - } - other => { - log::trace!("Cancelling simple select optimization because of expr: {other:?}"); - return None; - } - }) -} - -fn is_simple_select_placeholder(e: &Expr) -> bool { - match e { - Expr::Value(ValueWithSpan { - value: Value::Placeholder(_), - .. - }) => true, - Expr::Cast { - expr, - data_type: DataType::Text | DataType::Varchar(_) | DataType::Char(_), - format: None, - kind: CastKind::Cast, - .. - } if is_simple_select_placeholder(expr) => true, - _ => false, - } -} - -fn extract_set_variable( - stmt: &mut Statement, - params: &mut Vec, - db_info: &DbInfo, -) -> Option { - if let Statement::Set(Set::SingleAssignment { - variable: ObjectName(name), - values, - scope: None, - hivevar: false, - }) = stmt - && let ([ObjectNamePart::Identifier(ident)], [value]) = - (name.as_mut_slice(), values.as_mut_slice()) - { - let variable = if let Some(variable) = extract_ident_param(ident) { - variable - } else { - StmtParam::PostOrGet(std::mem::take(&mut ident.value)) - }; - let owned_expr = std::mem::replace(value, Expr::value(Value::Null)); - let mut params_iter = params.iter().cloned(); - if let Some(value) = expr_to_simple_select_val(&mut params_iter, &owned_expr) { - return Some(ParsedStatement::StaticSimpleSet { variable, value }); - } - - let mut select_stmt: Statement = expr_to_statement(owned_expr); - let delayed_functions = extract_toplevel_functions(&mut select_stmt); - if let Err(err) = validate_function_calls(&select_stmt) { - return Some(ParsedStatement::Error(err)); - } - let json_columns = extract_json_columns(&select_stmt, db_info.database_type); - let mut value = StmtWithParams { - query: select_stmt.to_string(), - query_position: extract_query_start(&select_stmt), - params: std::mem::take(params), - delayed_functions, - json_columns, - }; - transform_to_positional_placeholders(&mut value, db_info.kind); - return Some(ParsedStatement::SetVariable { variable, value }); - } - None -} - -const SQLPAGE_FUNCTION_NAMESPACE: &str = "sqlpage"; - -fn is_sqlpage_func(func_name_parts: &[ObjectNamePart]) -> bool { - if let [ - ObjectNamePart::Identifier(Ident { value, .. }), - ObjectNamePart::Identifier(Ident { .. }), - ] = func_name_parts - { - value == SQLPAGE_FUNCTION_NAMESPACE - } else { - false - } -} - -fn extract_sqlpage_function_name( - func_name_parts: &[ObjectNamePart], -) -> Option { - if let [ - ObjectNamePart::Identifier(Ident { - value: namespace, .. - }), - ObjectNamePart::Identifier(Ident { value, .. }), - ] = func_name_parts - && namespace == SQLPAGE_FUNCTION_NAMESPACE - { - return SqlPageFunctionName::from_str(value).ok(); - } - None -} - -fn sqlpage_func_name(func_name_parts: &[ObjectNamePart]) -> &str { - if let [ - ObjectNamePart::Identifier(Ident { .. }), - ObjectNamePart::Identifier(Ident { value, .. }), - ] = func_name_parts - { - value - } else { - debug_assert!( - false, - "sqlpage function name should have been checked by is_sqlpage_func" - ); - "" - } -} - -fn extract_json_columns(stmt: &Statement, dbms: SupportedDatabase) -> Vec { - // Only extract JSON columns for databases without native JSON support - if matches!(dbms, SupportedDatabase::Postgres | SupportedDatabase::Mssql) { - return Vec::new(); - } - - let mut json_columns = Vec::new(); - - if let Statement::Query(query) = stmt - && let SetExpr::Select(select) = query.body.as_ref() - { - for item in &select.projection { - if let SelectItem::ExprWithAlias { expr, alias } = item - && is_json_function(expr) - { - json_columns.push(alias.value.clone()); - log::trace!("Found JSON column: {alias}"); - } - } - } - - json_columns + .collect() } -fn is_json_function(expr: &Expr) -> bool { - match expr { +pub(super) fn is_json_expression(expression: &Expr) -> bool { + match expression { Expr::Function(function) => { - if let [ObjectNamePart::Identifier(Ident { value, .. })] = function.name.0.as_slice() { - [ - "json_object", - "json_array", - "json_build_object", - "json_build_array", - "to_json", - "to_jsonb", - "json_agg", - "jsonb_agg", - "json_arrayagg", - "json_objectagg", - "json_group_array", - "json_group_object", - "json", - "jsonb", - ] - .iter() - .any(|&func| value.eq_ignore_ascii_case(func)) - } else { - false - } - } - Expr::Cast { data_type, .. } => { - if matches!(data_type, DataType::JSON | DataType::JSONB) { - true - } else if let DataType::Custom(ObjectName(parts), _) = data_type { - if let [ObjectNamePart::Identifier(ident)] = parts.as_slice() { - ident.value.eq_ignore_ascii_case("json") - } else { - false - } - } else { - false - } + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return false; + }; + [ + "json_object", + "json_array", + "json_build_object", + "json_build_array", + "to_json", + "to_jsonb", + "json_agg", + "jsonb_agg", + "json_arrayagg", + "json_objectagg", + "json_group_array", + "json_group_object", + "json", + "jsonb", + ] + .iter() + .any(|candidate| name.value.eq_ignore_ascii_case(candidate)) } + Expr::Cast { data_type, .. } => matches!( + data_type, + sqlparser::ast::DataType::JSON | sqlparser::ast::DataType::JSONB + ), _ => false, } } -fn expr_to_statement(expr: Expr) -> Statement { +fn expression_to_query(expression: Expr) -> Statement { + if let Expr::Subquery(query) = expression { + return Statement::Query(query); + } Statement::Query(Box::new(sqlparser::ast::Query { with: None, - body: Box::new(sqlparser::ast::SetExpr::Select(Box::new( - sqlparser::ast::Select { - select_token: AttachedToken(TokenWithSpan::new( - Token::make_keyword("SELECT"), - expr.span(), - )), - distinct: None, - top: None, - projection: vec![SelectItem::ExprWithAlias { - expr, - alias: Ident::new("sqlpage_set_expr"), - }], - into: None, - from: vec![], - lateral_views: vec![], - selection: None, - group_by: sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]), - cluster_by: vec![], - distribute_by: vec![], - sort_by: vec![], - having: None, - named_window: vec![], - qualify: None, - top_before_distinct: false, - prewhere: None, - window_before_qualify: false, - value_table_mode: None, - connect_by: Vec::new(), - optimizer_hints: vec![], - select_modifiers: None, - flavor: SelectFlavor::Standard, - exclude: None, - }, - ))), + body: Box::new(SetExpr::Select(Box::new(sqlparser::ast::Select { + select_token: AttachedToken(TokenWithSpan::new( + Token::make_keyword("SELECT"), + expression.span(), + )), + distinct: None, + top: None, + projection: vec![SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::new("sqlpage_set_expr"), + }], + into: None, + from: vec![], + lateral_views: vec![], + selection: None, + group_by: sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]), + cluster_by: vec![], + distribute_by: vec![], + sort_by: vec![], + having: None, + named_window: vec![], + qualify: None, + top_before_distinct: false, + prewhere: None, + window_before_qualify: false, + value_table_mode: None, + connect_by: Vec::new(), + optimizer_hints: vec![], + select_modifiers: None, + flavor: SelectFlavor::Standard, + exclude: None, + }))), order_by: None, limit_clause: None, fetch: None, @@ -715,74 +338,16 @@ fn expr_to_statement(expr: Expr) -> Statement { } #[cfg(test)] -mod test { - use super::super::sqlpage_functions::functions::SqlPageFunctionName; - use super::super::syntax_tree::SqlPageFunctionCall; - +mod tests { use super::*; + use crate::webserver::database::sqlpage_expr::{ + ConcatNullBehavior, RowInputId, SqlPageExpr, VariableRef, VariableSource, + }; + use crate::webserver::database::sqlpage_functions::functions::SqlPageFunctionName; + use sqlparser::dialect::{MySqlDialect, PostgreSqlDialect}; + use sqlx::any::AnyKind; - fn parse_stmt(sql: &str, dialect: &dyn Dialect) -> Statement { - let mut ast = Parser::parse_sql(dialect, sql).unwrap(); - assert_eq!(ast.len(), 1); - ast.pop().unwrap() - } - - fn parse_postgres_stmt(sql: &str) -> Statement { - parse_stmt(sql, &PostgreSqlDialect {}) - } - - #[test] - fn test_statement_rewrite() { - let mut ast = - parse_postgres_stmt("select $a from t where $x > $a OR $x = sqlpage.cookie('cookoo')"); - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - // $a -> $1 - // $x -> $2 - // sqlpage.cookie(...) -> $3 - assert_eq!( - ast.to_string(), - "SELECT CAST($1 AS TEXT) FROM t WHERE CAST($2 AS TEXT) > CAST($1 AS TEXT) OR CAST($2 AS TEXT) = CAST($3 AS TEXT)" - ); - assert_eq!( - parameters, - [ - StmtParam::PostOrGet("a".to_string()), - StmtParam::PostOrGet("x".to_string()), - StmtParam::FunctionCall(SqlPageFunctionCall { - function: SqlPageFunctionName::cookie, - arguments: vec![StmtParam::Literal("cookoo".to_string())] - }), - ] - ); - } - - #[test] - fn test_statement_rewrite_sqlite() { - let mut ast = parse_stmt("select $x, :y from t", &SQLiteDialect {}); - let db_info = create_test_db_info(SupportedDatabase::Sqlite); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - assert_eq!( - ast.to_string(), - "SELECT CAST(?1 AS TEXT), CAST(?2 AS TEXT) FROM t" - ); - assert_eq!( - parameters, - [ - StmtParam::PostOrGet("x".to_string()), - StmtParam::Post("y".to_string()), - ] - ); - } - - const ALL_DIALECTS: &[(&dyn Dialect, SupportedDatabase)] = &[ - (&PostgreSqlDialect {}, SupportedDatabase::Postgres), - (&MsSqlDialect {}, SupportedDatabase::Mssql), - (&MySqlDialect {}, SupportedDatabase::MySql), - (&SQLiteDialect {}, SupportedDatabase::Sqlite), - ]; - - fn create_test_db_info(database_type: SupportedDatabase) -> DbInfo { + fn database(database_type: SupportedDatabase) -> DbInfo { let kind = match database_type { SupportedDatabase::Postgres => AnyKind::Postgres, SupportedDatabase::Mssql => AnyKind::Mssql, @@ -791,583 +356,452 @@ mod test { _ => AnyKind::Odbc, }; DbInfo { - dbms_name: database_type.display_name().to_string(), + dbms_name: database_type.display_name().to_owned(), database_type, kind, } } - #[test] - fn test_duckdb_odbc_dialect_selection() { - use std::any::Any; - - let dbms = SupportedDatabase::from_dbms_name("DuckDB"); - assert_eq!(dbms, SupportedDatabase::Duckdb); - let dialect = dialect_for_db(dbms); - assert_eq!(dialect.as_ref().type_id(), (DuckDbDialect {}).type_id()); + fn one(sql: &str) -> FileStatement { + one_for(SupportedDatabase::Postgres, sql) + } - let sql = "select {'a': 1, 'b': 2} as payload"; - let db_info = create_test_db_info(dbms); - let mut parsed = parse_sql(&db_info, dialect.as_ref(), sql).unwrap(); - let stmt = parsed.next().expect("expected one statement"); - assert!( - !matches!(stmt, ParsedStatement::Error(_)), - "duckdb dictionary literals should parse" - ); + fn one_for(database_type: SupportedDatabase, sql: &str) -> FileStatement { + let database = database(database_type); + parse_sql(&database, &PostgreSqlDialect {}, sql) + .unwrap() + .next() + .unwrap() + } - let pg_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&pg_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("expected one statement"); - assert!( - matches!(stmt, ParsedStatement::Error(_)), - "postgres should reject duckdb dictionary literals" - ); + fn rewrite_database(sql: &str) -> DatabaseQuery { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one(sql) + else { + panic!("expected database query"); + }; + query } - #[test] - fn test_extract_toplevel_delayed_functions() { - let mut ast = parse_stmt( - "select sqlpage.fetch($x) as x, sqlpage.persist_uploaded_file('a', 'b') as y from t", - &PostgreSqlDialect {}, - ); - let functions = extract_toplevel_functions(&mut ast); - assert_eq!( - ast.to_string(), - "SELECT $x AS \"_sqlpage_f0_a0\", 'a' AS \"_sqlpage_f1_a0\", 'b' AS \"_sqlpage_f1_a1\" FROM t" - ); - assert_eq!( - functions, - vec![ - DelayedFunctionCall { - function: SqlPageFunctionName::fetch, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "x".to_string() - }, - DelayedFunctionCall { - function: SqlPageFunctionName::persist_uploaded_file, - argument_col_names: vec![ - "_sqlpage_f1_a0".to_string(), - "_sqlpage_f1_a1".to_string() - ], - target_col_name: "y".to_string() - } - ] - ); + fn call( + function: SqlPageFunctionName, + arguments: [SqlPageExpr; N], + ) -> SqlPageExpr { + SqlPageExpr::Call { + function, + arguments: Box::new(arguments), + } } - #[test] - fn test_extract_toplevel_delayed_functions_parameter_order() { - // The order of the function arguments should be preserved - // Otherwise the statement parameters will be bound to the wrong arguments - let sql = "select $a as a, sqlpage.exec('xxx', x = $b) as b, $c as c from t"; - let mut ast = parse_postgres_stmt(sql); - let delayed_functions = extract_toplevel_functions(&mut ast); - assert_eq!( - ast.to_string(), - "SELECT $a AS a, 'xxx' AS \"_sqlpage_f0_a0\", x = $b AS \"_sqlpage_f0_a1\", $c AS c FROM t" - ); - assert_eq!( - delayed_functions, - &[DelayedFunctionCall { - function: SqlPageFunctionName::exec, - argument_col_names: vec![ - "_sqlpage_f0_a0".to_string(), - "_sqlpage_f0_a1".to_string() - ], - target_col_name: "b".to_string() - }] - ); + fn coalesce(arguments: [SqlPageExpr; N]) -> SqlPageExpr { + SqlPageExpr::Coalesce(Box::new(arguments)) } - #[test] - fn test_sqlpage_function_with_argument() { - for &(dialect, _kind) in ALL_DIALECTS { - let sql = "select sqlpage.fetch($x)"; - let mut ast = parse_stmt(sql, dialect); - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - assert_eq!( - parameters, - [StmtParam::FunctionCall(SqlPageFunctionCall { - function: SqlPageFunctionName::fetch, - arguments: vec![StmtParam::PostOrGet("x".to_string())] - })], - "Failed for dialect {dialect:?}" - ); + fn concat(arguments: [SqlPageExpr; N]) -> SqlPageExpr { + SqlPageExpr::Concat { + arguments: Box::new(arguments), + null_behavior: ConcatNullBehavior::PropagateNull, } } + fn variable(name: &str) -> SqlPageExpr { + SqlPageExpr::Variable(VariableRef { + name: name.into(), + source: VariableSource::SetOrUrl, + }) + } + + fn row(index: usize) -> SqlPageExpr { + SqlPageExpr::Input(RowInputId::new(index)) + } + + fn text(value: &str) -> SqlPageExpr { + SqlPageExpr::Literal(serde_json::Value::String(value.into())) + } + #[test] - fn test_parse_sql_unsupported_expr_in_sqlpage_arg() { - let sql = "SELECT sqlpage.link('x', json_build_object('k', c)) FROM (SELECT 1 AS c) t"; - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&db_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::Error(err) = stmt else { - panic!("expected ParsedStatement::Error: {stmt:?}"); + fn database_only_parent_forces_binding() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select upper(sqlpage.url_encode('x'))") + else { + panic!("expected database query"); }; - let err_msg = format!("{err:#}"); - assert!( - err_msg.contains("Unsupported sqlpage function argument:"), - "{err_msg}" - ); - assert!(err_msg.contains("\"c\" is an sql expression, which cannot be passed as a nested sqlpage function argument."), "{err_msg}"); + assert_eq!(query.bindings.len(), 1); + assert!(query.computed_columns.is_empty()); + assert!(query.sql.contains("upper(CAST($1 AS TEXT))")); } #[test] - fn test_parse_sql_unemulated_function_in_sqlpage_arg() { - let sql = "SELECT sqlpage.link('x', upper('a')) FROM (SELECT 1) t"; - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&db_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::Error(err) = stmt else { - panic!("expected ParsedStatement::Error: {stmt:?}"); + fn emulated_parent_keeps_nested_call_per_row() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select coalesce(sqlpage.url_encode(value), '') as encoded from t") + else { + panic!("expected database query"); }; - let err_msg = format!("{err:#}"); - assert!( - err_msg.contains("Unsupported sqlpage function argument:"), - "{err_msg}" - ); - assert!( - err_msg.contains("\"upper\" is not a supported sqlpage function"), - "{err_msg}" - ); + assert!(query.bindings.is_empty()); + assert_eq!(query.row_input_json.len(), 1); + assert_eq!(query.computed_columns.len(), 1); + assert!(!query.sql.contains("sqlpage.")); } #[test] - fn test_set_variable_to_other_variable() { - let sql = "set x = $y"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - match parse_single_statement(&mut parser, &db_info, sql) { - Some(ParsedStatement::StaticSimpleSet { variable, value }) => { - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!( - value, - SimpleSelectValue::Dynamic(StmtParam::PostOrGet("y".to_string())) - ); - } - other => panic!("Failed for dialect {dialect:?}: {other:#?}"), - } - } + fn parentheses_keep_nested_call_per_row() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select (sqlpage.url_encode(value)) as encoded from t") + else { + panic!("expected database query"); + }; + assert!(query.bindings.is_empty()); + assert_eq!(query.row_input_json.len(), 1); + assert_eq!(query.computed_columns.len(), 1); + assert!(!query.sql.contains("sqlpage.")); } #[test] - fn is_own_placeholder() { - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$1") - ); - - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![StmtParam::Get("x".to_string())], - extract_error: None, - } - .is_own_placeholder("$2") - ); - - assert!( - !ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$2") - ); + fn row_value_cannot_cross_database_only_parent() { + let FileStatement::Error(error) = one("select upper(sqlpage.url_encode(value)) from t") + else { + panic!("expected rewrite error"); + }; + assert!(format!("{error:#}").contains("required before the query")); + } - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Sqlite), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("?1") + #[test] + fn positional_bindings_follow_source_order() { + let database = database(SupportedDatabase::MySql); + let mut statements = parse_sql( + &database, + &MySqlDialect {}, + "select $a, upper(sqlpage.url_encode($b))", + ) + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statements.next().unwrap() + else { + panic!("expected database query"); + }; + assert_eq!(query.bindings.len(), 2); + assert_eq!(query.sql.matches('?').count(), 2); + assert_eq!( + query.bindings.as_ref(), + [ + variable("a"), + call(SqlPageFunctionName::url_encode, [variable("b")]) + ] ); + } - assert!( - !ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Sqlite), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$1") + #[test] + fn positional_bindings_follow_cte_rendering_order() { + let database = database(SupportedDatabase::MySql); + let statement = parse_sql( + &database, + &MySqlDialect {}, + "with c as (select $a as x) select $b as y from c", + ) + .unwrap() + .next() + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statement + else { + panic!("expected database query"); + }; + assert_eq!( + query.sql, + "WITH c AS (SELECT CAST(? AS CHAR) AS x) SELECT CAST(? AS CHAR) AS y FROM c" ); + assert_eq!(query.bindings.as_ref(), [variable("a"), variable("b")]); } #[test] - fn test_mssql_statement_rewrite() { - let mut ast = parse_stmt( - "select '' || $1 from [a schema].[a table]", - &MsSqlDialect {}, - ); - let db_info = create_test_db_info(SupportedDatabase::Mssql); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); + fn positional_bindings_follow_rendered_projection_order() { + let database = database(SupportedDatabase::MySql); + let statement = parse_sql( + &database, + &MySqlDialect {}, + "select $a as a, sqlpage.url_encode(upper(col || sqlpage.url_encode($b))) as b, $c as c from t", + ) + .unwrap() + .next() + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statement + else { + panic!("expected database query"); + }; assert_eq!( - ast.to_string(), - "SELECT CONCAT('', CAST(@p1 AS VARCHAR(MAX))) FROM [a schema].[a table]" + query.bindings.as_ref(), + [ + variable("a"), + variable("c"), + call(SqlPageFunctionName::url_encode, [variable("b")]), + ] ); - assert_eq!(parameters, [StmtParam::PostOrGet("1".to_string()),]); } #[test] - fn test_static_extract() { - use SimpleSelectValue::Static; + fn database_cannot_order_by_computed_column() { + let FileStatement::Error(error) = + one("select sqlpage.url_encode(value) as encoded from t order by encoded") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("ORDER BY")); + } - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt( - "select 'hello' as hello, 42 as answer, null as nothing, 'world' as hello" - ), - &[] - ), - Some(vec![ - ("hello".into(), Static("hello".into())), - ("answer".into(), Static(42.into())), - ("nothing".into(), Static(().into())), - ("hello".into(), Static("world".into())), - ]) - ); + #[test] + fn database_cannot_order_by_ordinal_with_computed_columns() { + let FileStatement::Error(error) = + one("select a, sqlpage.url_encode(b) as encoded, c from t order by 3") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("ORDER BY")); } #[test] - fn test_simple_select_with_sqlpage_pseudofunction() { - let sql = "select 'text' as component, $x as contents, $y as title"; - let dialects: &[&dyn Dialect] = &[ - &PostgreSqlDialect {}, - &SQLiteDialect {}, - &MySqlDialect {}, - &MsSqlDialect {}, - ]; - for &dialect in dialects { - use SimpleSelectValue::{Dynamic, Static}; - use StmtParam::PostOrGet; - use std::any::Any; + fn database_cannot_group_by_computed_column() { + let FileStatement::Error(error) = one( + "select coalesce(sqlpage.url_encode(name), '') as enc, count(*) from users group by enc", + ) else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("GROUP BY")); + } - let db_info = if dialect.type_id() == (PostgreSqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Postgres) - } else if dialect.type_id() == (SQLiteDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Sqlite) - } else if dialect.type_id() == (MySqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::MySql) - } else if dialect.type_id() == (MsSqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Mssql) - } else { - create_test_db_info(SupportedDatabase::Generic) - }; - let parsed: Vec = parse_sql(&db_info, dialect, sql).unwrap().collect(); - match &parsed[..] { - [ParsedStatement::StaticSimpleSelect { values: q, .. }] => assert_eq!( - q, - &[ - ("component".into(), Static("text".into())), - ("contents".into(), Dynamic(PostOrGet("x".into()))), - ("title".into(), Dynamic(PostOrGet("y".into()))), - ] - ), - other => panic!("failed to extract simple select in {dialect:?}: {other:?}"), - } - } + #[test] + fn database_cannot_group_by_ordinal_with_computed_columns() { + let FileStatement::Error(error) = + one("select a, sqlpage.url_encode(b) as encoded from t group by 1") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("GROUP BY")); } #[test] - fn test_simple_select_only_extraction() { - use SimpleSelectValue::{Dynamic, Static}; - use StmtParam::PostOrGet; - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'text' as component, $1 as contents"), - &[PostOrGet("cook".into())] - ), - Some(vec![ - ("component".into(), Static("text".into())), - ("contents".into(), Dynamic(PostOrGet("cook".into()))), - ]) - ); + fn database_cannot_filter_by_computed_column_in_having() { + let FileStatement::Error(error) = one( + "select sqlpage.url_encode(name) as enc, count(*) from users group by name having enc <> ''", + ) else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("HAVING")); } #[test] - fn test_extract_set_variable() { - let sql = "set x = CURRENT_TIMESTAMP"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - if let Some(ParsedStatement::SetVariable { - variable, - value: StmtWithParams { query, params, .. }, - }) = stmt - { - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!(query, "SELECT CURRENT_TIMESTAMP AS sqlpage_set_expr"); - assert!(params.is_empty()); - } else { - panic!("Failed for dialect {dialect:?}: {stmt:#?}"); - } - } + fn database_cannot_filter_by_computed_column_in_where() { + let FileStatement::Error(error) = + one("select sqlpage.url_encode(name) as enc from users where enc <> ''") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("WHERE")); } #[test] - fn test_extract_set_variable_static() { - let sql = "set x = 'hello'"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - match parse_single_statement(&mut parser, &db_info, sql) { - Some(ParsedStatement::StaticSimpleSet { - variable: StmtParam::PostOrGet(var_name), - value: SimpleSelectValue::Static(value), - }) => { - assert_eq!(var_name, "x"); - assert_eq!(value, "hello"); - } - other => panic!("Failed for dialect {dialect:?}: {other:#?}"), - } - } + fn database_cannot_group_by_computed_column_in_expression() { + let FileStatement::Error(error) = + one("select sqlpage.url_encode(name) as enc, count(*) from users group by lower(enc)") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("GROUP BY")); } #[test] - fn test_static_extract_doesnt_match() { - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer limit 0"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer order by 1"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer offset 1"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer where 1 = 0"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer FROM t"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select x'CAFEBABE' as hello, 42 as answer"), - &[] - ), - None - ); + fn placeholder_like_literal_is_not_rewritten() { + let database = database(SupportedDatabase::MySql); + let statement = parse_sql( + &database, + &MySqlDialect {}, + "select '@SQLPAGE_TEMP1' as value from t where id = $id", + ) + .unwrap() + .next() + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statement + else { + panic!("expected database query"); + }; + assert!(query.sql.contains("'@SQLPAGE_TEMP1'")); + assert_eq!(query.bindings.len(), 1); } #[test] - fn test_extract_json_columns() { - let sql = r" - WITH json_cte AS ( - SELECT json_build_object('a', x, 'b', y) AS cte_json - FROM generate_series(1, 3) x - JOIN generate_series(4, 6) y ON true - ) - SELECT - json_object('key', 'value') AS json_col1, - json_array(1, 2, 3) AS json_col2, - (SELECT json_build_object('nested', subq.val) - FROM (SELECT AVG(x) AS val FROM generate_series(1, 5) x) subq - ) AS json_col3, -- not supported because of the subquery - CASE - WHEN EXISTS (SELECT 1 FROM json_cte WHERE cte_json->>'a' = '2') - THEN to_json(ARRAY(SELECT cte_json FROM json_cte)) - ELSE json_build_array() - END AS json_col4, -- not supported because of the CASE - json_unknown_fn(regular_column) AS non_json_col, - CAST(json_col1 AS json) AS json_col6 - FROM some_table - CROSS JOIN json_cte - WHERE json_typeof(json_col1) = 'object' - "; + fn effectful_bindings_are_not_deduplicated() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select 1 as value where sqlpage.random_string(1) <> sqlpage.random_string(1)") + else { + panic!("expected database query"); + }; + assert_eq!(query.bindings.len(), 2); + } - let stmt = parse_postgres_stmt(sql); - let json_columns = extract_json_columns(&stmt, SupportedDatabase::Sqlite); + #[test] + fn nested_run_sql_requires_buffering() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select coalesce(sqlpage.run_sql(path), '') from files") + else { + panic!("expected database query"); + }; + assert!(query.must_buffer_rows()); + } - assert_eq!( - json_columns, - vec![ - "json_col1".to_string(), - "json_col2".to_string(), - "json_col6".to_string() - ] - ); + #[test] + fn standalone_projection_has_no_database_query() { + let FileStatement::Query(Query { + body: QueryBody::SingleRow(query), + .. + }) = one("select sqlpage.url_encode('a b') as value") + else { + panic!("expected a single SQLPage-owned row"); + }; + assert_eq!(query.columns.len(), 1); } #[test] - fn test_set_variable_with_sqlpage_function() { - let sql = "set x = sqlpage.url_encode(some_db_function())"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - let Some(ParsedStatement::SetVariable { - variable, - value: - StmtWithParams { - query, - params, - delayed_functions, - json_columns, - .. - }, - }) = stmt + fn concat_operator_uses_backend_null_behavior_in_sqlpage_expressions() { + for database_type in [SupportedDatabase::Oracle, SupportedDatabase::Mssql] { + let FileStatement::Query(Query { + body: QueryBody::SingleRow(query), + .. + }) = one_for(database_type, "select '/' || null as path") else { - panic!("for dialect {dialect:?}: {stmt:#?} instead of SetVariable"); + panic!("expected a single SQLPage-owned row"); + }; + assert!(matches!( + &query.columns[0].value, + SqlPageExpr::Concat { + null_behavior: ConcatNullBehavior::IgnoreNull, + .. + } + )); + + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one_for( + database_type, + "select sqlpage.url_encode('/' || nullable_col) as path from input_rows", + ) + else { + panic!("expected a database query"); + }; + let SqlPageExpr::Call { arguments, .. } = &query.computed_columns[0].value else { + panic!("expected a SQLPage function call"); }; - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!( - delayed_functions, - [DelayedFunctionCall { - function: SqlPageFunctionName::url_encode, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "sqlpage_set_expr".to_string() - }] - ); - assert_eq!(query, "SELECT some_db_function() AS \"_sqlpage_f0_a0\""); - assert_eq!(params, []); - assert_eq!(json_columns, Vec::::new()); + assert!(matches!( + &arguments[0], + SqlPageExpr::Concat { + null_behavior: ConcatNullBehavior::IgnoreNull, + .. + } + )); } } #[test] - fn test_extract_json_columns_from_literal() { - let sql = r#" - SELECT - 'Pro Plan' as title, - JSON('{"icon":"database","color":"blue","description":"1GB Database"}') as item, - JSON('{"icon":"headset","color":"green","description":"Priority Support"}') as item - "#; - - let stmt = parse_stmt(sql, &SQLiteDialect {}); - let json_columns = extract_json_columns(&stmt, SupportedDatabase::Sqlite); - - assert!(json_columns.contains(&"item".to_string())); - assert!(!json_columns.contains(&"title".to_string())); + fn unquoted_sqlpage_names_are_case_insensitive() { + let FileStatement::Query(Query { + body: QueryBody::SingleRow(query), + .. + }) = one("select SQLPAGE.URL_ENCODE('a b') as value") + else { + panic!("expected a single SQLPage-owned row"); + }; + assert_eq!(query.columns.len(), 1); } #[test] - fn test_positional_placeholders() { - let sql = "select \ - @SQLPAGE_TEMP10 as a1, \ - @SQLPAGE_TEMP9 as a2, \ - @SQLPAGE_TEMP8 as a3, \ - @SQLPAGE_TEMP7 as a4, \ - @SQLPAGE_TEMP6 as a5, \ - @SQLPAGE_TEMP5 as a6, \ - @SQLPAGE_TEMP4 as a7, \ - @SQLPAGE_TEMP3 as a8, \ - @SQLPAGE_TEMP2 as a9, \ - @SQLPAGE_TEMP1 as a10 \ - @SQLPAGE_TEMP10 as a1bis \ - from t"; - let mut stmt = StmtWithParams { - query: sql.to_string(), - query_position: SourceSpan { - start: SourceLocation { line: 1, column: 1 }, - end: SourceLocation { line: 1, column: 1 }, - }, - params: vec![ - StmtParam::PostOrGet("x1".to_string()), - StmtParam::PostOrGet("x2".to_string()), - StmtParam::PostOrGet("x3".to_string()), - StmtParam::PostOrGet("x4".to_string()), - StmtParam::PostOrGet("x5".to_string()), - StmtParam::PostOrGet("x6".to_string()), - StmtParam::PostOrGet("x7".to_string()), - StmtParam::PostOrGet("x8".to_string()), - StmtParam::PostOrGet("x9".to_string()), - StmtParam::PostOrGet("x10".to_string()), - ], - delayed_functions: vec![], - json_columns: vec![], - }; - transform_to_positional_placeholders(&mut stmt, AnyKind::MySql); + fn mixed_database_and_row_boundaries_are_rewritten_together() { assert_eq!( - stmt.query, - "select \ - ? as a1, \ - ? as a2, \ - ? as a3, \ - ? as a4, \ - ? as a5, \ - ? as a6, \ - ? as a7, \ - ? as a8, \ - ? as a9, \ - ? as a10 \ - ? as a1bis \ - from t" + rewrite_database( + "select coalesce(upper(sqlpage.url_encode($prefix)), sqlpage.url_encode(value)) as result from t" + ), + DatabaseQuery { + sql: "SELECT value AS \"__sqlpage_input_0\", upper(CAST($1 AS TEXT)) AS \"__sqlpage_input_1\" FROM t".into(), + bindings: Box::new([call(SqlPageFunctionName::url_encode, [variable("prefix")])]), + row_input_json: Box::new([false, false]), + computed_columns: Box::new([OutputColumn { + name: "result".into(), + value: coalesce([ + row(1), + call(SqlPageFunctionName::url_encode, [row(0)]), + ]), + }]), + json_columns: Box::new([]), + } ); + } + + #[test] + fn predicate_call_is_standalone_while_projection_call_is_per_row() { assert_eq!( - stmt.params, - vec![ - StmtParam::PostOrGet("x10".to_string()), - StmtParam::PostOrGet("x9".to_string()), - StmtParam::PostOrGet("x8".to_string()), - StmtParam::PostOrGet("x7".to_string()), - StmtParam::PostOrGet("x6".to_string()), - StmtParam::PostOrGet("x5".to_string()), - StmtParam::PostOrGet("x4".to_string()), - StmtParam::PostOrGet("x3".to_string()), - StmtParam::PostOrGet("x2".to_string()), - StmtParam::PostOrGet("x1".to_string()), - StmtParam::PostOrGet("x10".to_string()), - ] + rewrite_database( + "select sqlpage.url_encode(value) as encoded from t where sqlpage.url_encode($expected) = 'x'" + ), + DatabaseQuery { + sql: "SELECT value AS \"__sqlpage_input_0\" FROM t WHERE CAST($1 AS TEXT) = 'x'" + .into(), + bindings: Box::new([call( + SqlPageFunctionName::url_encode, + [variable("expected")] + )]), + row_input_json: Box::new([false]), + computed_columns: Box::new([OutputColumn { + name: "encoded".into(), + value: call(SqlPageFunctionName::url_encode, [row(0)]), + }]), + json_columns: Box::new([]), + } ); } #[test] - fn test_set_variable_error_handling() { - let sql = "set x = db_function(sqlpage.fetch(other_db_function()))"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - if let Some(ParsedStatement::Error(err)) = stmt { - assert!( - err.to_string() - .contains("Unsupported sqlpage function argument:"), - "Expected error for invalid function, got: {err}" - ); - } else { - panic!("Expected error for invalid function, got: {stmt:#?}"); + fn request_and_row_values_share_one_per_row_expression() { + assert_eq!( + rewrite_database( + "select coalesce(sqlpage.url_encode($prefix || value), '') as encoded from t" + ), + DatabaseQuery { + sql: "SELECT value AS \"__sqlpage_input_0\" FROM t".into(), + bindings: Box::new([]), + row_input_json: Box::new([false]), + computed_columns: Box::new([OutputColumn { + name: "encoded".into(), + value: coalesce([ + call( + SqlPageFunctionName::url_encode, + [concat([variable("prefix"), row(0)])], + ), + text(""), + ]), + }]), + json_columns: Box::new([]), } - } + ); } } diff --git a/src/webserver/database/sql/dialect.rs b/src/webserver/database/sql/dialect.rs new file mode 100644 index 00000000..edc676f1 --- /dev/null +++ b/src/webserver/database/sql/dialect.rs @@ -0,0 +1,40 @@ +//! Database-specific parser dialects and bind-placeholder syntax. + +use sqlparser::dialect::{ + Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, OracleDialect, + PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, +}; +use sqlx::any::AnyKind; + +use crate::webserver::database::SupportedDatabase; + +/// Native bind-placeholder forms supported by `sqlx` database backends. +#[derive(Clone, Copy)] +pub(super) enum PlaceholderStyle { + Numbered { prefix: &'static str }, + Positional { token: &'static str }, +} + +/// Returns the `sqlparser` dialect matching the configured database. +pub(super) fn parser_dialect(database: SupportedDatabase) -> Box { + match database { + SupportedDatabase::Duckdb => Box::new(DuckDbDialect {}), + SupportedDatabase::Oracle => Box::new(OracleDialect {}), + SupportedDatabase::Postgres => Box::new(PostgreSqlDialect {}), + SupportedDatabase::Generic => Box::new(GenericDialect {}), + SupportedDatabase::Mssql => Box::new(MsSqlDialect {}), + SupportedDatabase::MySql => Box::new(MySqlDialect {}), + SupportedDatabase::Sqlite => Box::new(SQLiteDialect {}), + SupportedDatabase::Snowflake => Box::new(SnowflakeDialect {}), + } +} + +/// Returns the native placeholder syntax emitted into rewritten SQL. +pub(super) fn placeholder_style(kind: AnyKind) -> PlaceholderStyle { + match kind { + AnyKind::Sqlite => PlaceholderStyle::Numbered { prefix: "?" }, + AnyKind::Postgres => PlaceholderStyle::Numbered { prefix: "$" }, + AnyKind::Mssql => PlaceholderStyle::Numbered { prefix: "@p" }, + AnyKind::MySql | AnyKind::Odbc => PlaceholderStyle::Positional { token: "?" }, + } +} diff --git a/src/webserver/database/sql/parameter_extraction.rs b/src/webserver/database/sql/parameter_extraction.rs deleted file mode 100644 index 57bb08c4..00000000 --- a/src/webserver/database/sql/parameter_extraction.rs +++ /dev/null @@ -1,591 +0,0 @@ -use super::super::{DbInfo, SupportedDatabase}; -use super::{is_sqlpage_func, sqlpage_func_name}; -use crate::webserver::database::sqlpage_functions::func_call_to_param; -use crate::webserver::database::syntax_tree::StmtParam; -use sqlparser::ast::{ - BinaryOperator, CastKind, CharacterLength, DataType, Expr, Function, FunctionArg, - FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, - Spanned, Statement, Value, ValueWithSpan, Visit, VisitMut, Visitor, VisitorMut, -}; -use sqlx::any::AnyKind; -use std::ops::ControlFlow; - -pub(super) struct ParameterExtractor { - pub(super) db_info: DbInfo, - pub(super) parameters: Vec, - pub(super) extract_error: Option, -} - -#[derive(Debug)] -pub(crate) enum DbPlaceHolder { - PrefixedNumber { prefix: &'static str }, - Positional { placeholder: &'static str }, -} - -pub(crate) const DB_PLACEHOLDERS: [(AnyKind, DbPlaceHolder); 5] = [ - ( - AnyKind::Sqlite, - DbPlaceHolder::PrefixedNumber { prefix: "?" }, - ), - ( - AnyKind::Postgres, - DbPlaceHolder::PrefixedNumber { prefix: "$" }, - ), - ( - AnyKind::MySql, - DbPlaceHolder::Positional { placeholder: "?" }, - ), - ( - AnyKind::Mssql, - DbPlaceHolder::PrefixedNumber { prefix: "@p" }, - ), - ( - AnyKind::Odbc, - DbPlaceHolder::Positional { placeholder: "?" }, - ), -]; - -/// For positional parameters, we use a temporary placeholder during parameter extraction, -/// And then replace it with the actual placeholder during statement rewriting. -pub(crate) const TEMP_PLACEHOLDER_PREFIX: &str = "@SQLPAGE_TEMP"; - -fn get_placeholder_prefix(kind: AnyKind) -> &'static str { - if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = DB_PLACEHOLDERS - .iter() - .find(|(placeholder_kind, _prefix)| *placeholder_kind == kind) - { - prefix - } else { - TEMP_PLACEHOLDER_PREFIX - } -} - -impl ParameterExtractor { - pub(super) fn extract_parameters( - sql_ast: &mut Statement, - db_info: DbInfo, - ) -> anyhow::Result> { - let mut this = Self { - db_info, - parameters: vec![], - extract_error: None, - }; - let _ = sql_ast.visit(&mut this); - if let Some(e) = this.extract_error { - return Err(e); - } - Ok(this.parameters) - } - - fn replace_with_placeholder(&mut self, value: &mut Expr, param: StmtParam) { - let placeholder = - if let Some(existing_idx) = self.parameters.iter().position(|p| *p == param) { - // Parameter already exists, use its index - self.make_placeholder_for_index(existing_idx + 1) - } else { - // New parameter, add it to the list - let placeholder = self.make_placeholder(); - log::trace!("Replacing {param} with {placeholder}"); - self.parameters.push(param); - placeholder - }; - *value = placeholder; - } - - fn make_placeholder_for_index(&self, index: usize) -> Expr { - let name = make_tmp_placeholder(self.db_info.kind, index); - let data_type = match self.db_info.database_type { - SupportedDatabase::MySql => DataType::Char(None), - SupportedDatabase::Mssql => DataType::Varchar(Some(CharacterLength::Max)), - SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, - SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { - length: 4000, - unit: None, - })), - _ => DataType::Varchar(None), - }; - let value = Expr::value(Value::Placeholder(name)); - Expr::Cast { - expr: Box::new(value), - data_type, - format: None, - kind: CastKind::Cast, - array: false, - } - } - - fn make_placeholder(&self) -> Expr { - self.make_placeholder_for_index(self.parameters.len() + 1) - } - - pub(super) fn is_own_placeholder(&self, param: &str) -> bool { - let prefix = get_placeholder_prefix(self.db_info.kind); - if let Some(param) = param.strip_prefix(prefix) - && let Ok(index) = param.parse::() - { - return index <= self.parameters.len() + 1; - } - false - } -} - -struct InvalidFunctionFinder; -impl Visitor for InvalidFunctionFinder { - type Break = (String, Vec); - fn pre_visit_expr(&mut self, value: &Expr) -> ControlFlow { - match value { - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if is_sqlpage_func(func_name_parts) => { - let func_name = sqlpage_func_name(func_name_parts); - let arguments = args.clone(); - return ControlFlow::Break((func_name.to_string(), arguments)); - } - _ => (), - } - ControlFlow::Continue(()) - } -} - -pub(super) fn validate_function_calls(stmt: &Statement) -> anyhow::Result<()> { - let mut finder = InvalidFunctionFinder; - if let ControlFlow::Break((func_name, mut args)) = stmt.visit(&mut finder) { - let ctx = ParamExtractContext { - parent_func: Some(func_name.clone()), - }; - function_args_to_stmt_params(&mut args, &ctx)?; - - let args_str = FormatArguments(&args); - let error_msg = format!( - "Invalid SQLPage function call: sqlpage.{func_name}({args_str})\n\n\ - Arbitrary SQL expressions as function arguments are not supported.\n\n\ - SQLPage functions can either:\n\ - 1. Run BEFORE the query (to provide input values)\n\ - 2. Run AFTER the query (to process the results)\n\ - But they can't run DURING the query - the database doesn't know how to call them!\n\n\ - To fix this, you can either:\n\ - 1. Store the function argument in a variable first:\n\ - SET {func_name}_arg = ...;\n\ - SET {func_name}_result = sqlpage.{func_name}(${func_name}_arg);\n\ - SELECT * FROM example WHERE xxx = ${func_name}_result;\n\n\ - 2. Or move the function to the top level to process results:\n\ - SELECT sqlpage.{func_name}(...) FROM example;" - ); - Err(anyhow::anyhow!(error_msg)) - } else { - Ok(()) - } -} - -/** This is a helper struct to format a list of arguments for an error message. */ -struct FormatArguments<'a>(&'a [FunctionArg]); -impl std::fmt::Display for FormatArguments<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut args = self.0.iter(); - if let Some(arg) = args.next() { - write!(f, "{arg}")?; - } - for arg in args { - write!(f, ", {arg}")?; - } - Ok(()) - } -} - -#[derive(Clone, Default)] -pub(crate) struct ParamExtractContext { - pub parent_func: Option, -} - -impl ParamExtractContext { - fn with_parent(parent: &str) -> Self { - Self { - parent_func: Some(parent.to_string()), - } - } - - fn build_error(&self, e: &ExprToParamError, arguments: &[FunctionArg]) -> SqlPageFunctionError { - let line = e.line.unwrap_or(0); - let func_name = self.parent_func.as_deref().unwrap_or("unknown").to_string(); - let arguments_str = FormatArguments(arguments).to_string(); - - let reason = match &e.kind { - ExprToParamErrorKind::UnsupportedExpr { summary } => { - format!( - "\"{summary}\" is an sql expression, which cannot be passed as a nested sqlpage function argument." - ) - } - ExprToParamErrorKind::UnemulatedFunction { name } => { - format!( - "\"{name}\" is not a supported sqlpage function. Only a few basic sql functions like concat or json_object can be used inside sqlpage functions." - ) - } - ExprToParamErrorKind::NamedArgs => "Named function arguments are not supported.\n\ - Please use positional arguments only." - .to_string(), - }; - - SqlPageFunctionError { - line, - func_name, - arguments_str, - reason, - } - } -} - -#[derive(Debug)] -pub(crate) struct SqlPageFunctionError { - pub line: u64, - pub func_name: String, - pub arguments_str: String, - pub reason: String, -} - -impl std::fmt::Display for SqlPageFunctionError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Unsupported sqlpage function argument:\n\ -sqlpage.{func}({args_str})\n\n\ -{reason}\n\n\ -SQLPage functions can either:\n\ -1. Run BEFORE the query (to provide input values)\n\ -2. Run AFTER the query (to process the results)\n\ -But they can't run DURING the query - the database doesn't know how to call them!\n\n\ -To fix this, you can either:\n\ -1. Store the function argument in a variable first:\n\ -SET {func}_arg = ...;\n\ -SET {func}_result = sqlpage.{func}(${func}_arg);\n\ -SELECT * FROM example WHERE xxx = ${func}_result;\n\n\ -2. Or move the function to the top level to process results:\n\ -SELECT sqlpage.{func}(...) FROM example;", - func = self.func_name, - args_str = self.arguments_str, - reason = self.reason - ) - } -} -impl std::error::Error for SqlPageFunctionError {} - -#[derive(Debug)] -struct ExprToParamError { - line: Option, - kind: ExprToParamErrorKind, -} - -#[derive(Debug)] -enum ExprToParamErrorKind { - UnsupportedExpr { summary: String }, - UnemulatedFunction { name: String }, - NamedArgs, -} - -fn expr_summary(expr: &Expr) -> String { - match expr { - Expr::CompoundIdentifier(idents) => { - let s = idents - .iter() - .map(|i| i.value.as_str()) - .collect::>() - .join("."); - format!("column/table reference '{s}'") - } - _ => format!("{expr}"), - } -} - -fn function_arg_to_stmt_param( - arg: &mut FunctionArg, - ctx: &ParamExtractContext, -) -> Result { - let expr = function_arg_expr(arg).ok_or(ExprToParamError { - line: None, - kind: ExprToParamErrorKind::NamedArgs, - })?; - expr_to_stmt_param(expr, ctx) -} - -pub(crate) fn function_args_to_stmt_params( - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, -) -> anyhow::Result> { - let mut params = Vec::with_capacity(arguments.len()); - // We iterate manually so we can pass the entire `arguments` slice to into_error on failure - for arg in arguments.iter_mut() { - match function_arg_to_stmt_param(arg, ctx) { - Ok(p) => params.push(p), - Err(e) => { - let func_err = ctx.build_error(&e, arguments); - return Err(anyhow::Error::new(func_err)); - } - } - } - Ok(params) -} - -fn emulated_func_args_to_param( - func_name: &str, - args: &mut [FunctionArg], - line: u64, -) -> Result { - let inner = ParamExtractContext::with_parent(func_name); - if func_name.eq_ignore_ascii_case("concat") { - let mut concat_args = Vec::with_capacity(args.len()); - for a in args { - concat_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::Concat(concat_args)) - } else if func_name.eq_ignore_ascii_case("json_object") - || func_name.eq_ignore_ascii_case("jsonb_object") - || func_name.eq_ignore_ascii_case("json_build_object") - || func_name.eq_ignore_ascii_case("jsonb_build_object") - { - let mut json_obj_args = Vec::with_capacity(args.len()); - for a in args { - json_obj_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::JsonObject(json_obj_args)) - } else if func_name.eq_ignore_ascii_case("json_array") - || func_name.eq_ignore_ascii_case("jsonb_array") - || func_name.eq_ignore_ascii_case("json_build_array") - || func_name.eq_ignore_ascii_case("jsonb_build_array") - { - let mut json_obj_args = Vec::with_capacity(args.len()); - for a in args { - json_obj_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::JsonArray(json_obj_args)) - } else if func_name.eq_ignore_ascii_case("coalesce") { - let mut coalesce_args = Vec::with_capacity(args.len()); - for a in args { - coalesce_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::Coalesce(coalesce_args)) - } else { - Err(ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnemulatedFunction { - name: func_name.to_string(), - }, - }) - } -} - -fn expr_to_stmt_param( - arg: &mut Expr, - ctx: &ParamExtractContext, -) -> Result { - let line = arg.span().start.line; - match arg { - Expr::Value(ValueWithSpan { - value: Value::Placeholder(placeholder), - .. - }) => Ok(map_param(std::mem::take(placeholder))), - Expr::Identifier(ident) => extract_ident_param(ident).ok_or_else(|| ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnsupportedExpr { - summary: expr_summary(arg), - }, - }), - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if is_sqlpage_func(func_name_parts) => Ok(func_call_to_param( - sqlpage_func_name(func_name_parts), - args.as_mut_slice(), - ctx, - )), - Expr::Value(ValueWithSpan { - value: Value::SingleQuotedString(param_value), - .. - }) => Ok(StmtParam::Literal(std::mem::take(param_value))), - Expr::Value(ValueWithSpan { - value: Value::Number(param_value, _is_long), - .. - }) => Ok(StmtParam::Literal(param_value.clone())), - Expr::Value(ValueWithSpan { - value: Value::Null, .. - }) => Ok(StmtParam::Null), - Expr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => { - let left = expr_to_stmt_param(left, ctx)?; - let right = expr_to_stmt_param(right, ctx)?; - Ok(StmtParam::Concat(vec![left, right])) - } - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if func_name_parts.len() == 1 => { - let func_name = func_name_parts[0] - .as_ident() - .map(|ident| ident.value.as_str()) - .unwrap_or_default(); - emulated_func_args_to_param(func_name, args.as_mut_slice(), line) - } - _ => Err(ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnsupportedExpr { - summary: expr_summary(arg), - }, - }), - } -} - -fn function_arg_expr(arg: &mut FunctionArg) -> Option<&mut Expr> { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Some(expr), - _ => None, - } -} - -#[inline] -#[must_use] -pub(super) fn make_tmp_placeholder(kind: AnyKind, arg_number: usize) -> String { - let prefix = if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = - DB_PLACEHOLDERS.iter().find(|(db_typ, _)| *db_typ == kind) - { - prefix - } else { - TEMP_PLACEHOLDER_PREFIX - }; - format!("{prefix}{arg_number}") -} - -pub(super) fn extract_ident_param(Ident { value, .. }: &mut Ident) -> Option { - if value.starts_with('$') || value.starts_with(':') { - let name = std::mem::take(value); - Some(map_param(name)) - } else { - None - } -} - -fn map_param(mut name: String) -> StmtParam { - if name.is_empty() { - return StmtParam::PostOrGet(name); - } - let prefix = name.remove(0); - match prefix { - '$' => StmtParam::PostOrGet(name), - ':' => StmtParam::Post(name), - _ => StmtParam::Get(name), - } -} - -impl VisitorMut for ParameterExtractor { - type Break = (); - fn pre_visit_expr(&mut self, value: &mut Expr) -> ControlFlow { - match value { - Expr::Identifier(ident) => { - if let Some(param) = extract_ident_param(ident) { - self.replace_with_placeholder(value, param); - } - } - Expr::Value(ValueWithSpan { - value: Value::Placeholder(param), - .. - }) if !self.is_own_placeholder(param) => - // this check is to avoid recursively replacing placeholders in the form of '?', or '$1', '$2', which we emit ourselves - { - let name = std::mem::take(param); - self.replace_with_placeholder(value, map_param(name)); - } - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - filter: None, - null_treatment: None, - over: None, - .. - }) if is_sqlpage_func(func_name_parts) => { - let func_name = sqlpage_func_name(func_name_parts); - log::trace!("Handling builtin function: {func_name}"); - let arguments = std::mem::take(args); - let ctx = ParamExtractContext { - parent_func: Some(func_name.to_string()), - }; - let mut arguments_clone = arguments.clone(); - let param = func_call_to_param(func_name, &mut arguments_clone, &ctx); - if let StmtParam::Error(msg) = ¶m { - log::trace!("Skipping extraction of {func_name} due to: {msg}"); - *args = arguments; - return ControlFlow::Continue(()); - } - self.replace_with_placeholder(value, param); - } - // Replace 'str1' || 'str2' with CONCAT('str1', 'str2') for MSSQL - Expr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } if self.db_info.database_type == SupportedDatabase::Mssql => { - let left = std::mem::replace(left.as_mut(), Expr::value(Value::Null)); - let right = std::mem::replace(right.as_mut(), Expr::value(Value::Null)); - *value = Expr::Function(Function { - name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new("CONCAT"))]), - args: FunctionArguments::List(FunctionArgumentList { - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(left)), - FunctionArg::Unnamed(FunctionArgExpr::Expr(right)), - ], - duplicate_treatment: None, - clauses: Vec::new(), - }), - parameters: FunctionArguments::None, - over: None, - filter: None, - null_treatment: None, - within_group: Vec::new(), - uses_odbc_syntax: false, - }); - } - Expr::Cast { - kind: kind @ CastKind::DoubleColon, - .. - } if ![ - SupportedDatabase::Postgres, - SupportedDatabase::Duckdb, - SupportedDatabase::Snowflake, - SupportedDatabase::Generic, - ] - .contains(&self.db_info.database_type) => - { - log::warn!( - "Casting with '::' is not supported on your database. \ - For backwards compatibility with older SQLPage versions, we will transform it to CAST(... AS ...)." - ); - *kind = CastKind::Cast; - } - _ => (), - } - ControlFlow::<()>::Continue(()) - } -} diff --git a/src/webserver/database/sql/rewrite.rs b/src/webserver/database/sql/rewrite.rs new file mode 100644 index 00000000..9ff49ea7 --- /dev/null +++ b/src/webserver/database/sql/rewrite.rs @@ -0,0 +1,1092 @@ +//! Compiler pass that partitions parsed SQL between the database and `SQLPage`. +//! +//! `SQLPage` files contain expressions owned by two runtimes: the configured +//! database handles ordinary SQL, while variables and `sqlpage.*` functions are +//! evaluated by `SQLPage`. This module makes that boundary explicit. Values +//! needed before a database statement become typed bindings; computed +//! projections run once per returned row, with any database-owned dependencies +//! appended to the SQL projection as a private trailing column suffix. Queries +//! that require no database work become single-row `SQLPage` plans instead. The +//! distinct standalone and row expression types prevent pre-query work from +//! depending on a row that does not exist yet. +//! +//! Rewriting also renders backend-appropriate placeholders and casts, preserves +//! selected DBMS semantics for operations `SQLPage` emulates, records which +//! returned values need JSON decoding, and rejects grouping, ordering, or other +//! relational clauses that would incorrectly depend on post-database computed +//! columns. Its output is the immutable query representation consumed by +//! `execute_queries`; no request values are resolved and no functions are +//! executed here. + +use std::ops::ControlFlow; +use std::str::FromStr as _; + +use anyhow::{Context as _, anyhow}; +use serde_json::Value as JsonValue; +use sqlparser::ast::{ + BinaryOperator, CastKind, CharacterLength, DataType, Expr as SqlExpr, Function, FunctionArg, + FunctionArgExpr, FunctionArgumentList, FunctionArguments, GroupByExpr, Ident, ObjectName, + ObjectNamePart, OrderByKind, SelectItem, SetExpr, Statement as SqlStatement, Value, + ValueWithSpan, VisitMut, VisitorMut, +}; +use sqlparser::tokenizer::Span; + +use super::dialect::{PlaceholderStyle, placeholder_style}; +use super::statement::{ + DatabaseQuery, OutputColumn, Query, QueryBody, SingleRowQuery, SourceLocation, SourceSpan, +}; +use super::{extract_json_columns, is_json_expression, is_sqlpage_func}; +use crate::webserver::database::sqlpage_expr::{ + NoRowInput, RowExpr, RowInputId, SqlPageExpr, StandaloneExpr, VariableRef, VariableSource, +}; +use crate::webserver::database::sqlpage_functions::functions::SqlPageFunctionName; +use crate::webserver::database::{DbInfo, SupportedDatabase}; + +const SQLPAGE_INPUT_PREFIX: &str = "__sqlpage_input_"; + +/// Mutable state used while rewriting one database query. +struct QueryRewriter<'a> { + database: &'a DbInfo, + bindings: Vec, + row_input_json: Vec, + private_projection: Vec, + error: Option, +} + +struct ComputedAliasFinder<'a> { + computed_columns: &'a [OutputColumn], +} + +impl sqlparser::ast::Visitor for ComputedAliasFinder<'_> { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { + let SqlExpr::Identifier(identifier) = expression else { + return ControlFlow::Continue(()); + }; + if self + .computed_columns + .iter() + .any(|column| identifier.value.eq_ignore_ascii_case(&column.name)) + { + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + } +} + +/// Result of partitioning one projected expression. +// Keeping the owned AST inline avoids one heap allocation for every ordinary +// projected expression. The enum is short-lived inside the rewriter. +#[allow(clippy::large_enum_variant)] +enum RewrittenProjection { + Database(SqlExpr), + PerRow(RowExpr), +} + +/// Defines how a SQL expression crossing into a SQLPage-owned expression is +/// represented at a particular evaluation site. +trait ExprEnvironment { + type Input; + + fn use_database_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result>; +} + +/// Rejects database-owned inputs because no returned row is available. +struct StandaloneEnvironment; +/// Projects database-owned inputs into the current returned row. +struct RowEnvironment; + +impl ExprEnvironment for StandaloneEnvironment { + type Input = NoRowInput; + + fn use_database_expr( + _rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result { + if let SqlExpr::Function(function) = &expression + && let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() + { + return Err(anyhow!( + "{} is not a supported sqlpage function and cannot be evaluated before the query", + name.value + )); + } + Err(anyhow!( + "{expression} is a database expression, but its value is required before the query can run" + )) + } +} + +impl ExprEnvironment for RowEnvironment { + type Input = RowInputId; + + fn use_database_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result { + let id = rewriter.add_row_input(expression)?; + Ok(SqlPageExpr::Input(id)) + } +} + +#[derive(Clone, Copy)] +/// SQL operations whose semantics are implemented by the shared `SQLPage` +/// expression evaluator. +enum EmulatedFunction { + Concat, + Coalesce, + JsonObject, + JsonArray, +} + +/// Rewrites one parsed statement into database SQL plus the `SQLPage` +/// expressions evaluated around it. +pub(super) fn rewrite_query( + mut statement: SqlStatement, + database: &DbInfo, + semicolon: bool, +) -> anyhow::Result { + let source_span = source_span(&statement); + let mut rewriter = QueryRewriter { + database, + bindings: Vec::new(), + row_input_json: Vec::new(), + private_projection: Vec::new(), + error: None, + }; + if let Some(single_row) = rewrite_single_row(&mut statement, &mut rewriter)? { + return Ok(Query { + body: QueryBody::SingleRow(single_row), + source_span, + }); + } + let computed_columns = rewrite_top_level_projection(&mut statement, &mut rewriter)?; + + let _ = statement.visit(&mut rewriter); + if let Some(error) = rewriter.error { + return Err(error); + } + + if let SqlStatement::Query(query) = &mut statement + && let SetExpr::Select(select) = query.body.as_mut() + && select.projection.is_empty() + { + select.projection.push(SelectItem::ExprWithAlias { + expr: SqlExpr::value(Value::Null), + alias: Ident::with_quote('"', format!("{SQLPAGE_INPUT_PREFIX}anchor")), + }); + rewriter.row_input_json.push(false); + } + + let json_columns = extract_json_columns(&statement, database.database_type) + .into_iter() + .filter(|name| !name.starts_with(SQLPAGE_INPUT_PREFIX)) + .collect(); + let bindings = rewriter.finish_bindings(&mut statement)?; + let sql = format!( + "{statement}{semicolon}", + semicolon = if semicolon { ";" } else { "" } + ); + + Ok(Query { + body: QueryBody::Database(DatabaseQuery { + sql, + bindings, + row_input_json: rewriter.row_input_json.into_boxed_slice(), + computed_columns: computed_columns.into_boxed_slice(), + json_columns, + }), + source_span, + }) +} + +/// Removes SQLPage-owned projection expressions from the database projection +/// and appends their private database inputs as a trailing suffix. +fn rewrite_top_level_projection( + statement: &mut SqlStatement, + rewriter: &mut QueryRewriter<'_>, +) -> anyhow::Result>> { + let SqlStatement::Query(query) = statement else { + return Ok(Vec::new()); + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(Vec::new()); + }; + if select.distinct.is_some() && select.projection.iter().any(select_item_contains_sqlpage) { + anyhow::bail!( + "SQLPage-computed projections cannot be used with SELECT DISTINCT because DISTINCT must be evaluated by the database" + ); + } + + let (mut database_projection, computed_columns) = + rewrite_projection_items(std::mem::take(&mut select.projection), rewriter)?; + database_projection.append(&mut rewriter.private_projection); + select.projection = database_projection; + + reject_computed_alias_references("WHERE", select.selection.as_ref(), &computed_columns)?; + reject_computed_group_by_references(&select.group_by, &computed_columns)?; + reject_computed_alias_references("HAVING", select.having.as_ref(), &computed_columns)?; + reject_computed_alias_references("QUALIFY", select.qualify.as_ref(), &computed_columns)?; + reject_computed_aliases_in_expressions("CLUSTER BY", &select.cluster_by, &computed_columns)?; + reject_computed_aliases_in_expressions( + "DISTRIBUTE BY", + &select.distribute_by, + &computed_columns, + )?; + reject_computed_ordering_references("SORT BY", &select.sort_by, &computed_columns)?; + reject_computed_order_by_references(query.order_by.as_ref(), &computed_columns)?; + Ok(computed_columns) +} + +fn rewrite_projection_items( + projection: Vec, + rewriter: &mut QueryRewriter<'_>, +) -> anyhow::Result<(Vec, Vec>)> { + let mut database_projection = Vec::with_capacity(projection.len()); + let mut computed_columns = Vec::new(); + for item in projection { + let (expression, alias) = match item { + SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias)), + SelectItem::UnnamedExpr(expr) => (expr, None), + item => { + database_projection.push(item); + continue; + } + }; + let name = alias + .as_ref() + .map_or_else(|| expression.to_string(), |alias| alias.value.clone()); + match rewriter.rewrite_projection(expression)? { + RewrittenProjection::Database(expression) => { + database_projection.push(match alias { + Some(alias) => SelectItem::ExprWithAlias { + expr: expression, + alias, + }, + None => SelectItem::UnnamedExpr(expression), + }); + } + RewrittenProjection::PerRow(value) => { + computed_columns.push(OutputColumn { name, value }); + } + } + } + Ok((database_projection, computed_columns)) +} + +fn reject_computed_order_by_references( + order_by: Option<&sqlparser::ast::OrderBy>, + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + if let Some(order_by) = order_by + && let OrderByKind::Expressions(expressions) = &order_by.kind + { + reject_computed_ordering_references("ORDER BY", expressions, computed_columns)?; + } + Ok(()) +} + +fn reject_computed_ordering_references( + clause: &str, + expressions: &[sqlparser::ast::OrderByExpr], + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + if expressions + .iter() + .any(|ordering| references_computed_projection(&ordering.expr, computed_columns, true)) + { + anyhow::bail!( + "{clause} cannot reference a SQLPage-computed column because ordering is performed by the database" + ); + } + Ok(()) +} + +fn reject_computed_group_by_references( + group_by: &GroupByExpr, + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + let GroupByExpr::Expressions(expressions, _) = group_by else { + return Ok(()); + }; + if expressions + .iter() + .any(|expression| references_computed_projection(expression, computed_columns, true)) + { + anyhow::bail!( + "GROUP BY cannot reference a SQLPage-computed column because grouping is performed by the database" + ); + } + Ok(()) +} + +fn reject_computed_alias_references( + clause: &str, + expression: Option<&SqlExpr>, + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + if expression.is_some_and(|expression| { + references_computed_projection(expression, computed_columns, false) + }) { + anyhow::bail!( + "{clause} cannot reference a SQLPage-computed column because it is evaluated by the database" + ); + } + Ok(()) +} + +fn reject_computed_aliases_in_expressions( + clause: &str, + expressions: &[SqlExpr], + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + if expressions + .iter() + .any(|expression| references_computed_projection(expression, computed_columns, false)) + { + anyhow::bail!( + "{clause} cannot reference a SQLPage-computed column because it is evaluated by the database" + ); + } + Ok(()) +} + +fn references_computed_projection( + expression: &SqlExpr, + computed_columns: &[OutputColumn], + reject_ordinal: bool, +) -> bool { + if computed_columns.is_empty() { + return false; + } + if reject_ordinal + && matches!( + expression, + SqlExpr::Value(ValueWithSpan { + value: Value::Number(_, _), + .. + }) + ) + { + return true; + } + + let mut finder = ComputedAliasFinder { computed_columns }; + sqlparser::ast::Visit::visit(expression, &mut finder).is_break() +} + +/// Rewrites a guaranteed one-row query directly as standalone expressions, +/// avoiding both a database round trip and an intermediate row-expression tree. +fn rewrite_single_row( + statement: &mut SqlStatement, + rewriter: &mut QueryRewriter<'_>, +) -> anyhow::Result> { + if !has_single_row_shape(statement) { + return Ok(None); + } + let SqlStatement::Query(query) = statement else { + return Ok(None); + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(None); + }; + for item in &select.projection { + let SelectItem::ExprWithAlias { expr, .. } = item else { + return Ok(None); + }; + if !can_build_standalone(expr)? { + return Ok(None); + } + } + + let mut columns = Vec::with_capacity(select.projection.len()); + for item in std::mem::take(&mut select.projection) { + let SelectItem::ExprWithAlias { expr, alias } = item else { + unreachable!("projection shape was checked") + }; + columns.push(OutputColumn { + name: alias.value, + value: build_sqlpage_expr::(rewriter, expr)?, + }); + } + Ok(Some(SingleRowQuery { + columns: columns.into_boxed_slice(), + })) +} + +fn has_single_row_shape(statement: &SqlStatement) -> bool { + let SqlStatement::Query(query) = statement else { + return false; + }; + if query.with.is_some() + || query.order_by.is_some() + || query.limit_clause.is_some() + || query.fetch.is_some() + || !query.locks.is_empty() + || query.for_clause.is_some() + || query.settings.is_some() + || query.format_clause.is_some() + || !query.pipe_operators.is_empty() + { + return false; + } + let SetExpr::Select(select) = query.body.as_ref() else { + return false; + }; + select.distinct.is_none() + && select.top.is_none() + && select.into.is_none() + && select.from.is_empty() + && select.lateral_views.is_empty() + && select.selection.is_none() + && select.group_by == sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]) + && select.cluster_by.is_empty() + && select.distribute_by.is_empty() + && select.sort_by.is_empty() + && select.having.is_none() + && select.named_window.is_empty() + && select.qualify.is_none() + && select.prewhere.is_none() + && select.connect_by.is_empty() + && select.optimizer_hints.is_empty() + && select.select_modifiers.is_none() + && select.exclude.is_none() +} + +/// Checks standalone support without consuming or cloning the AST, allowing +/// callers to select a rewrite path before moving any nodes. +fn can_build_standalone(expression: &SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Value(ValueWithSpan { + value: + Value::Boolean(_) + | Value::Number(_, _) + | Value::SingleQuotedString(_) + | Value::Null + | Value::Placeholder(_), + .. + }) => Ok(true), + SqlExpr::Identifier(identifier) => Ok(variable_from_ident(identifier).is_some()), + SqlExpr::Function(function) => { + if recognize_sqlpage_function(function)?.is_none() + && emulated_function(function).is_none() + { + return Ok(false); + } + let FunctionArguments::List(arguments) = &function.args else { + return Ok(false); + }; + for argument in &arguments.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { + return Ok(false); + }; + if !can_build_standalone(expression)? { + return Ok(false); + } + } + Ok(true) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(can_build_standalone(left)? && can_build_standalone(right)?), + SqlExpr::Nested(expression) => can_build_standalone(expression), + _ => Ok(false), + } +} + +impl QueryRewriter<'_> { + /// Splits a projected expression at SQLPage-supported operations while + /// leaving opaque database operations in the SQL AST. + fn rewrite_projection(&mut self, expression: SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Function(function) => { + if recognize_sqlpage_function(&function)?.is_some() { + return build_sqlpage_expr::(self, SqlExpr::Function(function)) + .map(RewrittenProjection::PerRow); + } + if let Some(kind) = emulated_function(&function) { + return self.rewrite_emulated_projection(function, kind); + } + let mut expression = SqlExpr::Function(function); + self.rewrite_database_expression(&mut expression)?; + Ok(RewrittenProjection::Database(expression)) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => { + let left = self.rewrite_projection(*left)?; + let right = self.rewrite_projection(*right)?; + match (left, right) { + (RewrittenProjection::Database(left), RewrittenProjection::Database(right)) => { + Ok(RewrittenProjection::Database(SqlExpr::BinaryOp { + left: Box::new(left), + op: BinaryOperator::StringConcat, + right: Box::new(right), + })) + } + (left, right) => Ok(RewrittenProjection::PerRow(SqlPageExpr::Concat { + arguments: vec![ + self.projection_into_row_expr(left)?, + self.projection_into_row_expr(right)?, + ] + .into_boxed_slice(), + null_behavior: self.database.database_type.concat_operator_null_behavior(), + })), + } + } + SqlExpr::Nested(expression) => match self.rewrite_projection(*expression)? { + RewrittenProjection::Database(expression) => Ok(RewrittenProjection::Database( + SqlExpr::Nested(Box::new(expression)), + )), + RewrittenProjection::PerRow(expression) => { + Ok(RewrittenProjection::PerRow(expression)) + } + }, + mut expression => { + self.rewrite_database_expression(&mut expression)?; + Ok(RewrittenProjection::Database(expression)) + } + } + } + + fn rewrite_emulated_projection( + &mut self, + function: Function, + kind: EmulatedFunction, + ) -> anyhow::Result { + let (arguments, original) = take_expression_arguments(function)?; + let mut rewritten = Vec::with_capacity(arguments.len()); + let mut has_per_row = false; + for argument in arguments { + let argument = self.rewrite_projection(argument)?; + has_per_row |= matches!(argument, RewrittenProjection::PerRow(_)); + rewritten.push(argument); + } + if !has_per_row { + let arguments = rewritten + .into_iter() + .map(|argument| match argument { + RewrittenProjection::Database(expression) => expression, + RewrittenProjection::PerRow(_) => unreachable!(), + }) + .collect(); + return Ok(RewrittenProjection::Database(rebuild_function( + original, arguments, + ))); + } + + let arguments = rewritten + .into_iter() + .map(|argument| self.projection_into_row_expr(argument)) + .collect::>>()?; + Ok(RewrittenProjection::PerRow(build_emulated( + kind, + arguments, + self.database.database_type, + )?)) + } + + /// Converts a database-owned projection fragment into a typed row input, + /// while preserving an already per-row expression unchanged. + fn projection_into_row_expr( + &mut self, + projection: RewrittenProjection, + ) -> anyhow::Result { + match projection { + RewrittenProjection::Database(expression) => { + build_sqlpage_expr::(self, expression) + } + RewrittenProjection::PerRow(expression) => Ok(expression), + } + } + + fn rewrite_database_expression(&mut self, expression: &mut SqlExpr) -> anyhow::Result<()> { + let _ = expression.visit(self); + self.error.take().map_or(Ok(()), Err) + } + + fn add_binding(&mut self, value: StandaloneExpr) -> SqlExpr { + let sequence = self.bindings.len(); + self.bindings.push(value); + let placeholder = match placeholder_style(self.database.kind) { + PlaceholderStyle::Numbered { prefix } => format!("{prefix}{}", sequence + 1), + PlaceholderStyle::Positional { .. } => format!("${}", sequence + 1), + }; + cast_placeholder(placeholder, self.database.database_type) + } + + fn add_row_input(&mut self, mut expression: SqlExpr) -> anyhow::Result { + let decode_as_json = is_json_expression(&expression); + self.rewrite_database_expression(&mut expression)?; + let index = self.row_input_json.len(); + let name = format!("{SQLPAGE_INPUT_PREFIX}{index}"); + self.private_projection.push(SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::with_quote('"', name), + }); + self.row_input_json.push(decode_as_json); + Ok(RowInputId::new(index)) + } + + fn finish_bindings( + &mut self, + statement: &mut SqlStatement, + ) -> anyhow::Result> { + let PlaceholderStyle::Positional { token } = placeholder_style(self.database.kind) else { + return Ok(std::mem::take(&mut self.bindings).into_boxed_slice()); + }; + let mut finalizer = PositionalBindingFinalizer { + token, + binding_count: self.bindings.len(), + order: Vec::with_capacity(self.bindings.len()), + error: None, + }; + let _ = statement.visit(&mut finalizer); + if let Some(error) = finalizer.error { + return Err(error); + } + let mut bindings = std::mem::take(&mut self.bindings) + .into_iter() + .map(Some) + .collect::>(); + finalizer + .order + .into_iter() + .map(|index| { + bindings[index].take().ok_or_else(|| { + anyhow!("Generated binding placeholder ${} was repeated", index + 1) + }) + }) + .collect::>>() + } +} + +struct PositionalBindingFinalizer { + token: &'static str, + binding_count: usize, + order: Vec, + error: Option, +} + +impl VisitorMut for PositionalBindingFinalizer { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { + let SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(name), + span, + }) = expression + else { + return ControlFlow::Continue(()); + }; + if *span != Span::empty() { + return ControlFlow::Continue(()); + } + let Some(index) = name + .strip_prefix('$') + .and_then(|number| number.parse::().ok()) + .and_then(|number| number.checked_sub(1)) + else { + return ControlFlow::Continue(()); + }; + if index >= self.binding_count { + self.error = Some(anyhow!("Invalid generated binding placeholder {name}")); + return ControlFlow::Break(()); + } + self.order.push(index); + self.token.clone_into(name); + ControlFlow::Continue(()) + } +} + +impl VisitorMut for QueryRewriter<'_> { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { + if self.error.is_some() { + return ControlFlow::Break(()); + } + + let replacement = match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + span, + }) if *span == Span::empty() => None, + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + .. + }) + | SqlExpr::Identifier(_) => variable_from_expr(expression) + .map(|variable| self.add_binding(SqlPageExpr::Variable(variable))), + SqlExpr::Function(function) => match recognize_sqlpage_function(function) { + Ok(Some(_)) => { + let owned = std::mem::replace(expression, SqlExpr::value(Value::Null)); + match build_sqlpage_expr::(self, owned) { + Ok(value) => Some(self.add_binding(value)), + Err(error) => { + self.error = Some(error.context( + "A SQLPage function used by the database could not be evaluated before the query", + )); + None + } + } + } + Ok(None) => None, + Err(error) => { + self.error = Some(error); + None + } + }, + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } if self.database.database_type == SupportedDatabase::Mssql => { + let left = std::mem::replace(left.as_mut(), SqlExpr::value(Value::Null)); + let right = std::mem::replace(right.as_mut(), SqlExpr::value(Value::Null)); + Some(make_function("CONCAT", vec![left, right])) + } + SqlExpr::Cast { + kind: kind @ CastKind::DoubleColon, + .. + } if ![ + SupportedDatabase::Postgres, + SupportedDatabase::Duckdb, + SupportedDatabase::Snowflake, + SupportedDatabase::Generic, + ] + .contains(&self.database.database_type) => + { + *kind = CastKind::Cast; + None + } + _ => None, + }; + + if let Some(replacement) = replacement { + *expression = replacement; + } + ControlFlow::Continue(()) + } +} + +/// Consumes an AST expression into the shared `SQLPage` expression type. The +/// environment determines whether opaque database fragments are illegal or +/// become private row inputs. +fn build_sqlpage_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, +) -> anyhow::Result> { + match expression { + SqlExpr::Value(ValueWithSpan { value, .. }) => match value { + Value::Placeholder(name) => Ok(SqlPageExpr::Variable(variable_from_placeholder(name))), + Value::SingleQuotedString(text) => Ok(SqlPageExpr::Literal(JsonValue::String(text))), + Value::Number(number, _) => Ok(SqlPageExpr::Literal(JsonValue::Number( + number.parse().context("Invalid numeric SQL literal")?, + ))), + Value::Boolean(value) => Ok(SqlPageExpr::Literal(JsonValue::Bool(value))), + Value::Null => Ok(SqlPageExpr::Literal(JsonValue::Null)), + _ => { + Environment::use_database_expr(rewriter, SqlExpr::Value(ValueWithSpan::from(value))) + } + }, + SqlExpr::Identifier(identifier) => { + if let Some(variable) = variable_from_ident(&identifier) { + Ok(SqlPageExpr::Variable(variable)) + } else { + Environment::use_database_expr(rewriter, SqlExpr::Identifier(identifier)) + } + } + SqlExpr::Function(function) => { + if let Some(function_name) = recognize_sqlpage_function(&function)? { + let (arguments, _) = take_expression_arguments(function)?; + let arguments = arguments + .into_iter() + .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .collect::>>()?; + Ok(SqlPageExpr::Call { + function: function_name, + arguments: arguments.into_boxed_slice(), + }) + } else if let Some(kind) = emulated_function(&function) { + let (arguments, _) = take_expression_arguments(function)?; + let arguments = arguments + .into_iter() + .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .collect::>>()?; + build_emulated(kind, arguments, rewriter.database.database_type) + } else { + Environment::use_database_expr(rewriter, SqlExpr::Function(function)) + } + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(SqlPageExpr::Concat { + arguments: vec![ + build_sqlpage_expr::(rewriter, *left)?, + build_sqlpage_expr::(rewriter, *right)?, + ] + .into_boxed_slice(), + null_behavior: rewriter + .database + .database_type + .concat_operator_null_behavior(), + }), + SqlExpr::Nested(expression) => build_sqlpage_expr::(rewriter, *expression), + expression => Environment::use_database_expr(rewriter, expression), + } +} + +fn build_emulated( + kind: EmulatedFunction, + arguments: Vec>, + database: SupportedDatabase, +) -> anyhow::Result> { + Ok(match kind { + EmulatedFunction::Concat => SqlPageExpr::Concat { + arguments: arguments.into_boxed_slice(), + null_behavior: database.concat_function_null_behavior(), + }, + EmulatedFunction::Coalesce => SqlPageExpr::Coalesce(arguments.into_boxed_slice()), + EmulatedFunction::JsonArray => SqlPageExpr::JsonArray(arguments.into_boxed_slice()), + EmulatedFunction::JsonObject => { + if !arguments.len().is_multiple_of(2) { + anyhow::bail!("JSON_OBJECT requires an even number of arguments"); + } + let mut arguments = arguments.into_iter(); + let mut entries = Vec::with_capacity(arguments.len() / 2); + while let Some(key) = arguments.next() { + let value = arguments.next().expect("argument count was checked"); + entries.push((key, value)); + } + SqlPageExpr::JsonObject(entries.into_boxed_slice()) + } + }) +} + +/// Recognizes and validates an unquoted `sqlpage.` call. A recognized +/// call is either rewritten or rejected and can never reach database SQL. +fn recognize_sqlpage_function(function: &Function) -> anyhow::Result> { + let ObjectName(parts) = &function.name; + if !is_sqlpage_func(parts) { + return Ok(None); + } + let [ + ObjectNamePart::Identifier(_), + ObjectNamePart::Identifier(name), + ] = parts.as_slice() + else { + unreachable!("is_sqlpage_func checked the name") + }; + if function.uses_odbc_syntax + || !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + anyhow::bail!( + "Modifiers are not supported on SQLPage function {}", + function.name + ); + } + let FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + clauses, + .. + }) = &function.args + else { + anyhow::bail!( + "Unsupported argument syntax for SQLPage function {}", + function.name + ); + }; + if !clauses.is_empty() { + anyhow::bail!( + "Argument clauses are not supported on SQLPage function {}", + function.name + ); + } + Ok(Some(SqlPageFunctionName::from_str(&name.value)?)) +} + +fn emulated_function(function: &Function) -> Option { + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return None; + }; + if !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + return None; + } + match name.value.to_ascii_lowercase().as_str() { + "concat" => Some(EmulatedFunction::Concat), + "coalesce" => Some(EmulatedFunction::Coalesce), + "json_object" | "jsonb_object" | "json_build_object" | "jsonb_build_object" => { + Some(EmulatedFunction::JsonObject) + } + "json_array" | "jsonb_array" | "json_build_array" | "jsonb_build_array" => { + Some(EmulatedFunction::JsonArray) + } + _ => None, + } +} + +/// Moves expression arguments out of a function while retaining its emptied +/// AST shell so database-owned functions can be rebuilt without cloning. +fn take_expression_arguments(mut function: Function) -> anyhow::Result<(Vec, Function)> { + let FunctionArguments::List(arguments) = &mut function.args else { + anyhow::bail!("Unsupported arguments to {}", function.name); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + anyhow::bail!("Unsupported arguments to {}", function.name); + } + let arguments = std::mem::take(&mut arguments.args) + .into_iter() + .map(|argument| match argument { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) => Ok(expression), + _ => Err(anyhow!( + "Named and wildcard function arguments are not supported" + )), + }) + .collect::>>()?; + Ok((arguments, function)) +} + +fn rebuild_function(mut function: Function, expressions: Vec) -> SqlExpr { + let FunctionArguments::List(arguments) = &mut function.args else { + unreachable!() + }; + arguments.args = expressions + .into_iter() + .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) + .collect(); + SqlExpr::Function(function) +} + +fn make_function(name: &str, expressions: Vec) -> SqlExpr { + SqlExpr::Function(Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), + args: FunctionArguments::List(FunctionArgumentList { + args: expressions + .into_iter() + .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) + .collect(), + duplicate_treatment: None, + clauses: Vec::new(), + }), + parameters: FunctionArguments::None, + over: None, + filter: None, + null_treatment: None, + within_group: Vec::new(), + uses_odbc_syntax: false, + }) +} + +fn variable_from_expr(expression: &SqlExpr) -> Option { + match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(name), + .. + }) => Some(variable_from_placeholder(name.clone())), + SqlExpr::Identifier(identifier) => variable_from_ident(identifier), + _ => None, + } +} + +fn variable_from_ident(identifier: &Ident) -> Option { + if identifier.quote_style.is_some() { + return None; + } + let prefix = identifier.value.chars().next()?; + matches!(prefix, '$' | ':' | '?').then(|| VariableRef { + name: identifier.value[prefix.len_utf8()..].to_owned(), + source: variable_source(prefix), + }) +} + +fn variable_from_placeholder(mut name: String) -> VariableRef { + let prefix = name.remove(0); + VariableRef { + name, + source: variable_source(prefix), + } +} + +fn variable_source(prefix: char) -> VariableSource { + match prefix { + '$' => VariableSource::SetOrUrl, + ':' => VariableSource::SetOrForm, + _ => VariableSource::Url, + } +} + +/// Wraps a generated placeholder in the backend-specific text cast expected +/// by `SQLPage`'s string-valued binding interface. +fn cast_placeholder(placeholder: String, database: SupportedDatabase) -> SqlExpr { + let data_type = match database { + SupportedDatabase::MySql => DataType::Char(None), + SupportedDatabase::Mssql => DataType::Varchar(Some(CharacterLength::Max)), + SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, + SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 4000, + unit: None, + })), + _ => DataType::Varchar(None), + }; + SqlExpr::Cast { + expr: Box::new(SqlExpr::value(Value::Placeholder(placeholder))), + data_type, + format: None, + kind: CastKind::Cast, + array: false, + } +} + +fn source_span(value: &impl sqlparser::ast::Spanned) -> SourceSpan { + let span = value.span(); + SourceSpan { + start: SourceLocation { + line: usize::try_from(span.start.line).unwrap_or(0), + column: usize::try_from(span.start.column).unwrap_or(0), + }, + end: SourceLocation { + line: usize::try_from(span.end.line).unwrap_or(0), + column: usize::try_from(span.end.column).unwrap_or(0), + }, + } +} + +fn select_item_contains_sqlpage(item: &SelectItem) -> bool { + struct Finder(bool); + impl sqlparser::ast::Visitor for Finder { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { + if let SqlExpr::Function(function) = expression + && is_sqlpage_func(&function.name.0) + { + self.0 = true; + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + } + } + let mut finder = Finder(false); + let _ = sqlparser::ast::Visit::visit(item, &mut finder); + finder.0 +} diff --git a/src/webserver/database/sql/statement.rs b/src/webserver/database/sql/statement.rs new file mode 100644 index 00000000..cb83ba7c --- /dev/null +++ b/src/webserver/database/sql/statement.rs @@ -0,0 +1,94 @@ +//! Immutable SQL-file statements consumed by the executor. + +use std::path::PathBuf; + +use super::super::csv_import::CsvImport; +use super::super::sqlpage_expr::{RowExpr, StandaloneExpr}; +use super::super::sqlpage_functions::functions::SqlPageFunctionName; + +/// A parsed and rewritten SQL file ready for repeated execution. +#[derive(Default)] +pub struct SqlFile { + pub(in crate::webserver::database) statements: Box<[FileStatement]>, + pub source_path: PathBuf, +} + +/// One statement in a SQL file. +#[derive(Debug)] +pub(in crate::webserver::database) enum FileStatement { + Query(Query), + SetVariable { target: VariableName, value: Query }, + CsvImport(CsvImport), + Error(anyhow::Error), +} + +/// A query and its original source location. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct Query { + pub body: QueryBody, + pub source_span: SourceSpan, +} + +/// The legal ways `SQLPage` obtains rows. +/// +/// Keeping output expressions inside each variant prevents a synthetic row +/// from containing an expression that requires a database row input. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) enum QueryBody { + Database(DatabaseQuery), + SingleRow(SingleRowQuery), +} + +/// A statement executed by the configured database. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct DatabaseQuery { + pub sql: String, + /// Evaluated once, in placeholder order, before executing `sql`. + pub bindings: Box<[StandaloneExpr]>, + /// JSON decoding flags for the trailing private input columns. + pub row_input_json: Box<[bool]>, + /// Evaluated once for every returned database row. + pub computed_columns: Box<[OutputColumn]>, + pub json_columns: Box<[String]>, +} + +impl DatabaseQuery { + /// Whether row evaluation needs the request's existing connection and + /// must therefore wait until the database stream is closed. + pub fn must_buffer_rows(&self) -> bool { + self.computed_columns + .iter() + .any(|column| column.value.contains_function(SqlPageFunctionName::run_sql)) + } +} + +/// Exactly one row generated without querying the database. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct SingleRowQuery { + pub columns: Box<[OutputColumn]>, +} + +/// A named SQLPage-owned output expression. +#[derive(Debug, PartialEq, Eq)] +pub(in crate::webserver::database) struct OutputColumn { + pub name: String, + pub value: Expr, +} + +/// A validated variable name used as the target of a `SET` statement. +#[derive(Debug, PartialEq, Eq)] +pub(in crate::webserver::database) struct VariableName(pub String); + +/// A location in the SQL source. +#[derive(Debug, PartialEq, Clone, Copy)] +pub(in crate::webserver::database) struct SourceSpan { + pub start: SourceLocation, + pub end: SourceLocation, +} + +/// A line and column in the SQL source. +#[derive(Debug, PartialEq, Clone, Copy)] +pub(in crate::webserver::database) struct SourceLocation { + pub line: usize, + pub column: usize, +} diff --git a/src/webserver/database/sql_to_json.rs b/src/webserver/database/sql_to_json.rs index 4c93dddd..72c0f40e 100644 --- a/src/webserver/database/sql_to_json.rs +++ b/src/webserver/database/sql_to_json.rs @@ -1,3 +1,20 @@ +//! Runtime codec from heterogeneous `sqlx::Any` rows to `SQLPage`'s JSON rows. +//! +//! Once a rewritten query returns data, this module decodes each physical SQL +//! value according to its driver-reported type and normalizes it into a +//! [`serde_json::Value`]. Public columns are collected into an object, duplicate +//! names become arrays, ODBC-folded identifiers are canonicalized where needed, +//! and database-specific numeric, temporal, JSON, binary, UUID, and range types +//! receive stable JSON-compatible representations. +//! +//! Rewritten queries may append private columns that carry database values into +//! SQLPage-computed projections. [`row_to_json_with_inputs`] splits that trailing +//! suffix by ordinal, not alias, so generated names cannot collide with user +//! columns; it returns the private values separately from the visible row after +//! decoding every physical value once. `execute_queries` then applies the +//! rewrite's JSON metadata and evaluates computed expressions. This module only +//! owns physical row decoding and does not decide where expressions execute. + use crate::utils::add_value_to_map; use crate::webserver::database::blob_to_data_url; use bigdecimal::BigDecimal; @@ -9,6 +26,7 @@ use sqlx::postgres::types::PgRange; use sqlx::{Column, Row, TypeInfo, ValueRef}; use sqlx::{Decode, Type}; +#[cfg(test)] pub fn row_to_json(row: &AnyRow) -> Value { use Value::Object; @@ -22,6 +40,32 @@ pub fn row_to_json(row: &AnyRow) -> Value { Object(map) } +/// Decodes a row into user-visible columns and a private trailing input suffix. +/// +/// Every SQL value is decoded exactly once. Private values are addressed by +/// ordinal, so their generated SQL aliases cannot collide with user columns. +pub fn row_to_json_with_inputs( + row: &AnyRow, + input_count: usize, +) -> anyhow::Result<(Value, Vec)> { + let columns = row.columns(); + let public_count = columns + .len() + .checked_sub(input_count) + .ok_or_else(|| anyhow::anyhow!("The query returned fewer columns than SQLPage expected"))?; + let mut map = Map::new(); + let mut inputs = Vec::with_capacity(input_count); + for column in &columns[..public_count] { + let key = canonical_col_name(column); + let value = sql_to_json(row, column); + map = add_value_to_map(map, (key, value)); + } + for column in &columns[public_count..] { + inputs.push(sql_to_json(row, column)); + } + Ok((Value::Object(map), inputs)) +} + fn canonical_col_name(col: &AnyColumn) -> String { // Some databases fold all unquoted identifiers to uppercase but SQLPage uses lowercase property names if matches!(col.type_info().0, AnyTypeInfoKind::Odbc(_)) @@ -156,16 +200,6 @@ pub fn sql_nonnull_to_json<'r>(mut get_ref: impl FnMut() -> sqlx::any::AnyValueR } } -/// Takes the first column of a row and converts it to a string. -pub fn row_to_string(row: &AnyRow) -> Option { - let col = row.columns().first()?; - match sql_to_json(row, col) { - serde_json::Value::String(s) => Some(s), - serde_json::Value::Null => None, - other => Some(other.to_string()), - } -} - #[cfg(test)] mod tests { use crate::app_config::tests::test_database_url; @@ -216,6 +250,24 @@ mod tests { Ok(()) } + #[actix_web::test] + async fn private_inputs_are_split_from_the_trailing_suffix() -> anyhow::Result<()> { + let db_url = test_database_url(); + let mut connection = sqlx::AnyConnection::connect(&db_url).await?; + let row = sqlx::query( + "SELECT 'public' AS \"__sqlpage_input_0\", 'private' AS \"__sqlpage_input_0\"", + ) + .fetch_one(&mut connection) + .await?; + let (public, inputs) = row_to_json_with_inputs(&row, 1)?; + expect_json_object_equal( + &public, + &serde_json::json!({ "__sqlpage_input_0": "public" }), + ); + assert_eq!(inputs, [serde_json::json!("private")]); + Ok(()) + } + #[actix_web::test] async fn test_postgres_types() -> anyhow::Result<()> { let Some(db_url) = db_specific_test("postgres") else { diff --git a/src/webserver/database/sqlpage_expr.rs b/src/webserver/database/sqlpage_expr.rs new file mode 100644 index 00000000..126a133f --- /dev/null +++ b/src/webserver/database/sqlpage_expr.rs @@ -0,0 +1,334 @@ +//! Expressions evaluated by `SQLPage` instead of the database. +//! +//! The input type records whether an expression may read values from a +//! returned database row. The expression implementation and evaluator remain +//! shared between standalone and per-row expressions. + +use std::borrow::Cow; + +use anyhow::Context as _; +use serde_json::Value; + +use super::execute_queries::DbConn; +use super::sqlpage_functions::functions::SqlPageFunctionName; +use crate::webserver::http_request_info::ExecutionContext; +use crate::webserver::single_or_vec::SingleOrVec; + +/// An expression evaluated by `SQLPage`. +/// +/// Nodes are not shared automatically because function calls may be +/// effectful. `Input` identifies values supplied by the evaluation site. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum SqlPageExpr { + Literal(Value), + Variable(VariableRef), + Input(Input), + Call { + function: SqlPageFunctionName, + arguments: Box<[Self]>, + }, + Concat { + arguments: Box<[Self]>, + null_behavior: ConcatNullBehavior, + }, + Coalesce(Box<[Self]>), + JsonObject(Box<[(Self, Self)]>), + JsonArray(Box<[Self]>), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ConcatNullBehavior { + IgnoreNull, + PropagateNull, +} + +/// Uninhabited input type for expressions that cannot read a database row. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum NoRowInput {} + +/// Identifies one private value projected by the database for `SQLPage`. +/// +/// The index is private and this type is intentionally not `Clone` or `Copy`. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct RowInputId(usize); + +impl RowInputId { + pub(super) fn new(index: usize) -> Self { + Self(index) + } + + pub(super) fn index(&self) -> usize { + self.0 + } +} + +/// An expression that can be evaluated without a returned database row. +pub(crate) type StandaloneExpr = SqlPageExpr; + +/// An expression evaluated once for each returned database row. +pub(crate) type RowExpr = SqlPageExpr; + +#[derive(Debug, PartialEq, Eq)] +/// A reference to request or previously assigned `SQLPage` state. +pub(crate) struct VariableRef { + pub name: String, + pub source: VariableSource, +} + +#[derive(Debug, PartialEq, Eq)] +/// Lookup precedence implied by `SQLPage`'s three variable syntaxes. +pub(crate) enum VariableSource { + /// Read only from URL parameters (`?name`). + Url, + /// Prefer a `SET` value, then read a URL parameter (`$name`). + SetOrUrl, + /// Prefer a `SET` value, then read a form field (`:name`). + SetOrForm, +} + +/// A possibly borrowed value produced by `SQLPage` expression evaluation. +pub(crate) enum SqlPageValue<'a> { + Null, + Text(Cow<'a, str>), + /// A number, boolean, array, or object. + Json(Cow<'a, Value>), +} + +impl<'a> SqlPageValue<'a> { + pub(crate) fn into_function_argument(self) -> Option> { + match self { + Self::Null => None, + Self::Text(text) => Some(text), + Self::Json(value) => Some(Cow::Owned(value.to_string())), + } + } + + pub(crate) fn into_json(self) -> Value { + match self { + Self::Null => Value::Null, + Self::Text(Cow::Borrowed(text)) => Value::String(text.to_owned()), + Self::Text(Cow::Owned(text)) => Value::String(text), + Self::Json(Cow::Borrowed(value)) => value.clone(), + Self::Json(Cow::Owned(value)) => value, + } + } +} + +/// Supplies external inputs to a `SQLPage` expression. +pub(crate) trait ExprInputs { + fn take(&mut self, input: &Input) -> anyhow::Result>; +} + +/// Input provider for standalone expressions. +pub(crate) struct NoInputs; + +impl ExprInputs for NoInputs { + fn take(&mut self, input: &NoRowInput) -> anyhow::Result> { + match *input {} + } +} + +/// Private values decoded from one returned database row. +pub(crate) struct RowInputs(Vec>); + +impl RowInputs { + pub(crate) fn new(values: Vec) -> Self { + Self(values.into_iter().map(Some).collect()) + } +} + +impl ExprInputs for RowInputs { + fn take(&mut self, input: &RowInputId) -> anyhow::Result> { + let value = self + .0 + .get_mut(input.index()) + .and_then(Option::take) + .with_context(|| { + format!( + "Row input {} is missing or was already consumed", + input.index() + ) + })?; + Ok(value_to_sqlpage_value(value)) + } +} + +fn value_to_sqlpage_value(value: Value) -> SqlPageValue<'static> { + match value { + Value::Null => SqlPageValue::Null, + Value::String(text) => SqlPageValue::Text(Cow::Owned(text)), + value => SqlPageValue::Json(Cow::Owned(value)), + } +} + +impl VariableRef { + fn evaluate<'a>(&self, request: &'a ExecutionContext) -> SqlPageValue<'a> { + let value = match self.source { + VariableSource::Url => request + .url_params + .get(&self.name) + .map(SingleOrVec::as_json_str), + VariableSource::SetOrForm => { + if let Some(value) = request.set_variables.borrow().get(&self.name) { + return value.as_ref().map_or(SqlPageValue::Null, |value| { + SqlPageValue::Text(Cow::Owned(value.as_json_str().into_owned())) + }); + } + request + .post_variables + .get(&self.name) + .map(SingleOrVec::as_json_str) + } + VariableSource::SetOrUrl => { + if let Some(value) = request.set_variables.borrow().get(&self.name) { + return value.as_ref().map_or(SqlPageValue::Null, |value| { + SqlPageValue::Text(Cow::Owned(value.as_json_str().into_owned())) + }); + } + let url_value = request.url_params.get(&self.name); + if request.post_variables.contains_key(&self.name) { + if url_value.is_some() { + log::warn!( + "Deprecation warning! There is both a URL parameter named '{}' and a form field named '{}'. SQLPage is using the URL parameter for ${}. Please use :{} to reference the form field explicitly.", + self.name, + self.name, + self.name, + self.name, + ); + } else { + log::warn!( + "Deprecation warning! ${} was used to reference a form field value (a POST variable). This now uses only URL parameters. Please use :{} instead.", + self.name, + self.name, + ); + } + } + url_value.map(SingleOrVec::as_json_str) + } + }; + value.map_or(SqlPageValue::Null, SqlPageValue::Text) + } +} + +impl SqlPageExpr { + /// Evaluates this expression from left to right. + pub(crate) async fn evaluate<'a>( + &'a self, + request: &'a ExecutionContext, + db_connection: &mut DbConn, + inputs: &mut impl ExprInputs, + ) -> anyhow::Result> { + match self { + Self::Literal(value) => Ok(match value { + Value::Null => SqlPageValue::Null, + Value::String(text) => SqlPageValue::Text(Cow::Borrowed(text)), + value => SqlPageValue::Json(Cow::Borrowed(value)), + }), + Self::Variable(variable) => Ok(variable.evaluate(request)), + Self::Input(input) => inputs.take(input).map(SqlPageValue::into_lifetime), + Self::Call { + function, + arguments, + } => { + let mut values = Vec::with_capacity(arguments.len()); + for argument in arguments { + values.push( + Box::pin(argument.evaluate(request, db_connection, inputs)) + .await? + .into_function_argument(), + ); + } + let result = function + .evaluate(request, db_connection, values) + .await + .with_context(|| format!("Error in function call {function}"))?; + Ok(result.map_or(SqlPageValue::Null, SqlPageValue::Text)) + } + Self::Concat { + arguments, + null_behavior, + } => { + let mut result = String::new(); + for argument in arguments { + let value = Box::pin(argument.evaluate(request, db_connection, inputs)).await?; + match value.into_function_argument() { + Some(value) => result.push_str(&value), + None if *null_behavior == ConcatNullBehavior::PropagateNull => { + return Ok(SqlPageValue::Null); + } + None => {} + } + } + Ok(SqlPageValue::Text(Cow::Owned(result))) + } + Self::Coalesce(arguments) => { + for argument in arguments { + let value = Box::pin(argument.evaluate(request, db_connection, inputs)).await?; + if !matches!(value, SqlPageValue::Null) { + return Ok(value); + } + } + Ok(SqlPageValue::Null) + } + Self::JsonObject(entries) => { + let mut object = serde_json::Map::with_capacity(entries.len()); + for (key, value) in entries { + let key = Box::pin(key.evaluate(request, db_connection, inputs)) + .await? + .into_function_argument() + .context("JSON object keys cannot be NULL")? + .into_owned(); + let value = Box::pin(value.evaluate(request, db_connection, inputs)) + .await? + .into_json(); + object.insert(key, value); + } + Ok(SqlPageValue::Json(Cow::Owned(Value::Object(object)))) + } + Self::JsonArray(elements) => { + let mut array = Vec::with_capacity(elements.len()); + for element in elements { + array.push( + Box::pin(element.evaluate(request, db_connection, inputs)) + .await? + .into_json(), + ); + } + Ok(SqlPageValue::Json(Cow::Owned(Value::Array(array)))) + } + } + } + + pub(crate) fn contains_function(&self, expected: SqlPageFunctionName) -> bool { + match self { + Self::Call { + function, + arguments, + } => { + *function == expected + || arguments + .iter() + .any(|argument| argument.contains_function(expected)) + } + Self::Concat { arguments, .. } + | Self::Coalesce(arguments) + | Self::JsonArray(arguments) => arguments + .iter() + .any(|argument| argument.contains_function(expected)), + Self::JsonObject(entries) => entries.iter().any(|(key, value)| { + key.contains_function(expected) || value.contains_function(expected) + }), + Self::Literal(_) | Self::Variable(_) | Self::Input(_) => false, + } + } +} + +impl SqlPageValue<'static> { + fn into_lifetime<'a>(self) -> SqlPageValue<'a> { + match self { + Self::Null => SqlPageValue::Null, + Self::Text(text) => SqlPageValue::Text(text), + Self::Json(value) => SqlPageValue::Json(value), + } + } +} diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index e6709bd8..46508eb1 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -57,7 +57,7 @@ impl ::std::str::FromStr for SqlPageFunctionName { SqlPageFunctionName::ALL .iter() .copied() - .find(|function| function.name() == name) + .find(|function| function.name().eq_ignore_ascii_case(name)) .ok_or_else(|| { anyhow::anyhow!( "Unknown function {name:?}. Supported functions:\n{}", diff --git a/src/webserver/database/sqlpage_functions/mod.rs b/src/webserver/database/sqlpage_functions/mod.rs index aea396bb..d4b70ec2 100644 --- a/src/webserver/database/sqlpage_functions/mod.rs +++ b/src/webserver/database/sqlpage_functions/mod.rs @@ -2,20 +2,3 @@ mod function_traits; pub(super) mod functions; mod http_fetch_request; mod url_parameters; - -use sqlparser::ast::FunctionArg; - -use super::sql::ParamExtractContext; -use super::syntax_tree::SqlPageFunctionCall; -use super::syntax_tree::StmtParam; - -pub(super) fn func_call_to_param( - func_name: &str, - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, -) -> StmtParam { - SqlPageFunctionCall::from_func_call(func_name, arguments, ctx).map_or_else( - |e| StmtParam::Error(format!("{e:#}")), - StmtParam::FunctionCall, - ) -} diff --git a/src/webserver/database/syntax_tree.rs b/src/webserver/database/syntax_tree.rs deleted file mode 100644 index 31a88159..00000000 --- a/src/webserver/database/syntax_tree.rs +++ /dev/null @@ -1,321 +0,0 @@ -/// This module contains the syntax tree for sqlpage statement parameters. -/// In a query like `SELECT sqlpage.some_function($my_param)`, -/// The stored database statement will be just `SELECT $1`, -/// and the `StmtParam` will contain a the following tree: -/// -/// ```text -/// StmtParam::FunctionCall( -/// SqlPageFunctionCall { -/// function: SqlPageFunctionName::some_function, -/// arguments: vec![StmtParam::Get("$my_param")] -/// } -/// ) -/// ``` -use std::borrow::Cow; -use std::str::FromStr; - -use sqlparser::ast::FunctionArg; - -use crate::webserver::http_request_info::ExecutionContext; -use crate::webserver::single_or_vec::SingleOrVec; - -use super::{ - execute_queries::DbConn, sql::ParamExtractContext, sql::function_args_to_stmt_params, - sqlpage_functions::functions::SqlPageFunctionName, -}; -use anyhow::Context as _; - -/// Represents a parameter to a SQL statement. -/// Objects of this type are created during SQL parsing. -/// Every time a SQL statement is executed, the parameters are evaluated to produce the actual values that are passed to the database. -/// Parameter evaluation can involve asynchronous operations, and extracting values from the request. -#[derive(Debug, PartialEq, Eq, Clone)] -pub(crate) enum StmtParam { - Get(String), - Post(String), - PostOrGet(String), - Error(String), - Literal(String), - Null, - Concat(Vec), - Coalesce(Vec), - JsonObject(Vec), - JsonArray(Vec), - FunctionCall(SqlPageFunctionCall), -} - -impl std::fmt::Display for StmtParam { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - StmtParam::Get(name) => write!(f, "?{name}"), - StmtParam::Post(name) => write!(f, ":{name}"), - StmtParam::PostOrGet(name) => write!(f, "${name}"), - StmtParam::Literal(x) => write!(f, "'{}'", x.replace('\'', "''")), - StmtParam::Null => write!(f, "NULL"), - StmtParam::Concat(items) => { - write!(f, "CONCAT(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::Coalesce(items) => { - write!(f, "COALESCE(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::JsonObject(items) => { - write!(f, "JSON_OBJECT(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::JsonArray(items) => { - write!(f, "JSON_ARRAY(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::FunctionCall(call) => write!(f, "{call}"), - StmtParam::Error(x) => { - if let Some((i, _)) = x.char_indices().nth(21) { - write!(f, "## {}... ##", &x[..i]) - } else { - write!(f, "## {x} ##") - } - } - } - } -} - -/// Represents a call to a `sqlpage.` function. -/// Objects of this type are created during SQL parsing and used to evaluate the function at runtime. -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct SqlPageFunctionCall { - pub function: SqlPageFunctionName, - pub arguments: Vec, -} - -impl SqlPageFunctionCall { - pub fn from_func_call( - func_name: &str, - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, - ) -> anyhow::Result { - let function = SqlPageFunctionName::from_str(func_name)?; - let arguments = function_args_to_stmt_params(arguments, ctx)?; - Ok(Self { - function, - arguments, - }) - } - - pub async fn evaluate<'a>( - &self, - request: &'a ExecutionContext, - db_connection: &mut DbConn, - ) -> anyhow::Result>> { - let mut params = Vec::with_capacity(self.arguments.len()); - for param in &self.arguments { - params.push(Box::pin(extract_req_param(param, request, db_connection)).await?); - } - log::trace!("Starting function call to {self}"); - let result = self - .function - .evaluate(request, db_connection, params) - .await?; - log::trace!( - "Function call to {self} returned: {}", - result.as_deref().unwrap_or("NULL") - ); - Ok(result) - } -} - -impl std::fmt::Display for SqlPageFunctionCall { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}(", self.function)?; - // interleave the arguments with commas - let mut it = self.arguments.iter(); - if let Some(x) = it.next() { - write!(f, "{x}")?; - } - for x in it { - write!(f, ", {x}")?; - } - write!(f, ")") - } -} - -/// Extracts the value of a parameter from the request. -/// Returns `Ok(None)` when NULL should be used as the parameter value. -pub(super) async fn extract_req_param<'a>( - param: &StmtParam, - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - Ok(match param { - // sync functions - StmtParam::Get(x) => request.url_params.get(x).map(SingleOrVec::as_json_str), - StmtParam::Post(x) => { - if let Some(val) = request.set_variables.borrow().get(x) { - val.as_ref() - .map(|v| Cow::Owned(v.as_json_str().into_owned())) - } else { - request.post_variables.get(x).map(SingleOrVec::as_json_str) - } - } - StmtParam::PostOrGet(x) => { - if let Some(val) = request.set_variables.borrow().get(x) { - val.as_ref() - .map(|v| Cow::Owned(v.as_json_str().into_owned())) - } else { - let url_val = request.url_params.get(x); - if request.post_variables.contains_key(x) { - if url_val.is_some() { - log::warn!( - "Deprecation warning! There is both a URL parameter named '{x}' and a form field named '{x}'. \ - SQLPage is using the URL parameter for ${x}. Please use :{x} to reference the form field explicitly." - ); - } else { - log::warn!( - "Deprecation warning! ${x} was used to reference a form field value (a POST variable). \ - This now uses only URL parameters. Please use :{x} instead." - ); - } - } - url_val.map(SingleOrVec::as_json_str) - } - } - StmtParam::Error(x) => anyhow::bail!("{x}"), - StmtParam::Literal(x) => Some(Cow::Owned(x.clone())), - StmtParam::Null => None, - StmtParam::Concat(args) => concat_params(&args[..], request, db_connection).await?, - StmtParam::JsonObject(args) => { - json_object_params(&args[..], request, db_connection).await? - } - StmtParam::JsonArray(args) => json_array_params(&args[..], request, db_connection).await?, - StmtParam::Coalesce(args) => coalesce_params(&args[..], request, db_connection).await?, - StmtParam::FunctionCall(func) => { - func.evaluate(request, db_connection) - .await - .with_context(|| { - format!( - "Error in function call {func}.\nExpected {:#}", - func.function - ) - })? - } - }) -} - -async fn concat_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - let mut result = String::new(); - for arg in args { - let Some(arg) = Box::pin(extract_req_param(arg, request, db_connection)).await? else { - return Ok(None); - }; - result.push_str(&arg); - } - Ok(Some(Cow::Owned(result))) -} - -async fn coalesce_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - for arg in args { - if let Some(arg) = Box::pin(extract_req_param(arg, request, db_connection)).await? { - return Ok(Some(arg)); - } - } - Ok(None) -} - -async fn json_object_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - use serde::{Serializer, ser::SerializeMap}; - let mut result = Vec::new(); - let mut ser = serde_json::Serializer::new(&mut result); - let mut map_ser = ser.serialize_map(Some(args.len()))?; - let mut it = args.iter(); - while let Some(key) = it.next() { - let key = Box::pin(extract_req_param(key, request, db_connection)).await?; - map_ser.serialize_key(&key)?; - let val = it - .next() - .ok_or_else(|| anyhow::anyhow!("Odd number of arguments in JSON_OBJECT"))?; - - match val { - StmtParam::JsonObject(args) => { - let raw_json = Box::pin(json_object_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - map_ser.serialize_value(&obj)?; - } - StmtParam::JsonArray(args) => { - let raw_json = Box::pin(json_array_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - map_ser.serialize_value(&obj)?; - } - val => { - let evaluated = Box::pin(extract_req_param(val, request, db_connection)).await?; - map_ser.serialize_value(&evaluated)?; - } - } - } - map_ser.end()?; - Ok(Some(Cow::Owned(String::from_utf8(result)?))) -} - -async fn json_array_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - use serde::{Serializer, ser::SerializeSeq}; - let mut result = Vec::new(); - let mut ser = serde_json::Serializer::new(&mut result); - let mut seq_ser = ser.serialize_seq(Some(args.len()))?; - for element in args { - match element { - StmtParam::JsonObject(args) => { - let raw_json = json_object_params(args, request, db_connection).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - seq_ser.serialize_element(&obj)?; - } - StmtParam::JsonArray(args) => { - let raw_json = Box::pin(json_array_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - seq_ser.serialize_element(&obj)?; - } - element => { - let evaluated = - Box::pin(extract_req_param(element, request, db_connection)).await?; - seq_ser.serialize_element(&evaluated)?; - } - } - } - seq_ser.end()?; - Ok(Some(Cow::Owned(String::from_utf8(result)?))) -} - -fn cow_to_raw_json<'a>( - raw_json: Option<&'a impl AsRef>, -) -> Option<&'a serde_json::value::RawValue> { - raw_json - .map(AsRef::as_ref) - .map(serde_json::from_str::<&'a serde_json::value::RawValue>) - .map(Result::unwrap) -} diff --git a/src/webserver/http.rs b/src/webserver/http.rs index e37410a5..0f17bb95 100644 --- a/src/webserver/http.rs +++ b/src/webserver/http.rs @@ -9,7 +9,7 @@ use crate::webserver::database::execute_queries::stop_at_first_error; use crate::webserver::database::{DbItem, execute_queries::stream_query_results_with_conn}; use crate::webserver::http_request_info::extract_request_info; use crate::webserver::server_timing::ServerTiming; -use crate::{AppConfig, AppState, DEFAULT_404_FILE, ParsedSqlFile}; +use crate::{AppConfig, AppState, DEFAULT_404_FILE, SqlFile}; use actix_web::dev::{ServiceFactory, ServiceRequest, fn_service}; use actix_web::error::{ErrorBadRequest, ErrorInternalServerError}; use actix_web::http::header::Accept; @@ -209,7 +209,7 @@ enum ResponseWithWriter { async fn render_sql( srv_req: &mut ServiceRequest, - sql_file: Arc, + sql_file: Arc, server_timing: ServerTiming, ) -> actix_web::Result { let app_state = srv_req diff --git a/src/webserver/oidc.rs b/src/webserver/oidc.rs index 9173aff1..195fe9e1 100644 --- a/src/webserver/oidc.rs +++ b/src/webserver/oidc.rs @@ -976,7 +976,7 @@ async fn execute_oidc_request_with_awc( let response = req.send_body(body).await.map_err(|e| { anyhow!(e.to_string()).context(format!( "Failed to send request: {} {}", - &req_head.method, &req_head.uri + req_head.method, req_head.uri )) })?; let head = response.headers(); @@ -995,7 +995,7 @@ async fn execute_oidc_request_with_awc( let body = response .body() .await - .with_context(|| format!("Couldnt read from {}", &req_head.uri))?; + .with_context(|| format!("Couldnt read from {}", req_head.uri))?; log::debug!("Received OIDC response body_len={} bytes", body.len()); let resp = resp_builder.body(body.to_vec())?; Ok(resp) diff --git a/src/webserver/routing.rs b/src/webserver/routing.rs index 0bc92bf6..25c644a6 100644 --- a/src/webserver/routing.rs +++ b/src/webserver/routing.rs @@ -77,7 +77,7 @@ //! ``` use crate::filesystem::{FileAccess, FileSystem}; -use crate::webserver::database::ParsedSqlFile; +use crate::webserver::database::SqlFile; use crate::{AppState, file_cache::FileCache}; use RoutingAction::{CustomNotFound, Execute, NotFound, Redirect, Serve}; use awc::http::uri::PathAndQuery; @@ -109,14 +109,14 @@ pub trait RoutingConfig { } pub(crate) struct AppFileStore<'a> { - cache: &'a FileCache, + cache: &'a FileCache, filesystem: &'a FileSystem, app_state: &'a AppState, } impl<'a> AppFileStore<'a> { pub fn new( - cache: &'a FileCache, + cache: &'a FileCache, filesystem: &'a FileSystem, app_state: &'a AppState, ) -> Self { diff --git a/tests/core/mod.rs b/tests/core/mod.rs index 229aeac2..4f63ae14 100644 --- a/tests/core/mod.rs +++ b/tests/core/mod.rs @@ -57,11 +57,24 @@ async fn test_routing_with_db_fs() { return; } - let drop_sql = "DROP TABLE IF EXISTS sqlpage_files"; - state.db.connection.execute(drop_sql).await.unwrap(); let create_table_sql = sqlpage::filesystem::DbFsQueries::get_create_table_sql(state.db.info.database_type); - state.db.connection.execute(create_table_sql).await.unwrap(); + // Other tests share this database, so never drop a table their initialized state may use. + if state + .db + .connection + .execute("SELECT 1 FROM sqlpage_files WHERE 1 = 0") + .await + .is_err() + { + state.db.connection.execute(create_table_sql).await.unwrap(); + } + state + .db + .connection + .execute("DELETE FROM sqlpage_files WHERE path = 'on_db.sql'") + .await + .unwrap(); let insert_sql = format!( "INSERT INTO sqlpage_files(path, contents) VALUES ('on_db.sql', {})", make_placeholder(state.db.info.kind, 1) diff --git a/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_column.sql b/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_column.sql new file mode 100644 index 00000000..15590e2c --- /dev/null +++ b/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_column.sql @@ -0,0 +1,3 @@ +set x = (select 1 as a, sqlpage.fetch(upper('invalid')) as b); + +select 'text' as component, 'It works !' as contents; diff --git a/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_row_nosqlite.sql b/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_row_nosqlite.sql new file mode 100644 index 00000000..e2a0706c --- /dev/null +++ b/tests/sql_test_files/component_rendering/error_set_scalar_query_returned_more_than_one_row_nosqlite.sql @@ -0,0 +1,3 @@ +set x = (select 1 union all select 2); + +select 'text' as component, 'It works !' as contents; diff --git a/tests/sql_test_files/component_rendering/error_too_many_nested_inclusions.sql b/tests/sql_test_files/component_rendering/error_too_many_nested_inclusions.sql index c35c0322..9573aece 100644 --- a/tests/sql_test_files/component_rendering/error_too_many_nested_inclusions.sql +++ b/tests/sql_test_files/component_rendering/error_too_many_nested_inclusions.sql @@ -1,4 +1,4 @@ select 'debug' as component, sqlpage.run_sql( 'tests/sql_test_files/component_rendering/error_too_many_nested_inclusions.sql' - ) as contents; \ No newline at end of file + ) as contents; diff --git a/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql b/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql index aaaaf714..6f284b18 100644 --- a/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql +++ b/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql @@ -1,4 +1,4 @@ -- Doesnt work on mssql because it does not support "create temporary table" create temporary table temp_t(x text); insert into temp_t(x) values ('It works !'); -select 'dynamic' as component, sqlpage.run_sql('tests/sql_test_files/select_temp_t.sql') AS properties; \ No newline at end of file +select 'dynamic' as component, sqlpage.run_sql('tests/sql_test_files/select_temp_t.sql') AS properties; diff --git a/tests/sql_test_files/data/coalesce_short_circuit.sql b/tests/sql_test_files/data/coalesce_short_circuit.sql new file mode 100644 index 00000000..4d4d6a4c --- /dev/null +++ b/tests/sql_test_files/data/coalesce_short_circuit.sql @@ -0,0 +1,7 @@ +select 'cached' as expected, + coalesce(cached_response, sqlpage.fetch('this invalid fetch must not run')) as actual +from (select 'cached' as cached_response) cache_row; + +select '%20' as expected, + coalesce(cached_response, sqlpage.url_encode(' ')) as actual +from (select null as cached_response) cache_row; diff --git a/tests/sql_test_files/data/computed_projection_binding_order.sql b/tests/sql_test_files/data/computed_projection_binding_order.sql new file mode 100644 index 00000000..4e99f145 --- /dev/null +++ b/tests/sql_test_files/data/computed_projection_binding_order.sql @@ -0,0 +1,18 @@ +set a = 'a'; +set b = 'b'; +set c = 'c'; + +select 'a' as "expected", $a as "actual", + sqlpage.url_encode(upper(col || sqlpage.url_encode($b))) as "encoded", + $c as "trailing" +from (select 'col' as col) values_table; + +select 'COLB' as "expected", $a as "leading", + sqlpage.url_encode(upper(col || sqlpage.url_encode($b))) as "actual", + $c as "trailing" +from (select 'col' as col) values_table; + +select 'c' as "expected", $a as "leading", + sqlpage.url_encode(upper(col || sqlpage.url_encode($b))) as "encoded", + $c as "actual" +from (select 'col' as col) values_table; diff --git a/tests/sql_test_files/data/concat_null_ignored_nogeneric_nomysql_nosnowflake.sql b/tests/sql_test_files/data/concat_null_ignored_nogeneric_nomysql_nosnowflake.sql new file mode 100644 index 00000000..c0b673b1 --- /dev/null +++ b/tests/sql_test_files/data/concat_null_ignored_nogeneric_nomysql_nosnowflake.sql @@ -0,0 +1,2 @@ +select 'x' as expected, concat('x', null) as actual; +select '%2F' as expected, sqlpage.url_encode(concat('/', null)) as actual; diff --git a/tests/sql_test_files/data/concat_null_propagated_noduckdb_nomssql_nooracle_nopostgres_nosqlite.sql b/tests/sql_test_files/data/concat_null_propagated_noduckdb_nomssql_nooracle_nopostgres_nosqlite.sql new file mode 100644 index 00000000..bda1a899 --- /dev/null +++ b/tests/sql_test_files/data/concat_null_propagated_noduckdb_nomssql_nooracle_nopostgres_nosqlite.sql @@ -0,0 +1,2 @@ +select null as expected, concat('x', null) as actual; +select null as expected, sqlpage.url_encode(concat('/', null)) as actual; diff --git a/tests/sql_test_files/data/concat_operator_null_ignored_noduckdb_nogeneric_nomysql_nopostgres_nosnowflake_nosqlite.sql b/tests/sql_test_files/data/concat_operator_null_ignored_noduckdb_nogeneric_nomysql_nopostgres_nosnowflake_nosqlite.sql new file mode 100644 index 00000000..0bfd8c2b --- /dev/null +++ b/tests/sql_test_files/data/concat_operator_null_ignored_noduckdb_nogeneric_nomysql_nopostgres_nosnowflake_nosqlite.sql @@ -0,0 +1,4 @@ +select '/' as expected, '/' || null as actual; + +select '%2F' as expected, sqlpage.url_encode('/' || nullable_col) as actual +from (select cast(null as varchar(1)) as nullable_col) input_rows; diff --git a/tests/sql_test_files/data/concat_str_in_pseudofunction.sql b/tests/sql_test_files/data/concat_str_in_pseudofunction.sql index 724e44b1..20e58d13 100644 --- a/tests/sql_test_files/data/concat_str_in_pseudofunction.sql +++ b/tests/sql_test_files/data/concat_str_in_pseudofunction.sql @@ -1,3 +1,2 @@ select '%2F1' as expected, sqlpage.url_encode('/' || $x) as actual; select '%2F1' as expected, sqlpage.url_encode(CONCAT('/', $x)) as actual; -select NULL as expected, sqlpage.url_encode(CONCAT('/', $thisisnull)) as actual; diff --git a/tests/sql_test_files/data/delayed_function_determinism.sql b/tests/sql_test_files/data/delayed_function_determinism.sql new file mode 100644 index 00000000..4215415a --- /dev/null +++ b/tests/sql_test_files/data/delayed_function_determinism.sql @@ -0,0 +1,11 @@ +set x = (select sqlpage.fetch('invalid') where 1 = 0); +select null as expected, $x as actual; + +set x = (select sqlpage.url_encode(' ') where 1 = 1); +select '%20' as expected, $x as actual; + +set x = (select sqlpage.url_encode(null) where 1 = 1); +select null as expected, $x as actual; + +select '%20' as expected, sqlpage.url_encode(' ') as actual +from (select 1 as n union all select 2 as n) result_rows; diff --git a/tests/sql_test_files/data/delayed_function_parentheses.sql b/tests/sql_test_files/data/delayed_function_parentheses.sql new file mode 100644 index 00000000..641fbf88 --- /dev/null +++ b/tests/sql_test_files/data/delayed_function_parentheses.sql @@ -0,0 +1,6 @@ +select + (sqlpage.url_encode(subquery.space)) as actual, + '%20' as expected +from ( + select ' ' as space +) subquery; diff --git a/tests/sql_test_files/data/nested_sqlpage_functions.sql b/tests/sql_test_files/data/nested_sqlpage_functions.sql index 49eeb48e..796abb50 100644 --- a/tests/sql_test_files/data/nested_sqlpage_functions.sql +++ b/tests/sql_test_files/data/nested_sqlpage_functions.sql @@ -1 +1,9 @@ select 'It%20works%20%21' as expected, sqlpage.url_encode(sqlpage.read_file_as_text('tests/it_works.txt')) as actual; + +select 'It%20works%20%21' as expected, + sqlpage.url_encode(sqlpage.read_file_as_text(path)) as actual +from (select 'tests/it_works.txt' as path) paths; + +select '%2Fvalue' as expected, + coalesce(sqlpage.url_encode('/' || value), '') as actual +from (select 'value' as value) value_rows; diff --git a/tests/sql_test_files/data/set_multiple_rows_noduckdb_nogeneric_nomssql_nomysql_nooracle_nopostgres_nosnowflake.sql b/tests/sql_test_files/data/set_multiple_rows_noduckdb_nogeneric_nomssql_nomysql_nooracle_nopostgres_nosnowflake.sql new file mode 100644 index 00000000..61a8c3b1 --- /dev/null +++ b/tests/sql_test_files/data/set_multiple_rows_noduckdb_nogeneric_nomssql_nomysql_nooracle_nopostgres_nosnowflake.sql @@ -0,0 +1,7 @@ +set actual = ( + select 'first' as value + union all + select 'second' as value +); + +select 'first' as expected, $actual as actual; diff --git a/tests/sql_test_files/data/set_variable_to_sqlpage_function_where_false.sql b/tests/sql_test_files/data/set_variable_to_sqlpage_function_where_false.sql new file mode 100644 index 00000000..e5f0b212 --- /dev/null +++ b/tests/sql_test_files/data/set_variable_to_sqlpage_function_where_false.sql @@ -0,0 +1,11 @@ +set my_var = ( + select sqlpage.fetch('this invalid fetch should never be executed') + where 1 = 0 +); +select null as expected, $my_var as actual; + +set norow = ( + select sqlpage.fetch('this invalid fetch should never be executed') + where 1 = 0 +); +select null as expected, $norow as actual; diff --git a/tests/sql_test_files/data/wildcard_before_computed_column.sql b/tests/sql_test_files/data/wildcard_before_computed_column.sql new file mode 100644 index 00000000..ea81793f --- /dev/null +++ b/tests/sql_test_files/data/wildcard_before_computed_column.sql @@ -0,0 +1,5 @@ +select + users.*, + 'Alice%20Smith' as expected, + sqlpage.url_encode(name) as actual +from (select 'first' as first, 'Alice Smith' as name) users; diff --git a/tests/sql_test_files/mod.rs b/tests/sql_test_files/mod.rs index a3b21e34..f4b32540 100644 --- a/tests/sql_test_files/mod.rs +++ b/tests/sql_test_files/mod.rs @@ -121,21 +121,9 @@ async fn run_sql_test( } fn format_error(obj: &serde_json::Map) -> Option { - if obj.get("component").and_then(|v| v.as_str()) != Some("error") { - return None; - } - let mut msg = String::new(); - if let Some(desc) = obj.get("description").and_then(|v| v.as_str()) { - msg.push_str(desc); - } - if let Some(bt) = obj.get("backtrace").and_then(|v| v.as_array()) { - for frame in bt { - if let Some(s) = frame.as_str() { - msg.push_str(&format!("\n {}", s)); - } - } - } - Some(msg) + obj.get("error") + .and_then(|value| value.as_str()) + .map(str::to_owned) } fn assert_json_test(body: &str, test_file: &std::path::Path) { @@ -155,7 +143,11 @@ fn assert_json_test(body: &str, test_file: &std::path::Path) { }; if let Some(err) = format_error(obj) { - panic!("Error in {}:\n{}", test_file.display(), err); + panic!( + "\n{}: response contains an error:\n\n{}", + test_file.display(), + err + ); } let actual = obj @@ -182,7 +174,7 @@ fn assert_json_test(body: &str, test_file: &std::path::Path) { if expected.is_empty() && expected_contains.is_empty() { panic!( - "No expected values found in {}: {}", + "{}: No `expected` column returned: \n{:#}", test_file.display(), row ); @@ -218,7 +210,20 @@ fn assert_html_test(body: &str, test_file: &std::path::Path, stem: &str) { ); if stem.starts_with("error_") { - let expected = stem.strip_prefix("error_").unwrap().replace('_', " "); + let mut expected = stem.strip_prefix("error_").unwrap().to_owned(); + for database in [ + "sqlite", + "duckdb", + "oracle", + "postgres", + "mysql", + "mssql", + "snowflake", + "generic", + ] { + expected = expected.replace(&format!("_no{database}"), ""); + } + let expected = expected.replace('_', " "); assert!( body.to_lowercase().contains(&expected.to_lowercase()), "Should contain '{}': {}", diff --git a/tests/uploads/upload_file_runsql_test.sql b/tests/uploads/upload_file_runsql_test.sql index 8f99a840..fe74c62a 100644 --- a/tests/uploads/upload_file_runsql_test.sql +++ b/tests/uploads/upload_file_runsql_test.sql @@ -1 +1 @@ -select 'dynamic' as component, sqlpage.run_sql('tests/uploads/upload_file_test.sql') as properties; \ No newline at end of file +select 'dynamic' as component, sqlpage.run_sql('tests/uploads/upload_file_test.sql') as properties;