From 0c8877fe36a8811a54d449665be4f1e216e0153c Mon Sep 17 00:00:00 2001 From: Jeff Roberts Date: Wed, 23 Sep 2026 11:31:38 -0500 Subject: [PATCH] Five SQL and name forms real applications write USE IN SELECT('alias'), ?name parameters in SELECT-SQL, INSERT INTO ... SELECT, UNION [ALL], and a quoted name after DO and CREATE TABLE (with its NAME clause). Every one was found in a 60,000-line CodeMine application, none of them rare there, and each was measured in Visual FoxPro 9 before being written: ?name is taken only by SELECT and is an ordinary expression, field first; INSERT ... SELECT goes by position to the fields named; a UNION's columns are the first SELECT's and its rows fold unless every UNION said ALL. Two faults in the VM came out on the way and are fixed here because the new forms could not be tested without them: a select list of nothing but `*` read blank rows from a freshly opened table, since no field read had fetched the page (SqlTouch now reads each starred source); and INTO ARRAY put the program back in the last source's work area rather than its own, because the area was noted after the sources were opened. The sql_forms golden is written from the product. Two gaps it exposed are recorded in the parity plan rather than fixed: a LOCAL shadows a field of the same name here where the product reads the field, and _TALLY is never set. Co-Authored-By: Claude Fable 5.1 --- crates/foxvm/src/ast.rs | 15 ++ crates/foxvm/src/bytecode.rs | 4 + crates/foxvm/src/compiler.rs | 197 +++++++++++++++++ crates/foxvm/src/parser.rs | 79 ++++++- crates/foxvm/src/vm.rs | 29 ++- crates/foxvm/tests/common/mod.rs | 38 ++-- .../foxvm/tests/programs/sql_forms.expected | 17 ++ crates/foxvm/tests/programs/sql_forms.prg | 72 ++++++ crates/foxvm/tests/sql_forms.rs | 205 ++++++++++++++++++ docs/parity-plan.md | 11 + 10 files changed, 649 insertions(+), 18 deletions(-) create mode 100644 crates/foxvm/tests/programs/sql_forms.expected create mode 100644 crates/foxvm/tests/programs/sql_forms.prg create mode 100644 crates/foxvm/tests/sql_forms.rs diff --git a/crates/foxvm/src/ast.rs b/crates/foxvm/src/ast.rs index d462eda..08d1b56 100644 --- a/crates/foxvm/src/ast.rs +++ b/crates/foxvm/src/ast.rs @@ -1042,6 +1042,9 @@ pub enum InsertSource { /// `FROM ARRAY a`, `FROM MEMVAR` and `FROM NAME oRec` read the record from the same three /// places GATHER does, which is what they are: an APPEND BLANK and a GATHER. From(ScatterWhere), + /// `INSERT INTO t [(fields)] SELECT ...`: a record per row of the query, the columns going + /// to the fields named, or to the table's fields in order when none are. + Query(Box), } /// What a copy makes. @@ -1210,9 +1213,21 @@ pub struct Query { pub having: Option, pub order_by: Vec, pub into: QueryInto, + /// `UNION [ALL] SELECT ...`: the next query whose rows go under this one's. The ORDER BY and + /// the INTO belong to the union as a whole and are kept on the first query, however many + /// SELECTs down the chain they were written after. + pub union: Option>, pub span: Span, } +/// One `UNION` step of a query. +#[derive(Debug, Clone, PartialEq)] +pub struct QueryUnion { + /// `UNION ALL`: rows that appear in both sides are kept twice. Without it they are folded. + pub all: bool, + pub query: Query, +} + /// One item of the select list. #[derive(Debug, Clone, PartialEq)] pub enum QueryColumn { diff --git a/crates/foxvm/src/bytecode.rs b/crates/foxvm/src/bytecode.rs index 2579168..9dcfcf0 100644 --- a/crates/foxvm/src/bytecode.rs +++ b/crates/foxvm/src/bytecode.rs @@ -667,6 +667,10 @@ pub enum Instr { /// by where it is in the FROM clause rather than by what it is called, because a source /// named by an expression is only called something once it has been opened. SelectSource(u16), + /// [] -> [] Selects the nth source and reads its current record into the buffer. A `*` + /// in the select list takes the record straight from there, and nothing else in a query + /// with only a `*` would have asked the host for the page. + SqlTouch(u16), /// [] -> [] The nth FROM source has no record matching the row being built, so it is /// parked past its last record and every field of it reads as .NULL. until it is moved /// again. That is what an outer join puts on the side that missed. diff --git a/crates/foxvm/src/compiler.rs b/crates/foxvm/src/compiler.rs index bb342dd..e07acde 100644 --- a/crates/foxvm/src/compiler.rs +++ b/crates/foxvm/src/compiler.rs @@ -305,6 +305,14 @@ fn leading_params(body: &Block) -> ParamDecl { /// takes, so both branches cannot be evaluated first. const COMPILED_FUNCTIONS: &[&str] = &["IIF"]; +/// Where a row copied out of a cursor goes: a table named outright, or one whose name a +/// temporary holds because the program worked it out. +#[derive(Clone, Copy)] +enum RowTarget { + Named(u32), + Worked(u32), +} + struct LoopCtx { breaks: Vec, continues: Vec, @@ -1973,6 +1981,10 @@ impl ModuleCompiler { fb.emit(Instr::FlushRecord); } StmtKind::Insert { alias, named, fields, source } => { + if let InsertSource::Query(q) = source { + self.insert_select(fb, alias, named.as_ref(), fields, q); + return; + } // An INSERT is an APPEND BLANK and a REPLACE of each value. The values are worked // out first, in the work area the statement was written in - `SCAN ... INSERT INTO // other VALUES (custno) ... ENDSCAN` reads custno from the table being scanned - @@ -2005,6 +2017,7 @@ impl ModuleCompiler { self.expr(fb, e); source_from = Some(2); } + InsertSource::Query(_) => unreachable!("INSERT ... SELECT was compiled above"), } // the name is worked out after the values, so it is read in the work area the // statement was written in, and then selects the one being written to @@ -2046,6 +2059,7 @@ impl ModuleCompiler { fb.emit(Instr::FlushRecord); } (InsertSource::From(_), None) => unreachable!("every FROM source has a number"), + (InsertSource::Query(_), _) => unreachable!("INSERT ... SELECT was compiled above"), } fb.emit(Instr::LoadLocal(saved)); fb.emit(Instr::SelectArea(None)); @@ -2269,6 +2283,9 @@ impl ModuleCompiler { // what happens to the rows once they are gathered, and that is the plan in `query.rs`. fn query(&mut self, fb: &mut FuncBuilder, q: &Query) { + if q.union.is_some() { + return self.union_query(fb, q); + } // `ORDER BY 2` and `GROUP BY 2` name the second column of the result, not the number // two, so they are put back to the expression that column is before anything else runs let resolved = by_column_number(q); @@ -2477,6 +2494,172 @@ impl ModuleCompiler { aliases } + /// `INSERT INTO t [(fields)] SELECT ...`. + /// + /// The query runs first, into a cursor of its own, and then each of its rows is a record + /// added to the table: the row taken whole as an array and put down with GATHER, which is + /// what makes the columns go by position - to the fields the statement names, in the order + /// it names them, or to the table's fields in order when it names none. That is what the + /// product does with it. The work area the statement was written in is selected again at + /// the end, as it is after every other INSERT. + fn insert_select(&mut self, fb: &mut FuncBuilder, alias: &Name, named: Option<&Expr>, fields: &[NameRef], q: &Query) { + let saved = fb.temp("AREA"); + let (select, _) = builtins::lookup("SELECT").expect("SELECT is a builtin"); + fb.emit(Instr::CallBuiltin { id: select, argc: 0 }); + fb.emit(Instr::StoreLocal(saved)); + // the table's name is worked out where the statement stands, before the query moves + let target = match named { + Some(e) => { + self.expr(fb, e); + let slot = fb.temp("INTO"); + fb.emit(Instr::StoreLocal(slot)); + RowTarget::Worked(slot) + } + None => RowTarget::Named(self.name(&alias.upper)), + }; + let mut names = Vec::new(); + for field in fields { + match field { + NameRef::Named(n) => names.push(n.clone()), + NameRef::Computed(e) => { + self.error(e.span, "INSERT ... SELECT names its fields outright; a field worked out from an expression is not supported"); + return; + } + } + } + self.sub_counter += 1; + let rows = format!("__INS{}", self.sub_counter); + let mut inner = q.clone(); + inner.into = QueryInto::Cursor(NameRef::Named(Name::new(rows.clone(), q.span))); + self.query(fb, &inner); + self.append_rows(fb, &rows, target, &names); + self.close_alias(fb, &rows); + fb.emit(Instr::LoadLocal(saved)); + fb.emit(Instr::SelectArea(None)); + } + + /// `SELECT ... UNION [ALL] SELECT ... [ORDER BY] [INTO]`. + /// + /// Each SELECT of the chain runs into a cursor of its own, the rows of the later ones are + /// added under the first's, and the whole is then one more query over that cursor: `SELECT + /// [DISTINCT] * FROM it`, carrying the ORDER BY and the INTO the union was written with. + /// The columns are named by the first SELECT, which is where the product takes them from + /// too. Rows the same on both sides are folded unless every UNION said ALL. + fn union_query(&mut self, fb: &mut FuncBuilder, q: &Query) { + let saved = fb.temp("AREA"); + let (select, _) = builtins::lookup("SELECT").expect("SELECT is a builtin"); + fb.emit(Instr::CallBuiltin { id: select, argc: 0 }); + fb.emit(Instr::StoreLocal(saved)); + + let mut parts: Vec<(&Query, bool)> = vec![(q, true)]; + let mut next = q.union.as_deref(); + while let Some(step) = next { + parts.push((&step.query, step.all)); + next = step.query.union.as_deref(); + } + self.sub_counter += 1; + let first = format!("__UNI{}", self.sub_counter); + let first_name = self.name(&first); + for (i, (part, _)) in parts.iter().enumerate() { + let alias = if i == 0 { first.clone() } else { format!("{first}_{i}") }; + let mut inner = (*part).clone(); + inner.union = None; + inner.order_by = Vec::new(); + inner.top = None; + inner.top_percent = false; + inner.into = QueryInto::Cursor(NameRef::Named(Name::new(alias.clone(), part.span))); + self.query(fb, &inner); + if i > 0 { + self.append_rows(fb, &alias, RowTarget::Named(first_name), &[]); + self.close_alias(fb, &alias); + } + } + // the statement's own work area is where the whole is asked from, so that what the + // query leaves selected afterwards is what any query leaves selected + fb.emit(Instr::LoadLocal(saved)); + fb.emit(Instr::SelectArea(None)); + let whole = Query { + distinct: parts.iter().any(|(_, all)| !all), + top: q.top.clone(), + top_percent: q.top_percent, + columns: vec![QueryColumn::All(None)], + from: vec![QuerySource { + table: first.clone(), + table_expr: None, + alias: Name::new(first.clone(), q.span), + join: JoinKind::Inner, + joined: false, + on: None, + span: q.span, + }], + where_: None, + group_by: Vec::new(), + having: None, + order_by: q.order_by.clone(), + into: q.into.clone(), + union: None, + span: q.span, + }; + self.query(fb, &whole); + self.close_alias(fb, &first); + } + + /// A record added to `target` for every record of the cursor `from`, each taken whole as an + /// array and put down by position. `fields` are the fields the row goes to, or none for the + /// table's own in order. The cursor `from` is selected afterwards, on EOF. + fn append_rows(&mut self, fb: &mut FuncBuilder, from: &str, target: RowTarget, fields: &[Name]) { + let row = fb.temp("ROW"); + let source = self.name(from); + fb.emit(Instr::SelectArea(Some(source))); + fb.emit(Instr::Go(GoTarget::Top)); + let top = fb.pc(); + self.emit_eof(fb); + let end = fb.emit(Instr::JumpIfTrue(0)); + fb.emit(Instr::Scatter { to: 0, except: false, count: 0, blank: false }); + fb.emit(Instr::StoreLocal(row)); + match target { + RowTarget::Named(n) => fb.emit(Instr::SelectArea(Some(n))), + RowTarget::Worked(slot) => { + fb.emit(Instr::LoadLocal(slot)); + fb.emit(Instr::SelectArea(None)) + } + }; + fb.emit(Instr::AppendBlank); + if fields.is_empty() { + // the table's fields in order, which is what GATHER FROM ARRAY does with a row + fb.emit(Instr::LoadLocal(row)); + fb.emit(Instr::Gather { to: 0, except: false, count: 0 }); + } else { + // the fields the statement names, in the order it names them - GATHER FIELDS would + // take the table's order, which is not the same thing when they differ + let array = Name::new(fb.locals[row as usize].clone(), fields[0].span); + for (i, field) in fields.iter().enumerate() { + let at = Expr::new(ExprKind::Num((i + 1) as f64, 1, 0), field.span); + let element = Expr::new( + ExprKind::Index { base: Box::new(Expr::new(ExprKind::Var(array.clone()), field.span)), args: vec![at] }, + field.span, + ); + self.expr(fb, &element); + let m = self.member(field); + fb.emit(Instr::ReplaceField { field: m, additive: false }); + } + } + fb.emit(Instr::FlushRecord); + fb.emit(Instr::SelectArea(Some(source))); + self.emit_skip_one(fb); + fb.emit(Instr::Jump(top)); + fb.patch_here(end); + } + + /// `USE IN alias`: the cursor is closed and the work area that was selected stays selected. + fn close_alias(&mut self, fb: &mut FuncBuilder, alias: &str) { + let area = self.constant(Constant::Str(alias.to_string())); + fb.emit(Instr::Const(area)); + let none = self.constant(Constant::Str(String::new())); + fb.emit(Instr::Const(none)); + fb.emit(Instr::Use { alias: None, named_alias: false, exclusive: false, online: false, in_area: true }); + } + /// Code that raises one of Visual FoxPro's own errors where it stands. It is what a statement /// the compiler can see is wrong compiles to when the product only complains about it as it /// runs, which is most of them. @@ -2654,6 +2837,20 @@ impl ModuleCompiler { /// The innermost body: test the WHERE clause, then push one row. fn query_row(&mut self, fb: &mut FuncBuilder, q: &Query, pushed: &[Pushed<'_>]) { + // a `*` reads each source's record out of its buffer when the row is made, and only a + // field read fills the buffer; a select list of nothing but `*` reads no field, so the + // sources it stands for are read here first. The innermost source is selected again + // afterwards, which is where the WHERE clause and the select list expect to stand. + let stars: Vec<&Option> = q.columns.iter().filter_map(|c| if let QueryColumn::All(a) = c { Some(a) } else { None }).collect(); + if !stars.is_empty() { + for (i, source) in q.from.iter().enumerate() { + let wanted = stars.iter().any(|a| a.as_ref().is_none_or(|a| a.upper.eq_ignore_ascii_case(&source.alias.upper))); + if wanted { + fb.emit(Instr::SqlTouch(i as u16)); + } + } + fb.emit(Instr::SelectSource((q.from.len() - 1) as u16)); + } let skip = q.where_.as_ref().map(|w| { self.expr(fb, w); fb.emit(Instr::JumpIfFalse(0)) diff --git a/crates/foxvm/src/parser.rs b/crates/foxvm/src/parser.rs index 2c0b0d2..50b5448 100644 --- a/crates/foxvm/src/parser.rs +++ b/crates/foxvm/src/parser.rs @@ -651,6 +651,11 @@ impl Parser { if matches!(self.peek_kind(), TokKind::Num(..)) { return Ok(NameRef::Computed(self.expr()?)); } + // `IN SELECT('orders')`: a name with a bracket after it is a call that answers with + // the work area, not an alias called SELECT + if matches!(self.peek_kind(), TokKind::Ident(_)) && matches!(self.peek_at(1).kind, TokKind::LParen) { + return Ok(NameRef::Computed(self.expr()?)); + } self.name_ref("alias") } @@ -2418,6 +2423,14 @@ impl Parser { let (args, in_prog) = self.do_clauses()?; return Ok(Some(StmtKind::DoExpr { name, args, in_prog })); } + // `DO "C:\tools\run.prg"`: a path with spaces or a drive in it is written in quotes, + // and a quoted name is the same thing as one worked out + if let TokKind::Str(text) = self.peek_kind() { + let tok = self.advance(); + let name = Expr::new(ExprKind::Str(text), tok.span); + let (args, in_prog) = self.do_clauses()?; + return Ok(Some(StmtKind::DoExpr { name, args, in_prog })); + } let name = self.file_name("program name")?; let (args, in_prog) = self.do_clauses()?; Ok(Some(StmtKind::Do { name, args, in_prog })) @@ -2800,6 +2813,11 @@ impl Parser { let e = self.expr()?; self.expect(&TokKind::RParen, "')'")?; e + } else if let TokKind::Str(text) = self.peek_kind() { + // `CREATE TABLE 'APPREG01.DBF' NAME 'APPREG01' (...)` is what a generated + // data-definition program writes: the file in quotes, then the long name + let tok = self.advance(); + Expr::new(ExprKind::Str(text), tok.span) } else { // a bare name, with the extension VFP lets it carry: CREATE TABLE people.dbf let mut name = self.expect_ident("table name")?.text; @@ -2809,6 +2827,12 @@ impl Parser { } Expr::new(ExprKind::Str(name), first.span) }; + // `NAME LongTableName` is what the database lists the table as. The container here + // lists a table by its file stem, which is what every program that writes this + // clause names it anyway, so the long name is read and not kept. + if self.eat_kw("NAME") { + let _ = self.name_ref("long table name")?; + } let free = self.eat_kw("FREE"); if let Some(from_array) = self.from_array_clause()? { self.skip_line_keep_newline(); @@ -3050,10 +3074,15 @@ impl Parser { } self.expect(&TokKind::RParen, "')'")?; } + // `INSERT INTO t (fields) SELECT ...`: the rows come from a query of the program's own + if self.is_kw("SELECT") { + let query = self.query(first.span)?; + return Ok(StmtKind::Insert { alias, named, fields, source: InsertSource::Query(Box::new(query)) }); + } if !self.eat_kw("VALUES") { let what = self.peek().ident().map(|s| s.to_ascii_uppercase()).unwrap_or_default(); self.skip_line_keep_newline(); - return Err(self.error(first.span, format!("INSERT ... {what} is not supported; only VALUES and FROM are"))); + return Err(self.error(first.span, format!("INSERT ... {what} is not supported; only VALUES, FROM and SELECT are"))); } self.expect(&TokKind::LParen, "'('")?; let mut values = Vec::new(); @@ -3612,6 +3641,7 @@ impl Parser { having: None, order_by: Vec::new(), into: QueryInto::Table(into), + union: None, span: first.span, })))) } @@ -5566,6 +5596,37 @@ impl Parser { } let having = if self.eat_kw("HAVING") { Some(self.expr()?) } else { None }; + // `UNION [ALL] SELECT ...`: the next query is read whole, and its ORDER BY and INTO - + // which the syntax puts after the last SELECT - belong to the union, so they come up + // here and the query that carried them is left with none + if self.eat_kw("UNION") { + let all = self.eat_kw("ALL"); + if !self.is_kw("SELECT") { + return Err(self.expected("SELECT")); + } + self.advance(); + let mut rest = self.query_body(start)?; + let order_by = std::mem::take(&mut rest.order_by); + let into = std::mem::replace(&mut rest.into, QueryInto::Browse); + if from.is_empty() { + return Err(self.error(start, "SELECT needs a FROM clause")); + } + return Ok(Query { + distinct, + top, + top_percent, + columns, + from, + where_, + group_by, + having, + order_by, + into, + union: Some(Box::new(QueryUnion { all, query: rest })), + span: start.to(self.prev_span()), + }); + } + let mut order_by = Vec::new(); if self.eat_kw("ORDER") { self.eat_kw("BY"); @@ -5622,6 +5683,7 @@ impl Parser { having, order_by, into, + union: None, span: start.to(self.prev_span()), }) } @@ -6045,6 +6107,9 @@ impl Parser { } match self.peek_kind() { TokKind::Num(..) => self.expr_or_recover(), + // `USE IN SELECT('orders')` is how a program closes a table it may or may not have + // open: a name with a bracket after it is a call, whose answer is the work area + TokKind::Ident(_) if matches!(self.peek_at(1).kind, TokKind::LParen) => self.expr_or_recover(), TokKind::Ident(name) => { self.advance(); Expr::new(ExprKind::Str(name), span) @@ -7600,6 +7665,18 @@ impl Parser { let tok = self.peek().clone(); let kind = match tok.kind.clone() { TokKind::Num(n, chars, decimals) => ExprKind::Num(n, chars, decimals), + // `WHERE clogon = ?lcUser` inside a SELECT-SQL: the mark is how a parameter is + // written for SQL pass-through, and a local query takes it too. Measured in Visual + // FoxPro 9: what follows it is an ordinary expression, and a name that is both a + // field of a source and a variable is the field, as it is without the mark - + // `?m.name` is how the variable is asked for. INSERT, UPDATE and DELETE - SQL do + // not take it: each is a syntax error in the product. + TokKind::Question if self.in_query > 0 => { + self.advance(); + let e = self.postfix_expr()?; + let span = tok.span.to(e.span); + return Ok(Expr::new(e.kind, span)); + } // `$` in front of a number is money written down; between two values it is the // substring operator, and that is the other place this token turns up TokKind::Dollar if matches!(self.peek_at(1).kind, TokKind::Num(..) | TokKind::Minus) => { diff --git a/crates/foxvm/src/vm.rs b/crates/foxvm/src/vm.rs index d15b58c..135edd7 100644 --- a/crates/foxvm/src/vm.rs +++ b/crates/foxvm/src/vm.rs @@ -176,6 +176,11 @@ struct Fiber { query_unwind: Option, /// Sources opened for a query that has not begun gathering yet. pending_sources: Vec, + /// The work area that was selected when the query's first source was opened, which is the + /// one the statement was written in. Opening a source selects it, so by the time the run + /// begins the current area is a source's, and INTO ARRAY has to put the program back where + /// it stood rather than there. + pending_area: Option, /// The routine a `RETURN TO` is on its way to, while the frames between here and it are /// being unwound. Empty text is `RETURN TO MASTER`: the program the run started in. return_to: Option, @@ -5525,6 +5530,20 @@ impl Vm { } self.sql_named = None; } + Instr::SqlTouch(which) => { + let area = fb + .query + .as_ref() + .and_then(|run| run.sources.get(*which as usize)) + .map(|source| source.area) + .ok_or_else(RtError::no_table_open)?; + self.data.select(&AreaRef::Number(area))?; + // a page that has to be fetched comes back to this same instruction, which + // selects the same source again and so hands the page to the cursor that asked + if let Some(req) = self.ensure_record(fb)? { + return Ok(self.ask(fb, req)); + } + } Instr::SelectSource(which) | Instr::JoinMiss(which) => { let area = fb .query @@ -8511,6 +8530,11 @@ impl Vm { /// Visual FoxPro does, and what keeps two cursors from answering to one alias - and its /// record pointer is put back where it was when the query finishes. fn sql_open(&mut self, fb: &mut Fiber, named: &str, alias: &str) -> Result, RtError> { + // where the statement stands, noted before the first source moves the selection; an + // open that has to wait comes back through here and must not note it twice + if fb.query.is_none() && fb.pending_sources.is_empty() && fb.pending_area.is_none() { + fb.pending_area = Some(self.data.current_area()); + } // a FROM that reaches into a container needs the container open first, as USE does if self.db_io.is_some() { let reply = fb.data_reply.take(); @@ -8600,6 +8624,9 @@ impl Vm { } } + // opening the sources selected the last of them; the program stood where the first + // open found it, and that is where INTO ARRAY leaves it again + let saved_area = fb.pending_area.take().unwrap_or_else(|| self.data.current_area()); fb.query = Some(QueryRun { plan: std::rc::Rc::new(plan), columns, @@ -8609,7 +8636,7 @@ impl Vm { having: None, top, sources, - saved_area: self.data.current_area(), + saved_area, frame: fb.frames.len().saturating_sub(1), into_alias: into, }); diff --git a/crates/foxvm/tests/common/mod.rs b/crates/foxvm/tests/common/mod.rs index 0a5e450..4b604db 100644 --- a/crates/foxvm/tests/common/mod.rs +++ b/crates/foxvm/tests/common/mod.rs @@ -110,6 +110,26 @@ pub fn args(a: &[Arg]) -> String { .join(", ") } +pub fn query(q: &Query) -> String { + format!( + "(select{}{} ({}) (from {}){}{}{}{}{})", + if q.distinct { " distinct" } else { "" }, + q.top.as_ref().map(|t| format!(" top {}", expr(t))).unwrap_or_default(), + q.columns.iter().map(query_column).collect::>().join(" "), + q.from.iter().map(|f| format!("{}:{}", f.table, f.alias.upper)).collect::>().join(" "), + q.where_.as_ref().map(|w| format!(" (where {})", expr(w))).unwrap_or_default(), + if q.group_by.is_empty() { String::new() } else { format!(" (group {})", q.group_by.iter().map(expr).collect::>().join(" ")) }, + q.union.as_ref().map(|u| format!(" (union{} {})", if u.all { "-all" } else { "" }, query(&u.query))).unwrap_or_default(), + if q.order_by.is_empty() { String::new() } else { format!(" (order {})", q.order_by.iter().map(|o| format!("{}{}", expr(&o.expr), if o.descending { " desc" } else { "" })).collect::>().join(" ")) }, + match &q.into { + QueryInto::Browse => String::new(), + QueryInto::Cursor(n) => format!(" (into-cursor {})", name_ref(n)), + QueryInto::Table(e) => format!(" (into-table {})", expr(e)), + QueryInto::Array(e) => format!(" (into-array {})", expr(e)), + }, + ) +} + pub fn stmt(s: &Stmt) -> String { match &s.kind { StmtKind::Assign { target, value } => format!("(= {} {})", expr(target), expr(value)), @@ -407,6 +427,7 @@ pub fn stmt(s: &Stmt) -> String { InsertSource::From(ScatterWhere::Memvar) => "from-memvar".into(), InsertSource::From(ScatterWhere::Array(e)) => format!("from-array {}", expr(e)), InsertSource::From(ScatterWhere::Name(e)) => format!("from-name {}", expr(e)), + InsertSource::Query(q) => query(q), } ), StmtKind::ClearAll { tables } => format!("(clear {})", if *tables { "all" } else { "memory" }), @@ -446,22 +467,7 @@ pub fn stmt(s: &Stmt) -> String { "(zap{})", area.as_ref().map(|a| format!(" in {}", name_ref(a))).unwrap_or_default() ), - StmtKind::Query(q) => format!( - "(select{}{} ({}) (from {}){}{}{}{})", - if q.distinct { " distinct" } else { "" }, - q.top.as_ref().map(|t| format!(" top {}", expr(t))).unwrap_or_default(), - q.columns.iter().map(query_column).collect::>().join(" "), - q.from.iter().map(|f| format!("{}:{}", f.table, f.alias.upper)).collect::>().join(" "), - q.where_.as_ref().map(|w| format!(" (where {})", expr(w))).unwrap_or_default(), - if q.group_by.is_empty() { String::new() } else { format!(" (group {})", q.group_by.iter().map(expr).collect::>().join(" ")) }, - if q.order_by.is_empty() { String::new() } else { format!(" (order {})", q.order_by.iter().map(|o| format!("{}{}", expr(&o.expr), if o.descending { " desc" } else { "" })).collect::>().join(" ")) }, - match &q.into { - QueryInto::Browse => String::new(), - QueryInto::Cursor(n) => format!(" (into-cursor {})", name_ref(n)), - QueryInto::Table(e) => format!(" (into-table {})", expr(e)), - QueryInto::Array(e) => format!(" (into-array {})", expr(e)), - }, - ), + StmtKind::Query(q) => query(q), StmtKind::Throw(None) => "(throw)".into(), StmtKind::Throw(Some(e)) => format!("(throw {})", expr(e)), StmtKind::NoDefault => "(nodefault)".into(), diff --git a/crates/foxvm/tests/programs/sql_forms.expected b/crates/foxvm/tests/programs/sql_forms.expected new file mode 100644 index 0000000..74075c7 --- /dev/null +++ b/crates/foxvm/tests/programs/sql_forms.expected @@ -0,0 +1,17 @@ +used before .T. +used after .F. +still here +param variable 1 2 Q1 +param m. 1 2 +insert select 2 PARTS + row BBB 2 + row CCC 3 +insert select whole 1 AAA 1 PARTS +union all 3 Q2 + row 1 top CCC + row 2 rest AAA + row 2 rest BBB +union 3 +union all again 5 +into array 2 AAA CCC PARTS +helper ran x diff --git a/crates/foxvm/tests/programs/sql_forms.prg b/crates/foxvm/tests/programs/sql_forms.prg new file mode 100644 index 0000000..cc3fa7e --- /dev/null +++ b/crates/foxvm/tests/programs/sql_forms.prg @@ -0,0 +1,72 @@ +* COVERS: SELECT - SQL, INSERT - SQL, UPDATE - SQL, DELETE - SQL, USE, DO, CREATE TABLE - SQL +* +* Five spellings a real application uses that the reference examples do not: USE IN +* SELECT('alias'), ?lcName parameters in local SQL, INSERT INTO ... SELECT, UNION [ALL], and a +* name in quotes after DO and CREATE TABLE. Every answer below was measured in Visual FoxPro 9. +CREATE CURSOR parts (code C(3), qty N(3)) +INSERT INTO parts VALUES ("AAA", 1) +INSERT INTO parts VALUES ("BBB", 2) +INSERT INTO parts VALUES ("CCC", 3) + +* USE IN SELECT(): closes the table when it is open, and does nothing when it is not +CREATE CURSOR other (a C(1)) +? "used before", USED("other") +USE IN SELECT("other") +? "used after", USED("other") +USE IN SELECT("other") +? "still here", ALIAS() + +* ?name is an expression, read as it would be without the mark; ?m.name says a variable. +* (Measured too, and not written here because this runtime lets a LOCAL shadow a field: a +* name that is both a field of a source and a variable is the field in the product.) +LOCAL lcCode +lcCode = "BBB" +SELECT qty FROM parts WHERE parts.code = ?lcCode INTO CURSOR q1 +? "param variable", TRANSFORM(RECCOUNT()), TRANSFORM(qty), ALIAS() +SELECT qty FROM parts WHERE parts.code = ?m.lcCode INTO CURSOR q1 +? "param m.", TRANSFORM(RECCOUNT()), TRANSFORM(qty) + +* INSERT ... SELECT: to the fields named, in the order named +CREATE CURSOR bigger (qty N(3), code C(3)) +SELECT parts +INSERT INTO bigger (code, qty) SELECT code, qty FROM parts WHERE qty > 1 +? "insert select", TRANSFORM(RECCOUNT("bigger")), ALIAS() +SELECT bigger +SCAN + ? " row", code, TRANSFORM(qty) +ENDSCAN + +* and without a field list, to the table's fields in order +CREATE CURSOR copy (code C(3), qty N(3)) +SELECT parts +INSERT INTO copy SELECT code, qty FROM parts WHERE qty = 1 +? "insert select whole", TRANSFORM(RECCOUNT("copy")), copy.code, TRANSFORM(copy.qty), ALIAS() + +* UNION ALL keeps every row; the columns are the first SELECT's +SELECT 1 AS norder, "top " AS label, code FROM parts WHERE qty = 3 ; + UNION ALL ; + SELECT 2 AS norder, "rest" AS label, code FROM parts WHERE qty < 3 ; + ORDER BY norder, code INTO CURSOR q2 +? "union all", TRANSFORM(RECCOUNT("q2")), ALIAS() +SCAN + ? " row", TRANSFORM(norder), label, code +ENDSCAN + +* UNION without ALL folds the rows the two sides have in common +SELECT code FROM parts WHERE NOT DELETED() UNION SELECT code FROM parts WHERE qty > 1 INTO CURSOR q3 +? "union", TRANSFORM(RECCOUNT("q3")) +SELECT code FROM parts WHERE NOT DELETED() UNION ALL SELECT code FROM parts WHERE qty > 1 INTO CURSOR q4 +? "union all again", TRANSFORM(RECCOUNT("q4")) + +* INTO ARRAY leaves the program where it stood +SELECT parts +SELECT code FROM parts WHERE qty = 1 UNION ALL SELECT code FROM parts WHERE qty = 3 INTO ARRAY laCodes +? "into array", TRANSFORM(ALEN(laCodes)), laCodes(1), laCodes(2), ALIAS() + +* a quoted name after DO +DO "helper" WITH "x" + +PROCEDURE helper + LPARAMETERS c + ? "helper ran", c +ENDPROC diff --git a/crates/foxvm/tests/sql_forms.rs b/crates/foxvm/tests/sql_forms.rs new file mode 100644 index 0000000..c2329d6 --- /dev/null +++ b/crates/foxvm/tests/sql_forms.rs @@ -0,0 +1,205 @@ +//! Five spellings real Visual FoxPro applications use that the reference examples do not: +//! `USE IN SELECT('alias')`, `?lcName` parameters in local SQL, `INSERT INTO ... SELECT`, +//! `UNION [ALL]`, and names in quotes after `DO` and `CREATE TABLE`. Every one was found in a +//! CodeMine application of 60,000 lines and none of them is rare there. + +mod common; +mod dbf_fixture; + +use dbf_fixture::table; +use foxvm::mock_host::{MockHost, run_program}; + +fn host() -> MockHost { + let parts = table( + &[("CODE", b'C', 3, 0), ("QTY", b'N', 3, 0)], + &[&["AAA", "1"], &["BBB", "2"], &["CCC", "3"]], + ); + let mut host = MockHost::new(); + host.tables.insert("PARTS.DBF".into(), parts); + host +} + +fn run(src: &str) -> Vec { + let mut host = host(); + match run_program(src, &mut host) { + Ok((_, output)) => output, + Err(e) => panic!("{} at line {}: {}", e.code, e.line, e.message), + } +} + +// ----- parsing ------------------------------------------------------------------------------- + +#[test] +fn use_in_select_of_alias_is_a_call() { + assert_eq!(common::ok("USE IN SELECT('orders')"), "(use - (in SELECT(\"orders\")))"); + assert_eq!(common::ok("USE IN (SELECT('orders'))"), "(use - (in SELECT(\"orders\")))"); + // a bare name is still an alias, as it always was + assert_eq!(common::ok("USE IN orders"), "(use - (in \"orders\"))"); +} + +#[test] +fn question_mark_is_a_parameter_inside_a_select() { + // measured: the mark is dropped and what follows is an ordinary expression + assert_eq!( + common::ok("SELECT a FROM t WHERE b = ?lcName INTO CURSOR q"), + "(select (A) (from t:T) (where (= B LCNAME)) (into-cursor Q))" + ); + assert_eq!( + common::ok("SELECT a FROM t WHERE b = ?m.lcName INTO CURSOR q"), + "(select (A) (from t:T) (where (= B M.LCNAME)) (into-cursor Q))" + ); + assert_eq!( + common::ok("SELECT a FROM t WHERE b = ?tblorder.ipkey INTO CURSOR q"), + "(select (A) (from t:T) (where (= B TBLORDER.IPKEY)) (into-cursor Q))" + ); + // measured: INSERT, UPDATE and DELETE - SQL refuse it, and so does an expression anywhere + // else, where `?` is the print command and nothing more + for src in ["INSERT INTO t (a) VALUES (?lcName)", "UPDATE t SET a = 1 WHERE b = ?lcName", "DELETE FROM t WHERE b = ?lcName", "x = 1 + ?y"] { + assert!(!foxvm::parser::parse_program(src).diagnostics.is_empty(), "{src}"); + } +} + +#[test] +fn insert_select_and_union_parse() { + assert_eq!( + common::ok("INSERT INTO t (a, b) SELECT x, y FROM s WHERE x > 1"), + "(insert T (A B) (select (X Y) (from s:S) (where (> X 1))))" + ); + assert_eq!( + common::ok("SELECT a FROM s UNION ALL SELECT b FROM t ORDER BY 1 INTO CURSOR q"), + "(select (A) (from s:S) (union-all (select (B) (from t:T))) (order 1) (into-cursor Q))" + ); + assert_eq!( + common::ok("SELECT a FROM s UNION SELECT b FROM t UNION ALL SELECT c FROM u INTO ARRAY laAll"), + "(select (A) (from s:S) (union (select (B) (from t:T) (union-all (select (C) (from u:U))))) (into-array LAALL))" + ); +} + +#[test] +fn quoted_names_after_do_and_create_table() { + assert_eq!(common::ok("DO \"C:\\tools\\run.prg\" WITH 1"), "(do-expr \"C:\\\\tools\\\\run.prg\" (with 1))"); + let created = common::ok("CREATE TABLE 'APPREG01.DBF' NAME 'APPREG01' (KEYNAME C(70) NOT NULL)"); + assert!(created.starts_with("(create-table \"APPREG01.DBF\" ("), "{created}"); +} + +// ----- running ------------------------------------------------------------------------------- + +#[test] +fn use_in_select_closes_the_table_and_ignores_one_that_is_not_open() { + let out = run("USE parts +? USED('parts') +USE IN SELECT('parts') +? USED('parts') +USE IN SELECT('parts') +? 'still here'"); + assert_eq!(out, [".T.", ".F.", "still here"]); +} + +#[test] +fn use_in_select_leaves_the_selected_area_alone() { + let out = run("USE parts +CREATE CURSOR other (a C(1)) +USE IN SELECT('parts') +? ALIAS()"); + assert_eq!(out, ["OTHER"]); +} + +#[test] +fn a_parameter_is_read_as_the_expression_it_marks() { + // measured: the mark changes nothing about how the name is read. (In the product a name + // that is both a field of a source and a variable is the field; this runtime lets a LOCAL + // shadow a field, which is a gap of its own and not exercised here.) + let out = run("LOCAL lcCode +lcCode = 'BBB' +SELECT qty FROM parts WHERE parts.code = ?lcCode INTO CURSOR q +? TRANSFORM(RECCOUNT()), TRANSFORM(qty) +SELECT qty FROM parts WHERE parts.code = ?m.lcCode INTO CURSOR q +? TRANSFORM(RECCOUNT()), TRANSFORM(qty)"); + assert_eq!(out, ["1 2", "1 2"]); +} + +#[test] +fn insert_select_copies_rows_to_the_fields_named_in_order() { + // the table's fields are the other way round from the select list + let out = run("CREATE CURSOR bigger (qty N(3), code C(3)) +INSERT INTO bigger (code, qty) SELECT code, qty FROM parts WHERE qty > 1 +? TRANSFORM(RECCOUNT('bigger')) +SCAN + ? code, TRANSFORM(qty) +ENDSCAN +? ALIAS(), USED('__ins1')"); + assert_eq!(out, ["2", "BBB 2", "CCC 3", "BIGGER .F."]); +} + +#[test] +fn insert_select_without_a_field_list_takes_the_table_in_order() { + let out = run("USE parts +CREATE CURSOR copy (code C(3), qty N(3)) +SELECT parts +INSERT INTO copy SELECT * FROM parts +? TRANSFORM(RECCOUNT('copy')), ALIAS() +SELECT copy +GO 3 +? code, TRANSFORM(qty)"); + assert_eq!(out, ["3 PARTS", "CCC 3"]); +} + +#[test] +fn insert_select_into_a_table_named_by_an_expression() { + let out = run("CREATE CURSOR copy (code C(3), qty N(3)) +lcTarget = 'copy' +INSERT INTO (lcTarget) SELECT code, qty FROM parts WHERE qty = 2 +? TRANSFORM(RECCOUNT('copy')), copy.code"); + assert_eq!(out, ["1 BBB"]); +} + +#[test] +fn union_all_stacks_the_rows_under_the_first_select() { + let out = run("SELECT 1 AS norder, 'top ' AS label, code FROM parts WHERE qty = 3 ; + UNION ALL ; + SELECT 2 AS norder, 'rest' AS label, code FROM parts WHERE qty < 3 ; + ORDER BY norder, code INTO CURSOR q +? TRANSFORM(RECCOUNT('q')), ALIAS() +SCAN + ? TRANSFORM(norder), label, code +ENDSCAN +? USED('__uni1'), USED('__uni1_1')"); + assert_eq!(out, ["3 Q", "1 top CCC", "2 rest AAA", "2 rest BBB", ".F. .F."]); +} + +#[test] +fn union_without_all_folds_rows_the_same_on_both_sides() { + let out = run("SELECT code FROM parts UNION SELECT code FROM parts WHERE qty > 1 INTO CURSOR q +? TRANSFORM(RECCOUNT('q')) +SELECT code FROM parts UNION ALL SELECT code FROM parts WHERE qty > 1 INTO CURSOR q +? TRANSFORM(RECCOUNT('q'))"); + assert_eq!(out, ["3", "5"]); +} + +#[test] +fn union_into_array_leaves_the_area_where_it_was() { + let out = run("USE parts +SELECT code FROM parts WHERE qty = 1 UNION ALL SELECT code FROM parts WHERE qty = 3 INTO ARRAY laCodes +? TRANSFORM(ALEN(laCodes)), laCodes(1), laCodes(2), ALIAS()"); + assert_eq!(out, ["2 AAA CCC PARTS"]); +} + +#[test] +fn a_query_into_an_array_puts_the_program_back_where_it_stood() { + // the source is open in another area; opening it for the query must not move the program + let out = run("USE parts +CREATE CURSOR other (a C(1)) +SELECT code FROM parts WHERE qty = 1 INTO ARRAY laCodes +? ALIAS()"); + assert_eq!(out, ["OTHER"]); +} + +#[test] +fn a_quoted_program_name_is_run() { + let out = run("DO \"helper\" WITH 'x' +PROCEDURE helper + LPARAMETERS c + ? 'ran', c +ENDPROC"); + assert_eq!(out, ["ran x"]); +} diff --git a/docs/parity-plan.md b/docs/parity-plan.md index c51b10d..cd08f38 100644 --- a/docs/parity-plan.md +++ b/docs/parity-plan.md @@ -334,6 +334,17 @@ missing feature; both are something already built getting it wrong. baseline only ever complained that *relations* were not imported. So this is the opening of the cursor when the form loads, not the import of it. +Two more, found while measuring the SQL forms a CodeMine application writes (the `sql_forms` +golden). Both belong to system 3, the resolver, and neither is fixed: + +- **A LOCAL shadows a field of the same name.** Measured: with `parts` open and `LOCAL code` + declared, the product reads `code` as the field, in a plain `? code` and in a SELECT-SQL + WHERE alike; `m.code` is how the variable is asked for. This runtime reads the local. The + golden avoids the case by not naming a variable after a field, and says so. +- **`_TALLY` is never set.** The product sets it after every SQL statement and after REPLACE, + DELETE, APPEND FROM and the other record-scoped commands; here it stays at zero. Nothing + writes it anywhere in the VM. + ## Milestones ### Where the product has been