diff --git a/crates/foxvm/src/builtins/string.rs b/crates/foxvm/src/builtins/string.rs index 0163051..755bf9b 100644 --- a/crates/foxvm/src/builtins/string.rs +++ b/crates/foxvm/src/builtins/string.rs @@ -53,17 +53,17 @@ fn find_all(hay: &[u8], needle: &[u8], case_insensitive: bool) -> Vec { /// Trim characters for ALLTRIM/LTRIM/RTRIM: numeric extra arguments are the VFP flags, which /// this runtime ignores, character ones add to the set. Defaults to the space. -fn trim_set(args: &[Value], from: usize) -> Result, RtError> { +fn trim_set(args: &[Value], from: usize) -> Result, RtError> { let mut set = Vec::new(); for v in args.iter().skip(from) { match v.deref() { Value::Number(..) | Value::Logical(_) => {} - Value::Str(s) => set.extend_from_slice(s.as_bytes()), + Value::Str(s) => set.extend(s.chars()), _ => return Err(RtError::function_arg_invalid()), } } if set.is_empty() { - set.push(b' '); + set.push(' '); } Ok(set) } @@ -84,6 +84,13 @@ fn trimmed(b: &[u8], set: &[u8], left: bool, right: bool) -> Vec { b[start..end].to_vec() } +// Work on decoded characters, not the low-byte carrier used by binary functions. +// Trimming must leave the retained text unchanged, including scalars above U+00FF. +fn trimmed_text<'a>(s: &'a str, set: &[char], left: bool, right: bool) -> &'a str { + let s = if left { s.trim_start_matches(|c| set.contains(&c)) } else { s }; + if right { s.trim_end_matches(|c| set.contains(&c)) } else { s } +} + // ------------------------------------------------------------------------------------------ // trimming and case // ------------------------------------------------------------------------------------------ @@ -92,21 +99,21 @@ fn f_alltrim(_c: &mut dyn BuiltinCtx, a: Vec) -> Result) -> Result { if any_null(&a) { return ok(Value::Null); } - ok(text(&trimmed(&bytes(&a, 0)?, &trim_set(&a, 1)?, true, false))) + ok(Value::str(trimmed_text(&arg_str(&a, 0)?, &trim_set(&a, 1)?, true, false))) } fn f_rtrim(_c: &mut dyn BuiltinCtx, a: Vec) -> Result { if any_null(&a) { return ok(Value::Null); } - ok(text(&trimmed(&bytes(&a, 0)?, &trim_set(&a, 1)?, false, true))) + ok(Value::str(trimmed_text(&arg_str(&a, 0)?, &trim_set(&a, 1)?, false, true))) } fn f_upper(_c: &mut dyn BuiltinCtx, a: Vec) -> Result { diff --git a/crates/foxvm/tests/builtins_string.rs b/crates/foxvm/tests/builtins_string.rs index 98ee0d7..1a55b6a 100644 --- a/crates/foxvm/tests/builtins_string.rs +++ b/crates/foxvm/tests/builtins_string.rs @@ -19,6 +19,38 @@ fn trimming() { assert_eq!(text("ALLTRIM", vec![s(" hi "), n(0.0)]), "hi"); } +#[test] +fn trimming_preserves_unicode() { + for input in ["Ąžuolas žąsis į ų ū ė č š", "Ελληνικά", "Українська", "العربية", "हिन्दी", "日本語", "😀", "Ġ"] + { + let padded = format!(" {input} "); + assert_eq!(text("ALLTRIM", vec![s(&padded)]), input); + assert_eq!(text("LTRIM", vec![s(&padded)]), format!("{input} ")); + for name in ["RTRIM", "TRIM"] { + assert_eq!(text(name, vec![s(&padded)]), format!(" {input}")); + } + } +} + +#[test] +fn trimming_unicode_parse_characters() { + assert_eq!(text("ALLTRIM", vec![s("ąąŽodisą"), s("ą")]), "Žodis"); + assert_eq!(text("LTRIM", vec![s("ąŽodisą"), s("ą")]), "Žodisą"); + assert_eq!(text("RTRIM", vec![s("ąŽodisą"), s("ą")]), "ąŽodis"); + // A scalar sharing a low byte with a parse character must not match it. + assert_eq!(text("ALLTRIM", vec![s("ĠwordĠ")]), "ĠwordĠ"); + assert_eq!(text("ALLTRIM", vec![s("ŁwordŁ"), s("A")]), "ŁwordŁ"); +} + +#[test] +fn trimming_preserves_byte_carriers_and_non_space_whitespace() { + let carrier: String = (0u8..=255).map(char::from).collect(); + for name in ["ALLTRIM", "LTRIM", "RTRIM", "TRIM"] { + assert_eq!(text(name, vec![s(&carrier)]), carrier); + assert_eq!(text(name, vec![s("\t\r\n\u{a0}hi\u{a0}\n\r\t")]), "\t\r\n\u{a0}hi\u{a0}\n\r\t"); + } +} + #[test] fn case_conversion() { assert_eq!(text("UPPER", vec![s("Hello, world!")]), "HELLO, WORLD!"); diff --git a/docs/unicode-trimming.md b/docs/unicode-trimming.md new file mode 100644 index 0000000..93b334d --- /dev/null +++ b/docs/unicode-trimming.md @@ -0,0 +1,22 @@ +# Unicode preservation in trimming + +ALLTRIM, LTRIM, RTRIM and its TRIM alias preserve decoded Unicode text. +Previously they truncated every scalar to its low byte before trimming, so +non-Latin-1 text became unrelated characters, even when no spaces were present. +Trimming now slices the original string at character boundaries. Explicit +single-character parse arguments use the same character representation. + +Default trimming still removes only ASCII space, not tabs, newlines or NBSP. +The U+0000–U+00FF byte-carrier representation is retained without re-encoding; +shared byte helpers and ALINES are unchanged. + +This is a text preservation fix, not full VFP trimming parity. Existing parsing +arguments remain a character set; multi-character parsing strings, case flags +and distinct Varbinary zero-byte trimming need separate compatibility work. +VFP documents those behaviors in [ALLTRIM help](https://www.vfphelp.com/help/html/767f9fa5-5271-4fe2-baf8-332cf8d15fb5.htm). +Other character built-ins may still truncate Unicode through the byte helper. + +Regression coverage includes eight script/character samples, single-sided and +custom-character trimming, low-byte collisions, all 256 byte-carrier values, +non-space whitespace and a compiled program running through the WASM bridge. +No native Windows VFP comparison was run for this change. diff --git a/tests/runtime/vmBridge.test.ts b/tests/runtime/vmBridge.test.ts index 5e92003..07d0503 100644 --- a/tests/runtime/vmBridge.test.ts +++ b/tests/runtime/vmBridge.test.ts @@ -102,6 +102,27 @@ describe('FoxVM bridge', () => { expect(host.lines).toEqual(['sum: 42', 'DONE']); }); + it('preserves decoded Unicode through trimming in a compiled program', () => { + const host = new StubHost(); + const { vm, scheduler } = makeSession(host); + const out = compileProgram([ + 'LOCAL label', + 'label = " Ąžuolas Ελληνικά हिन्दी 😀 "', + '? ALLTRIM(label)', + '? LTRIM(label)', + '? RTRIM(label)', + '? TRIM(label)', + ].join('\n'), 'unicode-trimming'); + expect(compileError(out)).toBeNull(); + scheduler.runProgram(vm.loadModule(out.bytes!)); + expect(host.lines).toEqual([ + 'Ąžuolas Ελληνικά हिन्दी 😀', + 'Ąžuolas Ελληνικά हिन्दी 😀 ', + ' Ąžuolas Ελληνικά हिन्दी 😀', + ' Ąžuolas Ελληνικά हिन्दी 😀', + ]); + }); + it('reports compile errors instead of a module', () => { const out = compileProgram('IF x\n', 'broken'); expect(out.bytes).toBeNull();