fix: preserve Unicode text in trimming functions

crates/foxvm/src/builtins/string.rs

@@ -53,17 +53,17 @@ fn find_all(hay: &[u8], needle: &[u8], case_insensitive: bool) -> Vec<usize> {
 
 /// Trim characters for ALLTRIM/LTRIM/RTRIM: numeric extra arguments are the VFP flags, which
 /// this runtime ignores, character ones add to the set. Defaults to the space.
-fn trim_set(args: &[Value], from: usize) -> Result<Vec<u8>, RtError> {
+fn trim_set(args: &[Value], from: usize) -> Result<Vec<char>, RtError> {
     let mut set = Vec::new();
     for v in args.iter().skip(from) {
         match v.deref() {
             Value::Number(..) | Value::Logical(_) => {}
-            Value::Str(s) => set.extend_from_slice(s.as_bytes()),
+            Value::Str(s) => set.extend(s.chars()),
             _ => return Err(RtError::function_arg_invalid()),
         }
     }
     if set.is_empty() {
-        set.push(b' ');
+        set.push(' ');
     }
     Ok(set)
 }
@@ -84,6 +84,13 @@ fn trimmed(b: &[u8], set: &[u8], left: bool, right: bool) -> Vec<u8> {
     b[start..end].to_vec()
 }
 
+// Work on decoded characters, not the low-byte carrier used by binary functions.
+// Trimming must leave the retained text unchanged, including scalars above U+00FF.
+fn trimmed_text<'a>(s: &'a str, set: &[char], left: bool, right: bool) -> &'a str {
+    let s = if left { s.trim_start_matches(|c| set.contains(&c)) } else { s };
+    if right { s.trim_end_matches(|c| set.contains(&c)) } else { s }
+}
+
 // ------------------------------------------------------------------------------------------
 // trimming and case
 // ------------------------------------------------------------------------------------------
@@ -92,21 +99,21 @@ fn f_alltrim(_c: &mut dyn BuiltinCtx, a: Vec<Value>) -> Result<BuiltinResult, Rt
     if any_null(&a) {
         return ok(Value::Null);
     }
-    ok(text(&trimmed(&bytes(&a, 0)?, &trim_set(&a, 1)?, true, true)))
+    ok(Value::str(trimmed_text(&arg_str(&a, 0)?, &trim_set(&a, 1)?, true, true)))
 }
 
 fn f_ltrim(_c: &mut dyn BuiltinCtx, a: Vec<Value>) -> Result<BuiltinResult, RtError> {
     if any_null(&a) {
         return ok(Value::Null);
     }
-    ok(text(&trimmed(&bytes(&a, 0)?, &trim_set(&a, 1)?, true, false)))
+    ok(Value::str(trimmed_text(&arg_str(&a, 0)?, &trim_set(&a, 1)?, true, false)))
 }
 
 fn f_rtrim(_c: &mut dyn BuiltinCtx, a: Vec<Value>) -> Result<BuiltinResult, RtError> {
     if any_null(&a) {
         return ok(Value::Null);
     }
-    ok(text(&trimmed(&bytes(&a, 0)?, &trim_set(&a, 1)?, false, true)))
+    ok(Value::str(trimmed_text(&arg_str(&a, 0)?, &trim_set(&a, 1)?, false, true)))
 }
 
 fn f_upper(_c: &mut dyn BuiltinCtx, a: Vec<Value>) -> Result<BuiltinResult, RtError> {

crates/foxvm/tests/builtins_string.rs

@@ -19,6 +19,38 @@ fn trimming() {
     assert_eq!(text("ALLTRIM", vec![s("  hi  "), n(0.0)]), "hi");
 }
 
+#[test]
+fn trimming_preserves_unicode() {
+    for input in ["Ąžuolas žąsis į ų ū ė č š", "Ελληνικά", "Українська", "العربية", "हिन्दी", "日本語", "😀", "Ġ"]
+    {
+        let padded = format!("  {input}  ");
+        assert_eq!(text("ALLTRIM", vec![s(&padded)]), input);
+        assert_eq!(text("LTRIM", vec![s(&padded)]), format!("{input}  "));
+        for name in ["RTRIM", "TRIM"] {
+            assert_eq!(text(name, vec![s(&padded)]), format!("  {input}"));
+        }
+    }
+}
+
+#[test]
+fn trimming_unicode_parse_characters() {
+    assert_eq!(text("ALLTRIM", vec![s("ąąŽodisą"), s("ą")]), "Žodis");
+    assert_eq!(text("LTRIM", vec![s("ąŽodisą"), s("ą")]), "Žodisą");
+    assert_eq!(text("RTRIM", vec![s("ąŽodisą"), s("ą")]), "ąŽodis");
+    // A scalar sharing a low byte with a parse character must not match it.
+    assert_eq!(text("ALLTRIM", vec![s("ĠwordĠ")]), "ĠwordĠ");
+    assert_eq!(text("ALLTRIM", vec![s("ŁwordŁ"), s("A")]), "ŁwordŁ");
+}
+
+#[test]
+fn trimming_preserves_byte_carriers_and_non_space_whitespace() {
+    let carrier: String = (0u8..=255).map(char::from).collect();
+    for name in ["ALLTRIM", "LTRIM", "RTRIM", "TRIM"] {
+        assert_eq!(text(name, vec![s(&carrier)]), carrier);
+        assert_eq!(text(name, vec![s("\t\r\n\u{a0}hi\u{a0}\n\r\t")]), "\t\r\n\u{a0}hi\u{a0}\n\r\t");
+    }
+}
+
 #[test]
 fn case_conversion() {
     assert_eq!(text("UPPER", vec![s("Hello, world!")]), "HELLO, WORLD!");

docs/unicode-trimming.md

@@ -0,0 +1,22 @@
+# Unicode preservation in trimming
+
+ALLTRIM, LTRIM, RTRIM and its TRIM alias preserve decoded Unicode text.
+Previously they truncated every scalar to its low byte before trimming, so
+non-Latin-1 text became unrelated characters, even when no spaces were present.
+Trimming now slices the original string at character boundaries. Explicit
+single-character parse arguments use the same character representation.
+
+Default trimming still removes only ASCII space, not tabs, newlines or NBSP.
+The U+0000–U+00FF byte-carrier representation is retained without re-encoding;
+shared byte helpers and ALINES are unchanged.
+
+This is a text preservation fix, not full VFP trimming parity. Existing parsing
+arguments remain a character set; multi-character parsing strings, case flags
+and distinct Varbinary zero-byte trimming need separate compatibility work.
+VFP documents those behaviors in [ALLTRIM help](https://www.vfphelp.com/help/html/767f9fa5-5271-4fe2-baf8-332cf8d15fb5.htm).
+Other character built-ins may still truncate Unicode through the byte helper.
+
+Regression coverage includes eight script/character samples, single-sided and
+custom-character trimming, low-byte collisions, all 256 byte-carrier values,
+non-space whitespace and a compiled program running through the WASM bridge.
+No native Windows VFP comparison was run for this change.

tests/runtime/vmBridge.test.ts

@@ -102,6 +102,27 @@ describe('FoxVM bridge', () => {
     expect(host.lines).toEqual(['sum:           42', 'DONE']);
   });
 
+  it('preserves decoded Unicode through trimming in a compiled program', () => {
+    const host = new StubHost();
+    const { vm, scheduler } = makeSession(host);
+    const out = compileProgram([
+      'LOCAL label',
+      'label = "  Ąžuolas Ελληνικά हिन्दी 😀  "',
+      '? ALLTRIM(label)',
+      '? LTRIM(label)',
+      '? RTRIM(label)',
+      '? TRIM(label)',
+    ].join('\n'), 'unicode-trimming');
+    expect(compileError(out)).toBeNull();
+    scheduler.runProgram(vm.loadModule(out.bytes!));
+    expect(host.lines).toEqual([
+      'Ąžuolas Ελληνικά हिन्दी 😀',
+      'Ąžuolas Ελληνικά हिन्दी 😀  ',
+      '  Ąžuolas Ελληνικά हिन्दी 😀',
+      '  Ąžuolas Ελληνικά हिन्दी 😀',
+    ]);
+  });
+
   it('reports compile errors instead of a module', () => {
     const out = compileProgram('IF x\n', 'broken');
     expect(out.bytes).toBeNull();