diff --git a/crates/foxvm/src/dbf/encoding.rs b/crates/foxvm/src/dbf/encoding.rs index 8a340ea..53dbc5c 100644 --- a/crates/foxvm/src/dbf/encoding.rs +++ b/crates/foxvm/src/dbf/encoding.rs @@ -1,8 +1,12 @@ //! Single-byte code page decoding for DBF text. //! -//! Only the code pages FoxPro tables realistically carry are implemented as tables: Windows-1252 -//! (the VFP default), and the DOS pages 437 and 850. Anything else falls back to 1252 so that -//! decoding a table never fails; bytes with no mapping become U+FFFD. +//! Windows 1250–1257, Thai 874, and DOS 437/737/850/852/857/860/861/863/865/866 +//! have explicit mappings. Unimplemented pages (including DBCS and Macintosh) retain +//! the legacy 1252 fallback; recognizing a header marker does not imply codec support. +//! Unassigned bytes decode to U+FFFD. + +#[path = "encoding_tables.rs"] +mod tables; /// Maps the DBF header language-driver byte (offset 29) to a code page number. /// @@ -13,7 +17,8 @@ pub fn codepage_for_language_id(id: u8) -> Option { 0x01 | 0x09 | 0x0B | 0x0D | 0x0F | 0x11 | 0x15 | 0x18 | 0x19 | 0x1B => 437, 0x02 | 0x0A | 0x0E | 0x10 | 0x12 | 0x14 | 0x16 | 0x1A | 0x1D | 0x25 | 0x37 => 850, 0x03 | 0x57 | 0x58 | 0x59 => 1252, - 0x04 | 0x98 => 10000, + 0x04 => 10000, + 0x98 => 10006, 0x08 | 0x17 | 0x66 => 865, 0x13 | 0x7B => 932, 0x1C | 0x6C => 863, @@ -71,6 +76,22 @@ fn high_table(codepage: Option) -> &'static [char; 128] { match codepage { Some(437) => &CP437, Some(850) => &CP850, + Some(1257) => &CP1257, + Some(737) => &tables::CP737, + Some(852) => &tables::CP852, + Some(857) => &tables::CP857, + Some(860) => &tables::CP860, + Some(861) => &tables::CP861, + Some(863) => &tables::CP863, + Some(865) => &tables::CP865, + Some(866) => &tables::CP866, + Some(874) => &tables::CP874, + Some(1250) => &tables::CP1250, + Some(1251) => &tables::CP1251, + Some(1253) => &tables::CP1253, + Some(1254) => &tables::CP1254, + Some(1255) => &tables::CP1255, + Some(1256) => &tables::CP1256, _ => &CP1252, } } @@ -137,3 +158,65 @@ static CP850: [char; 128] = [ '\u{00AD}', '\u{00B1}', '\u{2017}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}', '\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', ]; + +/// Windows-1257 (Baltic), bytes 0x80..=0xFF. Unassigned positions decode to U+FFFD. +#[rustfmt::skip] +static CP1257: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{FFFD}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{FFFD}', '\u{2030}', '\u{FFFD}', '\u{2039}', '\u{FFFD}', '\u{00A8}', '\u{02C7}', '\u{00B8}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{FFFD}', '\u{2122}', '\u{FFFD}', '\u{203A}', '\u{FFFD}', '\u{00AF}', '\u{02DB}', '\u{FFFD}', + '\u{00A0}', '\u{FFFD}', '\u{00A2}', '\u{00A3}', '\u{00A4}', '\u{FFFD}', '\u{00A6}', '\u{00A7}', + '\u{00D8}', '\u{00A9}', '\u{0156}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{00C6}', + '\u{00B0}', '\u{00B1}', '\u{00B2}', '\u{00B3}', '\u{00B4}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{00F8}', '\u{00B9}', '\u{0157}', '\u{00BB}', '\u{00BC}', '\u{00BD}', '\u{00BE}', '\u{00E6}', + '\u{0104}', '\u{012E}', '\u{0100}', '\u{0106}', '\u{00C4}', '\u{00C5}', '\u{0118}', '\u{0112}', + '\u{010C}', '\u{00C9}', '\u{0179}', '\u{0116}', '\u{0122}', '\u{0136}', '\u{012A}', '\u{013B}', + '\u{0160}', '\u{0143}', '\u{0145}', '\u{00D3}', '\u{014C}', '\u{00D5}', '\u{00D6}', '\u{00D7}', + '\u{0172}', '\u{0141}', '\u{015A}', '\u{016A}', '\u{00DC}', '\u{017B}', '\u{017D}', '\u{00DF}', + '\u{0105}', '\u{012F}', '\u{0101}', '\u{0107}', '\u{00E4}', '\u{00E5}', '\u{0119}', '\u{0113}', + '\u{010D}', '\u{00E9}', '\u{017A}', '\u{0117}', '\u{0123}', '\u{0137}', '\u{012B}', '\u{013C}', + '\u{0161}', '\u{0144}', '\u{0146}', '\u{00F3}', '\u{014D}', '\u{00F5}', '\u{00F6}', '\u{00F7}', + '\u{0173}', '\u{0142}', '\u{015B}', '\u{016B}', '\u{00FC}', '\u{017C}', '\u{017E}', '\u{02D9}', +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn baltic_letters_decode_and_encode() { + let bytes = [0xc0,0xc8,0xc6,0xcb,0xc1,0xd0,0xd8,0xdb,0xde, + 0xe0,0xe8,0xe6,0xeb,0xe1,0xf0,0xf8,0xfb,0xfe]; + let text = "ĄČĘĖĮŠŲŪŽąčęėįšųūž"; + let cp = codepage_for_language_id(0xcc); + assert_eq!(cp, Some(1257)); + assert_eq!(decode(&bytes, cp), text); + assert_eq!(encode(text, cp), bytes); + } + + #[test] + fn baltic_ascii_and_unsupported_characters() { + assert_eq!(decode(b"Hello 123", Some(1257)), "Hello 123"); + assert_eq!(encode("Hello 123 😀", Some(1257)), b"Hello 123 ?"); + assert_eq!(decode(&[0x81], Some(1257)), "\u{fffd}"); + } + + #[test] + fn baltic_defined_bytes_round_trip() { + for byte in 0u8..=255 { + let text = decode(&[byte], Some(1257)); + if text != "\u{fffd}" { + assert_eq!(encode(&text, Some(1257)), vec![byte]); + } + } + } + + #[test] + fn existing_pages_keep_their_mappings() { + assert_eq!(decode(&[0xe1], None), "á"); + assert_eq!(decode(&[0xe1], Some(9999)), "á"); + assert_eq!(decode(&[0x80], Some(437)), "Ç"); + assert_eq!(decode(&[0x80], Some(850)), "Ç"); + } +} diff --git a/crates/foxvm/src/dbf/encoding_tables.rs b/crates/foxvm/src/dbf/encoding_tables.rs new file mode 100644 index 0000000..385e96c --- /dev/null +++ b/crates/foxvm/src/dbf/encoding_tables.rs @@ -0,0 +1,302 @@ +// Generated from Unicode-hosted Microsoft mappings; see docs/codepage-mappings.md. +// Unassigned bytes use the existing U+FFFD policy. + +#[rustfmt::skip] +pub(super) static CP737: [char; 128] = [ + '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', '\u{0398}', + '\u{0399}', '\u{039A}', '\u{039B}', '\u{039C}', '\u{039D}', '\u{039E}', '\u{039F}', '\u{03A0}', + '\u{03A1}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', '\u{03A8}', '\u{03A9}', + '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{03B4}', '\u{03B5}', '\u{03B6}', '\u{03B7}', '\u{03B8}', + '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', '\u{03BD}', '\u{03BE}', '\u{03BF}', '\u{03C0}', + '\u{03C1}', '\u{03C3}', '\u{03C2}', '\u{03C4}', '\u{03C5}', '\u{03C6}', '\u{03C7}', '\u{03C8}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03C9}', '\u{03AC}', '\u{03AD}', '\u{03AE}', '\u{03CA}', '\u{03AF}', '\u{03CC}', '\u{03CD}', + '\u{03CB}', '\u{03CE}', '\u{0386}', '\u{0388}', '\u{0389}', '\u{038A}', '\u{038C}', '\u{038E}', + '\u{038F}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{03AA}', '\u{03AB}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP852: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{016F}', '\u{0107}', '\u{00E7}', + '\u{0142}', '\u{00EB}', '\u{0150}', '\u{0151}', '\u{00EE}', '\u{0179}', '\u{00C4}', '\u{0106}', + '\u{00C9}', '\u{0139}', '\u{013A}', '\u{00F4}', '\u{00F6}', '\u{013D}', '\u{013E}', '\u{015A}', + '\u{015B}', '\u{00D6}', '\u{00DC}', '\u{0164}', '\u{0165}', '\u{0141}', '\u{00D7}', '\u{010D}', + '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', '\u{0104}', '\u{0105}', '\u{017D}', '\u{017E}', + '\u{0118}', '\u{0119}', '\u{00AC}', '\u{017A}', '\u{010C}', '\u{015F}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{011A}', + '\u{015E}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{017B}', '\u{017C}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{0102}', '\u{0103}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', + '\u{0111}', '\u{0110}', '\u{010E}', '\u{00CB}', '\u{010F}', '\u{0147}', '\u{00CD}', '\u{00CE}', + '\u{011B}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{0162}', '\u{016E}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{0143}', '\u{0144}', '\u{0148}', '\u{0160}', '\u{0161}', + '\u{0154}', '\u{00DA}', '\u{0155}', '\u{0170}', '\u{00FD}', '\u{00DD}', '\u{0163}', '\u{00B4}', + '\u{00AD}', '\u{02DD}', '\u{02DB}', '\u{02C7}', '\u{02D8}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{02D9}', '\u{0171}', '\u{0158}', '\u{0159}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP857: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', + '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', '\u{00EE}', '\u{0131}', '\u{00C4}', '\u{00C5}', + '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{0130}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{015E}', '\u{015F}', + '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', '\u{00F1}', '\u{00D1}', '\u{011E}', '\u{011F}', + '\u{00BF}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{00C0}', + '\u{00A9}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{00A2}', '\u{00A5}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{00E3}', '\u{00C3}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', + '\u{00BA}', '\u{00AA}', '\u{00CA}', '\u{00CB}', '\u{00C8}', '\u{FFFD}', '\u{00CD}', '\u{00CE}', + '\u{00CF}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{00A6}', '\u{00CC}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{00D2}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{FFFD}', + '\u{00D7}', '\u{00DA}', '\u{00DB}', '\u{00D9}', '\u{00EC}', '\u{00FF}', '\u{00AF}', '\u{00B4}', + '\u{00AD}', '\u{00B1}', '\u{FFFD}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP860: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E3}', '\u{00E0}', '\u{00C1}', '\u{00E7}', + '\u{00EA}', '\u{00CA}', '\u{00E8}', '\u{00CD}', '\u{00D4}', '\u{00EC}', '\u{00C3}', '\u{00C2}', + '\u{00C9}', '\u{00C0}', '\u{00C8}', '\u{00F4}', '\u{00F5}', '\u{00F2}', '\u{00DA}', '\u{00F9}', + '\u{00CC}', '\u{00D5}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{20A7}', '\u{00D3}', + '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', + '\u{00BF}', '\u{00D2}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', + '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', + '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP861: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', + '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00D0}', '\u{00F0}', '\u{00DE}', '\u{00C4}', '\u{00C5}', + '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00FE}', '\u{00FB}', '\u{00DD}', + '\u{00FD}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', + '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', '\u{00C1}', '\u{00CD}', '\u{00D3}', '\u{00DA}', + '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', + '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', + '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP863: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00C2}', '\u{00E0}', '\u{00B6}', '\u{00E7}', + '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', '\u{00EE}', '\u{2017}', '\u{00C0}', '\u{00A7}', + '\u{00C9}', '\u{00C8}', '\u{00CA}', '\u{00F4}', '\u{00CB}', '\u{00CF}', '\u{00FB}', '\u{00F9}', + '\u{00A4}', '\u{00D4}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00D9}', '\u{00DB}', '\u{0192}', + '\u{00A6}', '\u{00B4}', '\u{00F3}', '\u{00FA}', '\u{00A8}', '\u{00B8}', '\u{00B3}', '\u{00AF}', + '\u{00CE}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00BE}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', + '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', + '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP865: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', + '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', '\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', + '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{20A7}', '\u{0192}', + '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', + '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00A4}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', + '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', + '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP866: [char; 128] = [ + '\u{0410}', '\u{0411}', '\u{0412}', '\u{0413}', '\u{0414}', '\u{0415}', '\u{0416}', '\u{0417}', + '\u{0418}', '\u{0419}', '\u{041A}', '\u{041B}', '\u{041C}', '\u{041D}', '\u{041E}', '\u{041F}', + '\u{0420}', '\u{0421}', '\u{0422}', '\u{0423}', '\u{0424}', '\u{0425}', '\u{0426}', '\u{0427}', + '\u{0428}', '\u{0429}', '\u{042A}', '\u{042B}', '\u{042C}', '\u{042D}', '\u{042E}', '\u{042F}', + '\u{0430}', '\u{0431}', '\u{0432}', '\u{0433}', '\u{0434}', '\u{0435}', '\u{0436}', '\u{0437}', + '\u{0438}', '\u{0439}', '\u{043A}', '\u{043B}', '\u{043C}', '\u{043D}', '\u{043E}', '\u{043F}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', + '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', + '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', + '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', + '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{0440}', '\u{0441}', '\u{0442}', '\u{0443}', '\u{0444}', '\u{0445}', '\u{0446}', '\u{0447}', + '\u{0448}', '\u{0449}', '\u{044A}', '\u{044B}', '\u{044C}', '\u{044D}', '\u{044E}', '\u{044F}', + '\u{0401}', '\u{0451}', '\u{0404}', '\u{0454}', '\u{0407}', '\u{0457}', '\u{040E}', '\u{045E}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{2116}', '\u{00A4}', '\u{25A0}', '\u{00A0}', +]; + +#[rustfmt::skip] +pub(super) static CP874: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{2026}', '\u{FFFD}', '\u{FFFD}', + '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{00A0}', '\u{0E01}', '\u{0E02}', '\u{0E03}', '\u{0E04}', '\u{0E05}', '\u{0E06}', '\u{0E07}', + '\u{0E08}', '\u{0E09}', '\u{0E0A}', '\u{0E0B}', '\u{0E0C}', '\u{0E0D}', '\u{0E0E}', '\u{0E0F}', + '\u{0E10}', '\u{0E11}', '\u{0E12}', '\u{0E13}', '\u{0E14}', '\u{0E15}', '\u{0E16}', '\u{0E17}', + '\u{0E18}', '\u{0E19}', '\u{0E1A}', '\u{0E1B}', '\u{0E1C}', '\u{0E1D}', '\u{0E1E}', '\u{0E1F}', + '\u{0E20}', '\u{0E21}', '\u{0E22}', '\u{0E23}', '\u{0E24}', '\u{0E25}', '\u{0E26}', '\u{0E27}', + '\u{0E28}', '\u{0E29}', '\u{0E2A}', '\u{0E2B}', '\u{0E2C}', '\u{0E2D}', '\u{0E2E}', '\u{0E2F}', + '\u{0E30}', '\u{0E31}', '\u{0E32}', '\u{0E33}', '\u{0E34}', '\u{0E35}', '\u{0E36}', '\u{0E37}', + '\u{0E38}', '\u{0E39}', '\u{0E3A}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{0E3F}', + '\u{0E40}', '\u{0E41}', '\u{0E42}', '\u{0E43}', '\u{0E44}', '\u{0E45}', '\u{0E46}', '\u{0E47}', + '\u{0E48}', '\u{0E49}', '\u{0E4A}', '\u{0E4B}', '\u{0E4C}', '\u{0E4D}', '\u{0E4E}', '\u{0E4F}', + '\u{0E50}', '\u{0E51}', '\u{0E52}', '\u{0E53}', '\u{0E54}', '\u{0E55}', '\u{0E56}', '\u{0E57}', + '\u{0E58}', '\u{0E59}', '\u{0E5A}', '\u{0E5B}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', +]; + +#[rustfmt::skip] +pub(super) static CP1250: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{FFFD}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{FFFD}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{015A}', '\u{0164}', '\u{017D}', '\u{0179}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{FFFD}', '\u{2122}', '\u{0161}', '\u{203A}', '\u{015B}', '\u{0165}', '\u{017E}', '\u{017A}', + '\u{00A0}', '\u{02C7}', '\u{02D8}', '\u{0141}', '\u{00A4}', '\u{0104}', '\u{00A6}', '\u{00A7}', + '\u{00A8}', '\u{00A9}', '\u{015E}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{017B}', + '\u{00B0}', '\u{00B1}', '\u{02DB}', '\u{0142}', '\u{00B4}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{00B8}', '\u{0105}', '\u{015F}', '\u{00BB}', '\u{013D}', '\u{02DD}', '\u{013E}', '\u{017C}', + '\u{0154}', '\u{00C1}', '\u{00C2}', '\u{0102}', '\u{00C4}', '\u{0139}', '\u{0106}', '\u{00C7}', + '\u{010C}', '\u{00C9}', '\u{0118}', '\u{00CB}', '\u{011A}', '\u{00CD}', '\u{00CE}', '\u{010E}', + '\u{0110}', '\u{0143}', '\u{0147}', '\u{00D3}', '\u{00D4}', '\u{0150}', '\u{00D6}', '\u{00D7}', + '\u{0158}', '\u{016E}', '\u{00DA}', '\u{0170}', '\u{00DC}', '\u{00DD}', '\u{0162}', '\u{00DF}', + '\u{0155}', '\u{00E1}', '\u{00E2}', '\u{0103}', '\u{00E4}', '\u{013A}', '\u{0107}', '\u{00E7}', + '\u{010D}', '\u{00E9}', '\u{0119}', '\u{00EB}', '\u{011B}', '\u{00ED}', '\u{00EE}', '\u{010F}', + '\u{0111}', '\u{0144}', '\u{0148}', '\u{00F3}', '\u{00F4}', '\u{0151}', '\u{00F6}', '\u{00F7}', + '\u{0159}', '\u{016F}', '\u{00FA}', '\u{0171}', '\u{00FC}', '\u{00FD}', '\u{0163}', '\u{02D9}', +]; + +#[rustfmt::skip] +pub(super) static CP1251: [char; 128] = [ + '\u{0402}', '\u{0403}', '\u{201A}', '\u{0453}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{20AC}', '\u{2030}', '\u{0409}', '\u{2039}', '\u{040A}', '\u{040C}', '\u{040B}', '\u{040F}', + '\u{0452}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{FFFD}', '\u{2122}', '\u{0459}', '\u{203A}', '\u{045A}', '\u{045C}', '\u{045B}', '\u{045F}', + '\u{00A0}', '\u{040E}', '\u{045E}', '\u{0408}', '\u{00A4}', '\u{0490}', '\u{00A6}', '\u{00A7}', + '\u{0401}', '\u{00A9}', '\u{0404}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{0407}', + '\u{00B0}', '\u{00B1}', '\u{0406}', '\u{0456}', '\u{0491}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{0451}', '\u{2116}', '\u{0454}', '\u{00BB}', '\u{0458}', '\u{0405}', '\u{0455}', '\u{0457}', + '\u{0410}', '\u{0411}', '\u{0412}', '\u{0413}', '\u{0414}', '\u{0415}', '\u{0416}', '\u{0417}', + '\u{0418}', '\u{0419}', '\u{041A}', '\u{041B}', '\u{041C}', '\u{041D}', '\u{041E}', '\u{041F}', + '\u{0420}', '\u{0421}', '\u{0422}', '\u{0423}', '\u{0424}', '\u{0425}', '\u{0426}', '\u{0427}', + '\u{0428}', '\u{0429}', '\u{042A}', '\u{042B}', '\u{042C}', '\u{042D}', '\u{042E}', '\u{042F}', + '\u{0430}', '\u{0431}', '\u{0432}', '\u{0433}', '\u{0434}', '\u{0435}', '\u{0436}', '\u{0437}', + '\u{0438}', '\u{0439}', '\u{043A}', '\u{043B}', '\u{043C}', '\u{043D}', '\u{043E}', '\u{043F}', + '\u{0440}', '\u{0441}', '\u{0442}', '\u{0443}', '\u{0444}', '\u{0445}', '\u{0446}', '\u{0447}', + '\u{0448}', '\u{0449}', '\u{044A}', '\u{044B}', '\u{044C}', '\u{044D}', '\u{044E}', '\u{044F}', +]; + +#[rustfmt::skip] +pub(super) static CP1253: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{FFFD}', '\u{2030}', '\u{FFFD}', '\u{2039}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{FFFD}', '\u{2122}', '\u{FFFD}', '\u{203A}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{00A0}', '\u{0385}', '\u{0386}', '\u{00A3}', '\u{00A4}', '\u{00A5}', '\u{00A6}', '\u{00A7}', + '\u{00A8}', '\u{00A9}', '\u{FFFD}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{2015}', + '\u{00B0}', '\u{00B1}', '\u{00B2}', '\u{00B3}', '\u{0384}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{0388}', '\u{0389}', '\u{038A}', '\u{00BB}', '\u{038C}', '\u{00BD}', '\u{038E}', '\u{038F}', + '\u{0390}', '\u{0391}', '\u{0392}', '\u{0393}', '\u{0394}', '\u{0395}', '\u{0396}', '\u{0397}', + '\u{0398}', '\u{0399}', '\u{039A}', '\u{039B}', '\u{039C}', '\u{039D}', '\u{039E}', '\u{039F}', + '\u{03A0}', '\u{03A1}', '\u{FFFD}', '\u{03A3}', '\u{03A4}', '\u{03A5}', '\u{03A6}', '\u{03A7}', + '\u{03A8}', '\u{03A9}', '\u{03AA}', '\u{03AB}', '\u{03AC}', '\u{03AD}', '\u{03AE}', '\u{03AF}', + '\u{03B0}', '\u{03B1}', '\u{03B2}', '\u{03B3}', '\u{03B4}', '\u{03B5}', '\u{03B6}', '\u{03B7}', + '\u{03B8}', '\u{03B9}', '\u{03BA}', '\u{03BB}', '\u{03BC}', '\u{03BD}', '\u{03BE}', '\u{03BF}', + '\u{03C0}', '\u{03C1}', '\u{03C2}', '\u{03C3}', '\u{03C4}', '\u{03C5}', '\u{03C6}', '\u{03C7}', + '\u{03C8}', '\u{03C9}', '\u{03CA}', '\u{03CB}', '\u{03CC}', '\u{03CD}', '\u{03CE}', '\u{FFFD}', +]; + +#[rustfmt::skip] +pub(super) static CP1254: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{02C6}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{02DC}', '\u{2122}', '\u{0161}', '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{FFFD}', '\u{0178}', + '\u{00A0}', '\u{00A1}', '\u{00A2}', '\u{00A3}', '\u{00A4}', '\u{00A5}', '\u{00A6}', '\u{00A7}', + '\u{00A8}', '\u{00A9}', '\u{00AA}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{00AF}', + '\u{00B0}', '\u{00B1}', '\u{00B2}', '\u{00B3}', '\u{00B4}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{00B8}', '\u{00B9}', '\u{00BA}', '\u{00BB}', '\u{00BC}', '\u{00BD}', '\u{00BE}', '\u{00BF}', + '\u{00C0}', '\u{00C1}', '\u{00C2}', '\u{00C3}', '\u{00C4}', '\u{00C5}', '\u{00C6}', '\u{00C7}', + '\u{00C8}', '\u{00C9}', '\u{00CA}', '\u{00CB}', '\u{00CC}', '\u{00CD}', '\u{00CE}', '\u{00CF}', + '\u{011E}', '\u{00D1}', '\u{00D2}', '\u{00D3}', '\u{00D4}', '\u{00D5}', '\u{00D6}', '\u{00D7}', + '\u{00D8}', '\u{00D9}', '\u{00DA}', '\u{00DB}', '\u{00DC}', '\u{0130}', '\u{015E}', '\u{00DF}', + '\u{00E0}', '\u{00E1}', '\u{00E2}', '\u{00E3}', '\u{00E4}', '\u{00E5}', '\u{00E6}', '\u{00E7}', + '\u{00E8}', '\u{00E9}', '\u{00EA}', '\u{00EB}', '\u{00EC}', '\u{00ED}', '\u{00EE}', '\u{00EF}', + '\u{011F}', '\u{00F1}', '\u{00F2}', '\u{00F3}', '\u{00F4}', '\u{00F5}', '\u{00F6}', '\u{00F7}', + '\u{00F8}', '\u{00F9}', '\u{00FA}', '\u{00FB}', '\u{00FC}', '\u{0131}', '\u{015F}', '\u{00FF}', +]; + +#[rustfmt::skip] +pub(super) static CP1255: [char; 128] = [ + '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{02C6}', '\u{2030}', '\u{FFFD}', '\u{2039}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{02DC}', '\u{2122}', '\u{FFFD}', '\u{203A}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{00A0}', '\u{00A1}', '\u{00A2}', '\u{00A3}', '\u{20AA}', '\u{00A5}', '\u{00A6}', '\u{00A7}', + '\u{00A8}', '\u{00A9}', '\u{00D7}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{00AF}', + '\u{00B0}', '\u{00B1}', '\u{00B2}', '\u{00B3}', '\u{00B4}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{00B8}', '\u{00B9}', '\u{00F7}', '\u{00BB}', '\u{00BC}', '\u{00BD}', '\u{00BE}', '\u{00BF}', + '\u{05B0}', '\u{05B1}', '\u{05B2}', '\u{05B3}', '\u{05B4}', '\u{05B5}', '\u{05B6}', '\u{05B7}', + '\u{05B8}', '\u{05B9}', '\u{FFFD}', '\u{05BB}', '\u{05BC}', '\u{05BD}', '\u{05BE}', '\u{05BF}', + '\u{05C0}', '\u{05C1}', '\u{05C2}', '\u{05C3}', '\u{05F0}', '\u{05F1}', '\u{05F2}', '\u{05F3}', + '\u{05F4}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', '\u{FFFD}', + '\u{05D0}', '\u{05D1}', '\u{05D2}', '\u{05D3}', '\u{05D4}', '\u{05D5}', '\u{05D6}', '\u{05D7}', + '\u{05D8}', '\u{05D9}', '\u{05DA}', '\u{05DB}', '\u{05DC}', '\u{05DD}', '\u{05DE}', '\u{05DF}', + '\u{05E0}', '\u{05E1}', '\u{05E2}', '\u{05E3}', '\u{05E4}', '\u{05E5}', '\u{05E6}', '\u{05E7}', + '\u{05E8}', '\u{05E9}', '\u{05EA}', '\u{FFFD}', '\u{FFFD}', '\u{200E}', '\u{200F}', '\u{FFFD}', +]; + +#[rustfmt::skip] +pub(super) static CP1256: [char; 128] = [ + '\u{20AC}', '\u{067E}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', + '\u{02C6}', '\u{2030}', '\u{0679}', '\u{2039}', '\u{0152}', '\u{0686}', '\u{0698}', '\u{0688}', + '\u{06AF}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', + '\u{06A9}', '\u{2122}', '\u{0691}', '\u{203A}', '\u{0153}', '\u{200C}', '\u{200D}', '\u{06BA}', + '\u{00A0}', '\u{060C}', '\u{00A2}', '\u{00A3}', '\u{00A4}', '\u{00A5}', '\u{00A6}', '\u{00A7}', + '\u{00A8}', '\u{00A9}', '\u{06BE}', '\u{00AB}', '\u{00AC}', '\u{00AD}', '\u{00AE}', '\u{00AF}', + '\u{00B0}', '\u{00B1}', '\u{00B2}', '\u{00B3}', '\u{00B4}', '\u{00B5}', '\u{00B6}', '\u{00B7}', + '\u{00B8}', '\u{00B9}', '\u{061B}', '\u{00BB}', '\u{00BC}', '\u{00BD}', '\u{00BE}', '\u{061F}', + '\u{06C1}', '\u{0621}', '\u{0622}', '\u{0623}', '\u{0624}', '\u{0625}', '\u{0626}', '\u{0627}', + '\u{0628}', '\u{0629}', '\u{062A}', '\u{062B}', '\u{062C}', '\u{062D}', '\u{062E}', '\u{062F}', + '\u{0630}', '\u{0631}', '\u{0632}', '\u{0633}', '\u{0634}', '\u{0635}', '\u{0636}', '\u{00D7}', + '\u{0637}', '\u{0638}', '\u{0639}', '\u{063A}', '\u{0640}', '\u{0641}', '\u{0642}', '\u{0643}', + '\u{00E0}', '\u{0644}', '\u{00E2}', '\u{0645}', '\u{0646}', '\u{0647}', '\u{0648}', '\u{00E7}', + '\u{00E8}', '\u{00E9}', '\u{00EA}', '\u{00EB}', '\u{0649}', '\u{064A}', '\u{00EE}', '\u{00EF}', + '\u{064B}', '\u{064C}', '\u{064D}', '\u{064E}', '\u{00F4}', '\u{064F}', '\u{0650}', '\u{00F7}', + '\u{0651}', '\u{00F9}', '\u{0652}', '\u{00FB}', '\u{00FC}', '\u{200E}', '\u{200F}', '\u{06D2}', +]; diff --git a/crates/foxvm/tests/codepage_reference.txt b/crates/foxvm/tests/codepage_reference.txt new file mode 100644 index 0000000..50e9792 --- /dev/null +++ b/crates/foxvm/tests/codepage_reference.txt @@ -0,0 +1,20 @@ +# Microsoft mappings hosted by Unicode; see docs/codepage-mappings.md. Columns: page marker then U+ values for bytes 80..FF. +437 01 00C7 00FC 00E9 00E2 00E4 00E0 00E5 00E7 00EA 00EB 00E8 00EF 00EE 00EC 00C4 00C5 00C9 00E6 00C6 00F4 00F6 00F2 00FB 00F9 00FF 00D6 00DC 00A2 00A3 00A5 20A7 0192 00E1 00ED 00F3 00FA 00F1 00D1 00AA 00BA 00BF 2310 00AC 00BD 00BC 00A1 00AB 00BB 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03B1 00DF 0393 03C0 03A3 03C3 00B5 03C4 03A6 0398 03A9 03B4 221E 03C6 03B5 2229 2261 00B1 2265 2264 2320 2321 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +737 6A 0391 0392 0393 0394 0395 0396 0397 0398 0399 039A 039B 039C 039D 039E 039F 03A0 03A1 03A3 03A4 03A5 03A6 03A7 03A8 03A9 03B1 03B2 03B3 03B4 03B5 03B6 03B7 03B8 03B9 03BA 03BB 03BC 03BD 03BE 03BF 03C0 03C1 03C3 03C2 03C4 03C5 03C6 03C7 03C8 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03C9 03AC 03AD 03AE 03CA 03AF 03CC 03CD 03CB 03CE 0386 0388 0389 038A 038C 038E 038F 00B1 2265 2264 03AA 03AB 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +850 02 00C7 00FC 00E9 00E2 00E4 00E0 00E5 00E7 00EA 00EB 00E8 00EF 00EE 00EC 00C4 00C5 00C9 00E6 00C6 00F4 00F6 00F2 00FB 00F9 00FF 00D6 00DC 00F8 00A3 00D8 00D7 0192 00E1 00ED 00F3 00FA 00F1 00D1 00AA 00BA 00BF 00AE 00AC 00BD 00BC 00A1 00AB 00BB 2591 2592 2593 2502 2524 00C1 00C2 00C0 00A9 2563 2551 2557 255D 00A2 00A5 2510 2514 2534 252C 251C 2500 253C 00E3 00C3 255A 2554 2569 2566 2560 2550 256C 00A4 00F0 00D0 00CA 00CB 00C8 0131 00CD 00CE 00CF 2518 250C 2588 2584 00A6 00CC 2580 00D3 00DF 00D4 00D2 00F5 00D5 00B5 00FE 00DE 00DA 00DB 00D9 00FD 00DD 00AF 00B4 00AD 00B1 2017 00BE 00B6 00A7 00F7 00B8 00B0 00A8 00B7 00B9 00B3 00B2 25A0 00A0 +852 64 00C7 00FC 00E9 00E2 00E4 016F 0107 00E7 0142 00EB 0150 0151 00EE 0179 00C4 0106 00C9 0139 013A 00F4 00F6 013D 013E 015A 015B 00D6 00DC 0164 0165 0141 00D7 010D 00E1 00ED 00F3 00FA 0104 0105 017D 017E 0118 0119 00AC 017A 010C 015F 00AB 00BB 2591 2592 2593 2502 2524 00C1 00C2 011A 015E 2563 2551 2557 255D 017B 017C 2510 2514 2534 252C 251C 2500 253C 0102 0103 255A 2554 2569 2566 2560 2550 256C 00A4 0111 0110 010E 00CB 010F 0147 00CD 00CE 011B 2518 250C 2588 2584 0162 016E 2580 00D3 00DF 00D4 0143 0144 0148 0160 0161 0154 00DA 0155 0170 00FD 00DD 0163 00B4 00AD 02DD 02DB 02C7 02D8 00A7 00F7 00B8 00B0 00A8 02D9 0171 0158 0159 25A0 00A0 +857 6B 00C7 00FC 00E9 00E2 00E4 00E0 00E5 00E7 00EA 00EB 00E8 00EF 00EE 0131 00C4 00C5 00C9 00E6 00C6 00F4 00F6 00F2 00FB 00F9 0130 00D6 00DC 00F8 00A3 00D8 015E 015F 00E1 00ED 00F3 00FA 00F1 00D1 011E 011F 00BF 00AE 00AC 00BD 00BC 00A1 00AB 00BB 2591 2592 2593 2502 2524 00C1 00C2 00C0 00A9 2563 2551 2557 255D 00A2 00A5 2510 2514 2534 252C 251C 2500 253C 00E3 00C3 255A 2554 2569 2566 2560 2550 256C 00A4 00BA 00AA 00CA 00CB 00C8 FFFD 00CD 00CE 00CF 2518 250C 2588 2584 00A6 00CC 2580 00D3 00DF 00D4 00D2 00F5 00D5 00B5 FFFD 00D7 00DA 00DB 00D9 00EC 00FF 00AF 00B4 00AD 00B1 FFFD 00BE 00B6 00A7 00F7 00B8 00B0 00A8 00B7 00B9 00B3 00B2 25A0 00A0 +860 24 00C7 00FC 00E9 00E2 00E3 00E0 00C1 00E7 00EA 00CA 00E8 00CD 00D4 00EC 00C3 00C2 00C9 00C0 00C8 00F4 00F5 00F2 00DA 00F9 00CC 00D5 00DC 00A2 00A3 00D9 20A7 00D3 00E1 00ED 00F3 00FA 00F1 00D1 00AA 00BA 00BF 00D2 00AC 00BD 00BC 00A1 00AB 00BB 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03B1 00DF 0393 03C0 03A3 03C3 00B5 03C4 03A6 0398 03A9 03B4 221E 03C6 03B5 2229 2261 00B1 2265 2264 2320 2321 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +861 67 00C7 00FC 00E9 00E2 00E4 00E0 00E5 00E7 00EA 00EB 00E8 00D0 00F0 00DE 00C4 00C5 00C9 00E6 00C6 00F4 00F6 00FE 00FB 00DD 00FD 00D6 00DC 00F8 00A3 00D8 20A7 0192 00E1 00ED 00F3 00FA 00C1 00CD 00D3 00DA 00BF 2310 00AC 00BD 00BC 00A1 00AB 00BB 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03B1 00DF 0393 03C0 03A3 03C3 00B5 03C4 03A6 0398 03A9 03B4 221E 03C6 03B5 2229 2261 00B1 2265 2264 2320 2321 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +863 6C 00C7 00FC 00E9 00E2 00C2 00E0 00B6 00E7 00EA 00EB 00E8 00EF 00EE 2017 00C0 00A7 00C9 00C8 00CA 00F4 00CB 00CF 00FB 00F9 00A4 00D4 00DC 00A2 00A3 00D9 00DB 0192 00A6 00B4 00F3 00FA 00A8 00B8 00B3 00AF 00CE 2310 00AC 00BD 00BC 00BE 00AB 00BB 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03B1 00DF 0393 03C0 03A3 03C3 00B5 03C4 03A6 0398 03A9 03B4 221E 03C6 03B5 2229 2261 00B1 2265 2264 2320 2321 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +865 66 00C7 00FC 00E9 00E2 00E4 00E0 00E5 00E7 00EA 00EB 00E8 00EF 00EE 00EC 00C4 00C5 00C9 00E6 00C6 00F4 00F6 00F2 00FB 00F9 00FF 00D6 00DC 00F8 00A3 00D8 20A7 0192 00E1 00ED 00F3 00FA 00F1 00D1 00AA 00BA 00BF 2310 00AC 00BD 00BC 00A1 00AB 00A4 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 03B1 00DF 0393 03C0 03A3 03C3 00B5 03C4 03A6 0398 03A9 03B4 221E 03C6 03B5 2229 2261 00B1 2265 2264 2320 2321 00F7 2248 00B0 2219 00B7 221A 207F 00B2 25A0 00A0 +866 65 0410 0411 0412 0413 0414 0415 0416 0417 0418 0419 041A 041B 041C 041D 041E 041F 0420 0421 0422 0423 0424 0425 0426 0427 0428 0429 042A 042B 042C 042D 042E 042F 0430 0431 0432 0433 0434 0435 0436 0437 0438 0439 043A 043B 043C 043D 043E 043F 2591 2592 2593 2502 2524 2561 2562 2556 2555 2563 2551 2557 255D 255C 255B 2510 2514 2534 252C 251C 2500 253C 255E 255F 255A 2554 2569 2566 2560 2550 256C 2567 2568 2564 2565 2559 2558 2552 2553 256B 256A 2518 250C 2588 2584 258C 2590 2580 0440 0441 0442 0443 0444 0445 0446 0447 0448 0449 044A 044B 044C 044D 044E 044F 0401 0451 0404 0454 0407 0457 040E 045E 00B0 2219 00B7 221A 2116 00A4 25A0 00A0 +874 7C 20AC FFFD FFFD FFFD FFFD 2026 FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD 2018 2019 201C 201D 2022 2013 2014 FFFD FFFD FFFD FFFD FFFD FFFD FFFD FFFD 00A0 0E01 0E02 0E03 0E04 0E05 0E06 0E07 0E08 0E09 0E0A 0E0B 0E0C 0E0D 0E0E 0E0F 0E10 0E11 0E12 0E13 0E14 0E15 0E16 0E17 0E18 0E19 0E1A 0E1B 0E1C 0E1D 0E1E 0E1F 0E20 0E21 0E22 0E23 0E24 0E25 0E26 0E27 0E28 0E29 0E2A 0E2B 0E2C 0E2D 0E2E 0E2F 0E30 0E31 0E32 0E33 0E34 0E35 0E36 0E37 0E38 0E39 0E3A FFFD FFFD FFFD FFFD 0E3F 0E40 0E41 0E42 0E43 0E44 0E45 0E46 0E47 0E48 0E49 0E4A 0E4B 0E4C 0E4D 0E4E 0E4F 0E50 0E51 0E52 0E53 0E54 0E55 0E56 0E57 0E58 0E59 0E5A 0E5B FFFD FFFD FFFD FFFD +1250 C8 20AC FFFD 201A FFFD 201E 2026 2020 2021 FFFD 2030 0160 2039 015A 0164 017D 0179 FFFD 2018 2019 201C 201D 2022 2013 2014 FFFD 2122 0161 203A 015B 0165 017E 017A 00A0 02C7 02D8 0141 00A4 0104 00A6 00A7 00A8 00A9 015E 00AB 00AC 00AD 00AE 017B 00B0 00B1 02DB 0142 00B4 00B5 00B6 00B7 00B8 0105 015F 00BB 013D 02DD 013E 017C 0154 00C1 00C2 0102 00C4 0139 0106 00C7 010C 00C9 0118 00CB 011A 00CD 00CE 010E 0110 0143 0147 00D3 00D4 0150 00D6 00D7 0158 016E 00DA 0170 00DC 00DD 0162 00DF 0155 00E1 00E2 0103 00E4 013A 0107 00E7 010D 00E9 0119 00EB 011B 00ED 00EE 010F 0111 0144 0148 00F3 00F4 0151 00F6 00F7 0159 016F 00FA 0171 00FC 00FD 0163 02D9 +1251 C9 0402 0403 201A 0453 201E 2026 2020 2021 20AC 2030 0409 2039 040A 040C 040B 040F 0452 2018 2019 201C 201D 2022 2013 2014 FFFD 2122 0459 203A 045A 045C 045B 045F 00A0 040E 045E 0408 00A4 0490 00A6 00A7 0401 00A9 0404 00AB 00AC 00AD 00AE 0407 00B0 00B1 0406 0456 0491 00B5 00B6 00B7 0451 2116 0454 00BB 0458 0405 0455 0457 0410 0411 0412 0413 0414 0415 0416 0417 0418 0419 041A 041B 041C 041D 041E 041F 0420 0421 0422 0423 0424 0425 0426 0427 0428 0429 042A 042B 042C 042D 042E 042F 0430 0431 0432 0433 0434 0435 0436 0437 0438 0439 043A 043B 043C 043D 043E 043F 0440 0441 0442 0443 0444 0445 0446 0447 0448 0449 044A 044B 044C 044D 044E 044F +1252 03 20AC FFFD 201A 0192 201E 2026 2020 2021 02C6 2030 0160 2039 0152 FFFD 017D FFFD FFFD 2018 2019 201C 201D 2022 2013 2014 02DC 2122 0161 203A 0153 FFFD 017E 0178 00A0 00A1 00A2 00A3 00A4 00A5 00A6 00A7 00A8 00A9 00AA 00AB 00AC 00AD 00AE 00AF 00B0 00B1 00B2 00B3 00B4 00B5 00B6 00B7 00B8 00B9 00BA 00BB 00BC 00BD 00BE 00BF 00C0 00C1 00C2 00C3 00C4 00C5 00C6 00C7 00C8 00C9 00CA 00CB 00CC 00CD 00CE 00CF 00D0 00D1 00D2 00D3 00D4 00D5 00D6 00D7 00D8 00D9 00DA 00DB 00DC 00DD 00DE 00DF 00E0 00E1 00E2 00E3 00E4 00E5 00E6 00E7 00E8 00E9 00EA 00EB 00EC 00ED 00EE 00EF 00F0 00F1 00F2 00F3 00F4 00F5 00F6 00F7 00F8 00F9 00FA 00FB 00FC 00FD 00FE 00FF +1253 CB 20AC FFFD 201A 0192 201E 2026 2020 2021 FFFD 2030 FFFD 2039 FFFD FFFD FFFD FFFD FFFD 2018 2019 201C 201D 2022 2013 2014 FFFD 2122 FFFD 203A FFFD FFFD FFFD FFFD 00A0 0385 0386 00A3 00A4 00A5 00A6 00A7 00A8 00A9 FFFD 00AB 00AC 00AD 00AE 2015 00B0 00B1 00B2 00B3 0384 00B5 00B6 00B7 0388 0389 038A 00BB 038C 00BD 038E 038F 0390 0391 0392 0393 0394 0395 0396 0397 0398 0399 039A 039B 039C 039D 039E 039F 03A0 03A1 FFFD 03A3 03A4 03A5 03A6 03A7 03A8 03A9 03AA 03AB 03AC 03AD 03AE 03AF 03B0 03B1 03B2 03B3 03B4 03B5 03B6 03B7 03B8 03B9 03BA 03BB 03BC 03BD 03BE 03BF 03C0 03C1 03C2 03C3 03C4 03C5 03C6 03C7 03C8 03C9 03CA 03CB 03CC 03CD 03CE FFFD +1254 CA 20AC FFFD 201A 0192 201E 2026 2020 2021 02C6 2030 0160 2039 0152 FFFD FFFD FFFD FFFD 2018 2019 201C 201D 2022 2013 2014 02DC 2122 0161 203A 0153 FFFD FFFD 0178 00A0 00A1 00A2 00A3 00A4 00A5 00A6 00A7 00A8 00A9 00AA 00AB 00AC 00AD 00AE 00AF 00B0 00B1 00B2 00B3 00B4 00B5 00B6 00B7 00B8 00B9 00BA 00BB 00BC 00BD 00BE 00BF 00C0 00C1 00C2 00C3 00C4 00C5 00C6 00C7 00C8 00C9 00CA 00CB 00CC 00CD 00CE 00CF 011E 00D1 00D2 00D3 00D4 00D5 00D6 00D7 00D8 00D9 00DA 00DB 00DC 0130 015E 00DF 00E0 00E1 00E2 00E3 00E4 00E5 00E6 00E7 00E8 00E9 00EA 00EB 00EC 00ED 00EE 00EF 011F 00F1 00F2 00F3 00F4 00F5 00F6 00F7 00F8 00F9 00FA 00FB 00FC 0131 015F 00FF +1255 7D 20AC FFFD 201A 0192 201E 2026 2020 2021 02C6 2030 FFFD 2039 FFFD FFFD FFFD FFFD FFFD 2018 2019 201C 201D 2022 2013 2014 02DC 2122 FFFD 203A FFFD FFFD FFFD FFFD 00A0 00A1 00A2 00A3 20AA 00A5 00A6 00A7 00A8 00A9 00D7 00AB 00AC 00AD 00AE 00AF 00B0 00B1 00B2 00B3 00B4 00B5 00B6 00B7 00B8 00B9 00F7 00BB 00BC 00BD 00BE 00BF 05B0 05B1 05B2 05B3 05B4 05B5 05B6 05B7 05B8 05B9 FFFD 05BB 05BC 05BD 05BE 05BF 05C0 05C1 05C2 05C3 05F0 05F1 05F2 05F3 05F4 FFFD FFFD FFFD FFFD FFFD FFFD FFFD 05D0 05D1 05D2 05D3 05D4 05D5 05D6 05D7 05D8 05D9 05DA 05DB 05DC 05DD 05DE 05DF 05E0 05E1 05E2 05E3 05E4 05E5 05E6 05E7 05E8 05E9 05EA FFFD FFFD 200E 200F FFFD +1256 7E 20AC 067E 201A 0192 201E 2026 2020 2021 02C6 2030 0679 2039 0152 0686 0698 0688 06AF 2018 2019 201C 201D 2022 2013 2014 06A9 2122 0691 203A 0153 200C 200D 06BA 00A0 060C 00A2 00A3 00A4 00A5 00A6 00A7 00A8 00A9 06BE 00AB 00AC 00AD 00AE 00AF 00B0 00B1 00B2 00B3 00B4 00B5 00B6 00B7 00B8 00B9 061B 00BB 00BC 00BD 00BE 061F 06C1 0621 0622 0623 0624 0625 0626 0627 0628 0629 062A 062B 062C 062D 062E 062F 0630 0631 0632 0633 0634 0635 0636 00D7 0637 0638 0639 063A 0640 0641 0642 0643 00E0 0644 00E2 0645 0646 0647 0648 00E7 00E8 00E9 00EA 00EB 0649 064A 00EE 00EF 064B 064C 064D 064E 00F4 064F 0650 00F7 0651 00F9 0652 00FB 00FC 200E 200F 06D2 +1257 CC 20AC FFFD 201A FFFD 201E 2026 2020 2021 FFFD 2030 FFFD 2039 FFFD 00A8 02C7 00B8 FFFD 2018 2019 201C 201D 2022 2013 2014 FFFD 2122 FFFD 203A FFFD 00AF 02DB FFFD 00A0 FFFD 00A2 00A3 00A4 FFFD 00A6 00A7 00D8 00A9 0156 00AB 00AC 00AD 00AE 00C6 00B0 00B1 00B2 00B3 00B4 00B5 00B6 00B7 00F8 00B9 0157 00BB 00BC 00BD 00BE 00E6 0104 012E 0100 0106 00C4 00C5 0118 0112 010C 00C9 0179 0116 0122 0136 012A 013B 0160 0143 0145 00D3 014C 00D5 00D6 00D7 0172 0141 015A 016A 00DC 017B 017D 00DF 0105 012F 0101 0107 00E4 00E5 0119 0113 010D 00E9 017A 0117 0123 0137 012B 013C 0161 0144 0146 00F3 014D 00F5 00F6 00F7 0173 0142 015B 016B 00FC 017C 017E 02D9 diff --git a/crates/foxvm/tests/dbf.rs b/crates/foxvm/tests/dbf.rs index fcec993..53d8677 100644 --- a/crates/foxvm/tests/dbf.rs +++ b/crates/foxvm/tests/dbf.rs @@ -572,3 +572,35 @@ fn code_pages_decode_high_bytes() { let odd = read_table(&b.build(&[(false, vec![0x81])]), None).expect("1252"); assert_eq!(odd.records[0].get(&odd, "TXT").unwrap().as_text(), "\u{fffd}"); } + +#[test] +fn windows_1257_character_and_memo_fields() { + let text = "ĄČĘĖĮŠŲŪŽąčęėįšųūž"; + let raw = [0xc0,0xc8,0xc6,0xcb,0xc1,0xd0,0xd8,0xdb,0xde, + 0xe0,0xe8,0xe6,0xeb,0xe1,0xf0,0xf8,0xfb,0xfe]; + let mut dbf = vec![0u8; 97 + 23]; + dbf[0] = 0x30; + dbf[4..8].copy_from_slice(&1u32.to_le_bytes()); + dbf[8..10].copy_from_slice(&97u16.to_le_bytes()); + dbf[10..12].copy_from_slice(&23u16.to_le_bytes()); + dbf[29] = 0xcc; + dbf[32..37].copy_from_slice(b"LABEL"); + dbf[43] = b'C'; + dbf[48] = 18; + dbf[64..74].copy_from_slice(b"PROPERTIES"); + dbf[75] = b'M'; + dbf[80] = 4; + dbf[96] = 0x0d; + dbf[97] = b' '; + dbf[98..116].copy_from_slice(&raw); + dbf[116..120].copy_from_slice(&1u32.to_le_bytes()); + let mut memo = vec![0u8; 1024]; + memo[6..8].copy_from_slice(&512u16.to_be_bytes()); + memo[512..516].copy_from_slice(&1u32.to_be_bytes()); + memo[516..520].copy_from_slice(&18u32.to_be_bytes()); + memo[520..538].copy_from_slice(&raw); + let table = read_table(&dbf, Some(&memo)).unwrap(); + assert_eq!(table.codepage, Some(1257)); + assert_eq!(table.records[0].get(&table, "LABEL").unwrap().as_text(), text); + assert_eq!(table.records[0].get(&table, "PROPERTIES").unwrap().as_text(), text); +} diff --git a/crates/foxvm/tests/dbf_codepages.rs b/crates/foxvm/tests/dbf_codepages.rs new file mode 100644 index 0000000..3741db8 --- /dev/null +++ b/crates/foxvm/tests/dbf_codepages.rs @@ -0,0 +1,161 @@ +//! Synthetic international text checks; no application fixtures. +use foxvm::dbf::{ + DbfField, + encoding::{codepage_for_language_id, decode, encode}, + read_table, + write::encode_field, +}; +use foxvm::value::Value; + +fn cases() -> Vec<(u16, u8, Vec)> { + include_str!("codepage_reference.txt") + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| { + let mut fields = l.split_whitespace(); + let cp = fields.next().unwrap().parse().unwrap(); + let marker = u8::from_str_radix(fields.next().unwrap(), 16).unwrap(); + let expected = fields.map(|s| char::from_u32(u32::from_str_radix(s, 16).unwrap()).unwrap()).collect(); + (cp, marker, expected) + }) + .collect() +} + +#[test] +fn reference_mappings_decode_and_encode_every_defined_byte() { + for (cp, marker, expected) in cases() { + assert_eq!(codepage_for_language_id(marker), Some(cp)); + assert_eq!(expected.len(), 128); + for byte in 0u8..=255 { + let want = if byte < 128 { byte as char } else { expected[(byte - 128) as usize] }; + assert_eq!(decode(&[byte], Some(cp)), want.to_string(), "CP{cp} byte {byte:02X}"); + if want != '\u{fffd}' { + assert_eq!(encode(&want.to_string(), Some(cp)), vec![byte], "CP{cp} byte {byte:02X}"); + } + } + assert_eq!(encode("😀", Some(cp)), b"?"); + } +} + +// A character field followed by a text/binary memo, using a real DBF/FPT layout. +fn fixture(marker: u8, raw: &[u8], binary: bool) -> (Vec, Vec) { + let width = raw.len(); + let mut dbf = vec![0u8; 97 + 1 + width + 4]; + dbf[0] = 0x30; + dbf[4..8].copy_from_slice(&1u32.to_le_bytes()); + dbf[8..10].copy_from_slice(&97u16.to_le_bytes()); + dbf[10..12].copy_from_slice(&((1 + width + 4) as u16).to_le_bytes()); + dbf[29] = marker; + dbf[32..37].copy_from_slice(b"LABEL"); + dbf[43] = b'C'; + dbf[48] = width as u8; + dbf[64..68].copy_from_slice(b"NOTE"); + dbf[75] = b'M'; + dbf[80] = 4; + dbf[96] = 0x0d; + dbf[97] = b' '; + dbf[98..98 + width].copy_from_slice(raw); + dbf[98 + width..].copy_from_slice(&1u32.to_le_bytes()); + let mut memo = vec![0u8; 1024]; + memo[0..4].copy_from_slice(&2u32.to_be_bytes()); + memo[6..8].copy_from_slice(&512u16.to_be_bytes()); + memo[512..516].copy_from_slice(&(if binary { 0u32 } else { 1u32 }).to_be_bytes()); + memo[516..520].copy_from_slice(&(width as u32).to_be_bytes()); + memo[520..520 + width].copy_from_slice(raw); + (dbf, memo) +} + +#[test] +fn character_write_reopen_and_memo_read_use_each_tables_page() { + for (cp, marker, expected) in cases() { + let pairs: Vec<_> = expected.iter().enumerate().filter(|(_, c)| **c != '\u{fffd}').collect(); + let raw: Vec<_> = pairs.iter().map(|(i, _)| *i as u8 + 128).collect(); + let text: String = pairs.iter().map(|(_, c)| **c).collect(); + let field = DbfField::new("LABEL", 'C', raw.len() as u8, 0); + let written = encode_field(&field, raw.len(), &Value::str(&text), Some(cp)).unwrap(); + assert_eq!(written, raw, "CP{cp} write bytes"); + let (dbf, memo) = fixture(marker, &written, false); + let table = read_table(&dbf, Some(&memo)).unwrap(); + for name in ["LABEL", "NOTE"] { + assert_eq!(table.records[0].get(&table, name).unwrap().as_text(), text, "CP{cp} {name}"); + } + let short = encode_field(&field, 2, &Value::str(&text), Some(cp)).unwrap(); + assert_eq!(short, &raw[..2]); + let padded = encode_field(&field, 3, &Value::str(text.chars().next().unwrap().to_string()), Some(cp)).unwrap(); + assert_eq!(padded, vec![raw[0], b' ', b' ']); + let (dbf, memo) = fixture(marker, &raw, true); + let table = read_table(&dbf, Some(&memo)).unwrap(); + assert!(matches!(table.records[0].get(&table,"NOTE").unwrap(),foxvm::dbf::DbfValue::Bytes(b) if b==&raw)); + } +} + +#[test] +fn unsupported_and_unmarked_pages_keep_the_existing_fallback() { + for cp in [None, Some(932), Some(936), Some(949), Some(950), Some(10000), Some(9999)] { + assert_eq!(decode(&[0xe9], cp), "é"); + assert_eq!(encode("é", cp), [0xe9]); + } + assert_eq!(codepage_for_language_id(0), None); +} + +#[test] +fn language_text_survives_vm_character_and_memo_write_reopen() { + use foxvm::mock_host::{MockHost, run_program}; + let samples = [ + (437, "Español"), + (737, "Ελληνικά"), + (850, "Português"), + (852, "Čeština"), + (857, "Türkçe"), + (860, "Português"), + (861, "Ísland"), + (863, "Québec"), + (865, "Norsk ø"), + (866, "Привет"), + (874, "ไทย"), + (1250, "Zażółć"), + (1251, "Привет"), + (1252, "Español"), + (1253, "Ελληνικά"), + (1254, "Türkçe"), + (1255, "שלום"), + (1256, "مرحبا"), + (1257, "Ąžuolas"), + ]; + for (cp, sample) in samples { + let marker = cases().into_iter().find(|c| c.0 == cp).unwrap().1; + let (dbf, memo) = fixture(marker, &[b' '; 40], false); + let mut host = MockHost::new(); + host.tables.insert("TEXT.DBF".into(), dbf); + host.memo_files.insert("TEXT.DBF".into(), memo); + let src = format!( + "USE text.dbf EXCLUSIVE\nREPLACE label WITH \"{sample}\", note WITH \"{sample}\"\nUSE\nUSE text.dbf\n? label\n? note\nUSE" + ); + let (_, output) = run_program(&src, &mut host).unwrap(); + assert_eq!( + output, + vec![format!("{sample}{}", " ".repeat(40 - sample.chars().count())), sample.to_string()], + "CP{cp}" + ); + let table = read_table(&host.tables["TEXT.DBF"], Some(&host.memo_files["TEXT.DBF"])).unwrap(); + assert_eq!(table.codepage, Some(cp)); + assert_eq!(table.records[0].get(&table, "NOTE").unwrap().as_text(), sample); + } +} + +#[test] +fn cpconvert_uses_the_requested_source_and_target_pages() { + use foxvm::mock_host::{MockHost, run_program}; + let (_, out) = run_program( + "? CPCONVERT(1251,866,CHR(192)+CHR(193))\n? CPCONVERT(737,1253,CHR(128))\n? CPCONVERT(1252,850,CHR(241))", + &mut MockHost::new(), + ) + .unwrap(); + assert_eq!(out, vec!["\u{80}\u{81}", "\u{c1}", "\u{a4}"]); +} + +#[test] +fn mac_greek_marker_is_distinct_from_mac_roman() { + assert_eq!(codepage_for_language_id(0x98), Some(10006)); + assert_eq!(codepage_for_language_id(0x04), Some(10000)); +} diff --git a/docs/codepage-mappings.md b/docs/codepage-mappings.md new file mode 100644 index 0000000..f59b9e5 --- /dev/null +++ b/docs/codepage-mappings.md @@ -0,0 +1,29 @@ +# Single-byte codec fixtures + +Independent reference mappings: https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/ + +The golden file records high-byte Unicode values parsed from the published +Microsoft mappings. Unassigned bytes use FFFD, matching the existing codec +policy. ASCII remains byte-preserving. No private files are used. + +| Page | Source | SHA-256 | +| --- | --- | --- | +| 437 | [PC/CP437.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT) | `6bad4dabcdf5940227c7d81fab130dcb18a77850b5d79de28b5dc4e047b0aaac` | +| 737 | [PC/CP737.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP737.TXT) | `9bf9ffe61e03bdeecd68e806e12af562fd638285a8f0fbabc8fc9cd137d65fbf` | +| 850 | [PC/CP850.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP850.TXT) | `ffdcc3c1c72f1aef600a63547100ef3dc452a09ad84923d382085519751c7479` | +| 852 | [PC/CP852.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP852.TXT) | `440d098e9f2b79eeacbe2bbc1814960b6554c885740615047f5b528c2947afb6` | +| 857 | [PC/CP857.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP857.TXT) | `6396619b918ce2fe3c366630e5652b25153bbd4bb01c8109ba8f7ed8a261658d` | +| 860 | [PC/CP860.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP860.TXT) | `1b3f983eac02d9ae9fc28106f2f3476ca1e4b337c7287f1a004372e14dd11e6a` | +| 861 | [PC/CP861.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP861.TXT) | `51c7f3ef5a435e69637237be2a414aef8dfc4f891365b686ff8d56b681f32074` | +| 863 | [PC/CP863.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP863.TXT) | `f467a2a652ce3f74bb3fa86c8767dd06cbde90edfb73bf3a5541ae4cbe806d7b` | +| 865 | [PC/CP865.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP865.TXT) | `e31eeb03a39a5fbdd5e23de60a22af4219c9987de2088386855ab20f273f470a` | +| 866 | [PC/CP866.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP866.TXT) | `abcc96dd4253321eb5e542c1ece3adab10df0cc20ec5d1124a0cec22d636c924` | +| 874 | [WINDOWS/CP874.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP874.TXT) | `f45277e205ff5aa8f0625c52b04614bbd9d8836c142e9b90235748b0a9921ec9` | +| 1250 | [WINDOWS/CP1250.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1250.TXT) | `e6535e3c81f4aff1a4b369c46e588a2423d6a75da341e01a6cea108c7542c19a` | +| 1251 | [WINDOWS/CP1251.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1251.TXT) | `d9d491808bd7956c26ef8ed07fa63015bfab32860720e07d5bf3a64a609927ee` | +| 1252 | [WINDOWS/CP1252.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT) | `f607ae328b4dff5e9bfef725f5fff0ae23f38797f8a5b95998a0d2735c0e8fad` | +| 1253 | [WINDOWS/CP1253.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1253.TXT) | `85d917854926ef09b22abefd19c7b61494e229fa391b648dc332f1fac0879ce3` | +| 1254 | [WINDOWS/CP1254.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1254.TXT) | `f93569399f81abce90441793c118d2524770b58d9c5774de3233c64d88c9fca0` | +| 1255 | [WINDOWS/CP1255.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1255.TXT) | `67ab4e3e7c088be41e0f230fa9c2c5b9a803395bec8027a5ee77df74432290c7` | +| 1256 | [WINDOWS/CP1256.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1256.TXT) | `fd5835c5e3be668d4fa3dc3d0d3a6491795646fc9629c0892f45dd08aa72f367` | +| 1257 | [WINDOWS/CP1257.TXT](https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1257.TXT) | `5ac72f382ea4f64fe61854e0dae7d311583bd1dd20797697d4dce32b175a268b` | diff --git a/docs/codepage-support.md b/docs/codepage-support.md new file mode 100644 index 0000000..b212dbf --- /dev/null +++ b/docs/codepage-support.md @@ -0,0 +1,45 @@ +# DBF text codepages + +The DBF language-driver marker selects the decoding and encoding used for +character fields and text memo blocks, including form/class property memos. + +Supported single-byte pages: + +- Windows: 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257. +- Thai: 874. +- DOS: 437, 737, 850, 852, 857, 860, 861, 863, 865, 866. + +The mapping references and their hashes are recorded in +`docs/codepage-mappings.md`. The test oracle covers every byte, +with undefined high-byte positions represented as U+FFFD. An unrepresentable +Unicode character is written as `?`, preserving the existing codec convention. +Undefined byte round trips are not guaranteed; this is a text codec, not a +lossless binary conversion. Binary memo blocks retain their original bytes. + +## Limits + +Unmarked, unknown and recognized-but-unimplemented pages retain the existing +CP1252 fallback. In particular, recognizing markers for East Asian or Macintosh +pages does not mean their codecs are implemented. The Mac Greek marker 0x98 +reports 10006; it still has no dedicated codec. + +This change does not implement full application locale semantics, collation, +RTL layout, keyboard input, font-specific legacy encodings or Unicode migration. +CPCURRENT/OEMTOANSI retain their existing fixed runtime assumptions. Plain text +source files and MEM files have separate encoding paths. + +Some string built-ins still operate on low bytes of Unicode codepoints. For +example, the current ALLTRIM implementation corrupts Greek text even after a DBF +has decoded correctly. That requires a separate string-semantics fix. The DBF +write/reopen tests inspect fields directly to isolate the codec contract. + +DBCS codecs must also handle character boundaries when writing fixed-width +fields; adding a decoder alone is insufficient. + +## Tests + +`cargo test -p foxvm --test dbf_codepages` covers canonical byte mappings, +character writes, memo reads, binary preservation, field width handling, +VM REPLACE/close/reopen for both character and memo fields, and CPCONVERT. +`tests/vfp/importForm.codepages.test.ts` imports synthetic form captions through +the built WASM module for all supported pages. diff --git a/tests/vfp/importForm.codepages.test.ts b/tests/vfp/importForm.codepages.test.ts new file mode 100644 index 0000000..5127c95 --- /dev/null +++ b/tests/vfp/importForm.codepages.test.ts @@ -0,0 +1,33 @@ +import { readFileSync } from 'node:fs'; +import { beforeAll, expect, it } from 'vitest'; +import { importFormTable } from '@shared/vfp/importForm'; +import { loadFoxVm, type FoxVmModule } from '../../src/wasm/foxvm/loader'; +import { buildDbf, buildFpt } from '../data/dbfFixture'; + +const cases = readFileSync('crates/foxvm/tests/codepage_reference.txt', 'utf8') + .trim().split('\n').filter((line) => !line.startsWith('#')).map((line) => { + const [page, marker, ...points] = line.split(/\s+/); + const letters = points.map((point, index) => ({ text: String.fromCodePoint(parseInt(point!, 16)), byte: index + 128 })) + .filter(({ text }) => /\p{L}/u.test(text)).slice(0, 3); + return { page: Number(page), marker: parseInt(marker!, 16), + expected: letters.map(({ text }) => text).join(''), + raw: String.fromCharCode(...letters.map(({ byte }) => byte)) }; + }); +let vm: FoxVmModule; +beforeAll(async () => { vm = await loadFoxVm(); }); + +it.each(cases)('imports a synthetic CP$page caption through WASM', ({ marker, raw, expected }) => { + const dbf = buildDbf([ + { name: 'OBJNAME', kind: 'C', width: 20 }, + { name: 'BASECLASS', kind: 'C', width: 20 }, + { name: 'CLASS', kind: 'C', width: 20 }, + { name: 'PARENT', kind: 'C', width: 20 }, + { name: 'PROPERTIES', kind: 'M', width: 4 }, + ], [['Demo', 'form', 'form', '', '1']]); + dbf[29] = marker; + const memo = buildFpt([`Caption = "Demo ${raw}"\r\n`], 512); + const table = vm.read_dbf(dbf, memo); + if (!table.ok) throw new Error(table.error); + const { doc } = importFormTable(table, 'demo'); + expect(doc.form.props['Caption']).toBe(`Demo ${expected}`); +});