From e2f46154cc5e90b552032d6e674dc82999484986 Mon Sep 17 00:00:00 2001 From: ArnauvGilotra Date: Tue, 13 Oct 2026 23:02:02 -0400 Subject: [PATCH] mmap a page-aligned dictionary cache Parsing the 10+ MB text dictionary dominated switch-in. Each dictionary now also gets a binary cache built for memory mapping, and Load() maps it read-only and answers queries in place instead of deserializing anything. Format (version 2, magic CANTOMAP): a header page (source size + mtime signature, counts, region offsets), a page-aligned index with one entry per data block (first page, page count, first key), then 4 KB data pages packed with sorted entries that never straddle a page, each page carrying its own entry count and offset table. An entry too big for one page gets a run of whole pages to itself, flagged by its page count in the index. Strings are UTF-16 so keys compare in place and candidates need no decode. Lookup binary-searches the index, then the one page. Prefix scans start there and walk forward across page boundaries. The header and whole index are validated on map, and every in-page offset and length is bounds-checked on access, so a stale, truncated, or damaged cache falls back to the text parse (which rewrites the cache) and never crashes the host app. Overrides move to a small in-memory overlay consulted on top of the base (same prepend + dedup semantics), since the mapping is read-only. Caches are written atomically (temp file + rename; a mapped old cache is renamed aside). The installer prebuilds both stock caches with the new canto-mkcache.exe, the options dialog prebuilds a locked-in custom dictionary per user, and CI runs canto-dicttest.exe, which checks every query on the mapped cache against the text parse plus stale/truncated/corrupt handling. --- .github/workflows/build.yml | 7 + CMakeLists.txt | 15 + README.md | 27 +- installer/setup.iss | 18 + src/Dictionary.cpp | 674 +++++++++++++++++++++++++++++++++--- src/Dictionary.h | 110 ++++-- src/OptionsDialog.cpp | 45 +++ src/TextService.cpp | 12 +- src/TextService.h | 2 +- src/dicttest.cpp | 303 ++++++++++++++++ src/mkcache.cpp | 41 +++ 11 files changed, 1181 insertions(+), 73 deletions(-) create mode 100644 src/dicttest.cpp create mode 100644 src/mkcache.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 799880c..0428abf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,12 @@ jobs: - name: Build (Release) run: cmake --build build --config Release + # Builds the mmap cache for both stock dictionaries (plus a synthetic one + # with a multi-page entry), maps it, and checks every query against the + # text parse, along with stale/truncated/corrupt cache handling. + - name: Test dictionary cache + run: build\Release\canto-dicttest.exe traditional.txt simplified.txt + # Inno Setup is not preinstalled on windows-latest; Chocolatey is. - name: Install Inno Setup run: choco install innosetup --no-progress -y @@ -48,6 +54,7 @@ jobs: build\Release\CantoIME-SG.dll, ` build\Release\traditional.txt, ` build\Release\simplified.txt, ` + build\Release\canto-mkcache.exe, ` installer\Output\CantoIME-Setup.exe dist\ - name: Upload artifact diff --git a/CMakeLists.txt b/CMakeLists.txt index b454fe3..73d69cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,3 +131,18 @@ add_canto_variant(TW) # Taiwan (Traditional) add_canto_variant(MO) # Macao SAR (Traditional) add_canto_variant(CN) # China (Simplified) add_canto_variant(SG) # Singapore (Simplified) + +# Command-line tools sharing Dictionary.cpp, built into the same Release folder. +# canto-mkcache ships with the app: the installer runs it for the stock +# dictionaries and the options dialog for a custom one, so the +# IME maps a prebuilt cache instead of parsing text. +# canto-dicttest is the cache check CI runs (not shipped). +add_executable(canto-mkcache src/mkcache.cpp src/Dictionary.cpp) +add_executable(canto-dicttest src/dicttest.cpp src/Dictionary.cpp) +foreach(tool canto-mkcache canto-dicttest) + target_include_directories(${tool} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + target_compile_definitions(${tool} PRIVATE UNICODE _UNICODE WIN32_LEAN_AND_MEAN NOMINMAX) + if(MSVC) + target_compile_options(${tool} PRIVATE /W4 /permissive-) + endif() +endforeach() diff --git a/README.md b/README.md index ee1f610..0406ad4 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,29 @@ The dictionary is parsed on a **background thread**, so switching to CantoIME (o changing the dictionary/overrides) never blocks the UI. Until the load finishes — a fraction of a second — typing simply produces no candidates yet. +### Dictionary cache (mmap) + +Parsing the text dictionary is the slow part of a load, so each dictionary also +gets a binary cache laid out for memory mapping. The IME maps it read-only and +answers lookups in place: a binary search over a small page index picks the one +4 KB page that holds a key, and only the pages actually touched are read from +disk. Layout (see `src/Dictionary.cpp`): + +- **Header page:** magic, format version, the source file's size + last-write + time (a mismatch means stale), entry/block/page counts, region offsets. +- **Page index:** one entry per data block (first page, page count, first key), + page-aligned. +- **Data pages:** sorted entries packed into 4 KB pages, never straddling a + page, each page with its own entry count and offset table. An entry too big + for one page gets a run of whole pages to itself. + +Stock dictionaries are cached next to the file (`traditional.txt.cache`) by the +installer via `canto-mkcache.exe`. Custom dictionaries are cached per user at +`%LOCALAPPDATA%\CantoIME\cache\.cache` (prebuilt when locked in through +the options dialog). A missing, stale, or damaged cache is never trusted: the +IME falls back to parsing the text and rewrites the cache for next time. +Overrides are kept in a small in-memory overlay consulted on top of the mapping. + --- ## Repository layout @@ -169,7 +192,9 @@ CantoIME/ ├── EditSession.h # reusable ITfEditSession adapter (lambda callback) ├── CandidateWindow.h/.cpp # owner-drawn GDI popup, sizable CJK font ├── LangBarButton.h/.cpp # language-bar buttons (粵 icon, 中/英 mode toggle) - ├── Dictionary.h/.cpp # UTF-8 loader + lookup + reload + ├── Dictionary.h/.cpp # UTF-8 loader, mmap cache, lookup + ├── mkcache.cpp # canto-mkcache.exe: prebuilds dictionary caches + ├── dicttest.cpp # canto-dicttest.exe: cache check run in CI ├── Settings.h/.cpp # per-user options in HKCU\Software\CantoIME ├── OptionsDialog.h/.cpp # "Keyboard options" dialog (ITfFnConfigure::Show) ├── resource.h # dialog/control IDs diff --git a/installer/setup.iss b/installer/setup.iss index a44ad4e..3779a74 100644 --- a/installer/setup.iss +++ b/installer/setup.iss @@ -57,6 +57,9 @@ Source: "..\build\Release\CantoIME-SG.dll"; DestDir: "{app}"; Check: IsVariant(' ; reads; simplified.txt ships for future use. Removed with the app on uninstall. Source: "..\traditional.txt"; DestDir: "{app}"; Flags: ignoreversion Source: "..\simplified.txt"; DestDir: "{app}"; Flags: ignoreversion +; Cache builder. Run post-install to prebuild the stock dictionaries' mmap +; caches, and by the options dialog when a custom dictionary is locked in. +Source: "..\build\Release\canto-mkcache.exe"; DestDir: "{app}"; Flags: ignoreversion 64bit [Registry] ; Clean up the per-user options key on uninstall (not created at install time; @@ -80,6 +83,21 @@ Filename: "{sys}\regsvr32.exe"; Parameters: "/s ""{app}\CantoIME-CN.dll"""; \ Filename: "{sys}\regsvr32.exe"; Parameters: "/s ""{app}\CantoIME-SG.dll"""; \ Check: IsVariant('SG'); StatusMsg: "Registering Singapore (Simplified) input method..."; \ Flags: runhidden waituntilterminated +; Prebuild the mmap cache for BOTH stock dictionaries. They are machine-wide, +; so the caches live next to them and serve every user; the IME maps them +; instead of parsing 10+ MB of text. Custom dictionaries are per-user and are +; cached under %LOCALAPPDATA% (by the options dialog, or on first load). +Filename: "{app}\canto-mkcache.exe"; \ + Parameters: """{app}\traditional.txt"" ""{app}\simplified.txt"""; \ + StatusMsg: "Preparing dictionaries..."; Flags: runhidden waituntilterminated + +[UninstallDelete] +; Caches are generated after install (not tracked as installed files), so remove +; them explicitly, including any ".old" left from replacing a mapped cache, plus +; the per-user cache folder. +Type: files; Name: "{app}\*.cache" +Type: files; Name: "{app}\*.cache.old" +Type: filesandordirs; Name: "{localappdata}\CantoIME" [UninstallRun] ; Unregister BEFORE files are removed. Only the variant DLL actually present in diff --git a/src/Dictionary.cpp b/src/Dictionary.cpp index e2d2ef0..d5256b9 100644 --- a/src/Dictionary.cpp +++ b/src/Dictionary.cpp @@ -9,6 +9,65 @@ #include #include #include +#include + +// ---- Cache file format ------------------------------------------------------ +// +// The cache is built for read-only memory mapping: a lookup binary-searches a +// small page index, then touches exactly one 4 KB data page, so a switch-in +// only faults in the pages it actually uses instead of reading the whole file. +// All integers are little-endian; strings are UTF-16 (wchar_t) with no +// terminator, so keys compare in place against std::wstring and candidates +// need no UTF-8 decode. Every region starts on a page boundary. +// +// page 0 CacheHeader (rest of the page is zero padding) +// page 1.. index: blockCount x IndexEntry, then the first-key blob +// (each block's first key, UTF-16), padded to a page boundary +// dataOffset.. data: dataPages pages, grouped into blocks +// +// A block is normally one page packed with as many whole entries (in sorted key +// order) as fit; an entry never straddles a page. An entry too big for an empty +// page gets a block of its own spanning as many whole pages as it needs (the +// index records pageCount > 1). Block layout: +// +// u16 count, u16 reserved, u16 offset[count] (offsets from block start) +// entry: u16 keyLen, u16 candCount, wchar_t key[keyLen], +// candCount x (u16 len, wchar_t chars[len]) +// +// The cache is a trust boundary (a stale or damaged file must never crash the +// host app): the header and the whole index are validated when mapped, and +// every in-block offset and length is bounds-checked on access. + +static const uint32_t kPage = 4096; +static const char kCacheMagic[8] = { 'C', 'A', 'N', 'T', 'O', 'M', 'A', 'P' }; +static const uint32_t kCacheVersion = 2; + +struct CacheHeader +{ + char magic[8]; + uint32_t version; + uint32_t pageSize; + uint64_t srcSize; // source .txt size ... + uint64_t srcMtime; // ... and last-write time; mismatch = stale + uint32_t entryCount; + uint32_t blockCount; + uint32_t dataPages; + uint32_t indexOffset; // always kPage + uint32_t indexKeyChars; // first-key blob length, in wchar_t + uint32_t dataOffset; // page-aligned + uint32_t maxKeyLen; + uint32_t reserved; +}; +static_assert(sizeof(CacheHeader) == 64, "cache header layout"); + +struct IndexEntry +{ + uint32_t firstPage; // block's first data page (relative to dataOffset) + uint16_t pageCount; // 1, or more for an oversized entry + uint16_t keyLen; // block's first key, in the first-key blob + uint32_t keyOffset; // in wchar_t +}; +static_assert(sizeof(IndexEntry) == 12, "cache index layout"); // Converts a UTF-8 byte buffer to a wide (UTF-16) string. static std::wstring Utf8ToWide(const char* data, int len) @@ -36,11 +95,68 @@ static bool IsSpace(wchar_t c) return c == L' ' || c == L'\t' || c == L'\r' || c == L'\n'; } -bool Dictionary::Load(const std::wstring& path) +// Same ordering as std::wstring::compare, on raw (mapped) characters. +static int CompareKeys(const wchar_t* a, size_t na, const wchar_t* b, size_t nb) { - m_path = path; - m_map.clear(); + int c = std::char_traits::compare(a, b, (std::min)(na, nb)); + if (c != 0) + return c; + return na < nb ? -1 : (na > nb ? 1 : 0); +} + +static uint16_t ReadU16(const uint8_t* p) +{ + uint16_t v; + std::memcpy(&v, p, 2); + return v; +} + +// Stock dictionaries: a machine-wide cache next to the file. +static std::wstring SiblingCachePath(const std::wstring& txt) +{ + return txt + L".cache"; +} +// Per-user cache: %LOCALAPPDATA%\CantoIME\cache\.cache, where is +// an FNV-1a-64 of the lowercased path. Creates the folders if `create` is set. +// Empty if %LOCALAPPDATA% cannot be resolved. +static std::wstring AppDataCachePath(const std::wstring& txt, bool create) +{ + wchar_t buf[MAX_PATH]; + DWORD n = ExpandEnvironmentStringsW(L"%LOCALAPPDATA%", buf, MAX_PATH); + if (n == 0 || n > MAX_PATH || buf[0] == L'%') + return std::wstring(); + + std::wstring dir = std::wstring(buf) + L"\\CantoIME"; + if (create) CreateDirectoryW(dir.c_str(), nullptr); + dir += L"\\cache"; + if (create) CreateDirectoryW(dir.c_str(), nullptr); + + unsigned long long h = 1469598103934665603ULL; + for (wchar_t c : txt) + { + h ^= static_cast(static_cast(towlower(c))); + h *= 1099511628211ULL; + } + const wchar_t* hex = L"0123456789abcdef"; + std::wstring name(16, L'0'); + for (int i = 15; i >= 0; --i) { name[i] = hex[h & 0xF]; h >>= 4; } + return dir + L"\\" + name + L".cache"; +} + +bool Dictionary::FileSignature(const std::wstring& path, Signature& sig) +{ + WIN32_FILE_ATTRIBUTE_DATA fad; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &fad)) + return false; + sig.size = (static_cast(fad.nFileSizeHigh) << 32) | fad.nFileSizeLow; + sig.mtime = (static_cast(fad.ftLastWriteTime.dwHighDateTime) << 32) | + fad.ftLastWriteTime.dwLowDateTime; + return true; +} + +bool Dictionary::ParseText(const std::wstring& path) +{ HANDLE hFile = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (hFile == INVALID_HANDLE_VALUE) @@ -135,40 +251,497 @@ bool Dictionary::Load(const std::wstring& path) } } - // Build the sorted key list used for prefix scans. Done once per load so - // KeysWithPrefix can binary-search instead of scanning the whole map. - RebuildIndex(); + // Sorted key list for prefix scans (and the cache writer's entry order). + m_sortedKeys.reserve(m_map.size()); + for (const auto& kv : m_map) + m_sortedKeys.push_back(kv.first); + std::sort(m_sortedKeys.begin(), m_sortedKeys.end()); + return true; +} + +static void PutU16(std::vector& b, size_t v) +{ + uint16_t x = static_cast(v); + b.insert(b.end(), reinterpret_cast(&x), reinterpret_cast(&x) + 2); +} + +static void PutChars(std::vector& b, const std::wstring& s) +{ + const uint8_t* p = reinterpret_cast(s.data()); + b.insert(b.end(), p, p + s.size() * sizeof(wchar_t)); +} + +bool Dictionary::WriteCacheFile(const std::wstring& cachePath, const Signature& sig) const +{ + if (cachePath.empty()) + return false; + + struct Block { uint32_t firstPage; uint32_t pageCount; const std::wstring* firstKey; }; + std::vector blocks; + std::vector data; // data pages + + // Entries waiting to be packed into the current page. + std::vector> page; + const std::wstring* pageFirst = nullptr; + size_t pageBytes = 4; // block header + + // Lays out a block: header, offset table, entries, zero padding. + auto emitBlock = [&](const std::vector>& entries, + uint32_t pageCount, const std::wstring* firstKey) + { + size_t base = data.size(); + data.resize(base + static_cast(pageCount) * kPage, 0); + uint8_t* p = &data[base]; + uint16_t n = static_cast(entries.size()); + std::memcpy(p, &n, 2); + size_t off = 4 + 2 * entries.size(); + for (size_t i = 0; i < entries.size(); ++i) + { + uint16_t o = static_cast(off); + std::memcpy(p + 4 + 2 * i, &o, 2); + std::memcpy(p + off, entries[i].data(), entries[i].size()); + off += entries[i].size(); + } + blocks.push_back({ static_cast(base / kPage), pageCount, firstKey }); + }; + auto flushPage = [&]() + { + if (page.empty()) + return; + emitBlock(page, 1, pageFirst); + page.clear(); + pageBytes = 4; + }; + + for (const std::wstring& key : m_sortedKeys) + { + auto it = m_map.find(key); + if (it == m_map.end()) + continue; + const std::vector& cands = it->second; + if (key.size() > 0xFFFF || cands.size() > 0xFFFF) + return false; + + std::vector e; + PutU16(e, key.size()); + PutU16(e, cands.size()); + PutChars(e, key); + for (const std::wstring& c : cands) + { + if (c.size() > 0xFFFF) + return false; + PutU16(e, c.size()); + PutChars(e, c); + } + + // Pack into the current page if the entry and its offset slot fit. + if (pageBytes + 2 + e.size() > kPage) + flushPage(); + if (pageBytes + 2 + e.size() <= kPage) + { + if (page.empty()) + pageFirst = &key; + pageBytes += 2 + e.size(); + page.push_back(std::move(e)); + continue; + } + + // Too big for an empty page: a run of whole pages holding only it. + size_t pages = (6 + e.size() + kPage - 1) / kPage; + if (pages > 0xFFFF) + return false; + std::vector> one; + one.push_back(std::move(e)); + emitBlock(one, static_cast(pages), &key); + } + flushPage(); + // Header page, then the index (entries + first-key blob), page-aligned. + std::vector file(kPage, 0); + std::vector keyBlob; + for (const Block& b : blocks) + { + IndexEntry ie; + ie.firstPage = b.firstPage; + ie.pageCount = static_cast(b.pageCount); + ie.keyLen = static_cast(b.firstKey->size()); + ie.keyOffset = static_cast(keyBlob.size() / sizeof(wchar_t)); + const uint8_t* p = reinterpret_cast(&ie); + file.insert(file.end(), p, p + sizeof(ie)); + PutChars(keyBlob, *b.firstKey); + } + file.insert(file.end(), keyBlob.begin(), keyBlob.end()); + file.resize((file.size() + kPage - 1) / kPage * kPage, 0); + size_t dataOffset = file.size(); + if (dataOffset + data.size() > 0x7FFFFFFF) + return false; + file.insert(file.end(), data.begin(), data.end()); + + CacheHeader h = {}; + std::memcpy(h.magic, kCacheMagic, sizeof(h.magic)); + h.version = kCacheVersion; + h.pageSize = kPage; + h.srcSize = sig.size; + h.srcMtime = sig.mtime; + h.entryCount = static_cast(m_map.size()); + h.blockCount = static_cast(blocks.size()); + h.dataPages = static_cast(data.size() / kPage); + h.indexOffset = kPage; + h.indexKeyChars = static_cast(keyBlob.size() / sizeof(wchar_t)); + h.dataOffset = static_cast(dataOffset); + h.maxKeyLen = static_cast(m_maxKeyLen); + std::memcpy(file.data(), &h, sizeof(h)); + + // Write a uniquely named temp file, then rename it over the cache, so a + // reader never sees a torn file and a process that has the old cache mapped + // keeps its (unchanged) view. + std::wstring tmp = cachePath + L"." + std::to_wstring(GetCurrentProcessId()) + + L"." + std::to_wstring(GetCurrentThreadId()) + L".tmp"; + HANDLE hf = CreateFileW(tmp.c_str(), GENERIC_WRITE, 0, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (hf == INVALID_HANDLE_VALUE) + return false; + DWORD written = 0; + BOOL ok = WriteFile(hf, file.data(), static_cast(file.size()), &written, nullptr); + CloseHandle(hf); + if (!ok || written != file.size()) + { + DeleteFileW(tmp.c_str()); + return false; + } + if (!MoveFileExW(tmp.c_str(), cachePath.c_str(), MOVEFILE_REPLACE_EXISTING)) + { + // Windows refuses to replace a file that another process has mapped. + // Renaming it aside is still allowed (readers open it with + // FILE_SHARE_DELETE), so move it out of the way and drop it once no one + // maps it any more (the delete simply fails until then). + std::wstring old = cachePath + L".old"; + DeleteFileW(old.c_str()); + if (!MoveFileExW(cachePath.c_str(), old.c_str(), MOVEFILE_REPLACE_EXISTING) || + !MoveFileExW(tmp.c_str(), cachePath.c_str(), 0)) + { + DeleteFileW(tmp.c_str()); + return false; + } + DeleteFileW(old.c_str()); + } return true; } -void Dictionary::RebuildIndex() +bool Dictionary::MapCacheFile(const std::wstring& cachePath, const Signature& sig) { + if (cachePath.empty()) + return false; + + HANDLE hf = CreateFileW(cachePath.c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (hf == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER sz; + if (!GetFileSizeEx(hf, &sz) || sz.QuadPart < kPage || sz.QuadPart % kPage != 0 || + sz.QuadPart > 0x7FFFFFFF) + { + CloseHandle(hf); + return false; + } + const uint64_t fileSize = static_cast(sz.QuadPart); + + HANDLE hm = CreateFileMappingW(hf, nullptr, PAGE_READONLY, 0, 0, nullptr); + CloseHandle(hf); + if (!hm) + return false; + const void* v = MapViewOfFile(hm, FILE_MAP_READ, 0, 0, 0); + CloseHandle(hm); // the view keeps the section alive + if (!v) + return false; + std::shared_ptr view(static_cast(v), + [](const uint8_t* p) { UnmapViewOfFile(p); }); + const uint8_t* base = view.get(); + + CacheHeader h; + std::memcpy(&h, base, sizeof(h)); + if (std::memcmp(h.magic, kCacheMagic, sizeof(h.magic)) != 0 || + h.version != kCacheVersion || h.pageSize != kPage) + return false; + if (h.srcSize != sig.size || h.srcMtime != sig.mtime) + return false; // the .txt changed since this cache was built + + // Region bounds (64-bit math so a hostile header cannot overflow). + uint64_t indexEnd = static_cast(h.indexOffset) + + static_cast(h.blockCount) * sizeof(IndexEntry) + + static_cast(h.indexKeyChars) * sizeof(wchar_t); + if (h.indexOffset != kPage || h.dataOffset % kPage != 0 || indexEnd > h.dataOffset || + static_cast(h.dataOffset) + static_cast(h.dataPages) * kPage != fileSize) + return false; + + // Validate the whole index once (it is small): blocks tile the data region + // in order, and first keys are in bounds and strictly increasing. + const uint8_t* index = base + h.indexOffset; + const wchar_t* keys = reinterpret_cast( + index + static_cast(h.blockCount) * sizeof(IndexEntry)); + uint64_t nextPage = 0; + const wchar_t* prevKey = nullptr; + size_t prevLen = 0; + for (uint32_t b = 0; b < h.blockCount; ++b) + { + IndexEntry ie; + std::memcpy(&ie, index + static_cast(b) * sizeof(IndexEntry), sizeof(ie)); + if (ie.pageCount == 0 || ie.firstPage != nextPage || ie.keyLen == 0 || + static_cast(ie.keyOffset) + ie.keyLen > h.indexKeyChars) + return false; + nextPage += ie.pageCount; + const wchar_t* k = keys + ie.keyOffset; + if (prevKey && CompareKeys(prevKey, prevLen, k, ie.keyLen) >= 0) + return false; + prevKey = k; + prevLen = ie.keyLen; + } + if (nextPage != h.dataPages) + return false; + + // Commit: drop any text-parse form and serve queries from the mapping. + m_map.clear(); m_sortedKeys.clear(); - m_sortedKeys.reserve(m_map.size()); - for (const auto& kv : m_map) - m_sortedKeys.push_back(kv.first); - std::sort(m_sortedKeys.begin(), m_sortedKeys.end()); + m_view = std::move(view); + m_index = index; + m_indexKeys = keys; + m_data = base + h.dataOffset; + m_blockCount = h.blockCount; + m_entryCount = h.entryCount; + m_maxKeyLen = (std::max)(static_cast(h.maxKeyLen), m_maxKeyLen); + return true; } -void Dictionary::ApplyOverrides( - const std::vector>>& overrides) +uint32_t Dictionary::BlockFor(const std::wstring& key) const +{ + // First block whose first key is > key, minus one. UINT32_MAX if key sorts + // before every block. + uint32_t lo = 0, hi = m_blockCount; + while (lo < hi) + { + uint32_t mid = lo + (hi - lo) / 2; + IndexEntry ie; + std::memcpy(&ie, m_index + static_cast(mid) * sizeof(IndexEntry), sizeof(ie)); + if (CompareKeys(m_indexKeys + ie.keyOffset, ie.keyLen, key.data(), key.size()) <= 0) + lo = mid + 1; + else + hi = mid; + } + return lo == 0 ? UINT32_MAX : lo - 1; +} + +uint32_t Dictionary::BlockEntryCount(uint32_t block) const +{ + IndexEntry ie; + std::memcpy(&ie, m_index + static_cast(block) * sizeof(IndexEntry), sizeof(ie)); + const uint8_t* p = m_data + static_cast(ie.firstPage) * kPage; + uint32_t n = ReadU16(p); + if (4 + 2 * static_cast(n) > static_cast(ie.pageCount) * kPage) + return 0; + return n; +} + +bool Dictionary::BlockEntry(uint32_t block, uint32_t i, Entry& e) const +{ + IndexEntry ie; + std::memcpy(&ie, m_index + static_cast(block) * sizeof(IndexEntry), sizeof(ie)); + const uint8_t* p = m_data + static_cast(ie.firstPage) * kPage; + const size_t bytes = static_cast(ie.pageCount) * kPage; + const size_t n = ReadU16(p); + if (i >= n || 4 + 2 * n > bytes) + return false; + size_t off = ReadU16(p + 4 + 2 * static_cast(i)); + if (off < 4 + 2 * n || (off & 1) || off + 4 > bytes) + return false; + e.keyLen = ReadU16(p + off); + e.candCount = ReadU16(p + off + 2); + if (off + 4 + 2 * static_cast(e.keyLen) > bytes) + return false; + e.key = reinterpret_cast(p + off + 4); + e.cands = p + off + 4 + 2 * static_cast(e.keyLen); + e.end = p + bytes; + return true; +} + +void Dictionary::DecodeCandidates(const Entry& e, std::vector& out) { - if (overrides.empty()) + out.clear(); + out.reserve(e.candCount); + const uint8_t* q = e.cands; + for (uint32_t c = 0; c < e.candCount; ++c) + { + if (e.end - q < 2) + break; + size_t len = ReadU16(q); + if (static_cast(e.end - q) < 2 + 2 * len) + break; // damaged entry: keep what decoded cleanly + out.emplace_back(reinterpret_cast(q + 2), len); + q += 2 + 2 * len; + } +} + +bool Dictionary::BaseLookup(const std::wstring& key, std::vector* out) const +{ + if (!m_view) + { + auto it = m_map.find(key); + if (it == m_map.end()) + return false; + if (out) + *out = it->second; + return true; + } + + uint32_t b = BlockFor(key); + if (b == UINT32_MAX) + return false; + uint32_t lo = 0, hi = BlockEntryCount(b); + while (lo < hi) + { + uint32_t mid = lo + (hi - lo) / 2; + Entry e; + if (!BlockEntry(b, mid, e)) + return false; + int c = CompareKeys(e.key, e.keyLen, key.data(), key.size()); + if (c == 0) + { + if (out) + DecodeCandidates(e, *out); + return true; + } + if (c < 0) + lo = mid + 1; + else + hi = mid; + } + return false; +} + +void Dictionary::BaseKeysWithPrefix(const std::wstring& lo, size_t maxKeys, + std::vector& out) const +{ + if (!m_view) + { + // First key >= prefix; every key sharing the prefix is contiguous from here. + auto it = std::lower_bound(m_sortedKeys.begin(), m_sortedKeys.end(), lo); + for (; it != m_sortedKeys.end() && out.size() < maxKeys; ++it) + { + if (it->compare(0, lo.size(), lo) != 0) + break; + out.push_back(*it); + } return; + } - bool changed = false; + // Start in the block that would hold the prefix itself, at its first entry + // >= prefix, then walk forward across block boundaries while keys match. + uint32_t b = BlockFor(lo); + if (b == UINT32_MAX) + b = 0; + uint32_t i = 0, hi = BlockEntryCount(b); + while (i < hi) + { + uint32_t mid = i + (hi - i) / 2; + Entry e; + if (!BlockEntry(b, mid, e)) + return; + if (CompareKeys(e.key, e.keyLen, lo.data(), lo.size()) < 0) + i = mid + 1; + else + hi = mid; + } + for (; b < m_blockCount; ++b, i = 0) + { + uint32_t n = BlockEntryCount(b); + for (; i < n; ++i) + { + Entry e; + if (!BlockEntry(b, i, e)) + return; + if (e.keyLen < lo.size() || + std::char_traits::compare(e.key, lo.data(), lo.size()) != 0) + return; + out.emplace_back(e.key, e.keyLen); + if (out.size() >= maxKeys) + return; + } + } +} + +bool Dictionary::Load(const std::wstring& path, bool machineWide) +{ + *this = Dictionary(); // drops any previous mapping, parse, and overrides + m_path = path; + + // Map the cache. Stock dictionaries carry a machine-wide sibling cache + // (built by the installer); custom ones use only the per-user cache. + Signature sig; + bool haveSig = FileSignature(path, sig); + if (haveSig) + { + if (machineWide && MapCacheFile(SiblingCachePath(path), sig)) + return true; + if (MapCacheFile(AppDataCachePath(path, false), sig)) + return true; + } + + // No usable cache: parse the text, then write the cache so the next load + // (in any process) maps it. Switch this instance over to the mapping too, + // which frees the parsed map's memory in the host process. + if (!ParseText(path)) + return false; + if (haveSig) + { + std::wstring sibling = machineWide ? SiblingCachePath(path) : std::wstring(); + if (WriteCacheFile(sibling, sig)) + MapCacheFile(sibling, sig); + else + { + std::wstring perUser = AppDataCachePath(path, true); + if (WriteCacheFile(perUser, sig)) + MapCacheFile(perUser, sig); + } + } + return true; +} + +bool Dictionary::PrebuildCache(const std::wstring& txtPath, bool machineWide) +{ + Signature sig; + if (!FileSignature(txtPath, sig)) + return false; + + Dictionary d; + if (machineWide && d.MapCacheFile(SiblingCachePath(txtPath), sig)) + return true; + if (d.MapCacheFile(AppDataCachePath(txtPath, false), sig)) + return true; + + if (!d.ParseText(txtPath)) + return false; + // Stock: prefer the sibling (machine-wide, readable by every user), and + // fall back to %LOCALAPPDATA% only if that folder is read-only. Custom: + // always per-user; never write next to the (per-user) dictionary. + if (machineWide && d.WriteCacheFile(SiblingCachePath(txtPath), sig)) + return true; + return d.WriteCacheFile(AppDataCachePath(txtPath, true), sig); +} + +void Dictionary::ApplyOverrides( + const std::vector>>& overrides) +{ + m_over.clear(); + m_overKeys.clear(); for (const auto& ov : overrides) { std::wstring key = ToLower(ov.first); if (key.empty() || ov.second.empty()) continue; - // Prepend before the stock candidates so the user's extras take the - // low positions (the first is pickable with the 1 key). Dedup: an - // override that duplicates a stock character surfaces only at the - // front, not twice. - std::vector& vec = m_map[key]; + // Later overrides for the same key go first, as when they were merged + // into the dictionary one by one. + std::vector& vec = m_over[key]; std::vector merged; std::unordered_set seen; for (const std::wstring& c : ov.second) @@ -181,32 +754,37 @@ void Dictionary::ApplyOverrides( if (key.size() > m_maxKeyLen) m_maxKeyLen = key.size(); - changed = true; } - - if (changed) - RebuildIndex(); -} - -bool Dictionary::Reload() -{ - if (m_path.empty()) - return false; - return Load(m_path); + for (const auto& kv : m_over) + m_overKeys.push_back(kv.first); + std::sort(m_overKeys.begin(), m_overKeys.end()); } -const std::vector& Dictionary::Lookup(const std::wstring& romanization) const +std::vector Dictionary::Lookup(const std::wstring& romanization) const { std::wstring key = ToLower(romanization); - auto it = m_map.find(key); - if (it == m_map.end()) - return m_empty; - return it->second; + std::vector base; + BaseLookup(key, &base); + + auto it = m_over.find(key); + if (it == m_over.end()) + return base; + + // Override candidates first so the user's extras take the low positions + // (the first is pickable with the 1 key); a stock character the override + // repeats surfaces only at the front, not twice. + std::vector merged = it->second; + std::unordered_set seen(merged.begin(), merged.end()); + for (std::wstring& c : base) + if (seen.insert(c).second) + merged.push_back(std::move(c)); + return merged; } bool Dictionary::HasExact(const std::wstring& romanization) const { - return m_map.find(ToLower(romanization)) != m_map.end(); + std::wstring key = ToLower(romanization); + return m_over.count(key) != 0 || BaseLookup(key, nullptr); } std::vector Dictionary::KeysWithPrefix(const std::wstring& prefix, @@ -217,16 +795,19 @@ std::vector Dictionary::KeysWithPrefix(const std::wstring& prefix, return out; std::wstring lo = ToLower(prefix); + BaseKeysWithPrefix(lo, maxKeys, out); + if (m_overKeys.empty()) + return out; - // First key >= prefix; every key sharing the prefix is contiguous from here. - auto it = std::lower_bound(m_sortedKeys.begin(), m_sortedKeys.end(), lo); - for (; it != m_sortedKeys.end() && out.size() < maxKeys; ++it) - { - // Stop as soon as a key no longer starts with the prefix. - if (it->compare(0, lo.size(), lo) != 0) - break; + // Merge in override-only keys. The base list already holds its first + // maxKeys matches, so the merged head is exact. + auto it = std::lower_bound(m_overKeys.begin(), m_overKeys.end(), lo); + for (; it != m_overKeys.end() && it->compare(0, lo.size(), lo) == 0; ++it) out.push_back(*it); - } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + if (out.size() > maxKeys) + out.resize(maxKeys); return out; } @@ -242,7 +823,8 @@ size_t Dictionary::LongestExactPrefixLen(const std::wstring& s, size_t start) co // Longest wins: jyutping keys are short, so a top-down probe is cheap. for (size_t len = maxLen; len >= 1; --len) { - if (m_map.find(s.substr(start, len)) != m_map.end()) + std::wstring sub = s.substr(start, len); + if (m_over.count(sub) != 0 || BaseLookup(sub, nullptr)) return len; } return 0; diff --git a/src/Dictionary.h b/src/Dictionary.h index 85a942e..3f89ef1 100644 --- a/src/Dictionary.h +++ b/src/Dictionary.h @@ -6,6 +6,13 @@ // romanizationchar1 char2 char3 ... // Lines beginning with '#' (after optional leading whitespace) are comments. // A UTF-8 BOM at the start of the file is tolerated. +// +// Parsing 10+ MB of text on every switch-in is slow, so a parsed dictionary is +// also written to a binary cache laid out for memory mapping (format described +// in Dictionary.cpp). Load() maps that cache read-only and answers queries in +// place, touching only the page index and the page an entry lives on. A +// missing, stale, or malformed cache falls back to the text parse (and rewrites +// the cache for next time), so a bad cache can never break the IME. #pragma once @@ -13,30 +20,39 @@ #include #include #include +#include +#include class Dictionary { public: Dictionary() = default; - // Loads (or reloads) the dictionary from the given UTF-8 file. - // Returns true if the file was opened and parsed (even if empty). - bool Load(const std::wstring& path); - - // Reloads from the last successfully loaded path. Returns false if we - // have never been given a path. - bool Reload(); + // Loads (or reloads) the dictionary for the given UTF-8 file: maps its + // cache if one is valid, else parses the text (and writes the cache). + // Returns true if the dictionary is usable (even if empty). Clears any + // overrides. machineWide selects where the cache lives: true (stock + // dictionaries) uses a sibling ".cache", falling back to + // %LOCALAPPDATA%\CantoIME if that folder is read-only; false (custom, + // per-user dictionaries) uses only %LOCALAPPDATA%\CantoIME. + bool Load(const std::wstring& path, bool machineWide = true); - // Merges user overrides on top of the currently-loaded dictionary. For each - // (key, candidates) pair the candidates are appended after the stock ones - // (deduped), and keys with no stock entry are added. Call after Load(). + // Sets the user overrides consulted on top of the loaded dictionary. They + // live in a small in-memory overlay (the mapped cache is read-only): for + // each key the override candidates come first, then the stock ones minus + // duplicates, and keys with no stock entry are added. Call after Load(). void ApplyOverrides( const std::vector>>& overrides); - // Looks up an exact romanization key. Returns the candidate list, or an empty - // vector if the key is not present. The comparison is case-insensitive - // (keys are stored lowercased). - const std::vector& Lookup(const std::wstring& romanization) const; + // Parses txtPath and writes its cache (see Load for the machineWide + // policy). A no-op if a valid cache already exists. Returns true if a valid + // cache exists afterward. Used by canto-mkcache (src/mkcache.cpp). + static bool PrebuildCache(const std::wstring& txtPath, bool machineWide = true); + + // Looks up an exact romanization key. Returns the candidate list, or an + // empty vector if the key is not present. Case-insensitive (keys are stored + // lowercased). Returned by value: candidates are decoded from the mapping. + std::vector Lookup(const std::wstring& romanization) const; // True if the exact key exists. bool HasExact(const std::wstring& romanization) const; @@ -52,18 +68,72 @@ class Dictionary size_t LongestExactPrefixLen(const std::wstring& s, size_t start) const; // True if at least one entry is loaded. - bool IsLoaded() const { return !m_map.empty(); } + bool IsLoaded() const { return m_entryCount != 0 || !m_map.empty() || !m_over.empty(); } + + // True if queries are served from the mapped cache (not a text parse). + bool IsMapped() const { return m_view != nullptr; } const std::wstring& Path() const { return m_path; } private: - // Rebuilds the sorted key list used for prefix scans from m_map. Called - // after a load or an override merge changes the key set. - void RebuildIndex(); + friend struct DictionaryTest; // src/dicttest.cpp + + // Source file fingerprint stored in the cache header; a mismatch means the + // text changed and the cache is stale. + struct Signature { uint64_t size = 0; uint64_t mtime = 0; }; + static bool FileSignature(const std::wstring& path, Signature& sig); + + // Parses the text file into m_map / m_sortedKeys / m_maxKeyLen. + bool ParseText(const std::wstring& path); + + // Writes m_map (a text parse) as a cache file, atomically. False if the + // dictionary cannot be represented (a field over 65535) or on I/O failure. + bool WriteCacheFile(const std::wstring& cachePath, const Signature& sig) const; + + // Maps and validates a cache file. On success the dictionary serves + // queries from it; on failure (missing, stale, malformed) nothing changes. + bool MapCacheFile(const std::wstring& cachePath, const Signature& sig); + + // Base-dictionary queries (mapped cache or text parse), without overrides. + bool BaseLookup(const std::wstring& key, std::vector* out) const; + void BaseKeysWithPrefix(const std::wstring& lo, size_t maxKeys, + std::vector& out) const; + + // Mapped-cache helpers. A "block" is one data page, or a multi-page run + // holding a single oversized entry. + struct Entry + { + const wchar_t* key; + uint16_t keyLen; + uint16_t candCount; + const uint8_t* cands; // candCount x (u16 len, wchar_t[len]) + const uint8_t* end; // end of the block, for bounds checks + }; + uint32_t BlockFor(const std::wstring& key) const; // last block whose first key <= key + uint32_t BlockEntryCount(uint32_t block) const; + bool BlockEntry(uint32_t block, uint32_t i, Entry& e) const; + static void DecodeCandidates(const Entry& e, std::vector& out); std::wstring m_path; + + // Text-parse form (used when no valid cache could be mapped). std::unordered_map> m_map; std::vector m_sortedKeys; // all keys, sorted, for prefix scans - size_t m_maxKeyLen = 0; // longest key length (for segmenter) - std::vector m_empty; // returned by Lookup on miss + + // Mapped form. The file and section handles are closed right after mapping + // (the view keeps the section alive), and the view is unmapped when the + // last owner goes away, so a Dictionary stays movable and reload/destroy + // unmaps. + std::shared_ptr m_view; + const uint8_t* m_index = nullptr; // block index entries + const wchar_t* m_indexKeys = nullptr; // first-key blob for the index + const uint8_t* m_data = nullptr; // first data page + uint32_t m_blockCount = 0; + uint32_t m_entryCount = 0; + + size_t m_maxKeyLen = 0; // longest key length, base or override (for segmenter) + + // Override overlay: key -> override candidates (deduped), plus sorted keys. + std::unordered_map> m_over; + std::vector m_overKeys; }; diff --git a/src/OptionsDialog.cpp b/src/OptionsDialog.cpp index a57aa79..bd9c2fb 100644 --- a/src/OptionsDialog.cpp +++ b/src/OptionsDialog.cpp @@ -123,6 +123,41 @@ static void OnBrowse(HWND hDlg) SetDlgItemTextW(hDlg, IDC_DICT_PATH, path); } +// Kicks off canto-mkcache.exe (ships next to the DLL) to prebuild the per-user +// mmap cache for a just-locked-in custom dictionary, so its first switch-in is +// not slowed by parsing text. Stock dictionaries are cached machine-wide at +// install time, so only custom dictionaries need this. Detached and best-effort: +// the IME still works (parsing once, then caching) if the tool is missing or +// fails. +static void LaunchCustomCachePrebuild(const std::wstring& customPath) +{ + std::wstring dir = GetModuleDirectory(); // includes a trailing separator + if (dir.empty()) + return; + std::wstring tool = dir + L"canto-mkcache.exe"; + + // Pass the expanded path: the runtime loads the expanded path, and the + // per-user cache file name is derived from it. + wchar_t buf[MAX_PATH] = { 0 }; + DWORD n = ExpandEnvironmentStringsW(customPath.c_str(), buf, MAX_PATH); + std::wstring dictPath = (n > 0 && n <= MAX_PATH) ? std::wstring(buf) : customPath; + if (dictPath.empty()) + return; + + std::wstring cmd = L"\"" + tool + L"\" --per-user \"" + dictPath + L"\""; + std::vector cmdBuf(cmd.begin(), cmd.end()); + cmdBuf.push_back(L'\0'); + + STARTUPINFOW si = { sizeof(si) }; + PROCESS_INFORMATION pi = {}; + if (CreateProcessW(tool.c_str(), cmdBuf.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) + { + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); // detached: do not wait + } +} + // --- Custom-candidate overrides ---------------------------------------------- // // The dialog edits a working copy of the overrides (a per-dictionary model held @@ -524,6 +559,8 @@ static INT_PTR CALLBACK DlgProc(HWND hDlg, UINT msg, WPARAM wParam, LPARAM lPara // Start from the stored settings so a remembered custom path survives // switching to a stock choice and back. CantoSettings s = LoadSettings(); + DictSource oldSrc = s.dictSource; + std::wstring oldPath = s.dictionaryPath; s.dictSource = IndexToSource(mode); if (s.dictSource == DictSource::Custom) @@ -547,6 +584,14 @@ static INT_PTR CALLBACK DlgProc(HWND hDlg, UINT msg, WPARAM wParam, LPARAM lPara if (OverrideModel* m = ModelState(hDlg)) for (const auto& kv : m->byDict) SaveOverrides(kv.first, kv.second); + + // If a custom dictionary was locked in, prebuild its per-user cache + // now (in the background) so its first switch-in maps a cache instead + // of parsing text. Stock dictionaries are cached at install time. + if (s.dictSource == DictSource::Custom && + (oldSrc != DictSource::Custom || s.dictionaryPath != oldPath)) + LaunchCustomCachePrebuild(s.dictionaryPath); + EndDialog(hDlg, IDOK); return TRUE; } diff --git a/src/TextService.cpp b/src/TextService.cpp index 747159c..0408174 100644 --- a/src/TextService.cpp +++ b/src/TextService.cpp @@ -1533,18 +1533,20 @@ void CTextService::ApplySettings() { m_overrides = std::move(overrides); m_dictPathLoaded = path; - BeginLoadDictionary(path); + // Stock dictionaries use a machine-wide sibling cache; a custom (per-user) + // dictionary uses the per-user cache only. + BeginLoadDictionary(path, s.dictSource != DictSource::Custom); } } -void CTextService::BeginLoadDictionary(const std::wstring& path) +void CTextService::BeginLoadDictionary(const std::wstring& path, bool machineWide) { HWND hwnd = m_hMsgWnd; if (!hwnd) { // No message window to hand the result back through: load synchronously. // This should not happen post-activation (the window is created first). - m_dict.Load(path); + m_dict.Load(path, machineWide); m_dict.ApplyOverrides(m_overrides); return; } @@ -1557,9 +1559,9 @@ void CTextService::BeginLoadDictionary(const std::wstring& path) // from under the thread if the service deactivates mid-load. DllAddRef(); Dictionary* d = new Dictionary(); - std::thread([d, path, hwnd]() + std::thread([d, path, hwnd, machineWide]() { - d->Load(path); + d->Load(path, machineWide); if (!PostMessageW(hwnd, WM_CANTO_DICT_READY, reinterpret_cast(d), 0)) delete d; DllRelease(); diff --git a/src/TextService.h b/src/TextService.h index c897163..18887fd 100644 --- a/src/TextService.h +++ b/src/TextService.h @@ -223,7 +223,7 @@ class CTextService final : // Loads the dictionary on a background thread so switching to CantoIME does // not block the UI thread. The parsed dictionary is handed back to the UI // thread via WM_CANTO_DICT_READY (see MsgWndProc / OnDictionaryLoaded). - void BeginLoadDictionary(const std::wstring& path); + void BeginLoadDictionary(const std::wstring& path, bool machineWide); void OnDictionaryLoaded(Dictionary* loaded); // UI-thread completion handler // Active service for the low-level mouse hook (one composition at a time). diff --git a/src/dicttest.cpp b/src/dicttest.cpp new file mode 100644 index 0000000..96c2f53 --- /dev/null +++ b/src/dicttest.cpp @@ -0,0 +1,303 @@ +// dicttest.cpp +// +// canto-dicttest.exe [ ...] +// +// The runnable check for the mmap dictionary cache (run in CI). For each +// dictionary, plus a synthetic one with an entry too big for a single page: +// - builds the cache from the text parse, maps it, and checks every key's +// candidates, exact-match and longest-prefix answers match the text parse; +// - checks prefix scans (unbounded per first character, which crosses every +// page boundary, and capped like the candidate engine for every 1-3 +// character prefix); +// - checks overrides give the same answers over either form; +// - checks stale, truncated, and corrupt headers/indexes are rejected, and +// that garbage inside data pages never crashes a query. +// Exit code 0 if everything passed. + +#include "Dictionary.h" + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include + +struct DictionaryTest +{ + static int failures; + + static void Fail(const char* what, const std::wstring& detail) + { + if (++failures <= 30) + wprintf(L"FAIL: %hs [%ls]\n", what, detail.c_str()); + } + + static std::wstring TempPath(const std::wstring& name) + { + wchar_t buf[MAX_PATH]; + GetTempPathW(MAX_PATH, buf); + return std::wstring(buf) + name; + } + + static double NowMs() + { + LARGE_INTEGER f, t; + QueryPerformanceFrequency(&f); + QueryPerformanceCounter(&t); + return 1000.0 * static_cast(t.QuadPart) / static_cast(f.QuadPart); + } + + static bool ReadAll(const std::wstring& path, std::vector& out) + { + HANDLE h = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, 0, nullptr); + if (h == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER sz; + GetFileSizeEx(h, &sz); + out.resize(static_cast(sz.QuadPart)); + DWORD rd = 0; + BOOL ok = ReadFile(h, out.data(), static_cast(out.size()), &rd, nullptr); + CloseHandle(h); + return ok && rd == out.size(); + } + + static bool WriteAll(const std::wstring& path, const void* data, size_t n) + { + HANDLE h = CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, 0, nullptr); + if (h == INVALID_HANDLE_VALUE) + return false; + DWORD wr = 0; + BOOL ok = n == 0 || WriteFile(h, data, static_cast(n), &wr, nullptr); + CloseHandle(h); + return ok && (n == 0 || wr == n); + } + + // Every query the candidate engine makes must answer the same on the + // mapped cache as on the text parse it was built from. + static void Compare(const Dictionary& text, const Dictionary& mapped) + { + std::set firsts; + std::set prefixes; + for (const std::wstring& key : text.m_sortedKeys) + { + if (mapped.Lookup(key) != text.Lookup(key)) + Fail("lookup differs", key); + if (!mapped.HasExact(key) || mapped.HasExact(key + L"\x7f")) + Fail("HasExact wrong", key); + std::wstring probe = L"x" + key + L"zzq"; + if (mapped.LongestExactPrefixLen(probe, 1) != text.LongestExactPrefixLen(probe, 1)) + Fail("LongestExactPrefixLen differs", key); + firsts.insert(key[0]); + for (size_t l = 1; l <= 3 && l <= key.size(); ++l) + prefixes.insert(key.substr(0, l)); + } + + size_t total = 0; + for (wchar_t c : firsts) + { + std::wstring p(1, c); + std::vector a = text.KeysWithPrefix(p, SIZE_MAX); + std::vector b = mapped.KeysWithPrefix(p, SIZE_MAX); + if (a != b) + Fail("full prefix scan differs", p); + total += b.size(); + } + if (total != text.m_sortedKeys.size()) + Fail("full prefix scans did not cover every key", std::to_wstring(total)); + + for (const std::wstring& p : prefixes) + if (text.KeysWithPrefix(p, 64) != mapped.KeysWithPrefix(p, 64)) + Fail("capped prefix scan differs", p); + + for (const wchar_t* miss : { L"\x01", L"\xffff", L"zzzzzzzzzzzzzzzz" }) + if (mapped.HasExact(miss) != text.HasExact(miss) || + mapped.KeysWithPrefix(miss, 64) != text.KeysWithPrefix(miss, 64)) + Fail("miss handling differs", miss); + } + + // Maps a damaged copy of the cache; returns whether it was accepted. + static bool MapsDamaged(const std::vector& bytes, size_t size, int tag, + const Dictionary::Signature& sig) + { + std::wstring bad = TempPath(L"canto-dicttest-bad" + std::to_wstring(tag) + L".cache"); + if (!WriteAll(bad, bytes.data(), size)) + { + Fail("could not write damaged copy", bad); + return false; + } + Dictionary d; + bool ok = d.MapCacheFile(bad, sig); + if (ok) + for (const wchar_t* k : { L"a", L"ngo", L"zz" }) + (void)d.Lookup(k); + return ok; + } + + static void Check(const std::wstring& txt, bool expectMultiPage) + { + int before = failures; + Dictionary text; + double t0 = NowMs(); + if (!text.ParseText(txt) || text.m_sortedKeys.empty()) + { + Fail("text parse failed", txt); + return; + } + double t1 = NowMs(); + + Dictionary::Signature sig; + Dictionary::FileSignature(txt, sig); + std::wstring cache = TempPath(L"canto-dicttest.cache"); + if (!text.WriteCacheFile(cache, sig)) + { + Fail("cache write failed", cache); + return; + } + + Dictionary mapped; + double t2 = NowMs(); + if (!mapped.MapCacheFile(cache, sig) || !mapped.IsMapped()) + { + Fail("cache map failed", cache); + return; + } + double t3 = NowMs(); + + std::vector bytes; + ReadAll(cache, bytes); + uint32_t multi = 0; + for (uint32_t b = 0; b < mapped.m_blockCount; ++b) + { + uint16_t pages; + std::memcpy(&pages, mapped.m_index + b * 12 + 4, 2); + multi += pages > 1; + } + wprintf(L"%ls: %u keys, %u blocks (%u multi-page), %zu bytes; parse %.1f ms, map %.3f ms\n", + txt.c_str(), mapped.m_entryCount, mapped.m_blockCount, multi, bytes.size(), + t1 - t0, t3 - t2); + if (mapped.m_entryCount != text.m_sortedKeys.size()) + Fail("entry count differs", txt); + if (expectMultiPage && multi == 0) + Fail("expected a multi-page block", txt); + + Compare(text, mapped); + + // Overrides: same answers over either form (prepend + dedup, new key). + const std::wstring& k0 = text.m_sortedKeys[text.m_sortedKeys.size() / 2]; + std::vector>> ov = { + { k0, { L"\x6E2C", text.Lookup(k0)[0], L"\x6E2C" } }, + { L"zzzcantotest", { L"\x7532", L"\x4E59" } }, + }; + text.ApplyOverrides(ov); + mapped.ApplyOverrides(ov); + for (const std::wstring& k : { k0, std::wstring(L"zzzcantotest") }) + if (mapped.Lookup(k) != text.Lookup(k) || mapped.Lookup(k).empty()) + Fail("override lookup differs", k); + if (mapped.Lookup(k0)[0] != L"\x6E2C") + Fail("override not first", k0); + for (const std::wstring& p : { k0.substr(0, 1), std::wstring(L"zzz") }) + if (mapped.KeysWithPrefix(p, 64) != text.KeysWithPrefix(p, 64)) + Fail("override prefix scan differs", p); + if (mapped.LongestExactPrefixLen(L"zzzcantotestx", 0) != 12) + Fail("override key not seen by segmenter", L"zzzcantotest"); + + // Stale: the source changed. + Dictionary::Signature stale = sig; + stale.mtime += 1; + if (Dictionary().MapCacheFile(cache, stale)) + Fail("accepted stale cache", txt); + + // Truncated anywhere. + const size_t n = bytes.size(); + const size_t cuts[] = { n - 1, n - 4096, n / 2, 4096, 64, 0 }; + int tag = 0; + for (size_t cut : cuts) + if (MapsDamaged(bytes, cut, tag++, sig)) + Fail("accepted truncated cache", std::to_wstring(cut)); + + // Corrupt header / index fields. Offsets follow CacheHeader and + // IndexEntry in Dictionary.cpp; the index starts at page 1. + struct Poke { size_t at; uint32_t value; int width; }; + const Poke pokes[] = { + { 8, 1, 4 }, // version + { 36, mapped.m_blockCount + 1, 4 }, // blockCount + { 40, 0xFFFFFFF, 4 }, // dataPages + { 4096 + 4, 0, 2 }, // first block pageCount = 0 + { 4096 + 8, 0x7FFFFFF, 4 }, // first block key offset out of range + }; + for (const Poke& pk : pokes) + { + std::vector c = bytes; + std::memcpy(&c[pk.at], &pk.value, pk.width); + if (MapsDamaged(c, c.size(), tag++, sig)) + Fail("accepted corrupt header/index", std::to_wstring(pk.at)); + } + + // Garbage inside data pages: still maps (the index is intact), and no + // query may crash or read out of bounds. + { + std::vector c = bytes; + size_t dataOff = static_cast(mapped.m_data - mapped.m_view.get()); + uint32_t seed = 12345; + for (size_t i = dataOff; i < c.size(); i += (i < dataOff + 3 * 4096) ? 1 : 97) + { + seed = seed * 1103515245 + 12345; + c[i] = static_cast(seed >> 16); + } + std::wstring bad = TempPath(L"canto-dicttest-garbage.cache"); + WriteAll(bad, c.data(), c.size()); + Dictionary d; + if (!d.MapCacheFile(bad, sig)) + Fail("rejected cache with intact index", bad); + size_t sink = 0; + for (const std::wstring& key : text.m_sortedKeys) + { + sink += d.Lookup(key).size(); + sink += d.LongestExactPrefixLen(key, 0); + } + for (wchar_t ch = L'a'; ch <= L'z'; ++ch) + sink += d.KeysWithPrefix(std::wstring(1, ch), SIZE_MAX).size(); + wprintf(L" garbage-page sweep survived (%zu)\n", sink); + } + + wprintf(L" %ls\n", failures == before ? L"OK" : L"FAILED"); + } + + // A small dictionary with one entry far bigger than a page, between + // ordinary ones, so a multi-page run sits mid-file. + static std::wstring WriteSynthetic() + { + std::wstring w = L"# synthetic\na \x4E00\n"; + for (int i = 0; i < 1500; ++i) + w += L"k" + std::to_wstring(1000 + i) + L" \x5B57 \x6587 \x8A9E\n"; + w += L"ji"; + for (int i = 0; i < 5000; ++i) + w += L" " + std::wstring(1, static_cast(0x4E00 + i)); + w += L"\njia \x5BB6\nzz \x5B57\n"; + + int n = WideCharToMultiByte(CP_UTF8, 0, w.c_str(), static_cast(w.size()), + nullptr, 0, nullptr, nullptr); + std::string u(n, '\0'); + WideCharToMultiByte(CP_UTF8, 0, w.c_str(), static_cast(w.size()), + &u[0], n, nullptr, nullptr); + std::wstring path = TempPath(L"canto-dicttest-synthetic.txt"); + WriteAll(path, u.data(), u.size()); + return path; + } +}; + +int DictionaryTest::failures = 0; + +int wmain(int argc, wchar_t** argv) +{ + DictionaryTest::Check(DictionaryTest::WriteSynthetic(), true); + for (int i = 1; i < argc; ++i) + DictionaryTest::Check(argv[i], false); + + wprintf(L"%ls (%d failures)\n", DictionaryTest::failures ? L"FAILED" : L"PASSED", + DictionaryTest::failures); + return DictionaryTest::failures ? 1 : 0; +} diff --git a/src/mkcache.cpp b/src/mkcache.cpp new file mode 100644 index 0000000..a711077 --- /dev/null +++ b/src/mkcache.cpp @@ -0,0 +1,41 @@ +// mkcache.cpp +// +// canto-mkcache.exe [--per-user] [ ...] +// +// Parses each dictionary and writes its page-aligned mmap cache (see +// Dictionary.cpp for the format), so the IME maps the cache instead of parsing +// text. A no-op for a dictionary whose cache is already valid. +// +// default machine-wide: writes ".cache" next to the dictionary +// (the installer uses this for the stock dictionaries). +// --per-user writes only under %LOCALAPPDATA%\CantoIME (the options dialog +// uses this for custom, per-user dictionaries). +// +// Exit code 0 if every dictionary ended up with a valid cache, 1 otherwise, +// 2 for usage. + +#include "Dictionary.h" + +#include + +int wmain(int argc, wchar_t** argv) +{ + bool machineWide = true; + bool any = false; + bool allOk = true; + + for (int i = 1; i < argc; ++i) + { + if (wcscmp(argv[i], L"--per-user") == 0) + { + machineWide = false; + continue; + } + any = true; + allOk &= Dictionary::PrebuildCache(argv[i], machineWide); + } + + if (!any) + return 2; + return allOk ? 0 : 1; +}