From 7b2ad52617df1eba03b4c9201613155fbb801807 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:11 +0800 Subject: [PATCH 001/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/CMakeLists.txt | 123 +++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 third_party/mecab/CMakeLists.txt diff --git a/third_party/mecab/CMakeLists.txt b/third_party/mecab/CMakeLists.txt new file mode 100644 index 00000000..9a53833f --- /dev/null +++ b/third_party/mecab/CMakeLists.txt @@ -0,0 +1,123 @@ +# MeCab morphological analyzer — vendored from https://github.com/taku910/mecab +# (BSD-3-Clause, see LICENSE). Used by the FTS extension for Japanese +# tokenization. The ipadic dictionary (EUC-JP CSV) is compiled to the native +# binary dictionary format (UTF-8) at build time via mecab-dict-index. + +set(MECAB_SRCS + src/char_property.cpp + src/connector.cpp + src/context_id.cpp + src/dictionary.cpp + src/dictionary_compiler.cpp + src/dictionary_generator.cpp + src/dictionary_rewriter.cpp + src/eval.cpp + src/feature_index.cpp + src/iconv_utils.cpp + src/lbfgs.cpp + src/learner.cpp + src/learner_tagger.cpp + src/libmecab.cpp + src/nbest_generator.cpp + src/param.cpp + src/string_buffer.cpp + src/tagger.cpp + src/tokenizer.cpp + src/utils.cpp + src/viterbi.cpp + src/writer.cpp) + +# Same defines the upstream Makefile.msvc uses; VERSION/DIC_VERSION come from +# configure.in (0.996, 102). DLL_EXPORT is omitted: we link statically. +set(MECAB_DEFS + _CRT_SECURE_NO_DEPRECATE + MECAB_USE_THREAD + HAVE_GETENV + HAVE_WINDOWS_H + DIC_VERSION=102 + VERSION=\"0.996\" + PACKAGE=\"mecab\" + # Default rc file: an empty file generated alongside the dictionary. + # (An empty string would make Param::load() try to open "" and fail.) + MECAB_DEFAULT_RC=\"${CMAKE_BINARY_DIR}/mecab-dict/ipadic/mecabrc\" + # Gate for StringBuffer::operator<<(unsigned long long) in string_buffer.h. + HAVE_UNSIGNED_LONG_LONG_INT) + +add_library(mecab STATIC ${MECAB_SRCS}) +target_include_directories(mecab PUBLIC src) +target_compile_definitions(mecab PRIVATE ${MECAB_DEFS}) +# mecab.h declares MECAB_DLL_EXTERN as __declspec(dllimport) on _WIN32 unless +# DLL_EXPORT is defined. We link statically, so export (not import) must be +# declared consistently in both the library and its consumers. +target_compile_definitions(mecab PUBLIC DLL_EXPORT) +if(MSVC) + # Upstream suppresses these in Makefile.msvc; the code is C++98-era. + target_compile_options(mecab PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Build-time tool that compiles dictionary CSV files into the binary dictionary +# format (sys.dic/sys.dat/sys.cha/unk.*/char.bin/matrix.bin). +add_executable(mecab-dict-index src/mecab-dict-index.cpp ${MECAB_SRCS}) +target_include_directories(mecab-dict-index PRIVATE src) +target_compile_definitions(mecab-dict-index PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-dict-index PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Compile the ipadic dictionary (EUC-JP CSV) into a UTF-8 binary dictionary. +# The dictionary is not vendored (54MB CSV): it is downloaded from +# https://github.com/taku910/mecab-ipadic at configure time. The compiled +# dictionary files are host-endian, so they are generated at build time. +# ipadic has no standalone repo; it lives in taku910/mecab's mecab-ipadic/ +# subdirectory, so we download the mecab archive and keep that subtree. +set(MECAB_IPADIC_DIR ${CMAKE_BINARY_DIR}/mecab-ipadic-src) +if(NOT EXISTS ${MECAB_IPADIC_DIR}/dicrc) + set(MECAB_IPADIC_ZIP ${CMAKE_BINARY_DIR}/mecab-ipadic.zip) + set(MECAB_IPADIC_EXTRACT ${CMAKE_BINARY_DIR}/mecab-ipadic-extract) + message(STATUS "Downloading mecab-ipadic dictionary sources (54MB CSV)...") + file(DOWNLOAD https://github.com/taku910/mecab/archive/refs/heads/master.zip + ${MECAB_IPADIC_ZIP} STATUS MECAB_DL_STATUS) + list(GET MECAB_DL_STATUS 0 MECAB_DL_CODE) + if(NOT MECAB_DL_CODE EQUAL 0) + list(GET MECAB_DL_STATUS 1 MECAB_DL_ERROR) + message(FATAL_ERROR "Failed to download mecab-ipadic: ${MECAB_DL_ERROR}") + endif() + file(MAKE_DIRECTORY ${MECAB_IPADIC_EXTRACT}) + file(ARCHIVE_EXTRACT INPUT ${MECAB_IPADIC_ZIP} DESTINATION ${MECAB_IPADIC_EXTRACT}) + set(MECAB_IPADIC_SUBDIR ${MECAB_IPADIC_EXTRACT}/mecab-master/mecab-ipadic) + if(NOT IS_DIRECTORY "${MECAB_IPADIC_SUBDIR}") + message(FATAL_ERROR "Unexpected mecab archive layout (mecab-ipadic dir missing)") + endif() + file(RENAME ${MECAB_IPADIC_SUBDIR} ${MECAB_IPADIC_DIR}) + file(REMOVE_RECURSE ${MECAB_IPADIC_EXTRACT}) + file(REMOVE ${MECAB_IPADIC_ZIP}) +endif() +set(MECAB_DICT_OUT_DIR ${CMAKE_BINARY_DIR}/mecab-dict/ipadic) +set(MECAB_DICT_OUTPUTS + ${MECAB_DICT_OUT_DIR}/sys.dic + ${MECAB_DICT_OUT_DIR}/unk.dic + ${MECAB_DICT_OUT_DIR}/char.bin + ${MECAB_DICT_OUT_DIR}/matrix.bin + ${MECAB_DICT_OUT_DIR}/dicrc + ${MECAB_DICT_OUT_DIR}/mecabrc) + +add_custom_command( + OUTPUT ${MECAB_DICT_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} + COMMAND $ -f euc-jp -t utf-8 + -d ${MECAB_IPADIC_DIR} -o ${MECAB_DICT_OUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${MECAB_IPADIC_DIR}/dicrc ${MECAB_DICT_OUT_DIR}/dicrc + COMMAND ${CMAKE_COMMAND} -E touch ${MECAB_DICT_OUT_DIR}/mecabrc + DEPENDS mecab-dict-index + COMMENT "Compiling ipadic dictionary (EUC-JP -> UTF-8)") + +add_custom_target(mecab-ipadic-dict + DEPENDS ${MECAB_DICT_OUT_DIR}/sys.dic) + +# CLI for ad-hoc tokenization checks during development. +add_executable(mecab-cli src/mecab.cpp ${MECAB_SRCS}) +target_include_directories(mecab-cli PRIVATE src) +target_compile_definitions(mecab-cli PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-cli PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() From 70f08f651a224d6971520be05b92e30939f48c32 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:12 +0800 Subject: [PATCH 002/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/LICENSE | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 third_party/mecab/LICENSE diff --git a/third_party/mecab/LICENSE b/third_party/mecab/LICENSE new file mode 100644 index 00000000..71d7d805 --- /dev/null +++ b/third_party/mecab/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2001-2008, Taku Kudo +Copyright (c) 2004-2008, Nippon Telegraph and Telephone Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are +permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + + * Neither the name of the Nippon Telegraph and Telegraph Corporation + nor the names of its contributors may be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 11d78d05ccba529e7d5deb38cfc342fabe1ab332 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:14 +0800 Subject: [PATCH 003/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/char_property.cpp | 279 ++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 third_party/mecab/src/char_property.cpp diff --git a/third_party/mecab/src/char_property.cpp b/third_party/mecab/src/char_property.cpp new file mode 100644 index 00000000..1029a11a --- /dev/null +++ b/third_party/mecab/src/char_property.cpp @@ -0,0 +1,279 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include +#include "char_property.h" +#include "common.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +namespace { +struct Range { + int low; + int high; + std::vector c; +}; + +int atohex(const char *s) { + int n = 0; + + CHECK_DIE(std::strlen(s) >= 3 + && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) + << "no hex value: " << s; + + const char *p = s; + s += 2; + while (*s) { + int r = 0; + if (*s >= '0' && *s <= '9') + r = *s - '0'; + else if (*s >= 'A' && *s <= 'F') + r = *s - 'A' + 10; + else if (*s >= 'a' && *s <= 'f') + r = *s - 'a' + 10; + else + CHECK_DIE(false) << "no hex value: " << p; + + n = 16 * n + r; + s++; + } + + return n; +} + +CharInfo encode(const std::vector &c, + std::map *category) { + CHECK_DIE(c.size()) << "category size is empty"; + + std::map::const_iterator it = category->find(c[0]); + CHECK_DIE(it != category->end()) + << "category [" << c[0] << "] is undefined"; + + CharInfo base = it->second; + for (size_t i = 0; i < c.size(); ++i) { + std::map::const_iterator it = + category->find(c[i]); + CHECK_DIE(it != category->end()) + << "category [" << c[i] << "] is undefined"; + base.type += (1 << it->second.default_type); + } + + return base; +} +} + +bool CharProperty::open(const Param ¶m) { + const std::string prefix = param.get("dicdir"); + const std::string filename = create_filename(prefix, CHAR_PROPERTY_FILE); + return open(filename.c_str()); +} + +bool CharProperty::open(const char *filename) { + std::ostringstream error; + CHECK_FALSE(cmmap_->open(filename, "r")); + + const char *ptr = cmmap_->begin(); + unsigned int csize; + read_static(&ptr, csize); + + size_t fsize = sizeof(unsigned int) + + (32 * csize) + sizeof(unsigned int) * 0xffff; + + CHECK_FALSE(fsize == cmmap_->size()) + << "invalid file size: " << filename; + + clist_.clear(); + for (unsigned int i = 0; i < csize; ++i) { + const char *s = read_ptr(&ptr, 32); + clist_.push_back(s); + } + + map_ = reinterpret_cast(ptr); + + return true; +} + +void CharProperty::close() { + cmmap_->close(); +} + +size_t CharProperty::size() const { return clist_.size(); } + +const char *CharProperty::name(size_t i) const { + return const_cast(clist_[i]); +} + +// this function must be rewritten. +void CharProperty::set_charset(const char *ct) { + charset_ = decode_charset(ct); +} + +int CharProperty::id(const char *key) const { + for (int i = 0; i < static_cast(clist_.size()); ++i) { + if (std::strcmp(key, clist_[i]) == 0) { + return i; + } + } + return -1; +} + +bool CharProperty::compile(const char *cfile, + const char *ufile, + const char *ofile) { + scoped_fixed_array line; + scoped_fixed_array col; + size_t id = 0; + std::vector range; + std::map category; + std::vector category_ary; + std::ifstream ifs(WPATH(cfile)); + std::istringstream iss(CHAR_PROPERTY_DEF_DEFAULT); + std::istream *is = &ifs; + + if (!ifs) { + std::cerr << cfile + << " is not found. minimum setting is used" << std::endl; + is = &iss; + } + + while (is->getline(line.get(), line.size())) { + if (std::strlen(line.get()) == 0 || line[0] == '#') { + continue; + } + const size_t size = tokenize2(line.get(), "\t ", col.get(), col.size()); + CHECK_DIE(size >= 2) << "format error: " << line.get(); + + // 0xFFFF..0xFFFF hoge hoge hgoe # + if (std::strncmp(col[0], "0x", 2) == 0) { + std::string low = col[0]; + std::string high; + size_t pos = low.find(".."); + + if (pos != std::string::npos) { + high = low.substr(pos + 2, low.size() - pos - 2); + low = low.substr(0, pos); + } else { + high = low; + } + + Range r; + r.low = atohex(low.c_str()); + r.high = atohex(high.c_str()); + + CHECK_DIE(r.low >= 0 && r.low < 0xffff && + r.high >= 0 && r.high < 0xffff && + r.low <= r.high) + << "range error: low=" << r.low << " high=" << r.high; + + for (size_t i = 1; i < size; ++i) { + if (col[i][0] == '#') { + break; // skip comments + } + CHECK_DIE(category.find(std::string(col[i])) != category.end()) + << "category [" << col[i] << "] is undefined"; + r.c.push_back(col[i]); + } + range.push_back(r); + } else { + CHECK_DIE(size >= 4) << "format error: " << line.get(); + + std::string key = col[0]; + CHECK_DIE(category.find(key) == category.end()) + << "category " << key << " is already defined"; + + CharInfo c; + std::memset(&c, 0, sizeof(c)); + c.invoke = std::atoi(col[1]); + c.group = std::atoi(col[2]); + c.length = std::atoi(col[3]); + c.default_type = id++; + + category.insert(std::pair(key, c)); + category_ary.push_back(key); + } + } + + CHECK_DIE(category.size() < 18) << "too many categories(>= 18)"; + + CHECK_DIE(category.find("DEFAULT") != category.end()) + << "category [DEFAULT] is undefined"; + + CHECK_DIE(category.find("SPACE") != category.end()) + << "category [SPACE] is undefined"; + + std::istringstream iss2(UNK_DEF_DEFAULT); + std::ifstream ifs2(WPATH(ufile)); + std::istream *is2 = &ifs2; + + if (!ifs2) { + std::cerr << ufile + << " is not found. minimum setting is used." << std::endl; + is2 = &iss2; + } + + std::set unk; + while (is2->getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col.get(), 2); + CHECK_DIE(n >= 1) << "format error: " << line.get(); + const std::string key = col[0]; + CHECK_DIE(category.find(key) != category.end()) + << "category [" << key << "] is undefined in " << cfile; + unk.insert(key); + } + + for (std::map::const_iterator it = category.begin(); + it != category.end(); + ++it) { + CHECK_DIE(unk.find(it->first) != unk.end()) + << "category [" << it->first << "] is undefined in " << ufile; + } + + std::vector table(0xffff); + { + std::vector tmp; + tmp.push_back("DEFAULT"); + const CharInfo c = encode(tmp, &category); + std::fill(table.begin(), table.end(), c); + } + + for (std::vector::const_iterator it = range.begin(); + it != range.end(); + ++it) { + const CharInfo c = encode(it->c, &category); + for (int i = it->low; i <= it->high; ++i) { + table[i] = c; + } + } + + // output binary table + { + std::ofstream ofs(WPATH(ofile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << ofile; + + unsigned int size = static_cast(category.size()); + ofs.write(reinterpret_cast(&size), sizeof(size)); + for (std::vector::const_iterator it = category_ary.begin(); + it != category_ary.end(); + ++it) { + char buf[32]; + std::fill(buf, buf + sizeof(buf), '\0'); + std::strncpy(buf, it->c_str(), sizeof(buf) - 1); + ofs.write(reinterpret_cast(buf), sizeof(buf)); + } + ofs.write(reinterpret_cast(&table[0]), + sizeof(CharInfo) * table.size()); + ofs.close(); + } + + return true; +} +} From 184989a5a1d286930084d86826534ab20c88c7c9 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:16 +0800 Subject: [PATCH 004/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/char_property.h | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 third_party/mecab/src/char_property.h diff --git a/third_party/mecab/src/char_property.h b/third_party/mecab/src/char_property.h new file mode 100644 index 00000000..9c904ba0 --- /dev/null +++ b/third_party/mecab/src/char_property.h @@ -0,0 +1,92 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CHARACTER_CATEGORY_H_ +#define MECAB_CHARACTER_CATEGORY_H_ + +#include "mmap.h" +#include "scoped_ptr.h" +#include "ucs.h" +#include "utils.h" + +namespace MeCab { +class Param; + +struct CharInfo { + unsigned int type: 18; + unsigned int default_type: 8; + unsigned int length: 4; + unsigned int group: 1; + unsigned int invoke: 1; + CharInfo() : type(0), default_type(0), length(0), group(0), invoke(0) {} + bool isKindOf(CharInfo c) const { return type & c.type; } +}; + +class CharProperty { + public: + bool open(const Param &); + bool open(const char*); + void close(); + size_t size() const; + void set_charset(const char *charset); + int id(const char *) const; + const char *name(size_t i) const; + const char *what() { return what_.str(); } + + inline const char *seekToOtherType(const char *begin, const char *end, + CharInfo c, CharInfo *fail, + size_t *mblen, size_t *clen) const { + const char *p = begin; + *clen = 0; + while (p != end && c.isKindOf(*fail = getCharInfo(p, end, mblen))) { + p += *mblen; + ++(*clen); + c = *fail; + } + return p; + } + + inline CharInfo getCharInfo(const char *begin, + const char *end, + size_t *mblen) const { + unsigned short int t = 0; +#ifndef MECAB_USE_UTF8_ONLY + switch (charset_) { + case EUC_JP: t = euc_to_ucs2(begin, end, mblen); break; + case CP932: t = cp932_to_ucs2(begin, end, mblen); break; + case UTF8: t = utf8_to_ucs2(begin, end, mblen); break; + case UTF16: t = utf16_to_ucs2(begin, end, mblen); break; + case UTF16LE: t = utf16le_to_ucs2(begin, end, mblen); break; + case UTF16BE: t = utf16be_to_ucs2(begin, end, mblen); break; + case ASCII: t = ascii_to_ucs2(begin, end, mblen); break; + default: t = utf8_to_ucs2(begin, end, mblen); break; + } +#else + switch (charset_) { + case UTF8: t = utf8_to_ucs2(begin, end, mblen); break; + case UTF16: t = utf16_to_ucs2(begin, end, mblen); break; + case UTF16LE: t = utf16le_to_ucs2(begin, end, mblen); break; + case UTF16BE: t = utf16be_to_ucs2(begin, end, mblen); break; + default: t = utf8_to_ucs2(begin, end, mblen); break; + } +#endif + return map_[t]; + } + + inline CharInfo getCharInfo(size_t id) const { return map_[id]; } + + static bool compile(const char *, const char *, const char*); + + CharProperty(): cmmap_(new Mmap), map_(0), charset_(0) {} + virtual ~CharProperty() { this->close(); } + + private: + scoped_ptr > cmmap_; + std::vector clist_; + const CharInfo *map_; + int charset_; + whatlog what_; +}; +} +#endif // MECAB_CHARACTER_CATEGORY_H_ From 34aeac5798bb98989747492c57a1b1a1deb813c1 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:18 +0800 Subject: [PATCH 005/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/common.h | 143 +++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 third_party/mecab/src/common.h diff --git a/third_party/mecab/src/common.h b/third_party/mecab/src/common.h new file mode 100644 index 00000000..2e452a76 --- /dev/null +++ b/third_party/mecab/src/common.h @@ -0,0 +1,143 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_COMMON_H_ +#define MECAB_COMMON_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __CYGWIN__ +#define _GLIBCXX_EXPORT_TEMPLATE +#endif + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#if defined(_MSC_VER) || defined(__CYGWIN__) +#define NOMINMAX +#define snprintf _snprintf +#endif + +#define COPYRIGHT "MeCab: Yet Another Part-of-Speech and Morphological Analyzer\n\ +\nCopyright(C) 2001-2012 Taku Kudo \nCopyright(C) 2004-2008 Nippon Telegraph and Telephone Corporation\n" + +#define SYS_DIC_FILE "sys.dic" +#define UNK_DEF_FILE "unk.def" +#define UNK_DIC_FILE "unk.dic" +#define MATRIX_DEF_FILE "matrix.def" +#define MATRIX_FILE "matrix.bin" +#define CHAR_PROPERTY_DEF_FILE "char.def" +#define CHAR_PROPERTY_FILE "char.bin" +#define FEATURE_FILE "feature.def" +#define REWRITE_FILE "rewrite.def" +#define LEFT_ID_FILE "left-id.def" +#define RIGHT_ID_FILE "right-id.def" +#define POS_ID_FILE "pos-id.def" +#define MODEL_DEF_FILE "model.def" +#define MODEL_FILE "model.bin" +#define DICRC "dicrc" +#define BOS_KEY "BOS/EOS" + +#define DEFAULT_MAX_GROUPING_SIZE 24 + +#define CHAR_PROPERTY_DEF_DEFAULT "DEFAULT 1 0 0\nSPACE 0 1 0\n0x0020 SPACE\n" +#define UNK_DEF_DEFAULT "DEFAULT,0,0,0,*\nSPACE,0,0,0,*\n" +#define MATRIX_DEF_DEFAULT "1 1\n0 0 0\n" + +#ifdef MECAB_USE_UTF8_ONLY +#define MECAB_DEFAULT_CHARSET "UTF-8" +#endif + +#ifndef MECAB_DEFAULT_CHARSET +#if defined(_WIN32) && !defined(__CYGWIN__) +#define MECAB_DEFAULT_CHARSET "SHIFT-JIS" +#else +#define MECAB_DEFAULT_CHARSET "EUC-JP" +#endif +#endif + +#define NBEST_MAX 512 +#define NODE_FREELIST_SIZE 512 +#define PATH_FREELIST_SIZE 2048 +#define MIN_INPUT_BUFFER_SIZE 8192 +#define MAX_INPUT_BUFFER_SIZE (8192*640) +#define BUF_SIZE 8192 + +#ifndef EXIT_FAILURE +#define EXIT_FAILURE 1 +#endif + +#ifndef EXIT_SUCCESS +#define EXIT_SUCCESS 0 +#endif + +#ifdef _WIN32 +#ifdef __GNUC__ +#define WPATH_FORCE(path) (MeCab::Utf8ToWide(path).c_str()) +#define WPATH(path) (path) +#else +// Upstream bug: the MSVC branch only defines WPATH, but call sites use +// WPATH_FORCE directly (e.g. mmap.h). Define it like the MinGW branch. +#define WPATH_FORCE(path) (MeCab::Utf8ToWide(path).c_str()) +#define WPATH(path) WPATH_FORCE(path) +#endif +#else +#define WPATH_FORCE(path) (path) +#define WPATH(path) (path) +#endif + +namespace MeCab { +class die { + public: + die() {} + ~die() { + std::cerr << std::endl; + exit(-1); + } + int operator&(std::ostream&) { return 0; } +}; + +struct whatlog { + std::ostringstream stream_; + std::string str_; + const char *str() { + str_ = stream_.str(); + return str_.c_str(); + } +}; + +class wlog { + public: + wlog(whatlog *what) : what_(what) { + what_->stream_.clear(); + } + bool operator&(std::ostream &) { + return false; + } + private: + whatlog *what_; +}; +} // MeCab + +#define WHAT what_.stream_ + +#define CHECK_FALSE(condition) \ + if (condition) {} else return \ + wlog(&what_) & what_.stream_ << \ + __FILE__ << "(" << __LINE__ << ") [" << #condition << "] " + +#define CHECK_DIE(condition) \ +(condition) ? 0 : die() & std::cerr << __FILE__ << \ +"(" << __LINE__ << ") [" << #condition << "] " + +#endif // MECAB_COMMON_H_ From 4942b95c4e3a6e4a532ef04ab98ee28eb5a3730e Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:19 +0800 Subject: [PATCH 006/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/connector.cpp | 113 ++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 third_party/mecab/src/connector.cpp diff --git a/third_party/mecab/src/connector.cpp b/third_party/mecab/src/connector.cpp new file mode 100644 index 00000000..56900221 --- /dev/null +++ b/third_party/mecab/src/connector.cpp @@ -0,0 +1,113 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "connector.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +bool Connector::open(const Param ¶m) { + const std::string filename = create_filename + (param.get("dicdir"), MATRIX_FILE); + return open(filename.c_str()); +} + +bool Connector::open(const char* filename, + const char *mode) { + CHECK_FALSE(cmmap_->open(filename, mode)) + << "cannot open: " << filename; + + matrix_ = cmmap_->begin(); + + CHECK_FALSE(matrix_) << "matrix is NULL" ; + CHECK_FALSE(cmmap_->size() >= 2) + << "file size is invalid: " << filename; + + lsize_ = static_cast((*cmmap_)[0]); + rsize_ = static_cast((*cmmap_)[1]); + + CHECK_FALSE(static_cast(lsize_ * rsize_ + 2) + == cmmap_->size()) + << "file size is invalid: " << filename; + + matrix_ = cmmap_->begin() + 2; + return true; +} + +void Connector::close() { + cmmap_->close(); +} + +bool Connector::openText(const char *filename) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + WHAT << "no such file or directory: " << filename; + return false; + } + char *column[2]; + scoped_fixed_array buf; + ifs.getline(buf.get(), buf.size()); + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " << buf.get(); + lsize_ = std::atoi(column[0]); + rsize_ = std::atoi(column[1]); + return true; +} + +bool Connector::compile(const char *ifile, const char *ofile) { + std::ifstream ifs(WPATH(ifile)); + std::istringstream iss(MATRIX_DEF_DEFAULT); + std::istream *is = &ifs; + + if (!ifs) { + std::cerr << ifile + << " is not found. minimum setting is used." << std::endl; + is = &iss; + } + + + char *column[4]; + scoped_fixed_array buf; + + is->getline(buf.get(), buf.size()); + + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " << buf.get(); + + const unsigned short lsize = std::atoi(column[0]); + const unsigned short rsize = std::atoi(column[1]); + std::vector matrix(lsize * rsize); + std::fill(matrix.begin(), matrix.end(), 0); + + std::cout << "reading " << ifile << " ... " + << lsize << "x" << rsize << std::endl; + + while (is->getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 3) == 3) + << "format error: " << buf.get(); + const size_t l = std::atoi(column[0]); + const size_t r = std::atoi(column[1]); + const int c = std::atoi(column[2]); + CHECK_DIE(l < lsize && r < rsize) << "index values are out of range"; + progress_bar("emitting matrix ", l + 1, lsize); + matrix[(l + lsize * r)] = static_cast(c); + } + + std::ofstream ofs(WPATH(ofile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << ofile; + ofs.write(reinterpret_cast(&lsize), sizeof(unsigned short)); + ofs.write(reinterpret_cast(&rsize), sizeof(unsigned short)); + ofs.write(reinterpret_cast(&matrix[0]), + lsize * rsize * sizeof(short)); + ofs.close(); + + return true; +} +} From 02c5bf23774c435c801d753cbc82577c9d8f52dc Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:21 +0800 Subject: [PATCH 007/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/connector.h | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 third_party/mecab/src/connector.h diff --git a/third_party/mecab/src/connector.h b/third_party/mecab/src/connector.h new file mode 100644 index 00000000..8a687170 --- /dev/null +++ b/third_party/mecab/src/connector.h @@ -0,0 +1,67 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CONNECTOR_H_ +#define MECAB_CONNECTOR_H_ + +#include "mecab.h" +#include "mmap.h" +#include "common.h" +#include "scoped_ptr.h" + +namespace MeCab { +class Param; + +class Connector { + private: + scoped_ptr > cmmap_; + short *matrix_; + unsigned short lsize_; + unsigned short rsize_; + whatlog what_; + + public: + + bool open(const Param ¶m); + void close(); + void clear() {} + + const char *what() { return what_.str(); } + + size_t left_size() const { return static_cast(lsize_); } + size_t right_size() const { return static_cast(rsize_); } + + void set_left_size(size_t lsize) { lsize_ = lsize; } + void set_right_size(size_t rsize) { rsize_ = rsize; } + + inline int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const { + return matrix_[rcAttr + lsize_ * lcAttr]; + } + + inline int cost(const Node *lNode, const Node *rNode) const { + return matrix_[lNode->rcAttr + lsize_ * rNode->lcAttr] + rNode->wcost; + } + + // access to raw matrix + short *mutable_matrix() { return &matrix_[0]; } + const short *matrix() const { return &matrix_[0]; } + + bool openText(const char *filename); + bool open(const char *filename, const char *mode = "r"); + + bool is_valid(size_t lid, size_t rid) const { + return (lid >= 0 && lid < rsize_ && rid >= 0 && rid < lsize_); + } + + static bool compile(const char *, const char *); + + explicit Connector(): + cmmap_(new Mmap), matrix_(0), lsize_(0), rsize_(0) {} + + virtual ~Connector() { this->close(); } +}; +} +#endif // MECAB_CONNECTOR_H_ From 4cf02ad749927cb70e414cb7c5442ecd15a28152 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:23 +0800 Subject: [PATCH 008/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/context_id.cpp | 107 +++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 third_party/mecab/src/context_id.cpp diff --git a/third_party/mecab/src/context_id.cpp b/third_party/mecab/src/context_id.cpp new file mode 100644 index 00000000..eeff28c4 --- /dev/null +++ b/third_party/mecab/src/context_id.cpp @@ -0,0 +1,107 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include "context_id.h" +#include "iconv_utils.h" +#include "utils.h" + +namespace { + +using namespace MeCab; + +bool open_map(const char *filename, + std::map *cmap, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + cmap->clear(); + char *col[2]; + std::string line; + while (std::getline(ifs, line)) { + CHECK_DIE(2 == tokenize2(const_cast(line.c_str()), + " \t", col, 2)) + << "format error: " << line; + std::string pos = col[1]; + if (iconv) { + iconv->convert(&pos); + } + cmap->insert(std::pair + (pos, std::atoi(col[0]))); + } + return true; +} + +bool build(std::map *cmap, + const std::string &bos) { + int id = 1; // for BOS/EOS + for (std::map::iterator it = cmap->begin(); + it != cmap->end(); + ++it) it->second = id++; + cmap->insert(std::make_pair(bos, 0)); + return true; +} + +bool save(const char* filename, + std::map *cmap) { + std::ofstream ofs(WPATH(filename)); + CHECK_DIE(ofs) << "permission denied: " << filename; + for (std::map::const_iterator it = cmap->begin(); + it != cmap->end(); ++it) { + ofs << it->second << " " << it->first << std::endl; + } + return true; +} +} + +namespace MeCab { + +void ContextID::clear() { + left_.clear(); + right_.clear(); + left_bos_.clear(); + right_bos_.clear(); +} + +void ContextID::add(const char *l, const char *r) { + left_.insert(std::make_pair(std::string(l), 1)); + right_.insert(std::make_pair(std::string(r), 1)); +} + +void ContextID::addBOS(const char *l, const char *r) { + left_bos_ = l; + right_bos_ = r; +} + +bool ContextID::save(const char* lfile, + const char* rfile) { + return (::save(lfile, &left_) && ::save(rfile, &right_)); +} + +bool ContextID::open(const char *lfile, + const char *rfile, + Iconv *iconv) { + return (::open_map(lfile, &left_, iconv) && + ::open_map(rfile, &right_, iconv)); +} + +bool ContextID::build() { + return (::build(&left_, left_bos_) && ::build(&right_, right_bos_)); +} + +int ContextID::lid(const char *l) const { + std::map::const_iterator it = left_.find(l); + CHECK_DIE(it != left_.end()) + << "cannot find LEFT-ID for " << l; + return it->second; +} + +int ContextID::rid(const char *r) const { + std::map::const_iterator it = right_.find(r); + CHECK_DIE(it != right_.end()) + << "cannot find RIGHT-ID for " << r; + return it->second; +} +} From 1a866237c4e2d79ddc7d18b00d2ef8ded8422bf6 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:25 +0800 Subject: [PATCH 009/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/context_id.h | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 third_party/mecab/src/context_id.h diff --git a/third_party/mecab/src/context_id.h b/third_party/mecab/src/context_id.h new file mode 100644 index 00000000..28622610 --- /dev/null +++ b/third_party/mecab/src/context_id.h @@ -0,0 +1,50 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CONTEXT_ID_H +#define MECAB_CONTEXT_ID_H + +#include +#include +#include + +namespace MeCab { + +class Param; +class Iconv; + +class ContextID { + private: + std::map left_; + std::map right_; + std::string left_bos_; + std::string right_bos_; + + public: + void clear(); + void add(const char *l, const char *r); + void addBOS(const char *l, const char *r); + bool save(const char* lfile, + const char* rfile); + bool build(); + bool open(const char *lfile, + const char *rfile, + Iconv *iconv = 0); + int lid(const char *l) const; + int rid(const char *r) const; + + size_t left_size() const { return left_.size(); } + size_t right_size() const { return right_.size(); } + + const std::map& left_ids() const { return left_; } + const std::map& right_ids() const { return right_; } + + bool is_valid(size_t lid, size_t rid) { + return (lid >= 0 && lid < left_size() && + rid >= 0 && rid < right_size()); + } +}; +} +#endif From 4fcbde4fec81cf72878cedb0402120303088ecaf Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:27 +0800 Subject: [PATCH 010/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/darts.h | 518 ++++++++++++++++++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 third_party/mecab/src/darts.h diff --git a/third_party/mecab/src/darts.h b/third_party/mecab/src/darts.h new file mode 100644 index 00000000..49bd3edf --- /dev/null +++ b/third_party/mecab/src/darts.h @@ -0,0 +1,518 @@ +/* + Darts -- Double-ARray Trie System + + + Copyright(C) 2001-2007 Taku Kudo +*/ +#ifndef DARTS_H_ +#define DARTS_H_ + +#define DARTS_VERSION "0.31" +#include +#include +#include + +#ifdef HAVE_ZLIB_H +namespace zlib { +#include +} +#define SH(p)((unsigned short)(unsigned char)((p)[0]) | ((unsigned short)(unsigned char)((p)[1]) << 8)) +#define LG(p)((unsigned long)(SH(p)) |((unsigned long)(SH((p)+2)) << 16)) +#endif + +namespace MeCab { + +namespace Darts { + +template inline T _max(T x, T y) { return(x > y) ? x : y; } +template inline T* _resize(T* ptr, size_t n, size_t l, T v) { + T *tmp = new T[l]; + for (size_t i = 0; i < n; ++i) tmp[i] = ptr[i]; + for (size_t i = n; i < l; ++i) tmp[i] = v; + delete [] ptr; + return tmp; +} + +template +class Length { + public: size_t operator()(const T *key) const + { size_t i; for (i = 0; key[i] != (T)0; ++i) {} return i; } +}; + +template <> class Length { + public: size_t operator()(const char *key) const + { return std::strlen(key); } +}; + +template > +class DoubleArrayImpl { + private: + + struct node_t { + array_u_type_ code; + size_t depth; + size_t left; + size_t right; + }; + + struct unit_t { + array_type_ base; + array_u_type_ check; + }; + + unit_t *array_; + unsigned char *used_; + size_t size_; + size_t alloc_size_; + node_type_ **key_; + size_t key_size_; + size_t *length_; + array_type_ *value_; + size_t progress_; + size_t next_check_pos_; + bool no_delete_; + int error_; + int (*progress_func_)(size_t, size_t); + + size_t resize(const size_t new_size) { + unit_t tmp; + tmp.base = 0; + tmp.check = 0; + array_ = _resize(array_, alloc_size_, new_size, tmp); + used_ = _resize(used_, alloc_size_, new_size, + static_cast(0)); + alloc_size_ = new_size; + return new_size; + } + + size_t fetch(const node_t &parent, std::vector &siblings) { + if (error_ < 0) return 0; + + array_u_type_ prev = 0; + + for (size_t i = parent.left; i < parent.right; ++i) { + if ((length_ ? length_[i] : length_func_()(key_[i])) < parent.depth) + continue; + + const node_u_type_ *tmp = reinterpret_cast(key_[i]); + + array_u_type_ cur = 0; + if ((length_ ? length_[i] : length_func_()(key_[i])) != parent.depth) + cur = (array_u_type_)tmp[parent.depth] + 1; + + if (prev > cur) { + error_ = -3; + return 0; + } + + if (cur != prev || siblings.empty()) { + node_t tmp_node; + tmp_node.depth = parent.depth + 1; + tmp_node.code = cur; + tmp_node.left = i; + if (!siblings.empty()) siblings[siblings.size()-1].right = i; + + siblings.push_back(tmp_node); + } + + prev = cur; + } + + if (!siblings.empty()) + siblings[siblings.size()-1].right = parent.right; + + return siblings.size(); + } + + size_t insert(const std::vector &siblings) { + if (error_ < 0) return 0; + + size_t begin = 0; + size_t pos = _max((size_t)siblings[0].code + 1, next_check_pos_) - 1; + size_t nonzero_num = 0; + int first = 0; + + if (alloc_size_ <= pos) resize(pos + 1); + + while (true) { + next: + ++pos; + + if (alloc_size_ <= pos) resize(pos + 1); + + if (array_[pos].check) { + ++nonzero_num; + continue; + } else if (!first) { + next_check_pos_ = pos; + first = 1; + } + + begin = pos - siblings[0].code; + if (alloc_size_ <= (begin + siblings[siblings.size()-1].code)) + resize(static_cast(alloc_size_ * + _max(1.05, 1.0 * key_size_ / progress_))); + + if (used_[begin]) continue; + + for (size_t i = 1; i < siblings.size(); ++i) + if (array_[begin + siblings[i].code].check != 0) goto next; + + break; + } + + // -- Simple heuristics -- + // if the percentage of non-empty contents in check between the index + // 'next_check_pos' and 'check' is greater than some constant + // value(e.g. 0.9), + // new 'next_check_pos' index is written by 'check'. + if (1.0 * nonzero_num/(pos - next_check_pos_ + 1) >= 0.95) + next_check_pos_ = pos; + + used_[begin] = 1; + size_ = _max(size_, + begin + + static_cast(siblings[siblings.size() - 1].code + 1)); + + for (size_t i = 0; i < siblings.size(); ++i) + array_[begin + siblings[i].code].check = begin; + + for (size_t i = 0; i < siblings.size(); ++i) { + std::vector new_siblings; + + if (!fetch(siblings[i], new_siblings)) { + array_[begin + siblings[i].code].base = + value_ ? + static_cast(-value_[siblings[i].left]-1) : + static_cast(-siblings[i].left-1); + + if (value_ && (array_type_)(-value_[siblings[i].left]-1) >= 0) { + error_ = -2; + return 0; + } + + ++progress_; + if (progress_func_)(*progress_func_)(progress_, key_size_); + + } else { + size_t h = insert(new_siblings); + array_[begin + siblings[i].code].base = h; + } + } + + return begin; + } + + public: + + typedef array_type_ value_type; + typedef node_type_ key_type; + typedef array_type_ result_type; // for compatibility + + struct result_pair_type { + value_type value; + size_t length; + }; + + explicit DoubleArrayImpl(): array_(0), used_(0), + size_(0), alloc_size_(0), + no_delete_(0), error_(0) {} + ~DoubleArrayImpl() { clear(); } + + void set_result(value_type& x, value_type r, size_t) const { + x = r; + } + + void set_result(result_pair_type& x, value_type r, size_t l) const { + x.value = r; + x.length = l; + } + + void set_array(void *ptr, size_t size = 0) { + clear(); + array_ = reinterpret_cast(ptr); + no_delete_ = true; + size_ = size; + } + + const void *array() const { + return const_cast(reinterpret_cast(array_)); + } + + void clear() { + if (!no_delete_) + delete [] array_; + delete [] used_; + array_ = 0; + used_ = 0; + alloc_size_ = 0; + size_ = 0; + no_delete_ = false; + } + + size_t unit_size() const { return sizeof(unit_t); } + size_t size() const { return size_; } + size_t total_size() const { return size_ * sizeof(unit_t); } + + size_t nonzero_size() const { + size_t result = 0; + for (size_t i = 0; i < size_; ++i) + if (array_[i].check) ++result; + return result; + } + + int build(size_t key_size, + key_type **key, + size_t *length = 0, + value_type *value = 0, + int (*progress_func)(size_t, size_t) = 0) { + if (!key_size || !key) return 0; + + progress_func_ = progress_func; + key_ = key; + length_ = length; + key_size_ = key_size; + value_ = value; + progress_ = 0; + + resize(8192); + + array_[0].base = 1; + next_check_pos_ = 0; + + node_t root_node; + root_node.left = 0; + root_node.right = key_size; + root_node.depth = 0; + + std::vector siblings; + fetch(root_node, siblings); + insert(siblings); + + size_ += (1 << 8 * sizeof(key_type)) + 1; + if (size_ >= alloc_size_) resize(size_); + + delete [] used_; + used_ = 0; + + return error_; + } + + int open(const char *file, + const char *mode = "rb", + size_t offset = 0, + size_t size = 0) { + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + if (std::fseek(fp, offset, SEEK_SET) != 0) return -1; + + if (!size) { + if (std::fseek(fp, 0L, SEEK_END) != 0) return -1; + size = std::ftell(fp); + if (std::fseek(fp, offset, SEEK_SET) != 0) return -1; + } + + clear(); + + size_ = size; + size_ /= sizeof(unit_t); + array_ = new unit_t[size_]; + if (size_ != std::fread(reinterpret_cast(array_), + sizeof(unit_t), size_, fp)) return -1; + std::fclose(fp); + + return 0; + } + + int save(const char *file, + const char *mode = "wb", + size_t offset = 0) { + if (!size_) return -1; + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + if (size_ != std::fwrite(reinterpret_cast(array_), + sizeof(unit_t), size_, fp)) + return -1; + std::fclose(fp); + return 0; + } + +#ifdef HAVE_ZLIB_H + int gzopen(const char *file, + const char *mode = "rb", + size_t offset = 0, + size_t size = 0) { + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + clear(); + + size_ = size; + if (!size_) { + if (-1L != static_cast(std::fseek(fp, -8, SEEK_END))) { + char buf[8]; + if (std::fread(static_cast(buf), + 1, 8, fp) != sizeof(buf)) { + std::fclose(fp); + return -1; + } + size_ = LG(buf+4); + size_ /= sizeof(unit_t); + } + } + std::fclose(fp); + + if (!size_) return -1; + + zlib::gzFile gzfp = zlib::gzopen(file, mode); + if (!gzfp) return -1; + array_ = new unit_t[size_]; + if (zlib::gzseek(gzfp, offset, SEEK_SET) != 0) return -1; + zlib::gzread(gzfp, reinterpret_cast(array_), + sizeof(unit_t) * size_); + zlib::gzclose(gzfp); + return 0; + } + + int gzsave(const char *file, const char *mode = "wb", + size_t offset = 0) { + zlib::gzFile gzfp = zlib::gzopen(file, mode); + if (!gzfp) return -1; + zlib::gzwrite(gzfp, reinterpret_cast(array_), + sizeof(unit_t) * size_); + zlib::gzclose(gzfp); + return 0; + } +#endif + + template + inline void exactMatchSearch(const key_type *key, + T & result, + size_t len = 0, + size_t node_pos = 0) const { + result = exactMatchSearch(key, len, node_pos); + return; + } + + template + inline T exactMatchSearch(const key_type *key, + size_t len = 0, + size_t node_pos = 0) const { + if (!len) len = length_func_()(key); + + T result; + set_result(result, -1, 0); + + array_type_ b = array_[node_pos].base; + array_u_type_ p; + + for (size_t i = 0; i < len; ++i) { + p = b +(node_u_type_)(key[i]) + 1; + if (static_cast(b) == array_[p].check) + b = array_[p].base; + else + return result; + } + + p = b; + array_type_ n = array_[p].base; + if (static_cast(b) == array_[p].check && n < 0) + set_result(result, -n-1, len); + + return result; + } + + template + size_t commonPrefixSearch(const key_type *key, + T* result, + size_t result_len, + size_t len = 0, + size_t node_pos = 0) const { + if (!len) len = length_func_()(key); + + array_type_ b = array_[node_pos].base; + size_t num = 0; + array_type_ n; + array_u_type_ p; + + for (size_t i = 0; i < len; ++i) { + p = b; // + 0; + n = array_[p].base; + if ((array_u_type_) b == array_[p].check && n < 0) { + // result[num] = -n-1; + if (num < result_len) set_result(result[num], -n-1, i); + ++num; + } + + p = b +(node_u_type_)(key[i]) + 1; + if ((array_u_type_) b == array_[p].check) + b = array_[p].base; + else + return num; + } + + p = b; + n = array_[p].base; + + if ((array_u_type_)b == array_[p].check && n < 0) { + if (num < result_len) set_result(result[num], -n-1, len); + ++num; + } + + return num; + } + + value_type traverse(const key_type *key, + size_t &node_pos, + size_t &key_pos, + size_t len = 0) const { + if (!len) len = length_func_()(key); + + array_type_ b = array_[node_pos].base; + array_u_type_ p; + + for (; key_pos < len; ++key_pos) { + p = b +(node_u_type_)(key[key_pos]) + 1; + if (static_cast(b) == array_[p].check) { + node_pos = p; + b = array_[p].base; + } else { + return -2; // no node + } + } + + p = b; + array_type_ n = array_[p].base; + if (static_cast(b) == array_[p].check && n < 0) + return -n-1; + + return -1; // found, but no value + } +}; + +#if 4 == 2 +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 4 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 4 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 8 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#endif +} +} +#endif From b4b0ea74a6b5a6e9156ad89014161184eb7b4b8c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:29 +0800 Subject: [PATCH 011/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/dictionary.cpp | 535 +++++++++++++++++++++++++++ 1 file changed, 535 insertions(+) create mode 100644 third_party/mecab/src/dictionary.cpp diff --git a/third_party/mecab/src/dictionary.cpp b/third_party/mecab/src/dictionary.cpp new file mode 100644 index 00000000..0b9141fc --- /dev/null +++ b/third_party/mecab/src/dictionary.cpp @@ -0,0 +1,535 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "connector.h" +#include "context_id.h" +#include "char_property.h" +#include "common.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "iconv_utils.h" +#include "mmap.h" +#include "param.h" +#include "scoped_ptr.h" +#include "utils.h" +#include "writer.h" + +namespace MeCab { +namespace { + +const unsigned int DictionaryMagicID = 0xef718f77u; + +int toInt(const char *str) { + if (!str || std::strlen(str) == 0) { + return INT_MAX; + } + return std::atoi(str); +} + +int calcCost(const std::string &w, const std::string &feature, + int factor, + DecoderFeatureIndex *fi, DictionaryRewriter *rewriter, + CharProperty *property) { + CHECK_DIE(fi); + CHECK_DIE(rewriter); + CHECK_DIE(property); + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + size_t mblen = 0; + const CharInfo cinfo = property->getCharInfo(w.c_str(), + w.c_str() + w.size(), + &mblen); + path.rnode->char_type = cinfo.default_type; + std::string ufeature, lfeature, rfeature; + rewriter->rewrite2(feature, &ufeature, &lfeature, &rfeature); + fi->buildUnigramFeature(&path, ufeature.c_str()); + fi->calcCost(&rnode); + return tocost(rnode.wcost, factor); +} + +int progress_bar_darts(size_t current, size_t total) { + return progress_bar("emitting double-array", current, total); +} + +// std::binary_function was removed in C++17; the functor works without it. +template +struct pair_1st_cmp { + bool operator()(const std::pair &x1, + const std::pair &x2) { + return x1.first < x2.first; + } +}; +} // namespace + +bool Dictionary::open(const char *file, const char *mode) { + close(); + filename_.assign(file); + CHECK_FALSE(dmmap_->open(file, mode)) + << "no such file or directory: " << file; + + CHECK_FALSE(dmmap_->size() >= 100) + << "dictionary file is broken: " << file; + + const char *ptr = dmmap_->begin(); + + unsigned int dsize; + unsigned int tsize; + unsigned int fsize; + unsigned int magic; + unsigned int dummy; + + read_static(&ptr, magic); + CHECK_FALSE((magic ^ DictionaryMagicID) == dmmap_->size()) + << "dictionary file is broken: " << file; + + read_static(&ptr, version_); + CHECK_FALSE(version_ == DIC_VERSION) + << "incompatible version: " << version_; + + read_static(&ptr, type_); + read_static(&ptr, lexsize_); + read_static(&ptr, lsize_); + read_static(&ptr, rsize_); + read_static(&ptr, dsize); + read_static(&ptr, tsize); + read_static(&ptr, fsize); + read_static(&ptr, dummy); + + charset_ = ptr; + ptr += 32; + da_.set_array(reinterpret_cast(const_cast(ptr))); + + ptr += dsize; + + token_ = reinterpret_cast(ptr); + ptr += tsize; + + feature_ = ptr; + ptr += fsize; + + CHECK_FALSE(ptr == dmmap_->end()) + << "dictionary file is broken: " << file; + + return true; +} + +void Dictionary::close() { + dmmap_->close(); +} + +#define DCONF(file) create_filename(dicdir, std::string(file)); + +bool Dictionary::assignUserDictionaryCosts( + const Param ¶m, + const std::vector &dics, + const char *output) { + Connector matrix; + DictionaryRewriter rewriter; + DecoderFeatureIndex fi; + ContextID cid; + CharProperty property; + + const std::string dicdir = param.get("dicdir"); + + const std::string matrix_file = DCONF(MATRIX_DEF_FILE); + const std::string matrix_bin_file = DCONF(MATRIX_FILE); + const std::string left_id_file = DCONF(LEFT_ID_FILE); + const std::string right_id_file = DCONF(RIGHT_ID_FILE); + const std::string rewrite_file = DCONF(REWRITE_FILE); + + const std::string from = param.get("dictionary-charset"); + + const int factor = param.get("cost-factor"); + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + + std::string config_charset = param.get("config-charset"); + if (config_charset.empty()) { + config_charset = from; + } + + CHECK_DIE(!from.empty()) << "input dictionary charset is empty"; + + Iconv config_iconv; + CHECK_DIE(config_iconv.open(config_charset.c_str(), from.c_str())) + << "iconv_open() failed with from=" << config_charset << " to=" << from; + + rewriter.open(rewrite_file.c_str(), &config_iconv); + CHECK_DIE(fi.open(param)) << "cannot open feature index"; + + CHECK_DIE(property.open(param)); + property.set_charset(from.c_str()); + + if (!matrix.openText(matrix_file.c_str()) && + !matrix.open(matrix_bin_file.c_str())) { + matrix.set_left_size(1); + matrix.set_right_size(1); + } + + cid.open(left_id_file.c_str(), + right_id_file.c_str(), &config_iconv); + CHECK_DIE(cid.left_size() == matrix.left_size() && + cid.right_size() == matrix.right_size()) + << "Context ID files(" + << left_id_file + << " or " + << right_id_file << " may be broken: " + << cid.left_size() << " " << matrix.left_size() << " " + << cid.right_size() << " " << matrix.right_size(); + + std::ofstream ofs(output); + CHECK_DIE(ofs) << "permission denied: " << output; + + for (size_t i = 0; i < dics.size(); ++i) { + std::ifstream ifs(WPATH(dics[i].c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << dics[i]; + std::cout << "reading " << dics[i] << " ... "; + scoped_fixed_array line; + while (ifs.getline(line.get(), line.size())) { + char *col[8]; + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + std::string w = col[0]; + const std::string feature = col[4]; + const int cost = calcCost(w, feature, factor, + &fi, &rewriter, &property); + std::string ufeature, lfeature, rfeature; + CHECK_DIE(rewriter.rewrite(feature, &ufeature, &lfeature, &rfeature)) + << "rewrite failed: " << feature; + const int lid = cid.lid(lfeature.c_str()); + const int rid = cid.rid(rfeature.c_str()); + CHECK_DIE(lid >= 0 && rid >= 0 && matrix.is_valid(lid, rid)) + << "invalid ids are found lid=" << lid << " rid=" << rid; + escape_csv_element(&w); + ofs << w << ',' << lid << ',' << rid << ',' + << cost << ',' << feature << '\n'; + } + } + + return true; +} + +bool Dictionary::compile(const Param ¶m, + const std::vector &dics, + const char *output) { + Connector matrix; + scoped_ptr rewrite; + scoped_ptr posid; + scoped_ptr fi; + scoped_ptr cid; + scoped_ptr writer; + scoped_ptr lattice; + scoped_ptr os; + scoped_ptr property; + Node node; + + const std::string dicdir = param.get("dicdir"); + + const std::string matrix_file = DCONF(MATRIX_DEF_FILE); + const std::string matrix_bin_file = DCONF(MATRIX_FILE); + const std::string left_id_file = DCONF(LEFT_ID_FILE); + const std::string right_id_file = DCONF(RIGHT_ID_FILE); + const std::string rewrite_file = DCONF(REWRITE_FILE); + const std::string pos_id_file = DCONF(POS_ID_FILE); + + std::vector > dic; + + size_t offset = 0; + unsigned int lexsize = 0; + std::string fbuf; + + const std::string from = param.get("dictionary-charset"); + const std::string to = param.get("charset"); + const bool wakati = param.get("wakati"); + const int type = param.get("type"); + const std::string node_format = param.get("node-format"); + const int factor = param.get("cost-factor"); + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + + // for backward compatibility + std::string config_charset = param.get("config-charset"); + if (config_charset.empty()) { + config_charset = from; + } + + CHECK_DIE(!from.empty()) << "input dictionary charset is empty"; + CHECK_DIE(!to.empty()) << "output dictionary charset is empty"; + + Iconv iconv; + CHECK_DIE(iconv.open(from.c_str(), to.c_str())) + << "iconv_open() failed with from=" << from << " to=" << to; + + Iconv config_iconv; + CHECK_DIE(config_iconv.open(config_charset.c_str(), from.c_str())) + << "iconv_open() failed with from=" << config_charset << " to=" << from; + + if (!node_format.empty()) { + writer.reset(new Writer); + lattice.reset(createLattice()); + os.reset(new StringBuffer); + memset(&node, 0, sizeof(node)); + } + + if (!matrix.openText(matrix_file.c_str()) && + !matrix.open(matrix_bin_file.c_str())) { + matrix.set_left_size(1); + matrix.set_right_size(1); + } + + posid.reset(new POSIDGenerator); + posid->open(pos_id_file.c_str(), &config_iconv); + + std::istringstream iss(UNK_DEF_DEFAULT); + + for (size_t i = 0; i < dics.size(); ++i) { + std::ifstream ifs(WPATH(dics[i].c_str())); + std::istream *is = &ifs; + if (!ifs) { + if (type == MECAB_UNK_DIC) { + std::cerr << dics[i] + << " is not found. minimum setting is used." << std::endl; + is = &iss; + } else { + CHECK_DIE(ifs) << "no such file or directory: " << dics[i]; + } + } + + std::cout << "reading " << dics[i] << " ... "; + + scoped_fixed_array line; + size_t num = 0; + + while (is->getline(line.get(), line.size())) { + char *col[8]; + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + + std::string w = col[0]; + int lid = toInt(col[1]); + int rid = toInt(col[2]); + int cost = toInt(col[3]); + std::string feature = col[4]; + const int pid = posid->id(feature.c_str()); + + if (cost == INT_MAX) { + CHECK_DIE(type == MECAB_USR_DIC) + << "cost field should not be empty in sys/unk dic."; + if (!rewrite.get()) { + rewrite.reset(new DictionaryRewriter); + rewrite->open(rewrite_file.c_str(), &config_iconv); + fi.reset(new DecoderFeatureIndex); + CHECK_DIE(fi->open(param)) << "cannot open feature index"; + property.reset(new CharProperty); + CHECK_DIE(property->open(param)); + property->set_charset(from.c_str()); + } + cost = calcCost(w, feature, factor, + fi.get(), rewrite.get(), property.get()); + } + + if (lid < 0 || rid < 0 || lid == INT_MAX || rid == INT_MAX) { + if (!rewrite.get()) { + rewrite.reset(new DictionaryRewriter); + rewrite->open(rewrite_file.c_str(), &config_iconv); + } + + std::string ufeature, lfeature, rfeature; + CHECK_DIE(rewrite->rewrite(feature, &ufeature, &lfeature, &rfeature)) + << "rewrite failed: " << feature; + + if (!cid.get()) { + cid.reset(new ContextID); + cid->open(left_id_file.c_str(), + right_id_file.c_str(), &config_iconv); + CHECK_DIE(cid->left_size() == matrix.left_size() && + cid->right_size() == matrix.right_size()) + << "Context ID files(" + << left_id_file + << " or " + << right_id_file << " may be broken"; + } + + lid = cid->lid(lfeature.c_str()); + rid = cid->rid(rfeature.c_str()); + } + + CHECK_DIE(lid >= 0 && rid >= 0 && matrix.is_valid(lid, rid)) + << "invalid ids are found lid=" << lid << " rid=" << rid; + + if (w.empty()) { + std::cerr << "empty word is found, discard this line" << std::endl; + continue; + } + + if (!iconv.convert(&feature)) { + std::cerr << "iconv conversion failed. skip this entry" + << std::endl; + continue; + } + + if (type != MECAB_UNK_DIC && !iconv.convert(&w)) { + std::cerr << "iconv conversion failed. skip this entry" + << std::endl; + continue; + } + + if (!node_format.empty()) { + node.surface = w.c_str(); + node.feature = feature.c_str(); + node.length = w.size(); + node.rlength = w.size(); + node.posid = pid; + node.stat = MECAB_NOR_NODE; + lattice->set_sentence(w.c_str()); + CHECK_DIE(os.get()); + CHECK_DIE(writer.get()); + os->clear(); + CHECK_DIE(writer->writeNode(lattice.get(), + node_format.c_str(), + &node, &*os)) << + "conversion error: " << feature << " with " << node_format; + *os << '\0'; + feature = os->str(); + } + + std::string key; + if (!wakati) { + key = feature + '\0'; + } + + Token* token = new Token; + token->lcAttr = lid; + token->rcAttr = rid; + token->posid = pid; + token->wcost = cost; + token->feature = offset; + token->compound = 0; + dic.push_back(std::pair(w, token)); + + // append to output buffer + if (!wakati) { + fbuf.append(key.data(), key.size()); + } + offset += key.size(); + + ++num; + ++lexsize; + } + + std::cout << num << std::endl; + } + + if (wakati) { + fbuf.append("\0", 1); + } + + std::stable_sort(dic.begin(), dic.end(), + pair_1st_cmp()); + + size_t bsize = 0; + size_t idx = 0; + std::string prev; + std::vector str; + std::vector len; + std::vector val; + + for (size_t i = 0; i < dic.size(); ++i) { + if (i != 0 && prev != dic[i].first) { + str.push_back(dic[idx].first.c_str()); + len.push_back(dic[idx].first.size()); + val.push_back(bsize +(idx << 8)); + bsize = 1; + idx = i; + } else { + ++bsize; + } + prev = dic[i].first; + } + str.push_back(dic[idx].first.c_str()); + len.push_back(dic[idx].first.size()); + val.push_back(bsize +(idx << 8)); + + CHECK_DIE(str.size() == len.size()); + CHECK_DIE(str.size() == val.size()); + + Darts::DoubleArray da; + CHECK_DIE(da.build(str.size(), const_cast(&str[0]), + &len[0], &val[0], &progress_bar_darts) == 0) + << "unknown error in building double-array"; + + std::string tbuf; + for (size_t i = 0; i < dic.size(); ++i) { + tbuf.append(reinterpret_cast(dic[i].second), + sizeof(Token)); + delete dic[i].second; + } + dic.clear(); + + // needs to be 8byte(64bit) aligned + while (tbuf.size() % 8 != 0) { + Token dummy; + memset(&dummy, 0, sizeof(Token)); + tbuf.append(reinterpret_cast(&dummy), sizeof(Token)); + } + + unsigned int dummy = 0; + unsigned int lsize = matrix.left_size(); + unsigned int rsize = matrix.right_size(); + unsigned int dsize = da.unit_size() * da.size(); + unsigned int tsize = tbuf.size(); + unsigned int fsize = fbuf.size(); + + unsigned int version = DIC_VERSION; + char charset[32]; + std::fill(charset, charset + sizeof(charset), '\0'); + std::strncpy(charset, to.c_str(), 31); + + std::ofstream bofs(WPATH(output), std::ios::binary|std::ios::out); + CHECK_DIE(bofs) << "permission denied: " << output; + + unsigned int magic = 0; + + // needs to be 64bit aligned + // 10*32 = 64*5 + bofs.write(reinterpret_cast(&magic), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&version), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&type), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&lexsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&lsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&rsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&dsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&tsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&fsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&dummy), sizeof(unsigned int)); + + // 32 * 8 = 64 * 4 + bofs.write(reinterpret_cast(charset), sizeof(charset)); + + bofs.write(reinterpret_cast(da.array()), + da.unit_size() * da.size()); + bofs.write(const_cast(tbuf.data()), tbuf.size()); + bofs.write(const_cast(fbuf.data()), fbuf.size()); + + // save magic id + magic = static_cast(bofs.tellp()); + magic ^= DictionaryMagicID; + bofs.seekp(0); + bofs.write(reinterpret_cast(&magic), sizeof(unsigned int)); + + bofs.close(); + + return true; +} +} From edc3bc759e643e151e248daf0c400012054b5929 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:31 +0800 Subject: [PATCH 012/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/dictionary.h | 99 ++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 third_party/mecab/src/dictionary.h diff --git a/third_party/mecab/src/dictionary.h b/third_party/mecab/src/dictionary.h new file mode 100644 index 00000000..b170b764 --- /dev/null +++ b/third_party/mecab/src/dictionary.h @@ -0,0 +1,99 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_DICTIONARY_H_ +#define MECAB_DICTIONARY_H_ + +#include "mecab.h" +#include "mmap.h" +#include "darts.h" +#include "char_property.h" + +namespace MeCab { + +class Param; + +struct Token { + unsigned short lcAttr; + unsigned short rcAttr; + unsigned short posid; + short wcost; + unsigned int feature; + unsigned int compound; +}; + +class Dictionary { + public: + typedef Darts::DoubleArray::result_pair_type result_type; + + bool open(const char *filename, const char *mode = "r"); + void close(); + + size_t commonPrefixSearch(const char* key, size_t len, + result_type *result, + size_t rlen) const { + return da_.commonPrefixSearch(key, result, rlen, len); + } + + result_type exactMatchSearch(const char* key) const { + result_type n; + da_.exactMatchSearch(key, n); + return n; + } + + bool isCompatible(const Dictionary &d) const { + return(version_ == d.version_ && + lsize_ == d.lsize_ && + rsize_ == d.rsize_ && + decode_charset(charset_) == + decode_charset(d.charset_)); + } + + const char *filename() const { return filename_.c_str(); } + const char *charset() const { return const_cast(charset_); } + unsigned short version() const { return version_; } + size_t size() const { return static_cast(lexsize_); } + int type() const { return static_cast(type_); } + size_t lsize() const { return static_cast(lsize_); } + size_t rsize() const { return static_cast(rsize_); } + + const Token *token(const result_type &n) const { + return token_ +(n.value >> 8); + } + size_t token_size(const result_type &n) const { return 0xff & n.value; } + const char *feature(const Token &t) const { return feature_ + t.feature; } + + static bool compile(const Param ¶m, + const std::vector &dics, + const char *output); // outputs + + static bool assignUserDictionaryCosts( + const Param ¶m, + const std::vector &dics, + const char *output); // outputs + + + const char *what() { return what_.str(); } + + explicit Dictionary(): dmmap_(new Mmap), token_(0), + feature_(0), charset_(0) {} + virtual ~Dictionary() { this->close(); } + + private: + scoped_ptr > dmmap_; + const Token *token_; + const char *feature_; + const char *charset_; + unsigned int version_; + unsigned int type_; + unsigned int lexsize_; + unsigned int lsize_; + unsigned int rsize_; + std::string filename_; + whatlog what_; + Darts::DoubleArray da_; +}; +} +#endif // MECAB_DICTIONARY_H_ From 230430ddb1d679a5b8e0001607b14306c3e1983a Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:32 +0800 Subject: [PATCH 013/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/dictionary_compiler.cpp | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 third_party/mecab/src/dictionary_compiler.cpp diff --git a/third_party/mecab/src/dictionary_compiler.cpp b/third_party/mecab/src/dictionary_compiler.cpp new file mode 100644 index 00000000..38a25f77 --- /dev/null +++ b/third_party/mecab/src/dictionary_compiler.cpp @@ -0,0 +1,156 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "char_property.h" +#include "connector.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "mecab.h" +#include "param.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace MeCab { + +class DictionaryComplier { + public: + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", "set DIR as dic dir (default \".\")" }, + { "outdir", 'o', ".", "DIR", + "set DIR as output dir (default \".\")" }, + { "model", 'm', 0, "FILE", "use FILE as model file" }, + { "userdic", 'u', 0, "FILE", "build user dictionary" }, + { "assign-user-dictionary-costs", 'a', 0, 0, + "only assign costs/ids to user dictionary" }, + { "build-unknown", 'U', 0, 0, + "build parameters for unknown words" }, + { "build-model", 'M', 0, 0, "build model file" }, + { "build-charcategory", 'C', 0, 0, "build character category maps" }, + { "build-sysdic", 's', 0, 0, "build system dictionary" }, + { "build-matrix", 'm', 0, 0, "build connection matrix" }, + { "charset", 'c', MECAB_DEFAULT_CHARSET, "ENC", + "make charset of binary dictionary ENC (default " + MECAB_DEFAULT_CHARSET ")" }, + { "charset", 't', MECAB_DEFAULT_CHARSET, "ENC", "alias of -c" }, + { "dictionary-charset", 'f', MECAB_DEFAULT_CHARSET, + "ENC", "assume charset of input CSVs as ENC (default " + MECAB_DEFAULT_CHARSET ")" }, + { "wakati", 'w', 0, 0, "build wakati-gaki only dictionary", }, + { "posid", 'p', 0, 0, "assign Part-of-speech id" }, + { "node-format", 'F', 0, "STR", + "use STR as the user defined node format" }, + { "version", 'v', 0, 0, "show the version and exit." }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + const std::string dicdir = param.get("dicdir"); + const std::string outdir = param.get("outdir"); + bool opt_unknown = param.get("build-unknown"); + bool opt_matrix = param.get("build-matrix"); + bool opt_charcategory = param.get("build-charcategory"); + bool opt_sysdic = param.get("build-sysdic"); + bool opt_model = param.get("build-model"); + bool opt_assign_user_dictionary_costs = param.get + ("assign-user-dictionary-costs"); + const std::string userdic = param.get("userdic"); + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() +#define OCONF(file) create_filename(outdir, std::string(file)).c_str() + + CHECK_DIE(param.load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + std::vector dic; + if (userdic.empty()) { + enum_csv_dictionaries(dicdir.c_str(), &dic); + } else { + dic = param.rest_args(); + } + + if (!userdic.empty()) { + CHECK_DIE(dic.size()) << "no dictionaries are specified"; + param.set("type", static_cast(MECAB_USR_DIC)); + if (opt_assign_user_dictionary_costs) { + Dictionary::assignUserDictionaryCosts(param, dic, + userdic.c_str()); + } else { + Dictionary::compile(param, dic, userdic.c_str()); + } + } else { + if (!opt_unknown && !opt_matrix && !opt_charcategory && + !opt_sysdic && !opt_model) { + opt_unknown = opt_matrix = opt_charcategory = + opt_sysdic = opt_model = true; + } + + if (opt_charcategory || opt_unknown) { + CharProperty::compile(DCONF(CHAR_PROPERTY_DEF_FILE), + DCONF(UNK_DEF_FILE), + OCONF(CHAR_PROPERTY_FILE)); + } + + if (opt_unknown) { + std::vector tmp; + tmp.push_back(DCONF(UNK_DEF_FILE)); + param.set("type", static_cast(MECAB_UNK_DIC)); + Dictionary::compile(param, tmp, OCONF(UNK_DIC_FILE)); + } + + if (opt_model) { + if (file_exists(DCONF(MODEL_DEF_FILE))) { + FeatureIndex::compile(param, + DCONF(MODEL_DEF_FILE), + OCONF(MODEL_FILE)); + } else { + std::cout << DCONF(MODEL_DEF_FILE) + << " is not found. skipped." << std::endl; + } + } + + if (opt_sysdic) { + CHECK_DIE(dic.size()) << "no dictionaries are specified"; + param.set("type", static_cast(MECAB_SYS_DIC)); + Dictionary::compile(param, dic, OCONF(SYS_DIC_FILE)); + } + + if (opt_matrix) { + Connector::compile(DCONF(MATRIX_DEF_FILE), + OCONF(MATRIX_FILE)); + } + } + + std::cout << "\ndone!\n"; + + return 0; + } +}; + +#undef DCONF +#undef OCONF +} + +int mecab_dict_index(int argc, char **argv) { + return MeCab::DictionaryComplier::run(argc, argv); +} From cecb6f2b6daf0f355883000a47b5855e7e29b59d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:34 +0800 Subject: [PATCH 014/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../mecab/src/dictionary_generator.cpp | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 third_party/mecab/src/dictionary_generator.cpp diff --git a/third_party/mecab/src/dictionary_generator.cpp b/third_party/mecab/src/dictionary_generator.cpp new file mode 100644 index 00000000..38fef058 --- /dev/null +++ b/third_party/mecab/src/dictionary_generator.cpp @@ -0,0 +1,295 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "char_property.h" +#include "common.h" +#include "context_id.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "mecab.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +void copy(const char *src, const char *dst) { + std::cout << "copying " << src << " to " << dst << std::endl; + Mmap mmap; + CHECK_DIE(mmap.open(src)) << mmap.what(); + std::ofstream ofs(WPATH(dst), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << dst; + ofs.write(reinterpret_cast(mmap.begin()), mmap.size()); + ofs.close(); +} + +class DictionaryGenerator { + private: + static void gencid_bos(const std::string &bos_feature, + DictionaryRewriter *rewrite, + ContextID *cid) { + std::string ufeature, lfeature, rfeature; + rewrite->rewrite2(bos_feature, &ufeature, &lfeature, &rfeature); + cid->addBOS(lfeature.c_str(), rfeature.c_str()); + } + + static void gencid(const char *filename, + DictionaryRewriter *rewrite, + ContextID *cid) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + scoped_fixed_array line; + std::cout << "reading " << filename << " ... " << std::flush; + size_t num = 0; + std::string feature, ufeature, lfeature, rfeature; + char *col[8]; + while (ifs.getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + feature = col[4]; + rewrite->rewrite2(feature, &ufeature, &lfeature, &rfeature); + cid->add(lfeature.c_str(), rfeature.c_str()); + ++num; + } + std::cout << num << std::endl; + ifs.close(); + } + + static bool genmatrix(const char *filename, + const ContextID &cid, + DecoderFeatureIndex *fi, + int factor) { + std::ofstream ofs(WPATH(filename)); + CHECK_DIE(ofs) << "permission denied: " << filename; + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + const std::map &left = cid.left_ids(); + const std::map &right = cid.right_ids(); + + CHECK_DIE(left.size() > 0) << "left id size is empty"; + CHECK_DIE(right.size() > 0) << "right id size is empty"; + + ofs << right.size() << ' ' << left.size() << std::endl; + + size_t l = 0; + for (std::map::const_iterator rit = right.begin(); + rit != right.end(); + ++rit) { + ++l; + progress_bar("emitting matrix ", l+1, right.size()); + for (std::map::const_iterator lit = left.begin(); + lit != left.end(); + ++lit) { + path.rnode->wcost = 0; + fi->buildBigramFeature(&path, rit->first.c_str(), lit->first.c_str()); + fi->calcCost(&path); + ofs << rit->second << ' ' << lit->second << ' ' + << tocost(path.cost, factor) << '\n'; + } + } + + return true; + } + + static void gendic(const char* ifile, + const char* ofile, + const CharProperty &property, + DictionaryRewriter *rewrite, + const ContextID &cid, + DecoderFeatureIndex *fi, + bool unk, + int factor) { + std::ifstream ifs(WPATH(ifile)); + CHECK_DIE(ifs) << "no such file or directory: " << ifile; + + std::ofstream ofs(WPATH(ofile)); + CHECK_DIE(ofs) << "permission denied: " << ofile; + + std::cout << "emitting " << ofile << " ... " << std::flush; + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + scoped_fixed_array line; + char *col[8]; + size_t num = 0; + + while (ifs.getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + + std::string w = std::string(col[0]); + const std::string feature = std::string(col[4]); + + std::string ufeature, lfeature, rfeature; + rewrite->rewrite2(feature, &ufeature, &lfeature, &rfeature); + const int lid = cid.lid(lfeature.c_str()); + const int rid = cid.rid(rfeature.c_str()); + + CHECK_DIE(lid > 0) << "CID is not found for " << lfeature; + CHECK_DIE(rid > 0) << "CID is not found for " << rfeature; + + if (unk) { + const int c = property.id(w.c_str()); + CHECK_DIE(c >= 0) << "unknown property [" << w << "]"; + path.rnode->char_type = static_cast(c); + } else { + size_t mblen = 0; + const CharInfo cinfo = property.getCharInfo(w.c_str(), + w.c_str() + w.size(), + &mblen); + path.rnode->char_type = cinfo.default_type; + } + + fi->buildUnigramFeature(&path, ufeature.c_str()); + fi->calcCost(&rnode); + CHECK_DIE(escape_csv_element(&w)) << "invalid character found: " << w; + + ofs << w << ',' << lid << ',' << rid << ',' + << tocost(rnode.wcost, factor) + << ',' << feature << '\n'; + ++num; + } + + std::cout << num << std::endl; + } + + public: + + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", "set DIR as dicdir(default \".\" )" }, + { "outdir", 'o', ".", "DIR", "set DIR as output dir" }, + { "model", 'm', 0, "FILE", "use FILE as model file" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) return 0; + + ContextID cid; + DecoderFeatureIndex fi; + DictionaryRewriter rewrite; + + const std::string dicdir = param.get("dicdir"); + const std::string outdir = param.get("outdir"); + const std::string model = param.get("model"); + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() +#define OCONF(file) create_filename(outdir, std::string(file)).c_str() + + CHECK_DIE(param.load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + std::string charset; + { + Dictionary dic; + CHECK_DIE(dic.open(DCONF(SYS_DIC_FILE), "r")); + charset = dic.charset(); + CHECK_DIE(!charset.empty()); + } + + CharProperty property; + CHECK_DIE(property.open(param)); + property.set_charset(charset.c_str()); + + const std::string bos = param.get("bos-feature"); + const int factor = param.get("cost-factor"); + + std::vector dic; + enum_csv_dictionaries(dicdir.c_str(), &dic); + + { + CHECK_DIE(dicdir != outdir) << + "output directory = dictionary directory! " + "Please specify different directory."; + CHECK_DIE(!outdir.empty()) << "output directory is empty"; + CHECK_DIE(!model.empty()) << "model file is empty"; + CHECK_DIE(fi.open(param)) << "cannot open feature index"; + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + CHECK_DIE(!bos.empty()) << "bos-feature is empty"; + CHECK_DIE(dic.size()) << "no dictionary is found in " << dicdir; + CHECK_DIE(rewrite.open(DCONF(REWRITE_FILE))); + } + + gencid_bos(bos, &rewrite, &cid); + gencid(DCONF(UNK_DEF_FILE), &rewrite, &cid); + + for (std::vector::const_iterator it = dic.begin(); + it != dic.end(); + ++it) { + gencid(it->c_str(), &rewrite, &cid); + } + + std::cout << "emitting " + << OCONF(LEFT_ID_FILE) << "/ " + << OCONF(RIGHT_ID_FILE) << std::endl; + + cid.build(); + cid.save(OCONF(LEFT_ID_FILE), OCONF(RIGHT_ID_FILE)); + + gendic(DCONF(UNK_DEF_FILE), OCONF(UNK_DEF_FILE), property, + &rewrite, cid, &fi, true, factor); + + for (std::vector::const_iterator it = dic.begin(); + it != dic.end(); + ++it) { + std::string file = *it; + remove_pathname(&file); + gendic(it->c_str(), OCONF(file.c_str()), property, + &rewrite, cid, &fi, false, factor); + } + + genmatrix(OCONF(MATRIX_DEF_FILE), cid, &fi, factor); + + copy(DCONF(CHAR_PROPERTY_DEF_FILE), OCONF(CHAR_PROPERTY_DEF_FILE)); + copy(DCONF(REWRITE_FILE), OCONF(REWRITE_FILE)); + copy(DCONF(DICRC), OCONF(DICRC)); + copy(DCONF(FEATURE_FILE), OCONF(FEATURE_FILE)); + copy(model.c_str(), OCONF(MODEL_DEF_FILE)); + +#undef OCONF +#undef DCONF + + std::cout << "\ndone!\n"; + + return 0; + } +}; +} + +// export functions +int mecab_dict_gen(int argc, char **argv) { + return MeCab::DictionaryGenerator::run(argc, argv); +} From 792f6e99d159eb7eb6f400a172c7f78c8dfee107 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:36 +0800 Subject: [PATCH 015/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/dictionary_rewriter.cpp | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 third_party/mecab/src/dictionary_rewriter.cpp diff --git a/third_party/mecab/src/dictionary_rewriter.cpp b/third_party/mecab/src/dictionary_rewriter.cpp new file mode 100644 index 00000000..ca6832fd --- /dev/null +++ b/third_party/mecab/src/dictionary_rewriter.cpp @@ -0,0 +1,242 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "dictionary_rewriter.h" +#include "iconv_utils.h" +#include "scoped_ptr.h" +#include "utils.h" + +namespace { + +using namespace MeCab; + +void append_rewrite_rule(RewriteRules *r, char* str) { + char *col[3]; + const size_t n = tokenize2(str, " \t", col, 3); + CHECK_DIE(n >= 2) << "format error: " << str; + r->resize(r->size() + 1); + std::string tmp; + if (n >= 3) { + tmp = col[1]; + tmp += ' '; + tmp += col[2]; + col[1] = const_cast(tmp.c_str()); + } + r->back().set_pattern(col[0], col[1]); +} + +bool match_rewrite_pattern(const char *pat, + const char *str) { + if (pat[0] == '*' || std::strcmp(pat, str) == 0) + return true; + + size_t len = std::strlen(pat); + if (len >= 3 && pat[0] == '(' && pat[len-1] == ')') { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(len < buf.size() - 3) << "too long parameter"; + std::strncpy(buf.get(), pat + 1, buf.size()); + buf[len-2] = '\0'; + const size_t n = tokenize(buf.get(), "|", col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long OR nodes"; + for (size_t i = 0; i < n; ++i) { + if (std::strcmp(str, col[i]) == 0) return true; + } + } + return false; +} +} + +namespace MeCab { + +bool RewritePattern::set_pattern(const char *src, + const char *dst) { + scoped_fixed_array buf; + spat_.clear(); + dpat_.clear(); + + std::strncpy(buf.get(), src, buf.size()); + tokenizeCSV(buf.get(), back_inserter(spat_), 512); + + std::strncpy(buf.get(), dst, buf.size()); + tokenizeCSV(buf.get(), back_inserter(dpat_), 512); + + return (spat_.size() && dpat_.size()); +} + +bool RewritePattern::rewrite(size_t size, + const char **input, + std::string *output) const { + if (spat_.size() > size) return false; + for (size_t i = 0; i < spat_.size(); ++i) { + if (!match_rewrite_pattern(spat_[i].c_str(), input[i])) + return false; + } + + output->clear(); + for (size_t i = 0; i < dpat_.size(); ++i) { + std::string elm; + const char *begin = dpat_[i].c_str(); + const char *end = begin + dpat_[i].size(); + for (const char *p = begin; p < end; ++p) { + if (*p == '$') { + ++p; + size_t n = 0; + for (; p < end; ++p) { + switch (*p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n + (*p - '0'); + break; + default: + goto NEXT; + } + } + NEXT: + CHECK_DIE(n > 0 && (n - 1) < size) + << " out of range: [" << dpat_[i] << "] " << n; + elm += input[n - 1]; + if (p < end) elm += *p; + } else { + elm += *p; + } + } + CHECK_DIE(escape_csv_element(&elm)); + *output += elm; + if (i + 1 != dpat_.size()) *output += ","; + } + + return true; +} + +bool RewriteRules::rewrite(size_t size, + const char **input, + std::string *output) const { + for (size_t i = 0; i < this->size(); ++i) { + if ((*this)[i].rewrite(size, input, output)) + return true; + } + return false; +} + +void DictionaryRewriter::clear() { cache_.clear(); } + +bool DictionaryRewriter::open(const char *filename, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + int append_to = 0; + std::string line; + while (std::getline(ifs, line)) { + if (iconv) iconv->convert(&line); + if (line.empty() || line[0] == '#') continue; + if (line == "[unigram rewrite]") { + append_to = 1; + } else if (line == "[left rewrite]") { + append_to = 2; + } else if (line == "[right rewrite]") { + append_to = 3; + } else { + CHECK_DIE(append_to != 0) << "no sections found"; + char *str = const_cast(line.c_str()); + switch (append_to) { + case 1: append_rewrite_rule(&unigram_rewrite_, str); break; + case 2: append_rewrite_rule(&left_rewrite_, str); break; + case 3: append_rewrite_rule(&right_rewrite_, str); break; + } + } + } + return true; +} + +// without cache +bool DictionaryRewriter::rewrite(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) const { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(feature.size() < buf.size() - 1) << "too long feature"; + std::strncpy(buf.get(), feature.c_str(), buf.size() - 1); + const size_t n = tokenizeCSV(buf.get(), col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long CSV entities"; + return (unigram_rewrite_.rewrite(n, const_cast(col.get()), + ufeature) && + left_rewrite_.rewrite(n, const_cast(col.get()), + lfeature) && + right_rewrite_.rewrite(n, const_cast(col.get()), + rfeature)); +} + +// with cache +bool DictionaryRewriter::rewrite2(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) { + std::map::const_iterator it = cache_.find(feature); + if (it == cache_.end()) { + if (!rewrite(feature, ufeature, lfeature, rfeature)) return false; + FeatureSet f; + f.ufeature = *ufeature; + f.lfeature = *lfeature; + f.rfeature = *rfeature; + cache_.insert(std::pair(feature, f)); + } else { + *ufeature = it->second.ufeature; + *lfeature = it->second.lfeature; + *rfeature = it->second.rfeature; + } + + return true; +} + +bool POSIDGenerator::open(const char *filename, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + std::cerr << filename + << " is not found. minimum setting is used" << std::endl; + rewrite_.resize(1); + rewrite_.back().set_pattern("*", "1"); + return true; + } + + std::string line; + char *col[2]; + while (std::getline(ifs, line)) { + if (iconv) iconv->convert(&line); + const size_t n = tokenize2(const_cast(line.c_str()), + " \t", col, 2); + CHECK_DIE(n == 2) << "format error: " << line; + for (char *p = col[1]; *p; ++p) { + CHECK_DIE(*p >= '0' && *p <= '9') << "not a number: " << col[1]; + } + rewrite_.resize(rewrite_.size() + 1); + rewrite_.back().set_pattern(col[0], col[1]); + } + return true; +} + +int POSIDGenerator::id(const char *feature) const { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(std::strlen(feature) < buf.size() - 1) << "too long feature"; + std::strncpy(buf.get(), feature, buf.size() - 1); + const size_t n = tokenizeCSV(buf.get(), col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long CSV entities"; + std::string tmp; + if (!rewrite_.rewrite(n, const_cast(col.get()), &tmp)) { + return -1; + } + return std::atoi(tmp.c_str()); +} +} From a3b9d993ffb357ce993f4f397342299db57ca3f6 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:38 +0800 Subject: [PATCH 016/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/dictionary_rewriter.h | 75 +++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 third_party/mecab/src/dictionary_rewriter.h diff --git a/third_party/mecab/src/dictionary_rewriter.h b/third_party/mecab/src/dictionary_rewriter.h new file mode 100644 index 00000000..9b06758b --- /dev/null +++ b/third_party/mecab/src/dictionary_rewriter.h @@ -0,0 +1,75 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_DICTIONARY_REWRITER_H +#define MECAB_DICTIONARY_REWRITER_H + +#include +#include +#include +#include "common.h" +#include "mecab.h" +#include "freelist.h" + +namespace MeCab { + +class Iconv; + +class RewritePattern { + private: + std::vector spat_; + std::vector dpat_; + public: + bool set_pattern(const char *src, const char *dst); + bool rewrite(size_t size, + const char **input, + std::string *output) const; +}; + +class RewriteRules: public std::vector { + public: + bool rewrite(size_t size, const char **input, + std::string *output) const; +}; + +struct FeatureSet { + std::string ufeature; + std::string lfeature; + std::string rfeature; +}; + +class DictionaryRewriter { + private: + RewriteRules unigram_rewrite_; + RewriteRules left_rewrite_; + RewriteRules right_rewrite_; + std::map cache_; + + public: + bool open(const char *filename, + Iconv *iconv = 0); + void clear(); + bool rewrite(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) const; + + bool rewrite2(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature); +}; + +class POSIDGenerator { + private: + RewriteRules rewrite_; + public: + bool open(const char *filename, + Iconv *iconv = 0); + void clear() { rewrite_.clear(); } + int id(const char *key) const; +}; +} +#endif From 77585be28bdbddef8c7133750129fa9be8dcfc31 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:39 +0800 Subject: [PATCH 017/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/eval.cpp | 268 +++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 third_party/mecab/src/eval.cpp diff --git a/third_party/mecab/src/eval.cpp b/third_party/mecab/src/eval.cpp new file mode 100644 index 00000000..922266f1 --- /dev/null +++ b/third_party/mecab/src/eval.cpp @@ -0,0 +1,268 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "common.h" +#include "mecab.h" +#include "param.h" +#include "stream_wrapper.h" +#include "scoped_ptr.h" +#include "utils.h" + +namespace MeCab { + +class Eval { + private: + static bool read(std::istream *is, + std::vector > *r, + const std::vector &level) { + if (!*is) { + return false; + } + + char *col[2]; + scoped_fixed_array buf; + scoped_fixed_array csv; + r->clear(); + while (is->getline(buf.get(), buf.size())) { + if (std::strcmp(buf.get(), "EOS") == 0) { + break; + } + CHECK_DIE(tokenize(buf.get(), "\t", col, 2) == 2) << "format error"; + csv[0] = col[0]; + size_t n = tokenizeCSV(col[1], csv.get() + 1, csv.size() - 1); + std::vector tmp; + for (size_t i = 0; i < level.size(); ++i) { + size_t m = level[i] < 0 ? n : level[i]; + CHECK_DIE(m <= n) << " out of range " << level[i]; + std::string output; + for (size_t j = 0; j <= m; ++j) { + output += csv[j]; + if (j != 0) { + output += "\t"; + } + } + tmp.push_back(output); + } + r->push_back(tmp); + } + + return true; + } + + static bool parseLevel(const char *level_str, + std::vector *level) { + scoped_fixed_array buf; + scoped_fixed_array col; + std::strncpy(buf.get(), level_str, buf.size()); + level->clear(); + size_t n = tokenize2(buf.get(), "\t ", col.get(), col.size()); + for (size_t i = 0; i < n; ++i) { + level->push_back(std::atoi(col[i])); + } + return true; + } + + static void printeval(std::ostream *os, size_t c, size_t p, size_t r) { + double pr = (p == 0) ? 0 : 100.0 * c/p; + double re = (r == 0) ? 0 : 100.0 * c/r; + double F = ((pr + re) == 0.0) ? 0 : 2 * pr * re /(pr + re); + scoped_fixed_array buf; + sprintf(buf.get(), "%4.4f(%d/%d) %4.4f(%d/%d) %4.4f\n", + pr, + static_cast(c), + static_cast(p), + re, + static_cast(c), + static_cast(r), + F); + *os << buf.get(); + } + + public: + static bool eval(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "level", 'l', "0 -1", "STR", "set level of evaluations" }, + { "output", 'o', 0, "FILE", "set the output file name" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + MeCab::Param param; + param.open(argc, argv, long_options); + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) return 0; + + const std::vector &files = param.rest_args(); + if (files.size() < 2) { + std::cout << "Usage: " << + param.program_name() << " output answer" << std::endl; + return -1; + } + + std::string output = param.get("output"); + if (output.empty()) output = "-"; + MeCab::ostream_wrapper ofs(output.c_str()); + CHECK_DIE(*ofs) << "no such file or directory: " << output; + + const std::string system = files[0]; + const std::string answer = files[1]; + + const std::string level_str = param.get("level"); + + std::ifstream ifs1(WPATH(files[0].c_str())); + std::ifstream ifs2(WPATH(files[1].c_str())); + + CHECK_DIE(ifs1) << "no such file or directory: " << files[0].c_str(); + CHECK_DIE(ifs2) << "no such file or directory: " << files[0].c_str(); + CHECK_DIE(!level_str.empty()) << "level_str is NULL"; + + std::vector level; + parseLevel(level_str.c_str(), &level); + CHECK_DIE(level.size()) << "level_str is empty: " << level_str; + std::vector result_tbl(level.size()); + std::fill(result_tbl.begin(), result_tbl.end(), 0); + + size_t prec = 0; + size_t recall = 0; + + std::vector > r1; + std::vector > r2; + + while (true) { + if (!read(&ifs1, &r1, level) || !read(&ifs2, &r2, level)) + break; + + size_t i1 = 0; + size_t i2 = 0; + size_t p1 = 0; + size_t p2 = 0; + + while (i1 < r1.size() && i2 < r2.size()) { + if (p1 == p2) { + for (size_t i = 0; i < result_tbl.size(); ++i) { + if (r1[i1][i] == r2[i2][i]) { + result_tbl[i]++; + } + } + p1 += r1[i1][0].size(); + p2 += r2[i2][0].size(); + ++i1; + ++i2; + ++prec; + ++recall; + } else if (p1 < p2) { + p1 += r1[i1][0].size(); + ++i1; + ++prec; + } else { + p2 += r2[i2][0].size(); + ++i2; + ++recall; + } + } + + while (i1 < r1.size()) { + ++prec; + ++i1; + } + + while (i2 < r2.size()) { + ++recall; + ++i2; + } + } + + *ofs << " precision recall F" + << std::endl; + for (size_t i = 0; i < result_tbl.size(); ++i) { + if (level[i] == -1) { + *ofs << "LEVEL ALL: "; + } else { + *ofs << "LEVEL " << level[i] << ": "; + } + printeval(&*ofs, result_tbl[i], prec, recall); + } + + return true; + } +}; + +class TestSentenceGenerator { + public: + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "output", 'o', 0, "FILE", "set the output filename" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + MeCab::Param param; + param.open(argc, argv, long_options); + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + const std::vector &tmp = param.rest_args(); + std::vector files = tmp; + if (files.empty()) { + files.push_back("-"); + } + + std::string output = param.get("output"); + if (output.empty()) output = "-"; + MeCab::ostream_wrapper ofs(output.c_str()); + CHECK_DIE(*ofs) << "permission denied: " << output; + + scoped_fixed_array buf; + char *col[2]; + std::string str; + for (size_t i = 0; i < files.size(); ++i) { + MeCab::istream_wrapper ifs(files[i].c_str()); + CHECK_DIE(*ifs) << "no such file or directory: " << files[i]; + while (ifs->getline(buf.get(), buf.size())) { + const size_t n = tokenize(buf.get(), "\t ", col, 2); + CHECK_DIE(n <= 2) << "format error: " << buf.get(); + if (std::strcmp(col[0], "EOS") == 0 && !str.empty()) { + *ofs << str << std::endl; + str.clear(); + } else { + str += col[0]; + } + } + } + + return 0; + } +}; +} + +// exports +int mecab_system_eval(int argc, char **argv) { + return MeCab::Eval::eval(argc, argv); +} + +int mecab_test_gen(int argc, char **argv) { + return MeCab::TestSentenceGenerator::run(argc, argv); +} From c24d55922e1cd4a82bf25be6da2c85693309d8be Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:41 +0800 Subject: [PATCH 018/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/feature_index.h | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 third_party/mecab/src/feature_index.h diff --git a/third_party/mecab/src/feature_index.h b/third_party/mecab/src/feature_index.h new file mode 100644 index 00000000..9e08caab --- /dev/null +++ b/third_party/mecab/src/feature_index.h @@ -0,0 +1,115 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_FEATUREINDEX_H_ +#define MECAB_FEATUREINDEX_H_ + +#include +#include +#include "mecab.h" +#include "mmap.h" +#include "darts.h" +#include "freelist.h" +#include "common.h" +#include "learner_node.h" +#include "string_buffer.h" +#include "dictionary_rewriter.h" + +namespace MeCab { + +class Param; + +class FeatureIndex { + public: + virtual bool open(const Param ¶m) = 0; + virtual void clear() = 0; + virtual void close() = 0; + virtual bool buildFeature(LearnerPath *path) = 0; + + void set_alpha(const double *alpha); + + size_t size() const { return maxid_; } + + bool buildUnigramFeature(LearnerPath *, const char *); + bool buildBigramFeature(LearnerPath *, const char *, const char*); + + void calcCost(LearnerPath *path); + void calcCost(LearnerNode *node); + + const char *strdup(const char *str); + + static bool convert(const Param ¶m, + const char *text_filename, std::string *output); + static bool compile(const Param ¶m, + const char *text_filename, const char *binary_filename); + + explicit FeatureIndex(): feature_freelist_(8192 * 32), + char_freelist_(8192 * 32), + maxid_(0), alpha_(0) {} + virtual ~FeatureIndex() {} + + protected: + std::vector feature_; + ChunkFreeList feature_freelist_; + ChunkFreeList char_freelist_; + std::vector unigram_templs_; + std::vector bigram_templs_; + DictionaryRewriter rewrite_; + StringBuffer os_; + size_t maxid_; + const double *alpha_; + + virtual int id(const char *key) = 0; + const char* getIndex(char **, char **, size_t); + bool openTemplate(const Param ¶m); +}; + +class EncoderFeatureIndex: public FeatureIndex { + public: + bool open(const Param ¶m); + void close(); + void clear(); + + bool reopen(const char *filename, + const char *charset, + std::vector *alpha, + Param *param); + + bool save(const char *filename, const char *header) const; + void shrink(size_t freq, + std::vector *observed); + bool buildFeature(LearnerPath *path); + void clearcache(); + + private: + std::map dic_; + std::map > feature_cache_; + int id(const char *key); +}; + +class DecoderFeatureIndex: public FeatureIndex { + public: + bool open(const Param ¶m); + void clear(); + void close(); + bool buildFeature(LearnerPath *path); + + const char *charset() const { + return charset_; + } + + private: + bool openFromArray(const char *begin, const char *end); + bool openBinaryModel(const Param ¶m); + bool openTextModel(const Param ¶m); + int id(const char *key); + + Mmap mmap_; + std::string model_buffer_; + const uint64_t *key_; + const char *charset_; +}; +} +#endif From 0269b6db3397d0543a33f17c7e5838323cee47af Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:42 +0800 Subject: [PATCH 019/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/freelist.h | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 third_party/mecab/src/freelist.h diff --git a/third_party/mecab/src/freelist.h b/third_party/mecab/src/freelist.h new file mode 100644 index 00000000..85de6344 --- /dev/null +++ b/third_party/mecab/src/freelist.h @@ -0,0 +1,85 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_FREELIST_H +#define MECAB_FREELIST_H + +#include +#include +#include "utils.h" +#include "common.h" + +namespace MeCab { + +template class FreeList { + private: + std::vector freeList; + size_t pi_; + size_t li_; + size_t size; + + public: + void free() { li_ = pi_ = 0; } + + T* alloc() { + if (pi_ == size) { + li_++; + pi_ = 0; + } + if (li_ == freeList.size()) freeList.push_back(new T[size]); + return freeList[li_] + (pi_++); + } + + explicit FreeList(size_t _size): pi_(0), li_(0), size(_size) {} + + virtual ~FreeList() { + for (li_ = 0; li_ < freeList.size(); li_++) + delete [] freeList[li_]; + } +}; + +template class ChunkFreeList { + private: + std::vector > freelist_; + size_t pi_; + size_t li_; + size_t default_size; + + public: + void free() { li_ = pi_ = 0; } + + T* alloc(T *src) { + T* n = alloc(1); + *n = *src; + return n; + } + + T* alloc(size_t req = 1) { + while (li_ < freelist_.size()) { + if ((pi_ + req) < freelist_[li_].first) { + T *r = freelist_[li_].second + pi_; + pi_ += req; + return r; + } + li_++; + pi_ = 0; + } + size_t _size = std::max(req, default_size); + freelist_.push_back(std::make_pair(_size, new T[_size])); + li_ = freelist_.size() - 1; + pi_ += req; + return freelist_[li_].second; + } + + explicit ChunkFreeList(size_t _size): + pi_(0), li_(0), default_size(_size) {} + + virtual ~ChunkFreeList() { + for (li_ = 0; li_ < freelist_.size(); li_++) + delete [] freelist_[li_].second; + } +}; +} +#endif From 6a13db8dc787f5bdf148a6be01e395c540e50c95 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:44 +0800 Subject: [PATCH 020/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/iconv_utils.cpp | 203 ++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 third_party/mecab/src/iconv_utils.cpp diff --git a/third_party/mecab/src/iconv_utils.cpp b/third_party/mecab/src/iconv_utils.cpp new file mode 100644 index 00000000..1a815664 --- /dev/null +++ b/third_party/mecab/src/iconv_utils.cpp @@ -0,0 +1,203 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "scoped_ptr.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "char_property.h" +#include "iconv_utils.h" + +#if defined(_WIN32) && !defined(__CYGWIN__) +#include "windows.h" +#endif + +namespace { + +#ifdef HAVE_ICONV +const char * decode_charset_iconv(const char *str) { + const int charset = MeCab::decode_charset(str); + switch (charset) { + case MeCab::UTF8: + return "UTF-8"; + case MeCab::EUC_JP: + return "EUC-JP"; + case MeCab::CP932: + return "SHIFT-JIS"; + case MeCab::UTF16: + return "UTF-16"; + case MeCab::UTF16LE: + return "UTF-16LE"; + case MeCab::UTF16BE: + return "UTF-16BE"; + default: + std::cerr << "charset " << str + << " is not defined, use " MECAB_DEFAULT_CHARSET; + return MECAB_DEFAULT_CHARSET; + } + return MECAB_DEFAULT_CHARSET; +} +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +DWORD decode_charset_win32(const char *str) { + const int charset = MeCab::decode_charset(str); + switch (charset) { + case MeCab::UTF8: + return CP_UTF8; + case MeCab::UTF16: + return 1200; + case MeCab::UTF16LE: + return 1200; + case MeCab::UTF16BE: + return 1201; + case MeCab::EUC_JP: + // return 51932; + return 20932; + case MeCab::CP932: + return 932; + default: + std::cerr << "charset " << str + << " is not defined, use 'CP_THREAD_ACP'"; + return CP_THREAD_ACP; + } + return 0; +} +#endif +} // namespace + +namespace MeCab { +bool Iconv::open(const char* from, const char* to) { + ic_ = 0; +#if defined HAVE_ICONV + const char *from2 = decode_charset_iconv(from); + const char *to2 = decode_charset_iconv(to); + if (std::strcmp(from2, to2) == 0) { + return true; + } + ic_ = 0; + ic_ = iconv_open(to2, from2); + if (ic_ == (iconv_t)(-1)) { + ic_ = 0; + return false; + } +#else +#if defined(_WIN32) && !defined(__CYGWIN__) + from_cp_ = decode_charset_win32(from); + to_cp_ = decode_charset_win32(to); + if (from_cp_ == to_cp_) { + return true; + } + ic_ = from_cp_; +#else + std::cerr << "iconv_open is not supported" << std::endl; +#endif +#endif + + return true; +} + +bool Iconv::convert(std::string *str) { + if (str->empty()) { + return true; + } + if (ic_ == 0) { + return true; + } + +#if defined HAVE_ICONV + size_t ilen = 0; + size_t olen = 0; + ilen = str->size(); + olen = ilen * 4; + std::string tmp; + tmp.reserve(olen); + char *ibuf = const_cast(str->data()); + char *obuf_org = const_cast(tmp.data()); + char *obuf = obuf_org; + std::fill(obuf, obuf + olen, 0); + size_t olen_org = olen; + iconv(ic_, 0, &ilen, 0, &olen); // reset iconv state + while (ilen != 0) { + if (iconv(ic_, (ICONV_CONST char **)&ibuf, &ilen, &obuf, &olen) + == (size_t) -1) { + return false; + } + } + str->assign(obuf_org, olen_org - olen); +#else +#if defined(_WIN32) && !defined(__CYGWIN__) + // covert it to wide character first + const size_t wide_len = ::MultiByteToWideChar(from_cp_, 0, + str->c_str(), + -1, NULL, 0); + if (wide_len == 0) { + return false; + } + + scoped_array wide_str(new wchar_t[wide_len + 1]); + + if (!wide_str.get()) { + return false; + }; + + if (::MultiByteToWideChar(from_cp_, 0, str->c_str(), -1, + wide_str.get(), wide_len + 1) == 0) { + return false; + } + + if (to_cp_ == 1200 || to_cp_ == 1201) { + str->resize(2 * wide_len); + std::memcpy(const_cast(str->data()), + reinterpret_cast(wide_str.get()), wide_len * 2); + if (to_cp_ == 1201) { + char *buf = const_cast(str->data()); + for (size_t i = 0; i < 2 * wide_len; i += 2) { + std::swap(buf[i], buf[i+1]); + } + } + return true; + } + + const size_t output_len = ::WideCharToMultiByte(to_cp_, 0, + wide_str.get(), + -1, + NULL, 0, NULL, NULL); + + if (output_len == 0) { + return false; + } + + scoped_array encoded(new char[output_len + 1]); + if (::WideCharToMultiByte(to_cp_, 0, wide_str.get(), wide_len, + encoded.get(), output_len + 1, + NULL, NULL) == 0) { + return false; + } + + str->assign(encoded.get()); + +#endif +#endif + + return true; +} + +Iconv::Iconv() : ic_(0) {} + +Iconv::~Iconv() { +#if defined HAVE_ICONV + if (ic_ != 0) iconv_close(ic_); +#endif +} +} From f2a56683b56eb405204592b395489383938e2032 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:46 +0800 Subject: [PATCH 021/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/iconv_utils.h | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 third_party/mecab/src/iconv_utils.h diff --git a/third_party/mecab/src/iconv_utils.h b/third_party/mecab/src/iconv_utils.h new file mode 100644 index 00000000..69b5a029 --- /dev/null +++ b/third_party/mecab/src/iconv_utils.h @@ -0,0 +1,40 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_ICONV_H +#define MECAB_ICONV_H + +#if defined HAVE_ICONV +#include +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +#include "windows.h" +#endif + +namespace MeCab { + +class Iconv { + private: +#ifdef HAVE_ICONV + iconv_t ic_; +#else + int ic_; +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) + DWORD from_cp_; + DWORD to_cp_; +#endif + + public: + explicit Iconv(); + virtual ~Iconv(); + bool open(const char *from, const char *to); + bool convert(std::string *); +}; +} + +#endif From 28fa16663fafa5762c046888d2ebf1f14b56d17c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:47 +0800 Subject: [PATCH 022/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/lbfgs.cpp | 572 ++++++++++++++++++++++++++++++++ 1 file changed, 572 insertions(+) create mode 100644 third_party/mecab/src/lbfgs.cpp diff --git a/third_party/mecab/src/lbfgs.cpp b/third_party/mecab/src/lbfgs.cpp new file mode 100644 index 00000000..b9ed45f2 --- /dev/null +++ b/third_party/mecab/src/lbfgs.cpp @@ -0,0 +1,572 @@ +// MeCab: Yet Another Part-of-Speech and Morphological Analyzer +// +// +// lbfgs.c was ported from the FORTRAN code of lbfgs.m to C +// using f2c converter +// +// http://www.ece.northwestern.edu/~nocedal/lbfgs.html +// +// Software for Large-scale Unconstrained Optimization +// L-BFGS is a limited-memory quasi-Newton code for unconstrained +// optimization. +// The code has been developed at the Optimization Technology Center, +// a joint venture of Argonne National Laboratory and Northwestern University. +// +// Authors +// Jorge Nocedal +// +// References +// - J. Nocedal. Updating Quasi-Newton Matrices with Limited Storage(1980), +// Mathematics of Computation 35, pp. 773-782. +// - D.C. Liu and J. Nocedal. On the Limited Memory Method for +// Large Scale Optimization(1989), +// Mathematical Programming B, 45, 3, pp. 503-528. +#include +#include +#include +#include "lbfgs.h" +#include "common.h" + +namespace { +static const double ftol = 1e-4; +static const double xtol = 1e-16; +static const double eps = 1e-7; +static const double lb3_1_gtol = 0.9; +static const double lb3_1_stpmin = 1e-20; +static const double lb3_1_stpmax = 1e20; +static const int lb3_1_mp = 6; +static const int lb3_1_lp = 6; + +inline double sigma(double x) { + if (x > 0) { + return 1.0; + } else if (x < 0) { + return -1.0; + } + return 0.0; +} + +inline double pi(double x, double y) { + return sigma(x) == sigma(y) ? x : 0.0; +} + +inline void daxpy_(int n, double da, const double *dx, double *dy) { + for (int i = 0; i < n; ++i) { + dy[i] += da * dx[i]; + } +} + +inline double ddot_(int size, const double *dx, const double *dy) { + return std::inner_product(dx, dx + size, dy, 0.0); +} + +void mcstep(double *stx, double *fx, double *dx, + double *sty, double *fy, double *dy, + double *stp, double fp, double dp, + int *brackt, + double stpmin, double stpmax, + int *info) { + bool bound = true; + double p, q, s, d1, d2, d3, r, gamma, theta, stpq, stpc, stpf; + *info = 0; + + if (*brackt && ((*stp <= std::min(*stx, *sty) || + *stp >= std::max(*stx, *sty)) || + *dx * (*stp - *stx) >= 0.0 || stpmax < stpmin)) { + return; + } + + double sgnd = dp * (*dx / std::abs(*dx)); + + if (fp > *fx) { + *info = 1; + bound = true; + theta =(*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dx / s *(dp / s)); + if (*stp < *stx) { + gamma = -gamma; + } + p = gamma - *dx + theta; + q = gamma - *dx + gamma + dp; + r = p / q; + stpc = *stx + r * (*stp - *stx); + stpq = *stx + *dx / ((*fx - fp) / + (*stp - *stx) + *dx) / 2 * (*stp - *stx); + if ((d1 = stpc - *stx, std::abs(d1)) < (d2 = stpq - *stx, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpc + (stpq - stpc) / 2; + } + *brackt = true; + } else if (sgnd < 0.0) { + *info = 2; + bound = false; + theta = (*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dx / s * (dp / s)); + if (*stp > *stx) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma - dp + gamma + *dx; + r = p / q; + stpc = *stp + r *(*stx - *stp); + stpq = *stp + dp /(dp - *dx) * (*stx - *stp); + if ((d1 = stpc - *stp, std::abs(d1)) > (d2 = stpq - *stp, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + *brackt = true; + } else if (std::abs(dp) < std::abs(*dx)) { + *info = 3; + bound = true; + theta = (*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d3 = theta / s; + d1 = 0.0; + d2 = d3 * d3 - *dx / s *(dp / s); + gamma = s * std::sqrt((std::max(d1, d2))); + if (*stp > *stx) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma + (*dx - dp) + gamma; + r = p / q; + if (r < 0.0 && gamma != 0.0) { + stpc = *stp + r *(*stx - *stp); + } else if (*stp > *stx) { + stpc = stpmax; + } else { + stpc = stpmin; + } + stpq = *stp + dp /(dp - *dx) * (*stx - *stp); + if (*brackt) { + if ((d1 = *stp - stpc, std::abs(d1)) < + (d2 = *stp - stpq, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + } else { + if ((d1 = *stp - stpc, std::abs(d1)) > + (d2 = *stp - stpq, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + } + } else { + *info = 4; + bound = false; + if (*brackt) { + theta =(fp - *fy) * 3 / (*sty - *stp) + *dy + dp; + d1 = std::abs(theta); + d2 = std::abs(*dy); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dy / s * (dp / s)); + if (*stp > *sty) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma - dp + gamma + *dy; + r = p / q; + stpc = *stp + r * (*sty - *stp); + stpf = stpc; + } else if (*stp > *stx) { + stpf = stpmax; + } else { + stpf = stpmin; + } + } + + if (fp > *fx) { + *sty = *stp; + *fy = fp; + *dy = dp; + } else { + if (sgnd < 0.0) { + *sty = *stx; + *fy = *fx; + *dy = *dx; + } + *stx = *stp; + *fx = fp; + *dx = dp; + } + + stpf = std::min(stpmax, stpf); + stpf = std::max(stpmin, stpf); + *stp = stpf; + if (*brackt && bound) { + if (*sty > *stx) { + d1 = *stx + (*sty - *stx) * 0.66; + *stp = std::min(d1, *stp); + } else { + d1 = *stx + (*sty - *stx) * 0.66; + *stp = std::max(d1, *stp); + } + } + + return; +} +} + +namespace MeCab { + +class LBFGS::Mcsrch { + private: + int infoc, stage1, brackt; + double finit, dginit, dgtest, width, width1; + double stx, fx, dgx, sty, fy, dgy, stmin, stmax; + + public: + Mcsrch(): + infoc(0), + stage1(0), + brackt(0), + finit(0.0), dginit(0.0), dgtest(0.0), width(0.0), width1(0.0), + stx(0.0), fx(0.0), dgx(0.0), sty(0.0), fy(0.0), dgy(0.0), + stmin(0.0), stmax(0.0) {} + + void mcsrch(int size, + double *x, + double f, const double *g, double *s, + double *stp, + int *info, int *nfev, double *wa, bool orthant, double C) { + const double p5 = 0.5; + const double p66 = 0.66; + const double xtrapf = 4.0; + const int maxfev = 20; + + /* Parameter adjustments */ + --wa; + --s; + --g; + --x; + + if (*info == -1) { + goto L45; + } + infoc = 1; + + if (size <= 0 || *stp <= 0.0) { + return; + } + + dginit = ddot_(size, &g[1], &s[1]); + if (dginit >= 0.0) { + return; + } + + brackt = false; + stage1 = true; + *nfev = 0; + finit = f; + dgtest = ftol * dginit; + width = lb3_1_stpmax - lb3_1_stpmin; + width1 = width / p5; + for (int j = 1; j <= size; ++j) { + wa[j] = x[j]; + } + + stx = 0.0; + fx = finit; + dgx = dginit; + sty = 0.0; + fy = finit; + dgy = dginit; + + while (true) { + if (brackt) { + stmin = std::min(stx, sty); + stmax = std::max(stx, sty); + } else { + stmin = stx; + stmax = *stp + xtrapf * (*stp - stx); + } + + *stp = std::max(*stp, lb3_1_stpmin); + *stp = std::min(*stp, lb3_1_stpmax); + + if ((brackt && ((*stp <= stmin || *stp >= stmax) || + *nfev >= maxfev - 1 || infoc == 0)) || + (brackt && (stmax - stmin <= xtol * stmax))) { + *stp = stx; + } + + if (orthant) { + for (int j = 1; j <= size; ++j) { + double grad_neg = 0.0; + double grad_pos = 0.0; + double grad = 0.0; + if (wa[j] == 0.0) { + grad_neg = g[j] - 1.0 / C; + grad_pos = g[j] + 1.0 / C; + } else { + grad_pos = grad_neg = g[j] + 1.0 * sigma(wa[j]) / C; + } + if (grad_neg > 0.0) { + grad = grad_neg; + } else if (grad_pos < 0.0) { + grad = grad_pos; + } else { + grad = 0.0; + } + const double p = pi(s[j], -grad); + const double xi = wa[j] == 0.0 ? sigma(-grad) : sigma(wa[j]); + x[j] = pi(wa[j] + *stp * p, xi); + } + } else { + for (int j = 1; j <= size; ++j) { + x[j] = wa[j] + *stp * s[j]; + } + } + *info = -1; + return; + + L45: + *info = 0; + ++(*nfev); + double dg = ddot_(size, &g[1], &s[1]); + double ftest1 = finit + *stp * dgtest; + + if (brackt && ((*stp <= stmin || *stp >= stmax) || infoc == 0)) { + *info = 6; + } + if (*stp == lb3_1_stpmax && f <= ftest1 && dg <= dgtest) { + *info = 5; + } + if (*stp == lb3_1_stpmin && (f > ftest1 || dg >= dgtest)) { + *info = 4; + } + if (*nfev >= maxfev) { + *info = 3; + } + if (brackt && stmax - stmin <= xtol * stmax) { + *info = 2; + } + if (f <= ftest1 && std::abs(dg) <= lb3_1_gtol * (-dginit)) { + *info = 1; + } + + if (*info != 0) { + return; + } + + if (stage1 && f <= ftest1 && dg >= std::min(ftol, lb3_1_gtol) * dginit) { + stage1 = false; + } + + if (stage1 && f <= fx && f > ftest1) { + double fm = f - *stp * dgtest; + double fxm = fx - stx * dgtest; + double fym = fy - sty * dgtest; + double dgm = dg - dgtest; + double dgxm = dgx - dgtest; + double dgym = dgy - dgtest; + mcstep(&stx, &fxm, &dgxm, &sty, &fym, &dgym, stp, fm, dgm, &brackt, + stmin, stmax, &infoc); + fx = fxm + stx * dgtest; + fy = fym + sty * dgtest; + dgx = dgxm + dgtest; + dgy = dgym + dgtest; + } else { + mcstep(&stx, &fx, &dgx, &sty, &fy, &dgy, stp, f, dg, &brackt, + stmin, stmax, &infoc); + } + + if (brackt) { + double d1 = 0.0; + if ((d1 = sty - stx, std::abs(d1)) >= p66 * width1) { + *stp = stx + p5 * (sty - stx); + } + width1 = width; + width = (d1 = sty - stx, std::abs(d1)); + } + } + + return; + } +}; + +void LBFGS::clear() { + iflag_ = iscn = nfev = iycn = point = npt = + iter = info = ispt = isyt = iypt = 0; + stp = stp1 = 0.0; + diag_.clear(); + w_.clear(); + delete mcsrch_; + mcsrch_ = 0; +} + +void LBFGS::lbfgs_optimize(int size, + int msize, + double *x, + double f, + const double *g, + double *diag, + double *w, + bool orthant, + double C, + int *iflag) { + double yy = 0.0; + double ys = 0.0; + int bound = 0; + int cp = 0; + + --diag; + --g; + --x; + --w; + + if (!mcsrch_) { + mcsrch_ = new Mcsrch; + } + + if (*iflag == 1) { + goto L172; + } + if (*iflag == 2) { + goto L100; + } + + // initialization + if (*iflag == 0) { + point = 0; + for (int i = 1; i <= size; ++i) { + diag[i] = 1.0; + } + ispt = size + (msize << 1); + iypt = ispt + size * msize; + for (int i = 1; i <= size; ++i) { + w[ispt + i] = -g[i] * diag[i]; + } + stp1 = 1.0 / std::sqrt(ddot_(size, &g[1], &g[1])); + } + + // MAIN ITERATION LOOP + while (true) { + ++iter; + info = 0; + if (iter == 1) goto L165; + if (iter > size) bound = size; + + // COMPUTE -H*G USING THE FORMULA GIVEN IN: Nocedal, J. 1980, + // "Updating quasi-Newton matrices with limited storage", + // Mathematics of Computation, Vol.24, No.151, pp. 773-782. + ys = ddot_(size, &w[iypt + npt + 1], &w[ispt + npt + 1]); + yy = ddot_(size, &w[iypt + npt + 1], &w[iypt + npt + 1]); + for (int i = 1; i <= size; ++i) { + diag[i] = ys / yy; + } + + L100: + cp = point; + if (point == 0) cp = msize; + w[size + cp] = 1.0 / ys; + + for (int i = 1; i <= size; ++i) { + w[i] = -g[i]; + } + + bound = std::min(iter - 1, msize); + + cp = point; + for (int i = 1; i <= bound; ++i) { + --cp; + if (cp == -1) cp = msize - 1; + double sq = ddot_(size, &w[ispt + cp * size + 1], &w[1]); + int inmc = size + msize + cp + 1; + iycn = iypt + cp * size; + w[inmc] = w[size + cp + 1] * sq; + double d = -w[inmc]; + daxpy_(size, d, &w[iycn + 1], &w[1]); + } + + for (int i = 1; i <= size; ++i) { + w[i] = diag[i] * w[i]; + } + + for (int i = 1; i <= bound; ++i) { + double yr = ddot_(size, &w[iypt + cp * size + 1], &w[1]); + double beta = w[size + cp + 1] * yr; + int inmc = size + msize + cp + 1; + beta = w[inmc] - beta; + iscn = ispt + cp * size; + daxpy_(size, beta, &w[iscn + 1], &w[1]); + ++cp; + if (cp == msize) { + cp = 0; + } + } + + // STORE THE NEW SEARCH DIRECTION + for (int i = 1; i <= size; ++i) { + w[ispt + point * size + i] = w[i]; + } + + L165: + // OBTAIN THE ONE-DIMENSIONAL MINIMIZER OF THE FUNCTION + // BY USING THE LINE SEARCH ROUTINE MCSRCH + nfev = 0; + stp = 1.0; + if (iter == 1) { + stp = stp1; + } + for (int i = 1; i <= size; ++i) { + w[i] = g[i]; + } + + L172: + mcsrch_->mcsrch(size, &x[1], f, &g[1], &w[ispt + point * size + 1], + &stp, &info, &nfev, &diag[1], orthant, C); + if (info == -1) { + *iflag = 1; // next value + return; + } + if (info != 1) { + std::cerr << "The line search routine mcsrch failed: error code:" + << info << std::endl; + *iflag = -1; + return; + } + + // COMPUTE THE NEW STEP AND GRADIENT CHANGE + npt = point * size; + for (int i = 1; i <= size; ++i) { + w[ispt + npt + i] = stp * w[ispt + npt + i]; + w[iypt + npt + i] = g[i] - w[i]; + } + ++point; + if (point == msize) { + point = 0; + } + + double gnorm = std::sqrt(ddot_(size, &g[1], &g[1])); + double xnorm = std::max(1.0, std::sqrt(ddot_(size, &x[1], &x[1]))); + if (gnorm / xnorm <= eps) { + *iflag = 0; // OK terminated + return; + } + } +} +} From 9d495fded989eda60587026e991b276b31a24570 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:49 +0800 Subject: [PATCH 023/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/lbfgs.h | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 third_party/mecab/src/lbfgs.h diff --git a/third_party/mecab/src/lbfgs.h b/third_party/mecab/src/lbfgs.h new file mode 100644 index 00000000..64eb0f2b --- /dev/null +++ b/third_party/mecab/src/lbfgs.h @@ -0,0 +1,71 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_LBFGS_H_ +#define MECAB_LBFGS_H_ + +#include +#include + +namespace MeCab { + +class LBFGS { + public: + explicit LBFGS(): iflag_(0), iscn(0), nfev(0), iycn(0), + point(0), npt(0), iter(0), info(0), + ispt(0), isyt(0), iypt(0), maxfev(0), + stp(0.0), stp1(0.0), mcsrch_(0) {} + virtual ~LBFGS() { clear(); } + + void clear(); + + int optimize(size_t size, double *x, double f, double *g, + bool orthant, double C) { + static const int msize = 5; + if (w_.empty()) { + iflag_ = 0; + w_.resize(size * (2 * msize + 1) + 2 * msize); + diag_.resize(size); + } else if (diag_.size() != size) { + std::cerr << "size of array is different" << std::endl; + return -1; + } + + lbfgs_optimize(static_cast(size), + msize, x, f, g, &diag_[0], &w_[0], orthant, C, &iflag_); + + if (iflag_ < 0) { + std::cerr << "routine stops with unexpected error" << std::endl; + return -1; + } + + if (iflag_ == 0) { + clear(); + return 0; // terminate + } + + return 1; // evaluate next f and g + } + + private: + class Mcsrch; + int iflag_, iscn, nfev, iycn, point, npt; + int iter, info, ispt, isyt, iypt, maxfev; + double stp, stp1; + std::vector diag_; + std::vector w_; + Mcsrch *mcsrch_; + + void lbfgs_optimize(int size, + int msize, + double *x, + double f, + const double *g, + double *diag, + double *w, bool orthant, double C, int *iflag); +}; +} + +#endif From cb040760384e80f2da0575817c2811715229ad06 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:51 +0800 Subject: [PATCH 024/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/learner.cpp | 320 ++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 third_party/mecab/src/learner.cpp diff --git a/third_party/mecab/src/learner.cpp b/third_party/mecab/src/learner.cpp new file mode 100644 index 00000000..a04a6727 --- /dev/null +++ b/third_party/mecab/src/learner.cpp @@ -0,0 +1,320 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include "common.h" +#include "feature_index.h" +#include "freelist.h" +#include "lbfgs.h" +#include "learner_tagger.h" +#include "param.h" +#include "string_buffer.h" +#include "thread.h" +#include "utils.h" + +namespace MeCab { +namespace { + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() + +#ifdef MECAB_USE_THREAD +class learner_thread: public thread { + public: + unsigned short start_i; + unsigned short thread_num; + size_t size; + size_t micro_p; + size_t micro_r; + size_t micro_c; + size_t err; + double f; + EncoderLearnerTagger **x; + std::vector expected; + void run() { + micro_p = micro_r = micro_c = err = 0; + f = 0.0; + std::fill(expected.begin(), expected.end(), 0.0); + for (size_t i = start_i; i < size; i += thread_num) { + f += x[i]->gradient(&expected[0]); + err += x[i]->eval(µ_c, µ_p, µ_r); + } + } +}; +#endif + +class CRFLearner { + public: + static int run(Param *param) { + const std::string dicdir = param->get("dicdir"); + CHECK_DIE(param->load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + const std::vector &files = param->rest_args(); + if (files.size() != 2) { + std::cout << "Usage: " << + param->program_name() << " corpus model" << std::endl; + return -1; + } + + const std::string ifile = files[0]; + const std::string model = files[1]; + const std::string old_model = param->get("old-model"); + + EncoderFeatureIndex feature_index; + std::vector expected; + std::vector observed; + std::vector alpha; + std::vector old_alpha; + std::vector x; + Tokenizer tokenizer; + Allocator allocator; + + CHECK_DIE(tokenizer.open(*param)) << "cannot open tokenizer"; + CHECK_DIE(feature_index.open(*param)) << "cannot open feature index"; + + if (!old_model.empty()) { + std::cout << "Using previous model: " << old_model << std::endl; + std::cout << "--cost --freq and --eta options are overwritten." + << std::endl; + CHECK_DIE(tokenizer.dictionary_info()); + const char *dic_charset = tokenizer.dictionary_info()->charset; + feature_index.reopen(old_model.c_str(), + dic_charset, &old_alpha, param); + } + + const double C = param->get("cost"); + const double eta = param->get("eta"); + const size_t eval_size = param->get("eval-size"); + const size_t unk_eval_size = param->get("unk-eval-size"); + const size_t thread_num = param->get("thread"); + const size_t freq = param->get("freq"); + + CHECK_DIE(C > 0) << "cost parameter is out of range: " << C; + CHECK_DIE(eta > 0) "eta is out of range: " << eta; + CHECK_DIE(eval_size > 0) << "eval-size is out of range: " << eval_size; + CHECK_DIE(unk_eval_size > 0) << + "unk-eval-size is out of range: " << unk_eval_size; + CHECK_DIE(freq > 0) << + "freq is out of range: " << unk_eval_size; + CHECK_DIE(thread_num > 0 && thread_num <= 512) + << "# thread is invalid: " << thread_num; + + std::cout.setf(std::ios::fixed, std::ios::floatfield); + std::cout.precision(5); + + std::cout << "reading corpus ..." << std::flush; + + std::ifstream ifs(WPATH(ifile.c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << ifile; + + while (ifs) { + EncoderLearnerTagger *tagger = new EncoderLearnerTagger(); + + CHECK_DIE(tagger->open(&tokenizer, + &allocator, + &feature_index, + eval_size, + unk_eval_size)); + + CHECK_DIE(tagger->read(&ifs, &observed)); + + if (!tagger->empty()) { + x.push_back(tagger); + } else { + delete tagger; + } + + if (x.size() % 100 == 0) { + std::cout << x.size() << "... " << std::flush; + } + } + + feature_index.shrink(freq, &observed); + feature_index.clearcache(); + + const size_t psize = feature_index.size(); + observed.resize(psize); + expected.resize(psize); + alpha.resize(psize); + old_alpha.resize(psize); + alpha = old_alpha; + + feature_index.set_alpha(&alpha[0]); + + std::cout << std::endl; + std::cout << "Number of sentences: " << x.size() << std::endl; + std::cout << "Number of features: " << psize << std::endl; + std::cout << "eta: " << eta << std::endl; + std::cout << "freq: " << freq << std::endl; + std::cout << "eval-size: " << eval_size << std::endl; + std::cout << "unk-eval-size: " << unk_eval_size << std::endl; +#ifdef MECAB_USE_THREAD + std::cout << "threads: " << thread_num << std::endl; +#endif + std::cout << "charset: " << + tokenizer.dictionary_info()->charset << std::endl; + std::cout << "C(sigma^2): " << C << std::endl + << std::endl; + +#ifdef MECAB_USE_THREAD + std::vector thread; + if (thread_num > 1) { + thread.resize(thread_num); + for (size_t i = 0; i < thread_num; ++i) { + thread[i].start_i = i; + thread[i].size = x.size(); + thread[i].thread_num = thread_num; + thread[i].x = &x[0]; + thread[i].expected.resize(expected.size()); + } + } +#endif + + int converge = 0; + double prev_obj = 0.0; + LBFGS lbfgs; + + for (size_t itr = 0; ; ++itr) { + std::fill(expected.begin(), expected.end(), 0.0); + double obj = 0.0; + size_t err = 0; + size_t micro_p = 0; + size_t micro_r = 0; + size_t micro_c = 0; + +#ifdef MECAB_USE_THREAD + if (thread_num > 1) { + for (size_t i = 0; i < thread_num; ++i) { + thread[i].start(); + } + + for (size_t i = 0; i < thread_num; ++i) { + thread[i].join(); + } + + for (size_t i = 0; i < thread_num; ++i) { + obj += thread[i].f; + err += thread[i].err; + micro_r += thread[i].micro_r; + micro_p += thread[i].micro_p; + micro_c += thread[i].micro_c; + for (size_t k = 0; k < psize; ++k) { + expected[k] += thread[i].expected[k]; + } + } + } else +#endif + { + for (size_t i = 0; i < x.size(); ++i) { + obj += x[i]->gradient(&expected[0]); + err += x[i]->eval(µ_c, µ_p, µ_r); + } + } + + const double p = 1.0 * micro_c / micro_p; + const double r = 1.0 * micro_c / micro_r; + const double micro_f = 2 * p * r / (p + r); + + for (size_t i = 0; i < psize; ++i) { + const double penalty = (alpha[i] - old_alpha[i]); + obj += (penalty * penalty / (2.0 * C)); + expected[i] = expected[i] - observed[i] + penalty / C; + } + + const double diff = (itr == 0 ? 1.0 : + std::fabs(1.0 * (prev_obj - obj)) / prev_obj); + std::cout << "iter=" << itr + << " err=" << 1.0 * err/x.size() + << " F=" << micro_f + << " target=" << obj + << " diff=" << diff << std::endl; + prev_obj = obj; + + if (diff < eta) { + converge++; + } else { + converge = 0; + } + + if (converge == 3) { + break; // 3 is ad-hoc + } + + const int ret = lbfgs.optimize(psize, + &alpha[0], obj, + &expected[0], false, C); + + CHECK_DIE(ret >= 0) << "unexpected error in LBFGS routin"; + + if (ret == 0) { + break; + } + } + + std::cout << "\nDone! writing model file ... " << std::endl; + + std::ostringstream oss; + + oss << "eta: " << eta << std::endl; + oss << "freq: " << freq << std::endl; + oss << "C: " << C << std::endl; + oss.setf(std::ios::fixed, std::ios::floatfield); + oss.precision(16); + oss << "eval-size: " << eval_size << std::endl; + oss << "unk-eval-size: " << unk_eval_size << std::endl; + oss << "charset: " << tokenizer.dictionary_info()->charset << std::endl; + + const std::string header = oss.str(); + + CHECK_DIE(feature_index.save(model.c_str(), header.c_str())) + << "permission denied: " << model; + + return 0; + } +}; + +class Learner { + public: + static bool run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", + "set DIR as dicdir(default \".\" )" }, + { "old-model", 'M', 0, "FILE", + "set FILE as old CRF model file" }, + { "cost", 'c', "1.0", "FLOAT", + "set FLOAT for cost C for constraints violatoin" }, + { "freq", 'f', "1", "INT", + "set the frequency cut-off (default 1)" }, + { "eta", 'e', "0.00005", "DIR", + "set FLOAT for tolerance of termination criterion" }, + { "thread", 'p', "1", "INT", "number of threads(default 1)" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + return CRFLearner::run(¶m); + } +}; +} +} + +int mecab_cost_train(int argc, char **argv) { + return MeCab::Learner::run(argc, argv); +} From e277b5e62c6827fe2e1b69b79f0a7a01ccd72c27 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:52 +0800 Subject: [PATCH 025/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/learner_node.h | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 third_party/mecab/src/learner_node.h diff --git a/third_party/mecab/src/learner_node.h b/third_party/mecab/src/learner_node.h new file mode 100644 index 00000000..db0ac329 --- /dev/null +++ b/third_party/mecab/src/learner_node.h @@ -0,0 +1,134 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_LEARNER_NODE_H_ +#define MECAB_LEARNER_NODE_H_ + +#include +#include "mecab.h" +#include "common.h" +#include "utils.h" + +struct mecab_learner_path_t { + struct mecab_learner_node_t* rnode; + struct mecab_learner_path_t* rnext; + struct mecab_learner_node_t* lnode; + struct mecab_learner_path_t* lnext; + double cost; + const int *fvector; +}; + +struct mecab_learner_node_t { + struct mecab_learner_node_t *prev; + struct mecab_learner_node_t *next; + struct mecab_learner_node_t *enext; + struct mecab_learner_node_t *bnext; + struct mecab_learner_path_t *rpath; + struct mecab_learner_path_t *lpath; + struct mecab_learner_node_t *anext; + const char *surface; + const char *feature; + unsigned int id; + unsigned short length; + unsigned short rlength; + unsigned short rcAttr; + unsigned short lcAttr; + unsigned short posid; + unsigned char char_type; + unsigned char stat; + unsigned char isbest; + double alpha; + double beta; + short wcost2; + double wcost; + double cost; + const int *fvector; + struct mecab_token_t *token; +}; + +namespace MeCab { + +typedef struct mecab_learner_path_t LearnerPath; +typedef struct mecab_learner_node_t LearnerNode; + +template T1 repeat_find_if(T1 b, T1 e, + const T2& v, size_t n) { + T1 r = b; + for (size_t i = 0; i < n; ++i) { + r = std::find(b, e, v); + if (r == e) return e; + b = r + 1; + } + return r; +} + +// NOTE: first argment: answer, +// second argment: system output +inline bool node_cmp_eq(const LearnerNode &node1, + const LearnerNode &node2, + size_t size, size_t unk_size) { + if (node1.length == node2.length && + strncmp(node1.surface, node2.surface, node1.length) == 0) { + const char *p1 = node1.feature; + const char *p2 = node2.feature; + // There is NO case when node1 becomes MECAB_UNK_NODE + if (node2.stat == MECAB_UNK_NODE) + size = unk_size; // system cannot output other extra information + const char *r1 = repeat_find_if(p1, p1 + std::strlen(p1), ',', size); + const char *r2 = repeat_find_if(p2, p2 + std::strlen(p2), ',', size); + if (static_cast(r1 - p1) == static_cast(r2 - p2) && + std::strncmp(p1, p2, static_cast(r1 - p1)) == 0) { + return true; + } + } + + return false; +} + +inline bool is_empty(LearnerPath *path) { + return ((!path->rnode->rpath && path->rnode->stat != MECAB_EOS_NODE) || + (!path->lnode->lpath && path->lnode->stat != MECAB_BOS_NODE) ); +} + +inline void calc_expectation(LearnerPath *path, double *expected, double Z) { + if (is_empty(path)) { + return; + } + + const double c = std::exp(path->lnode->alpha + + path->cost + + path->rnode->beta - Z); + + for (const int *f = path->fvector; *f != -1; ++f) { + expected[*f] += c; + } + + if (path->rnode->stat != MECAB_EOS_NODE) { + for (const int *f = path->rnode->fvector; *f != -1; ++f) { + expected[*f] += c; + } + } +} + +inline void calc_alpha(LearnerNode *n) { + n->alpha = 0.0; + for (LearnerPath *path = n->lpath; path; path = path->lnext) { + n->alpha = logsumexp(n->alpha, + path->cost + path->lnode->alpha, + path == n->lpath); + } +} + +inline void calc_beta(LearnerNode *n) { + n->beta = 0.0; + for (LearnerPath *path = n->rpath; path; path = path->rnext) { + n->beta = logsumexp(n->beta, + path->cost + path->rnode->beta, + path == n->rpath); + } +} +} + +#endif // MECAB_LEARNER_NODE_H_ From dae6abc3a3f97a0d0d5e5fcbc3e2e051cdc842e9 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:54 +0800 Subject: [PATCH 026/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/learner_tagger.cpp | 418 +++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 third_party/mecab/src/learner_tagger.cpp diff --git a/third_party/mecab/src/learner_tagger.cpp b/third_party/mecab/src/learner_tagger.cpp new file mode 100644 index 00000000..ff4228a5 --- /dev/null +++ b/third_party/mecab/src/learner_tagger.cpp @@ -0,0 +1,418 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "common.h" +#include "learner_node.h" +#include "learner_tagger.h" +#include "utils.h" + +namespace MeCab { +namespace { +char *mystrdup(const char *str) { + const size_t l = std::strlen(str); + char *r = new char[l + 1]; + std::strncpy(r, str, l+1); + return r; +} + +char *mystrdup(const std::string &str) { + return mystrdup(str.c_str()); +} +} // namespace + +bool EncoderLearnerTagger::open(Tokenizer *tokenizer, + Allocator *allocator, + FeatureIndex *feature_index, + size_t eval_size, + size_t unk_eval_size) { + close(); + tokenizer_ = tokenizer; + allocator_ = allocator; + feature_index_ = feature_index; + eval_size_ = eval_size; + unk_eval_size_ = unk_eval_size; + return true; +} + +bool DecoderLearnerTagger::open(const Param ¶m) { + close(); + allocator_data_.reset(new Allocator()); + tokenizer_data_.reset(new Tokenizer()); + feature_index_data_.reset(new DecoderFeatureIndex); + allocator_ = allocator_data_.get(); + tokenizer_ = tokenizer_data_.get(); + feature_index_ = feature_index_data_.get(); + + CHECK_DIE(tokenizer_->open(param)) << tokenizer_->what(); + CHECK_DIE(feature_index_->open(param)); + + return true; +} + +bool EncoderLearnerTagger::read(std::istream *is, + std::vector *observed) { + scoped_fixed_array line; + char *column[8]; + std::string sentence; + std::vector corpus; + ans_path_list_.clear(); + + bool eos = false; + + for (;;) { + if (!is->getline(line.get(), line.size())) { + is->clear(std::ios::eofbit|std::ios::badbit); + return true; + } + + eos = (std::strcmp(line.get(), "EOS") == 0 || line[0] == '\0'); + + LearnerNode *m = new LearnerNode; + std::memset(m, 0, sizeof(LearnerNode)); + + if (eos) { + m->stat = MECAB_EOS_NODE; + } else { + const size_t size = tokenize(line.get(), "\t", column, 2); + CHECK_DIE(size == 2) << "format error: " << line.get(); + m->stat = MECAB_NOR_NODE; + m->surface = mystrdup(column[0]); + m->feature = mystrdup(column[1]); + m->length = m->rlength = std::strlen(column[0]); + } + + corpus.push_back(m); + + if (eos) { + break; + } + + sentence.append(column[0]); + } + + CHECK_DIE(!sentence.empty()) << "empty sentence"; + + CHECK_DIE(eos) << "\"EOS\" is not found"; + + begin_data_.reset_string(sentence); + begin_ = begin_data_.get(); + + initList(); + + size_t pos = 0; + for (size_t i = 0; corpus[i]->stat != MECAB_EOS_NODE; ++i) { + LearnerNode *found = 0; + for (LearnerNode *node = lookup(pos); node; node = node->bnext) { + if (node_cmp_eq(*(corpus[i]), *node, eval_size_, unk_eval_size_)) { + found = node; + break; + } + } + + // cannot find node even using UNKNOWN WORD PROSESSING + if (!found) { + LearnerNode *node = allocator_->newNode(); + node->surface = begin_ + pos; + node->length = node->rlength = std::strlen(corpus[i]->surface); + node->feature = feature_index_->strdup(corpus[i]->feature); + node->stat = MECAB_NOR_NODE; + node->fvector = 0; + node->wcost = 0.0; + node->bnext = begin_node_list_[pos]; + begin_node_list_[pos] = node; + std::cout << "adding virtual node: " << node->feature << std::endl; + } + + pos += corpus[i]->length; + } + + buildLattice(); + + LearnerNode* prev = end_node_list_[0]; // BOS + prev->anext = 0; + pos = 0; + + for (size_t i = 0; i < corpus.size(); ++i) { + LearnerNode *rNode = 0; + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + if (corpus[i]->stat == MECAB_EOS_NODE || + node_cmp_eq(*(corpus[i]), *node, eval_size_, unk_eval_size_)) { + rNode = node; // take last node + } + } + + LearnerPath *lpath = 0; + for (LearnerPath *path = rNode->lpath; path; path = path->lnext) { + if (prev == path->lnode) { + lpath = path; + break; + } + } + + CHECK_DIE(lpath->fvector) << "lpath is NULL"; + for (const int *f = lpath->fvector; *f != -1; ++f) { + if (*f >= static_cast(observed->size())) { + observed->resize(*f + 1); + } + ++(*observed)[*f]; + } + + if (lpath->rnode->stat != MECAB_EOS_NODE) { + for (const int *f = lpath->rnode->fvector; *f != -1; ++f) { + if (*f >= static_cast(observed->size())) { + observed->resize(*f + 1); + } + ++(*observed)[*f]; + } + } + + ans_path_list_.push_back(lpath); + + prev->anext = rNode; + prev = rNode; + + if (corpus[i]->stat == MECAB_EOS_NODE) { + break; + } + + pos += std::strlen(corpus[i]->surface); + } + + prev->anext = begin_node_list_[len_]; // connect to EOS + begin_node_list_[len_]->anext = 0; + + for (size_t i = 0 ; i < corpus.size(); ++i) { + delete [] corpus[i]->surface; + delete [] corpus[i]->feature; + delete corpus[i]; + } + + return true; +} + +int EncoderLearnerTagger::eval(size_t *crr, + size_t *prec, size_t *recall) const { + int zeroone = 0; + + LearnerNode *res = end_node_list_[0]->next; + LearnerNode *ans = end_node_list_[0]->anext; + + size_t resp = 0; + size_t ansp = 0; + + while (ans->anext && res->next) { + if (resp == ansp) { + if (node_cmp_eq(*ans, *res, eval_size_, unk_eval_size_)) { + ++(*crr); // same + } else { + zeroone = 1; + } + ++(*prec); + ++(*recall); + res = res->next; + ans = ans->anext; + resp += res->rlength; + ansp += ans->rlength; + } else if (resp < ansp) { + res = res->next; + resp += res->rlength; + zeroone = 1; + ++(*recall); + } else { + ans = ans->anext; + ansp += ans->rlength; + zeroone = 1; + ++(*prec); + } + } + + while (ans->anext) { + ++(*prec); + ans = ans->anext; + } + + while (res->next) { + ++(*recall); + res = res->next; + } + + return zeroone; +} + +bool DecoderLearnerTagger::parse(std::istream* is, std::ostream *os) { + allocator_->free(); + feature_index_->clear(); + + if (!begin_) { + begin_data_.reset(new char[BUF_SIZE * 16]); + begin_ = begin_data_.get(); + } + + if (!is->getline(const_cast(begin_), BUF_SIZE * 16)) { + is->clear(std::ios::eofbit|std::ios::badbit); + return false; + } + + initList(); + buildLattice(); + viterbi(); + + for (LearnerNode *node = end_node_list_[0]->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature << '\n'; + } + *os << "EOS\n"; + + return true; +} + +LearnerNode *LearnerTagger::lookup(size_t pos) { + if (begin_node_list_[pos]) { + return begin_node_list_[pos]; + } + LearnerNode *m = tokenizer_->lookup(begin_ + pos, end_, allocator_, 0); + begin_node_list_[pos] = m; + return m; +} + +bool LearnerTagger::connect(size_t pos, LearnerNode *_rNode) { + for (LearnerNode *rNode = _rNode ; rNode; rNode = rNode->bnext) { + for (LearnerNode *lNode = end_node_list_[pos]; lNode; + lNode = lNode->enext) { + LearnerPath *path = allocator_->newPath(); + std::memset(path, 0, sizeof(Path)); + path->rnode = rNode; + path->lnode = lNode; + path->fvector = 0; + path->cost = 0.0; + path->rnode = rNode; + path->lnode = lNode; + path->lnext = rNode->lpath; + rNode->lpath = path; + path->rnext = lNode->rpath; + lNode->rpath = path; + CHECK_DIE(feature_index_->buildFeature(path)); + CHECK_DIE(path->fvector); + } + const size_t x = rNode->rlength + pos; + rNode->enext = end_node_list_[x]; + end_node_list_[x] = rNode; + } + + return true; +} + +bool LearnerTagger::initList() { + if (!begin_) { + return false; + } + + len_ = std::strlen(begin_); + end_ = begin_ + len_; + + end_node_list_.resize(len_ + 2); + std::fill(end_node_list_.begin(), end_node_list_.end(), + reinterpret_cast(0)); + + begin_node_list_.resize(len_ + 2); + std::fill(begin_node_list_.begin(), begin_node_list_.end(), + reinterpret_cast(0)); + + end_node_list_[0] = tokenizer_->getBOSNode(allocator_); + end_node_list_[0]->surface = begin_; + begin_node_list_[len_] = tokenizer_->getEOSNode(allocator_); + + return true; +} + +bool LearnerTagger::buildLattice() { + for (int pos = 0; pos <= static_cast(len_); pos++) { + if (!end_node_list_[pos]) { + continue; + } + connect(pos, lookup(pos)); + } + + if (!end_node_list_[len_]) { + begin_node_list_[len_] = lookup(len_); + for (size_t pos = len_; static_cast(pos) >= 0; pos--) { + if (end_node_list_[pos]) { + connect(pos, begin_node_list_[len_]); + break; + } + } + } + + return true; +} + +bool LearnerTagger::viterbi() { + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + double bestc = -1e37; + LearnerNode *best = 0; + feature_index_->calcCost(node); + for (LearnerPath *path = node->lpath; path; path = path->lnext) { + feature_index_->calcCost(path); + double cost = path->cost + path->lnode->cost; + if (cost > bestc) { + bestc = cost; + best = path->lnode; + } + } + + node->prev = best; + node->cost = bestc; + } + } + + LearnerNode *node = begin_node_list_[len_]; // EOS + for (LearnerNode *prev; node->prev;) { + prev = node->prev; + prev->next = node; + node = prev; + } + + return true; +} + +double EncoderLearnerTagger::gradient(double *expected) { + viterbi(); + + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + calc_alpha(node); + } + } + + for (int pos = static_cast(len_); pos >=0; --pos) { + for (LearnerNode *node = end_node_list_[pos]; node; node = node->enext) { + calc_beta(node); + } + } + + double Z = begin_node_list_[len_]->alpha; // alpha of EOS + + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + for (LearnerPath *path = node->lpath; path; path = path->lnext) { + calc_expectation(path, expected, Z); + } + } + } + + for (size_t i = 0; i < ans_path_list_.size(); ++i) { + Z -= ans_path_list_[i]->cost; + } + + return Z; +} +} From d171ad935542c8db40b9fce18962311d13c6fe5c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:56 +0800 Subject: [PATCH 027/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/learner_tagger.h | 80 ++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 third_party/mecab/src/learner_tagger.h diff --git a/third_party/mecab/src/learner_tagger.h b/third_party/mecab/src/learner_tagger.h new file mode 100644 index 00000000..abea4594 --- /dev/null +++ b/third_party/mecab/src/learner_tagger.h @@ -0,0 +1,80 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_TAGGER_H +#define MECAB_TAGGER_H + +#include +#include "mecab.h" +#include "freelist.h" +#include "feature_index.h" +#include "tokenizer.h" +#include "scoped_ptr.h" + +namespace MeCab { + +class FeatureIndex; + +class LearnerTagger { + public: + bool empty() const { return (len_ == 0); } + void close() {} + void clear() {} + + explicit LearnerTagger(): tokenizer_(0), path_allocator_(0), + feature_index_(0), begin_(0), end_(0), len_(0) {} + virtual ~LearnerTagger() {} + + protected: + Tokenizer *tokenizer_; + Allocator *allocator_; + FreeList *path_allocator_; + FeatureIndex *feature_index_; + scoped_string begin_data_; + const char *begin_; + const char *end_; + size_t len_; + std::vector begin_node_list_; + std::vector end_node_list_; + + LearnerNode *lookup(size_t); + bool connect(size_t, LearnerNode *); + bool viterbi(); + bool buildLattice(); + bool initList(); +}; + +class EncoderLearnerTagger: public LearnerTagger { + public: + bool open(Tokenizer *tokenzier, + Allocator *allocator, + FeatureIndex *feature_index, + size_t eval_size, size_t unk_eval_size); + bool read(std::istream *, std::vector *); + int eval(size_t *, size_t *, size_t *) const; + double gradient(double *expected); + explicit EncoderLearnerTagger(): eval_size_(1024), unk_eval_size_(1024) {} + virtual ~EncoderLearnerTagger() { close(); } + + private: + size_t eval_size_; + size_t unk_eval_size_; + std::vector ans_path_list_; +}; + +class DecoderLearnerTagger: public LearnerTagger { + public: + bool open(const Param &); + bool parse(std::istream *, std::ostream *); + virtual ~DecoderLearnerTagger() { close(); } + + private: + scoped_ptr > tokenizer_data_; + scoped_ptr > allocator_data_; + scoped_ptr feature_index_data_; +}; +} + +#endif From 311259c75b686b1e6e0139aeedd4a95703cf13df Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:58 +0800 Subject: [PATCH 028/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/libmecab.cpp | 496 +++++++++++++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 third_party/mecab/src/libmecab.cpp diff --git a/third_party/mecab/src/libmecab.cpp b/third_party/mecab/src/libmecab.cpp new file mode 100644 index 00000000..413c4754 --- /dev/null +++ b/third_party/mecab/src/libmecab.cpp @@ -0,0 +1,496 @@ +// +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#if defined(_WIN32) && !defined(__CYGWIN__) +#include +#endif + +#include "mecab.h" +#include "tokenizer.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace { +const char kUnknownError[] = "Unknown Error"; +const size_t kErrorBufferSize = 256; +} + +#if defined(_WIN32) && !defined(__CYGWIN__) +namespace { +DWORD g_tls_index = TLS_OUT_OF_INDEXES; +} + +const char *getGlobalError() { + LPVOID data = ::TlsGetValue(g_tls_index); + return data == NULL ? kUnknownError : reinterpret_cast(data); +} + +void setGlobalError(const char *str) { + char *data = reinterpret_cast(::TlsGetValue(g_tls_index)); + if (data == NULL) { + return; + } + strncpy(data, str, kErrorBufferSize - 1); + data[kErrorBufferSize - 1] = '\0'; +} + +HINSTANCE DllInstance = 0; + +extern "C" { + BOOL WINAPI DllMain(HINSTANCE hinst, DWORD dwReason, LPVOID) { + LPVOID data = 0; + if (!DllInstance) { + DllInstance = hinst; + } + switch (dwReason) { + case DLL_PROCESS_ATTACH: + if ((g_tls_index = ::TlsAlloc()) == TLS_OUT_OF_INDEXES) { + return FALSE; + } + // Not break in order to initialize the TLS. + case DLL_THREAD_ATTACH: + data = (LPVOID)::LocalAlloc(LPTR, kErrorBufferSize); + if (data) { + ::TlsSetValue(g_tls_index, data); + } + break; + case DLL_THREAD_DETACH: + data = ::TlsGetValue(g_tls_index); + if (data) { + ::LocalFree((HLOCAL)data); + } + break; + case DLL_PROCESS_DETACH: + data = ::TlsGetValue(g_tls_index); + if (data) { + ::LocalFree((HLOCAL)data); + } + ::TlsFree(g_tls_index); + g_tls_index = TLS_OUT_OF_INDEXES; + break; + default: + break; + } + return TRUE; + } +} +#else // _WIN32 +namespace { +#ifdef HAVE_TLS_KEYWORD +__thread char kErrorBuffer[kErrorBufferSize]; +#else +char kErrorBuffer[kErrorBufferSize]; +#endif +} + +const char *getGlobalError() { + return kErrorBuffer; +} + +void setGlobalError(const char *str) { + strncpy(kErrorBuffer, str, kErrorBufferSize - 1); + kErrorBuffer[kErrorBufferSize - 1] = '\0'; +} +#endif + +mecab_t* mecab_new(int argc, char **argv) { + MeCab::Tagger *tagger = MeCab::createTagger(argc, argv); + if (!tagger) { + MeCab::deleteTagger(tagger); + return 0; + } + return reinterpret_cast(tagger); +} + +mecab_t* mecab_new2(const char *arg) { + MeCab::Tagger *tagger = MeCab::createTagger(arg); + if (!tagger) { + MeCab::deleteTagger(tagger); + return 0; + } + return reinterpret_cast(tagger); +} + +const char *mecab_version() { + return MeCab::Tagger::version(); +} + +const char* mecab_strerror(mecab_t *tagger) { + if (!tagger) { + return MeCab::getLastError(); + } + return reinterpret_cast(tagger)->what(); +} + +void mecab_destroy(mecab_t *tagger) { + MeCab::Tagger *ptr = reinterpret_cast(tagger); + MeCab::deleteTagger(ptr); + ptr = 0; +} + +int mecab_get_partial(mecab_t *tagger) { + return reinterpret_cast(tagger)->partial(); +} + +void mecab_set_partial(mecab_t *tagger, int partial) { + reinterpret_cast(tagger)->set_partial(partial); +} + +float mecab_get_theta(mecab_t *tagger) { + return reinterpret_cast(tagger)->theta(); +} + +void mecab_set_theta(mecab_t *tagger, float theta) { + reinterpret_cast(tagger)->set_theta(theta); +} + +int mecab_get_lattice_level(mecab_t *tagger) { + return reinterpret_cast(tagger)->lattice_level(); +} + +void mecab_set_lattice_level(mecab_t *tagger, int level) { + reinterpret_cast(tagger)->set_lattice_level(level); +} + +int mecab_get_all_morphs(mecab_t *tagger) { + return static_cast( + reinterpret_cast(tagger)->all_morphs()); +} + +void mecab_set_all_morphs(mecab_t *tagger, int all_morphs) { + reinterpret_cast(tagger)->set_all_morphs(all_morphs); +} + +const char* mecab_sparse_tostr(mecab_t *tagger, const char *str) { + return reinterpret_cast(tagger)->parse(str); +} + +const char* mecab_sparse_tostr2(mecab_t *tagger, const char *str, size_t len) { + return reinterpret_cast(tagger)->parse(str, len); +} + +char* mecab_sparse_tostr3(mecab_t *tagger, const char *str, size_t len, + char *out, size_t len2) { + return const_cast( + reinterpret_cast(tagger)->parse( + str, len, out, len2)); +} + +const mecab_node_t* mecab_sparse_tonode(mecab_t *tagger, const char *str) { + return reinterpret_cast( + reinterpret_cast(tagger)->parseToNode(str)); +} + +const mecab_node_t* mecab_sparse_tonode2(mecab_t *tagger, + const char *str, size_t len) { + return reinterpret_cast( + reinterpret_cast(tagger)->parseToNode(str, len)); +} + +const char* mecab_nbest_sparse_tostr(mecab_t *tagger, size_t N, + const char *str) { + return reinterpret_cast(tagger)->parseNBest(N, str); +} + +const char* mecab_nbest_sparse_tostr2(mecab_t *tagger, size_t N, + const char* str, size_t len) { + return reinterpret_cast( + tagger)->parseNBest(N, str, len); +} + +char* mecab_nbest_sparse_tostr3(mecab_t *tagger, size_t N, + const char *str, size_t len, + char *out, size_t len2) { + return const_cast( + reinterpret_cast( + tagger)->parseNBest(N, str, len, out, len2)); +} + +int mecab_nbest_init(mecab_t *tagger, const char *str) { + return reinterpret_cast< + MeCab::Tagger *>(tagger)->parseNBestInit(str); +} + +int mecab_nbest_init2(mecab_t *tagger, const char *str, size_t len) { + return reinterpret_cast< + MeCab::Tagger *>(tagger)->parseNBestInit(str, len); +} + +const char* mecab_nbest_next_tostr(mecab_t *tagger) { + return reinterpret_cast(tagger)->next(); +} + +char* mecab_nbest_next_tostr2(mecab_t *tagger, char *out, size_t len2) { + return const_cast( + reinterpret_cast(tagger)->next(out, len2)); +} + +const mecab_node_t* mecab_nbest_next_tonode(mecab_t *tagger) { + return reinterpret_cast( + reinterpret_cast(tagger)->nextNode()); +} + +const char* mecab_format_node(mecab_t *tagger, const mecab_node_t* n) { + return reinterpret_cast(tagger)->formatNode(n); +} + +const mecab_dictionary_info_t *mecab_dictionary_info(mecab_t *tagger) { + return reinterpret_cast( + reinterpret_cast(tagger)->dictionary_info()); +} + +int mecab_parse_lattice(mecab_t *mecab, mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(mecab)->parse( + reinterpret_cast(lattice))); +} + +mecab_lattice_t *mecab_lattice_new() { + return reinterpret_cast(MeCab::createLattice()); +} + +void mecab_lattice_destroy(mecab_lattice_t *lattice) { + MeCab::Lattice *ptr = reinterpret_cast(lattice); + MeCab::deleteLattice(ptr); + ptr = 0; +} + +void mecab_lattice_clear(mecab_lattice_t *lattice) { + reinterpret_cast(lattice)->clear(); +} + +int mecab_lattice_is_available(mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(lattice)->is_available()); +} +mecab_node_t *mecab_lattice_get_bos_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->bos_node()); +} + +mecab_node_t *mecab_lattice_get_eos_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->eos_node()); +} + +mecab_node_t **mecab_lattice_get_all_begin_nodes(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->begin_nodes()); +} + +mecab_node_t **mecab_lattice_get_all_end_nodes(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->end_nodes()); +} + +mecab_node_t *mecab_lattice_get_begin_nodes(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + reinterpret_cast(lattice)->begin_nodes(pos)); +} + +mecab_node_t *mecab_lattice_get_end_nodes(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + reinterpret_cast(lattice)->end_nodes(pos)); +} + +const char *mecab_lattice_get_sentence(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->sentence(); +} + +void mecab_lattice_set_sentence(mecab_lattice_t *lattice, + const char *sentence) { + reinterpret_cast(lattice)->set_sentence(sentence); +} + +void mecab_lattice_set_sentence2(mecab_lattice_t *lattice, + const char *sentence, size_t len) { + reinterpret_cast(lattice)->set_sentence( + sentence, len); +} + +size_t mecab_lattice_get_size(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->size(); +} + +double mecab_lattice_get_z(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->Z(); +} + +void mecab_lattice_set_z(mecab_lattice_t *lattice, double Z) { + reinterpret_cast(lattice)->set_Z(Z); +} + +double mecab_lattice_get_theta(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->theta(); +} + +void mecab_lattice_set_theta(mecab_lattice_t *lattice, double theta) { + reinterpret_cast(lattice)->set_theta(theta); +} + +int mecab_lattice_next(mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(lattice)->next()); +} + +int mecab_lattice_get_request_type(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->request_type(); +} + +int mecab_lattice_has_request_type(mecab_lattice_t *lattice, + int request_type) { + return reinterpret_cast( + lattice)->has_request_type(request_type); +} + +void mecab_lattice_set_request_type(mecab_lattice_t *lattice, + int request_type) { + reinterpret_cast( + lattice)->set_request_type(request_type); +} + +void mecab_lattice_add_request_type(mecab_lattice_t *lattice, + int request_type) { + reinterpret_cast( + lattice)->add_request_type(request_type); +} + +void mecab_lattice_remove_request_type(mecab_lattice_t *lattice, + int request_type) { + return reinterpret_cast( + lattice)->remove_request_type(request_type); +} + +mecab_node_t *mecab_lattice_new_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->newNode()); +} + +const char *mecab_lattice_tostr(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->toString(); +} + +const char *mecab_lattice_tostr2(mecab_lattice_t *lattice, + char *buf, size_t size) { + return reinterpret_cast( + lattice)->toString(buf, size); +} +const char *mecab_lattice_nbest_tostr(mecab_lattice_t *lattice, + size_t N) { + return reinterpret_cast( + lattice)->enumNBestAsString(N); +} +const char *mecab_lattice_nbest_tostr2(mecab_lattice_t *lattice, + size_t N, char *buf, size_t size) { + return reinterpret_cast( + lattice)->enumNBestAsString(N, buf, size); +} + +int mecab_lattice_has_constraint(mecab_lattice_t *lattice) { + return static_cast(reinterpret_cast( + lattice)->has_constraint()); +} + +int mecab_lattice_get_boundary_constraint(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + lattice)->boundary_constraint(pos); +} + +const char *mecab_lattice_get_feature_constraint(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + lattice)->feature_constraint(pos); +} + +void mecab_lattice_set_boundary_constraint(mecab_lattice_t *lattice, + size_t pos, int boundary_type) { + return reinterpret_cast( + lattice)->set_boundary_constraint(pos, boundary_type); +} + +void mecab_lattice_set_feature_constraint(mecab_lattice_t *lattice, + size_t begin_pos, size_t end_pos, + const char *feature) { + return reinterpret_cast( + lattice)->set_feature_constraint(begin_pos, end_pos, feature); +} + +void mecab_lattice_set_result(mecab_lattice_t *lattice, + const char *result) { + return reinterpret_cast(lattice)->set_result(result); +} + +const char *mecab_lattice_strerror(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->what(); +} + +mecab_model_t *mecab_model_new(int argc, char **argv) { + MeCab::Model *model = MeCab::createModel(argc, argv); + if (!model) { + MeCab::deleteModel(model); + return 0; + } + return reinterpret_cast(model); +} + +mecab_model_t *mecab_model_new2(const char *arg) { + MeCab::Model *model = MeCab::createModel(arg); + if (!model) { + MeCab::deleteModel(model); + return 0; + } + return reinterpret_cast(model); +} + +void mecab_model_destroy(mecab_model_t *model) { + MeCab::Model *ptr = reinterpret_cast(model); + MeCab::deleteModel(ptr); + ptr = 0; +} + +mecab_t *mecab_model_new_tagger(mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->createTagger()); +} + +mecab_lattice_t *mecab_model_new_lattice(mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->createLattice()); +} + +int mecab_model_swap(mecab_model_t *model, mecab_model_t *new_model) { + return static_cast( + reinterpret_cast(model)->swap( + reinterpret_cast(new_model))); +} + +const mecab_dictionary_info_t* mecab_model_dictionary_info( + mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->dictionary_info()); +} + +int mecab_model_transition_cost(mecab_model_t *model, + unsigned short rcAttr, + unsigned short lcAttr) { + return reinterpret_cast(model)->transition_cost( + rcAttr, lcAttr); +} + +mecab_node_t *mecab_model_lookup(mecab_model_t *model, + const char *begin, + const char *end, + mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(model)->lookup( + begin, end, + reinterpret_cast(lattice))); +} From 9b0d81ada37e85feb481687265d5282d7b0e68cc Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:39:59 +0800 Subject: [PATCH 029/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/make.bat | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 third_party/mecab/src/make.bat diff --git a/third_party/mecab/src/make.bat b/third_party/mecab/src/make.bat new file mode 100644 index 00000000..f782e521 --- /dev/null +++ b/third_party/mecab/src/make.bat @@ -0,0 +1,8 @@ +Set PATH=c:\Program Files\Microsoft Visual Studio 8\VC\bin;%PATH% +Set INCLUDE=c:\Program Files\Microsoft Visual Studio 8\VC\include;c:\Program Files\Microsoft Platform SDK\Include;%INCLUDE% +Set LIB=c:\Program Files\Microsoft Visual Studio 8\VC\lib;c:\Program Files\Microsoft Platform SDK\Lib;%LIB% +Set COMSPEC=cmd.exe +rem nmake -f Makefile.msvc clean +nmake -f Makefile.msvc + + From 28cd47021d14a20647a8e57984737591f816c378 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:01 +0800 Subject: [PATCH 030/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/Makefile.am | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 third_party/mecab/src/Makefile.am diff --git a/third_party/mecab/src/Makefile.am b/third_party/mecab/src/Makefile.am new file mode 100644 index 00000000..0b9342fb --- /dev/null +++ b/third_party/mecab/src/Makefile.am @@ -0,0 +1,41 @@ +AUTOMAKE_OPTIONS = no-dependencies +lib_LTLIBRARIES = libmecab.la +EXTRA_DIST = Makefile.msvc.in make.bat +pkglibexecdir = ${libexecdir}/mecab +INCLUDES = -DDIC_VERSION=$(DIC_VERSION) $(MECAB_NO_TLS) $(MECAB_USE_UTF8_ONLY) -DMECAB_DEFAULT_RC="\"$(MECAB_DEFAULT_RC)\"" +libmecab_la_LDFLAGS = -no-undefined -version-info $(LTVERSION) +libmecab_la_SOURCES = viterbi.cpp tagger.cpp utils.cpp utils.h eval.cpp iconv_utils.cpp iconv_utils.h \ + dictionary_rewriter.h dictionary_rewriter.cpp dictionary_generator.cpp \ + dictionary_compiler.cpp context_id.h context_id.cpp \ + winmain.h thread.h connector.cpp nbest_generator.h nbest_generator.cpp connector.h \ + writer.h writer.cpp mmap.h ucs.h \ + string_buffer.h string_buffer.cpp \ + tokenizer.h stream_wrapper.h common.h darts.h char_property.h ucstable.h \ + freelist.h viterbi.h param.cpp tokenizer.cpp \ + ucstable.h char_property.cpp dictionary.h scoped_ptr.h \ + param.h mecab.h dictionary.cpp \ + feature_index.cpp feature_index.h lbfgs.cpp \ + lbfgs.h learner_tagger.cpp learner_tagger.h learner.cpp \ + learner_node.h libmecab.cpp + +include_HEADERS = mecab.h +bin_PROGRAMS = mecab +pkglibexec_PROGRAMS = mecab-dict-index mecab-dict-gen mecab-cost-train mecab-system-eval mecab-test-gen + +mecab_dict_index_SOURCES = mecab-dict-index.cpp +mecab_dict_index_LDADD = libmecab.la + +mecab_dict_gen_SOURCES = mecab-dict-gen.cpp +mecab_dict_gen_LDADD = libmecab.la + +mecab_system_eval_SOURCES = mecab-system-eval.cpp +mecab_system_eval_LDADD = libmecab.la + +mecab_cost_train_SOURCES = mecab-cost-train.cpp +mecab_cost_train_LDADD = libmecab.la + +mecab_test_gen_SOURCES = mecab-test-gen.cpp +mecab_test_gen_LDADD = libmecab.la + +mecab_SOURCES = mecab.cpp +mecab_LDADD = libmecab.la From e95f535073a222a2f7e796c6d4d0712b95008b20 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:02 +0800 Subject: [PATCH 031/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/Makefile.msvc.in | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 third_party/mecab/src/Makefile.msvc.in diff --git a/third_party/mecab/src/Makefile.msvc.in b/third_party/mecab/src/Makefile.msvc.in new file mode 100644 index 00000000..eec583df --- /dev/null +++ b/third_party/mecab/src/Makefile.msvc.in @@ -0,0 +1,53 @@ +CC = cl.exe +CXXC = cl.exe +LINK=link.exe + +CFLAGS = /EHsc /O2 /GL /GA /Ob2 /nologo /W3 /MT /Zi /wd4800 /wd4305 /wd4244 +LDFLAGS = /nologo /OPT:REF /OPT:ICF /LTCG /NXCOMPAT /DYNAMICBASE /MACHINE:X86 ADVAPI32.LIB +DEFS = -D_CRT_SECURE_NO_DEPRECATE -DMECAB_USE_THREAD \ + -DDLL_EXPORT -DHAVE_GETENV -DHAVE_WINDOWS_H -DDIC_VERSION=@DIC_VERSION@ \ + -DVERSION="\"@VERSION@\"" -DPACKAGE="\"mecab\"" \ + -DUNICODE -D_UNICODE \ + -DMECAB_DEFAULT_RC="\"c:\\Program Files\\mecab\\etc\\mecabrc\"" +INC = -I. -I.. +DEL = del + +OBJ = feature_index.obj param.obj learner.obj string_buffer.obj \ + char_property.obj learner_tagger.obj tagger.obj \ + connector.obj tokenizer.obj \ + context_id.obj dictionary.obj utils.obj \ + dictionary_compiler.obj viterbi.obj \ + dictionary_generator.obj writer.obj iconv_utils.obj \ + dictionary_rewriter.obj lbfgs.obj eval.obj nbest_generator.obj + +.c.obj: + $(CC) $(CFLAGS) $(INC) $(DEFS) -c $< + +.cpp.obj: + $(CC) $(CFLAGS) $(INC) $(DEFS) -c $< + +all: libmecab mecab mecab-dict-index mecab-dict-gen mecab-cost-train mecab-system-eval mecab-test-gen + +mecab: $(OBJ) mecab.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab.obj libmecab.lib + +mecab-dict-index: $(OBJ) mecab-dict-index.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-dict-index.obj libmecab.lib + +mecab-dict-gen: $(OBJ) mecab-dict-gen.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-dict-gen.obj libmecab.lib + +mecab-cost-train: $(OBJ) mecab-cost-train.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-cost-train.obj libmecab.lib + +mecab-system-eval: $(OBJ) mecab-system-eval.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-system-eval.obj libmecab.lib + +mecab-test-gen: mecab-test-gen.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-test-gen.obj libmecab.lib + +libmecab: $(OBJ) libmecab.obj + $(LINK) $(LDFLAGS) /out:$@.dll $(OBJ) libmecab.obj /dll + +clean: + $(DEL) *.exe *.obj *.dll *.a *.lib *.o *.exp *.def From b88a6feed3045d103616b5188c291c125702d7be Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:04 +0800 Subject: [PATCH 032/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab-cost-train.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 third_party/mecab/src/mecab-cost-train.cpp diff --git a/third_party/mecab/src/mecab-cost-train.cpp b/third_party/mecab/src/mecab-cost-train.cpp new file mode 100644 index 00000000..fca181aa --- /dev/null +++ b/third_party/mecab/src/mecab-cost-train.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_cost_train(argc, argv); +} + From 9e7a1e137d08f5273d5eaa809bb1b8d4856cd9e0 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:06 +0800 Subject: [PATCH 033/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab-dict-gen.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 third_party/mecab/src/mecab-dict-gen.cpp diff --git a/third_party/mecab/src/mecab-dict-gen.cpp b/third_party/mecab/src/mecab-dict-gen.cpp new file mode 100644 index 00000000..c354b82d --- /dev/null +++ b/third_party/mecab/src/mecab-dict-gen.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_dict_gen(argc, argv); +} + From 3100c76c2218a2a996af6f88477d0825abe711c2 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:07 +0800 Subject: [PATCH 034/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab-dict-index.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 third_party/mecab/src/mecab-dict-index.cpp diff --git a/third_party/mecab/src/mecab-dict-index.cpp b/third_party/mecab/src/mecab-dict-index.cpp new file mode 100644 index 00000000..6ace3eda --- /dev/null +++ b/third_party/mecab/src/mecab-dict-index.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_dict_index(argc, argv); +} + From 0f73b3f1db4f10feebc6c9a7d47c224909b3a969 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:09 +0800 Subject: [PATCH 035/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab-system-eval.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 third_party/mecab/src/mecab-system-eval.cpp diff --git a/third_party/mecab/src/mecab-system-eval.cpp b/third_party/mecab/src/mecab-system-eval.cpp new file mode 100644 index 00000000..3b650573 --- /dev/null +++ b/third_party/mecab/src/mecab-system-eval.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_system_eval(argc, argv); +} + From 1c292ee2af453d62728c4351bee3d7c165830323 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:11 +0800 Subject: [PATCH 036/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab-test-gen.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 third_party/mecab/src/mecab-test-gen.cpp diff --git a/third_party/mecab/src/mecab-test-gen.cpp b/third_party/mecab/src/mecab-test-gen.cpp new file mode 100644 index 00000000..e8c3f189 --- /dev/null +++ b/third_party/mecab/src/mecab-test-gen.cpp @@ -0,0 +1,11 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_test_gen(argc, argv); +} From 9091aac9474a5e8d85212ef05e4fd5654f7fb647 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:13 +0800 Subject: [PATCH 037/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mecab.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 third_party/mecab/src/mecab.cpp diff --git a/third_party/mecab/src/mecab.cpp b/third_party/mecab/src/mecab.cpp new file mode 100644 index 00000000..2ec2c23c --- /dev/null +++ b/third_party/mecab/src/mecab.cpp @@ -0,0 +1,11 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_do (argc, argv); +} From a018da3b6fbb045db429d494ea34027c175508cd Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:15 +0800 Subject: [PATCH 038/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/mmap.h | 212 +++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 third_party/mecab/src/mmap.h diff --git a/third_party/mecab/src/mmap.h b/third_party/mecab/src/mmap.h new file mode 100644 index 00000000..3a174f59 --- /dev/null +++ b/third_party/mecab/src/mmap.h @@ -0,0 +1,212 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_MMAP_H +#define MECAB_MMAP_H + +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +extern "C" { + +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#ifdef HAVE_STRING_H +#include +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +#ifdef HAVE_WINDOWS_H +#include +#endif +#else + +#ifdef HAVE_SYS_MMAN_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif +#endif +} + +#include "common.h" +#include "utils.h" + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +namespace MeCab { + +template class Mmap { + private: + T *text; + size_t length; + std::string fileName; + whatlog what_; + +#if defined(_WIN32) && !defined(__CYGWIN__) + HANDLE hFile; + HANDLE hMap; +#else + int fd; + int flag; +#endif + + public: + T& operator[](size_t n) { return *(text + n); } + const T& operator[](size_t n) const { return *(text + n); } + T* begin() { return text; } + const T* begin() const { return text; } + T* end() { return text + size(); } + const T* end() const { return text + size(); } + size_t size() { return length/sizeof(T); } + const char *what() { return what_.str(); } + const char *file_name() { return fileName.c_str(); } + size_t file_size() { return length; } + bool empty() { return(length == 0); } + + // This code is imported from sufary, develoved by + // TATUO Yamashita Thanks! +#if defined(_WIN32) && !defined(__CYGWIN__) + bool open(const char *filename, const char *mode = "r") { + this->close(); + unsigned long mode1, mode2, mode3; + fileName = std::string(filename); + + if (std::strcmp(mode, "r") == 0) { + mode1 = GENERIC_READ; + mode2 = PAGE_READONLY; + mode3 = FILE_MAP_READ; + } else if (std::strcmp(mode, "r+") == 0) { + mode1 = GENERIC_READ | GENERIC_WRITE; + mode2 = PAGE_READWRITE; + mode3 = FILE_MAP_ALL_ACCESS; + } else { + CHECK_FALSE(false) << "unknown open mode:" << filename; + } + + hFile = ::CreateFileW(WPATH_FORCE(filename), mode1, FILE_SHARE_READ, 0, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + CHECK_FALSE(hFile != INVALID_HANDLE_VALUE) + << "CreateFile() failed: " << filename; + + length = ::GetFileSize(hFile, 0); + + hMap = ::CreateFileMapping(hFile, 0, mode2, 0, 0, 0); + CHECK_FALSE(hMap) << "CreateFileMapping() failed: " << filename; + + text = reinterpret_cast(::MapViewOfFile(hMap, mode3, 0, 0, 0)); + CHECK_FALSE(text) << "MapViewOfFile() failed: " << filename; + + return true; + } + + void close() { + if (text) { ::UnmapViewOfFile(text); } + if (hFile != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile); + hFile = INVALID_HANDLE_VALUE; + } + if (hMap) { + ::CloseHandle(hMap); + hMap = 0; + } + text = 0; + } + + Mmap(): text(0), hFile(INVALID_HANDLE_VALUE), hMap(0) {} + +#else + + bool open(const char *filename, const char *mode = "r") { + this->close(); + struct stat st; + fileName = std::string(filename); + + if (std::strcmp(mode, "r") == 0) + flag = O_RDONLY; + else if (std::strcmp(mode, "r+") == 0) + flag = O_RDWR; + else + CHECK_FALSE(false) << "unknown open mode: " << filename; + + CHECK_FALSE((fd = ::open(filename, flag | O_BINARY)) >= 0) + << "open failed: " << filename; + + CHECK_FALSE(::fstat(fd, &st) >= 0) + << "failed to get file size: " << filename; + + length = st.st_size; + +#ifdef HAVE_MMAP + int prot = PROT_READ; + if (flag == O_RDWR) prot |= PROT_WRITE; + char *p; + CHECK_FALSE((p = reinterpret_cast + (::mmap(0, length, prot, MAP_SHARED, fd, 0))) + != MAP_FAILED) + << "mmap() failed: " << filename; + + text = reinterpret_cast(p); +#else + text = new T[length]; + CHECK_FALSE(::read(fd, text, length) >= 0) + << "read() failed: " << filename; +#endif + ::close(fd); + fd = -1; + + return true; + } + + void close() { + if (fd >= 0) { + ::close(fd); + fd = -1; + } + + if (text) { +#ifdef HAVE_MMAP + ::munmap(reinterpret_cast(text), length); + text = 0; +#else + if (flag == O_RDWR) { + int fd2; + if ((fd2 = ::open(fileName.c_str(), O_RDWR)) >= 0) { + ::write(fd2, text, length); + ::close(fd2); + } + } + delete [] text; +#endif + } + + text = 0; + } + + Mmap() : text(0), fd(-1) {} +#endif + + virtual ~Mmap() { this->close(); } +}; +} +#endif From 57c6e7f5e6a953b181df2e2d13f4a818ae60dda2 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:16 +0800 Subject: [PATCH 039/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/nbest_generator.cpp | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 third_party/mecab/src/nbest_generator.cpp diff --git a/third_party/mecab/src/nbest_generator.cpp b/third_party/mecab/src/nbest_generator.cpp new file mode 100644 index 00000000..d30796ec --- /dev/null +++ b/third_party/mecab/src/nbest_generator.cpp @@ -0,0 +1,52 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include "mecab.h" +#include "nbest_generator.h" + +namespace MeCab { + +bool NBestGenerator::set(Lattice *lattice) { + freelist_.free(); + while (!agenda_.empty()) { + agenda_.pop(); // make empty + } + QueueElement *eos = freelist_.alloc(); + eos->node = lattice->eos_node(); + eos->next = 0; + eos->fx = eos->gx = 0; + agenda_.push(eos); + return true; +} + +bool NBestGenerator::next() { + while (!agenda_.empty()) { + QueueElement *top = agenda_.top(); + agenda_.pop(); + Node *rnode = top->node; + + if (rnode->stat == MECAB_BOS_NODE) { // BOS + for (QueueElement *n = top; n->next; n = n->next) { + n->node->next = n->next->node; // change next & prev + n->next->node->prev = n->node; + // TODO: rewrite costs; + } + return true; + } + + for (Path *path = rnode->lpath; path; path = path->lnext) { + QueueElement *n = freelist_.alloc(); + n->node = path->lnode; + n->gx = path->cost + top->gx; + n->fx = path->lnode->cost + path->cost + top->gx; + n->next = top; + agenda_.push(n); + } + } + + return false; +} +} From f8e61b501c2f99fa78d7b10f1adc5f41b30211cd Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:18 +0800 Subject: [PATCH 040/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/nbest_generator.h | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 third_party/mecab/src/nbest_generator.h diff --git a/third_party/mecab/src/nbest_generator.h b/third_party/mecab/src/nbest_generator.h new file mode 100644 index 00000000..e09e8ac7 --- /dev/null +++ b/third_party/mecab/src/nbest_generator.h @@ -0,0 +1,43 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_NBEST_GENERATOR_H_ +#define MECAB_NBEST_GENERATOR_H_ + +#include +#include "mecab.h" +#include "freelist.h" + +namespace MeCab { + +class NBestGenerator { + private: + struct QueueElement { + Node *node; + QueueElement *next; + long fx; // f(x) = h(x) + g(x): cost function for A* search + long gx; // g(x) + }; + + class QueueElementComp { + public: + const bool operator()(QueueElement *q1, QueueElement *q2) { + return (q1->fx > q2->fx); + } + }; + + std::priority_queue, + QueueElementComp> agenda_; + FreeList freelist_; + + public: + explicit NBestGenerator() : freelist_(512) {} + virtual ~NBestGenerator() {} + bool set(Lattice *lattice); + bool next(); +}; +} + +#endif // MECAB_NBEST_GENERATOR_H_ From 2483d3da2f8c563d78540cdc5398c8dafaa97469 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:20 +0800 Subject: [PATCH 041/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/param.cpp | 223 ++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 third_party/mecab/src/param.cpp diff --git a/third_party/mecab/src/param.cpp b/third_party/mecab/src/param.cpp new file mode 100644 index 00000000..322a74f5 --- /dev/null +++ b/third_party/mecab/src/param.cpp @@ -0,0 +1,223 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "param.h" +#include "string_buffer.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace MeCab { +namespace { +void init_param(std::string *help, + std::string *version, + const std::string &system_name, + const Option *opts) { + *help = std::string(COPYRIGHT) + "\nUsage: " + + system_name + " [options] files\n"; + + *version = std::string(PACKAGE) + " of " + VERSION + '\n'; + + size_t max = 0; + for (size_t i = 0; opts[i].name; ++i) { + size_t l = 1 + std::strlen(opts[i].name); + if (opts[i].arg_description) + l += (1 + std::strlen(opts[i].arg_description)); + max = std::max(l, max); + } + + for (size_t i = 0; opts[i].name; ++i) { + size_t l = std::strlen(opts[i].name); + if (opts[i].arg_description) + l += (1 + std::strlen(opts[i].arg_description)); + *help += " -"; + *help += opts[i].short_name; + *help += ", --"; + *help += opts[i].name; + if (opts[i].arg_description) { + *help += '='; + *help += opts[i].arg_description; + } + for (; l <= max; l++) *help += ' '; + *help += opts[i].description; + *help += '\n'; + } + + *help += '\n'; + return; +} +} // namespace + +void Param::dump_config(std::ostream *os) const { + for (std::map::const_iterator it = conf_.begin(); + it != conf_.end(); + ++it) { + *os << it->first << ": " << it->second << std::endl; + } +} + +bool Param::load(const char *filename) { + std::ifstream ifs(WPATH(filename)); + + CHECK_FALSE(ifs) << "no such file or directory: " << filename; + + std::string line; + while (std::getline(ifs, line)) { + if (!line.size() || + (line.size() && (line[0] == ';' || line[0] == '#'))) continue; + + size_t pos = line.find('='); + CHECK_FALSE(pos != std::string::npos) << "format error: " << line; + + size_t s1, s2; + for (s1 = pos+1; s1 < line.size() && isspace(line[s1]); s1++); + for (s2 = pos-1; static_cast(s2) >= 0 && isspace(line[s2]); s2--); + const std::string value = line.substr(s1, line.size() - s1); + const std::string key = line.substr(0, s2 + 1); + set(key.c_str(), value, false); + } + + return true; +} + +bool Param::open(int argc, char **argv, const Option *opts) { + int ind = 0; + int _errno = 0; + +#define GOTO_ERROR(n) { \ + _errno = n; \ + goto ERROR; } while (0) + + if (argc <= 0) { + system_name_ = "unknown"; + return true; // this is not error + } + + system_name_ = std::string(argv[0]); + + init_param(&help_, &version_, system_name_, opts); + + for (size_t i = 0; opts[i].name; ++i) { + if (opts[i].default_value) set + (opts[i].name, opts[i].default_value); + } + + for (ind = 1; ind < argc; ind++) { + if (argv[ind][0] == '-') { + // long options + if (argv[ind][1] == '-') { + char *s; + for (s = &argv[ind][2]; *s != '\0' && *s != '='; s++); + size_t len = (size_t)(s - &argv[ind][2]); + if (len == 0) return true; // stop the scanning + + bool hit = false; + size_t i = 0; + for (i = 0; opts[i].name; ++i) { + size_t nlen = std::strlen(opts[i].name); + if (nlen == len && std::strncmp(&argv[ind][2], + opts[i].name, len) == 0) { + hit = true; + break; + } + } + + if (!hit) GOTO_ERROR(0); + + if (opts[i].arg_description) { + if (*s == '=') { + set(opts[i].name, s+1); + } else { + if (argc == (ind+1)) GOTO_ERROR(1); + set(opts[i].name, argv[++ind]); + } + } else { + if (*s == '=') GOTO_ERROR(2); + set(opts[i].name, 1); + } + + // short options + } else if (argv[ind][1] != '\0') { + size_t i = 0; + bool hit = false; + for (i = 0; opts[i].name; ++i) { + if (opts[i].short_name == argv[ind][1]) { + hit = true; + break; + } + } + + if (!hit) GOTO_ERROR(0); + + if (opts[i].arg_description) { + if (argv[ind][2] != '\0') { + set(opts[i].name, &argv[ind][2]); + } else { + if (argc == (ind+1)) GOTO_ERROR(1); + set(opts[i].name, argv[++ind]); + } + } else { + if (argv[ind][2] != '\0') GOTO_ERROR(2); + set(opts[i].name, 1); + } + } + } else { + rest_.push_back(std::string(argv[ind])); // others + } + } + + return true; + +ERROR: + switch (_errno) { + case 0: WHAT << "unrecognized option `" << argv[ind] << "`"; break; + case 1: WHAT << "`" << argv[ind] << "` requires an argument"; break; + case 2: WHAT << "`" << argv[ind] << "` doesn't allow an argument"; break; + } + return false; +} + +void Param::clear() { + conf_.clear(); + rest_.clear(); +} + +bool Param::open(const char *arg, const Option *opts) { + scoped_fixed_array str; + std::strncpy(str.get(), arg, str.size()); + char* ptr[64]; + unsigned int size = 1; + ptr[0] = const_cast(PACKAGE); + + for (char *p = str.get(); *p;) { + while (isspace(*p)) *p++ = '\0'; + if (*p == '\0') break; + ptr[size++] = p; + if (size == sizeof(ptr)) break; + while (*p && !isspace(*p)) p++; + } + + return open(size, ptr, opts); +} + +int Param::help_version() const { + if (get("help")) { + std::cout << help(); + return 0; + } + + if (get("version")) { + std::cout << version(); + return 0; + } + + return 1; +} +} From aa8782003e428e847392efa5ea125ae35df13054 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:22 +0800 Subject: [PATCH 042/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/param.h | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 third_party/mecab/src/param.h diff --git a/third_party/mecab/src/param.h b/third_party/mecab/src/param.h new file mode 100644 index 00000000..89a449fc --- /dev/null +++ b/third_party/mecab/src/param.h @@ -0,0 +1,92 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_PARAM_H +#define MECAB_PARAM_H + +#include +#include +#include +#include +#include "scoped_ptr.h" +#include "common.h" + +namespace { +template +Target lexical_cast(Source arg) { + std::stringstream interpreter; + Target result; + if (!(interpreter << arg) || !(interpreter >> result) || + !(interpreter >> std::ws).eof()) { + MeCab::scoped_ptr r(new Target()); // return default value + return *r; + } + return result; +} + +template <> +std::string lexical_cast(std::string arg) { + return arg; +} +} + +namespace MeCab { + +struct Option { + const char *name; + char short_name; + const char *default_value; + const char *arg_description; + const char *description; +}; + +class Param { + private: + std::map conf_; + std::vector rest_; + std::string system_name_; + std::string help_; + std::string version_; + whatlog what_; + + public: + bool open(int argc, char **argv, const Option *opt); + bool open(const char *arg, const Option *opt); + bool load(const char *filename); + void clear(); + const std::vector& rest_args() const { return rest_; } + + const char* program_name() const { return system_name_.c_str(); } + const char *what() { return what_.str(); } + const char* help() const { return help_.c_str(); } + const char* version() const { return version_.c_str(); } + int help_version() const; + + template + T get(const char *key) const { + std::map::const_iterator it = conf_.find(key); + if (it == conf_.end()) { + scoped_ptr r(new T()); + return *r; + } + return lexical_cast(it->second); + } + + template + void set(const char* key, const T &value, + bool rewrite = true) { + std::string key2 = std::string(key); + if (rewrite || (!rewrite && conf_.find(key2) == conf_.end())) + conf_[key2] = lexical_cast(value); + } + + void dump_config(std::ostream *os) const; + + explicit Param() {} + virtual ~Param() {} +}; +} + +#endif From 0c788c2298f69b168d22fed6f2811bc44c752d9d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:23 +0800 Subject: [PATCH 043/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/scoped_ptr.h | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 third_party/mecab/src/scoped_ptr.h diff --git a/third_party/mecab/src/scoped_ptr.h b/third_party/mecab/src/scoped_ptr.h new file mode 100644 index 00000000..325f5a7e --- /dev/null +++ b/third_party/mecab/src/scoped_ptr.h @@ -0,0 +1,95 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_SCOPED_PTR_H +#define MECAB_SCOPED_PTR_H + +#include +#include + +namespace MeCab { + +template class scoped_ptr { + private: + T * ptr_; + scoped_ptr(scoped_ptr const &); + scoped_ptr & operator= (scoped_ptr const &); + typedef scoped_ptr this_type; + + public: + typedef T element_type; + explicit scoped_ptr(T * p = 0): ptr_(p) {} + virtual ~scoped_ptr() { delete ptr_; } + void reset(T * p = 0) { + delete ptr_; + ptr_ = p; + } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } +}; + +template class scoped_array { + private: + T * ptr_; + scoped_array(scoped_array const &); + scoped_array & operator= (scoped_array const &); + typedef scoped_array this_type; + + public: + typedef T element_type; + explicit scoped_array(T * p = 0): ptr_(p) {} + virtual ~scoped_array() { delete [] ptr_; } + void reset(T * p = 0) { + delete [] ptr_; + ptr_ = p; + } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } + T & operator[](size_t i) const { return ptr_[i]; } +}; + +template class scoped_fixed_array { + private: + T * ptr_; + size_t size_; + scoped_fixed_array(scoped_fixed_array const &); + scoped_fixed_array & operator= (scoped_fixed_array const &); + typedef scoped_fixed_array this_type; + + public: + typedef T element_type; + explicit scoped_fixed_array() + : ptr_(new T[N]), size_(N) {} + virtual ~scoped_fixed_array() { delete [] ptr_; } + size_t size() const { return size_; } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } + T & operator[](size_t i) const { return ptr_[i]; } +}; + +class scoped_string: public scoped_array { + public: + explicit scoped_string() { reset_string(""); } + explicit scoped_string(const std::string &str) { + reset_string(str); + } + + void reset_string(const std::string &str) { + char *p = new char[str.size() + 1]; + std::strcpy(p, str.c_str()); + reset(p); + } + + void reset_string(const char *str) { + char *p = new char[std::strlen(str) + 1]; + std::strcpy(p, str); + reset(p); + } +}; +} +#endif From eac96c40eecda46c1e8734ec4d7ca41e53c20f63 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:25 +0800 Subject: [PATCH 044/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/stream_wrapper.h | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 third_party/mecab/src/stream_wrapper.h diff --git a/third_party/mecab/src/stream_wrapper.h b/third_party/mecab/src/stream_wrapper.h new file mode 100644 index 00000000..f4ce1a19 --- /dev/null +++ b/third_party/mecab/src/stream_wrapper.h @@ -0,0 +1,55 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_STREAM_WRAPPER_H_ +#define MECAB_STREAM_WRAPPER_H_ + +#include +#include +#include +#include "utils.h" + +namespace MeCab { + +class istream_wrapper { + private: + std::istream* is_; + public: + std::istream &operator*() const { return *is_; } + std::istream *operator->() const { return is_; } + explicit istream_wrapper(const char* filename): is_(0) { + if (std::strcmp(filename, "-") == 0) { + is_ = &std::cin; + } else { + is_ = new std::ifstream(WPATH(filename)); + } + } + + virtual ~istream_wrapper() { + if (is_ != &std::cin) delete is_; + } +}; + +class ostream_wrapper { + private: + std::ostream* os_; + public: + std::ostream &operator*() const { return *os_; } + std::ostream *operator->() const { return os_; } + explicit ostream_wrapper(const char* filename): os_(0) { + if (std::strcmp(filename, "-") == 0) { + os_ = &std::cout; + } else { + os_ = new std::ofstream(WPATH(filename)); + } + } + + virtual ~ostream_wrapper() { + if (os_ != &std::cout) delete os_; + } +}; +} + +#endif // MECAB_STREAM_WRAPPER_H_ From 86fb7b83662ceb64c804ae00cf95150c0212a692 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:26 +0800 Subject: [PATCH 045/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/string_buffer.cpp | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 third_party/mecab/src/string_buffer.cpp diff --git a/third_party/mecab/src/string_buffer.cpp b/third_party/mecab/src/string_buffer.cpp new file mode 100644 index 00000000..6a65b310 --- /dev/null +++ b/third_party/mecab/src/string_buffer.cpp @@ -0,0 +1,65 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "string_buffer.h" + +#define DEFAULT_ALLOC_SIZE BUF_SIZE + +namespace MeCab { + +bool StringBuffer::reserve(size_t length) { + if (!is_delete_) { + error_ = (size_ + length >= alloc_size_); + return (!error_); + } + + if (size_ + length >= alloc_size_) { + if (alloc_size_ == 0) { + alloc_size_ = DEFAULT_ALLOC_SIZE; + ptr_ = new char[alloc_size_]; + } + size_t len = size_ + length; + do { + alloc_size_ *= 2; + } while (len >= alloc_size_); + char *new_ptr = new char[alloc_size_]; + std::memcpy(new_ptr, ptr_, size_); + delete [] ptr_; + ptr_ = new_ptr; + } + + return true; +} + +StringBuffer::~StringBuffer() { + if (is_delete_) { + delete [] ptr_; + ptr_ = 0; + } +} + +StringBuffer& StringBuffer::write(char str) { + if (reserve(1)) { + ptr_[size_] = str; + ++size_; + } + return *this; +} + +StringBuffer& StringBuffer::write(const char* str) { + return this->write(str, std::strlen(str)); +} + +StringBuffer& StringBuffer::write(const char* str, size_t length) { + if (reserve(length)) { + std::memcpy(ptr_ + size_ , str, length); + size_ += length; + } + return *this; +} +} From 01d9a9ba528be4d237027f01be5cb555d1effa6e Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:28 +0800 Subject: [PATCH 046/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/string_buffer.h | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 third_party/mecab/src/string_buffer.h diff --git a/third_party/mecab/src/string_buffer.h b/third_party/mecab/src/string_buffer.h new file mode 100644 index 00000000..8fc8a682 --- /dev/null +++ b/third_party/mecab/src/string_buffer.h @@ -0,0 +1,74 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_STRINGBUFFER_H +#define MECAB_STRINGBUFFER_H + +#include +#include "common.h" +#include "utils.h" + +namespace MeCab { + +#define _ITOA(n) do { char fbuf[64]; itoa(n, fbuf); return this->write(fbuf); } while (0) +#define _UITOA(n) do { char fbuf[64]; uitoa(n, fbuf); return this->write(fbuf);} while (0) +#define _DTOA(n) do { char fbuf[64]; dtoa(n, fbuf); return this->write(fbuf); } while (0) + +class StringBuffer { + private: + size_t size_; + size_t alloc_size_; + char *ptr_; + bool is_delete_; + bool error_; + bool reserve(size_t); + + public: + explicit StringBuffer(): size_(0), alloc_size_(0), + ptr_(0), is_delete_(true), error_(false) {} + explicit StringBuffer(char *_s, size_t _l): + size_(0), alloc_size_(_l), ptr_(_s), + is_delete_(false), error_(false) {} + + virtual ~StringBuffer(); + + StringBuffer& write(char); + StringBuffer& write(const char*, size_t); + StringBuffer& write(const char*); + StringBuffer& operator<<(double n) { _DTOA(n); } + StringBuffer& operator<<(short int n) { _ITOA(n); } + StringBuffer& operator<<(int n) { _ITOA(n); } + StringBuffer& operator<<(long int n) { _ITOA(n); } + StringBuffer& operator<<(unsigned short int n) { _UITOA(n); } + StringBuffer& operator<<(unsigned int n) { _UITOA(n); } + StringBuffer& operator<<(unsigned long int n) { _UITOA(n); } +#ifdef HAVE_UNSIGNED_LONG_LONG_INT + StringBuffer& operator<<(unsigned long long int n) { _UITOA(n); } +#endif + + StringBuffer& operator<< (char n) { + return this->write(n); + } + + StringBuffer& operator<< (unsigned char n) { + return this->write(n); + } + + StringBuffer& operator<< (const char* n) { + return this->write(n); + } + + StringBuffer& operator<< (const std::string& n) { + return this->write(n.c_str()); + } + + void clear() { size_ = 0; } + const char *str() const { + return error_ ? 0 : const_cast(ptr_); + } +}; +} + +#endif From e735e272f68f1bebeff9f1155f1de93eef06d56a Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:29 +0800 Subject: [PATCH 047/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/thread.h | 189 +++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 third_party/mecab/src/thread.h diff --git a/third_party/mecab/src/thread.h b/third_party/mecab/src/thread.h new file mode 100644 index 00000000..35282a4f --- /dev/null +++ b/third_party/mecab/src/thread.h @@ -0,0 +1,189 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_THREAD_H +#define MECAB_THREAD_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_PTHREAD_H +#include +#else +#ifdef _WIN32 +#include +#include +#endif +#endif + +#if defined HAVE_GCC_ATOMIC_OPS || defined HAVE_OSX_ATOMIC_OPS +#include +#endif + +#if defined HAVE_OSX_ATOMIC_OPS +#include +#endif + +#if defined HAVE_PTHREAD_H +#define MECAB_USE_THREAD 1 +#endif + +#if (defined(_WIN32) && !defined(__CYGWIN__)) +#define MECAB_USE_THREAD 1 +#define BEGINTHREAD(src, stack, func, arg, flag, id) \ + (HANDLE)_beginthreadex((void *)(src), (unsigned)(stack), \ + (unsigned(_stdcall *)(void *))(func), (void *)(arg), \ + (unsigned)(flag), (unsigned *)(id)) +#endif + +namespace MeCab { + +#if (defined(_WIN32) && !defined(__CYGWIN__)) +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) ::InterlockedExchangeAdd(a, b) +#define compare_and_swap(a, b, c) ::InterlockedCompareExchange(a, c, b) +#define yield_processor() YieldProcessor() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_GCC_ATOMIC_OPS +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) __sync_add_and_fetch(a, b) +#define compare_and_swap(a, b, c) __sync_val_compare_and_swap(a, b, c) +#define yield_processor() sched_yield() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_OSX_ATOMIC_OPS +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) OSAtomicAdd32(b, a) +#define compare_and_swap(a, b, c) OSAtomicCompareAndSwapInt(b, c, a) +#define yield_processor() sched_yield() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_ATOMIC_OPS +// This is a simple non-scalable writer-preference lock. +// Slightly modified the following paper. +// "Scalable Reader-Writer Synchronization for Shared-Memory Multiprocessors" +// PPoPP '91. John M. Mellor-Crummey and Michael L. Scott. T +class read_write_mutex { + public: + inline void write_lock() { + atomic_add(&write_pending_, 1); + while (compare_and_swap(&l_, 0, kWaFlag)) { + yield_processor(); + } + } + inline void read_lock() { + while (write_pending_ > 0) { + yield_processor(); + } + atomic_add(&l_, kRcIncr); + while ((l_ & kWaFlag) != 0) { + yield_processor(); + } + } + inline void write_unlock() { + atomic_add(&l_, -kWaFlag); + atomic_add(&write_pending_, -1); + } + inline void read_unlock() { + atomic_add(&l_, -kRcIncr); + } + + read_write_mutex(): l_(0), write_pending_(0) {} + + private: + static const int kWaFlag = 0x1; + static const int kRcIncr = 0x2; +#ifdef HAVE_OSX_ATOMIC_OPS + volatile int l_; + volatile int write_pending_; +#else + long l_; + long write_pending_; +#endif +}; + +class scoped_writer_lock { + public: + scoped_writer_lock(read_write_mutex *mutex) : mutex_(mutex) { + mutex_->write_lock(); + } + ~scoped_writer_lock() { + mutex_->write_unlock(); + } + private: + read_write_mutex *mutex_; +}; + +class scoped_reader_lock { + public: + scoped_reader_lock(read_write_mutex *mutex) : mutex_(mutex) { + mutex_->read_lock(); + } + ~scoped_reader_lock() { + mutex_->read_unlock(); + } + private: + read_write_mutex *mutex_; +}; +#endif // HAVE_ATOMIC_OPS + +class thread { + private: +#ifdef HAVE_PTHREAD_H + pthread_t hnd; +#else +#ifdef _WIN32 + HANDLE hnd; +#endif +#endif + + public: + static void* wrapper(void *ptr) { + thread *p = static_cast(ptr); + p->run(); + return 0; + } + + virtual void run() {} + + void start() { +#ifdef HAVE_PTHREAD_H + pthread_create(&hnd, 0, &thread::wrapper, + static_cast(this)); + +#else +#ifdef _WIN32 + DWORD id; + hnd = BEGINTHREAD(0, 0, &thread::wrapper, this, 0, &id); +#endif +#endif + } + + void join() { +#ifdef HAVE_PTHREAD_H + pthread_join(hnd, 0); +#else +#ifdef _WIN32 + WaitForSingleObject(hnd, INFINITE); + CloseHandle(hnd); +#endif +#endif + } + + virtual ~thread() {} +}; +} +#endif From 9e4a4a70676073541a312a123a9f568b2db6a822 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:31 +0800 Subject: [PATCH 048/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/tokenizer.cpp | 393 ++++++++++++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 third_party/mecab/src/tokenizer.cpp diff --git a/third_party/mecab/src/tokenizer.cpp b/third_party/mecab/src/tokenizer.cpp new file mode 100644 index 00000000..bb59ebc9 --- /dev/null +++ b/third_party/mecab/src/tokenizer.cpp @@ -0,0 +1,393 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "common.h" +#include "connector.h" +#include "darts.h" +#include "learner_node.h" +#include "param.h" +#include "scoped_ptr.h" +#include "tokenizer.h" +#include "utils.h" +#include "viterbi.h" + +namespace MeCab { +namespace { + +void inline read_node_info(const Dictionary &dic, + const Token &token, + LearnerNode **node) { + (*node)->lcAttr = token.lcAttr; + (*node)->rcAttr = token.rcAttr; + (*node)->posid = token.posid; + (*node)->wcost2 = token.wcost; + (*node)->feature = dic.feature(token); +} + +void inline read_node_info(const Dictionary &dic, + const Token &token, + Node **node) { + (*node)->lcAttr = token.lcAttr; + (*node)->rcAttr = token.rcAttr; + (*node)->posid = token.posid; + (*node)->wcost = token.wcost; + (*node)->feature = dic.feature(token); +} +} // namespace + +template class Tokenizer; +template class Tokenizer; +template Tokenizer::Tokenizer(); +template void Tokenizer::close(); +template const DictionaryInfo +*Tokenizer::dictionary_info() const; +template Node* Tokenizer::getBOSNode(Allocator *) const; +template Node* Tokenizer::getEOSNode(Allocator *) const; +template Node* Tokenizer::lookup( + const char *, + const char *, + Allocator *, + Lattice *) const; +template Node* Tokenizer::lookup( + const char *, + const char *, + Allocator *, + Lattice *) const; +template bool Tokenizer::open(const Param &); +template Tokenizer::Tokenizer(); +template void Tokenizer::close(); +template const DictionaryInfo +*Tokenizer::dictionary_info() const; +template LearnerNode * Tokenizer::getEOSNode( + Allocator *) const; +template LearnerNode * Tokenizer::getBOSNode( + Allocator *) const; +template LearnerNode *Tokenizer::lookup( + const char *, + const char *, + Allocator *, Lattice *) const; +template bool Tokenizer::open(const Param &); + +template +Tokenizer::Tokenizer() + : dictionary_info_freelist_(4), + dictionary_info_(0), + max_grouping_size_(0) {} + +template +N *Tokenizer::getBOSNode(Allocator *allocator) const { + N *bos_node = allocator->newNode(); + bos_node->surface = const_cast(BOS_KEY); // dummy + bos_node->feature = bos_feature_.get(); + bos_node->isbest = 1; + bos_node->stat = MECAB_BOS_NODE; + return bos_node; +} + +template +N *Tokenizer::getEOSNode(Allocator *allocator) const { + N *eos_node = getBOSNode(allocator); // same + eos_node->stat = MECAB_EOS_NODE; + return eos_node; +} + +template +bool Tokenizer::open(const Param ¶m) { + close(); + + const std::string prefix = param.template get("dicdir"); + + CHECK_FALSE(unkdic_.open(create_filename + (prefix, UNK_DIC_FILE).c_str())) + << unkdic_.what(); + CHECK_FALSE(property_.open(param)) << property_.what(); + + Dictionary *sysdic = new Dictionary; + + CHECK_FALSE(sysdic->open + (create_filename(prefix, SYS_DIC_FILE).c_str())) + << sysdic->what(); + + CHECK_FALSE(sysdic->type() == 0) + << "not a system dictionary: " << prefix; + + property_.set_charset(sysdic->charset()); + dic_.push_back(sysdic); + + const std::string userdic = param.template get("userdic"); + if (!userdic.empty()) { + scoped_fixed_array buf; + scoped_fixed_array dicfile; + std::strncpy(buf.get(), userdic.c_str(), buf.size()); + const size_t n = tokenizeCSV(buf.get(), dicfile.get(), dicfile.size()); + for (size_t i = 0; i < n; ++i) { + Dictionary *d = new Dictionary; + CHECK_FALSE(d->open(dicfile[i])) << d->what(); + CHECK_FALSE(d->type() == 1) + << "not a user dictionary: " << dicfile[i]; + CHECK_FALSE(sysdic->isCompatible(*d)) + << "incompatible dictionary: " << dicfile[i]; + dic_.push_back(d); + } + } + + dictionary_info_ = 0; + dictionary_info_freelist_.free(); + for (int i = static_cast(dic_.size() - 1); i >= 0; --i) { + DictionaryInfo *d = dictionary_info_freelist_.alloc(); + d->next = dictionary_info_; + d->filename = dic_[i]->filename(); + d->charset = dic_[i]->charset(); + d->size = dic_[i]->size(); + d->lsize = dic_[i]->lsize(); + d->rsize = dic_[i]->rsize(); + d->type = dic_[i]->type(); + d->version = dic_[i]->version(); + dictionary_info_ = d; + } + + unk_tokens_.clear(); + for (size_t i = 0; i < property_.size(); ++i) { + const char *key = property_.name(i); + const Dictionary::result_type n = unkdic_.exactMatchSearch(key); + CHECK_FALSE(n.value != -1) << "cannot find UNK category: " << key; + const Token *token = unkdic_.token(n); + size_t size = unkdic_.token_size(n); + unk_tokens_.push_back(std::make_pair(token, size)); + } + + space_ = property_.getCharInfo(0x20); // ad-hoc + + bos_feature_.reset_string(param.template get("bos-feature")); + + const std::string tmp = param.template get("unk-feature"); + unk_feature_.reset(0); + if (!tmp.empty()) { + unk_feature_.reset_string(tmp); + } + + CHECK_FALSE(*bos_feature_ != '\0') + << "bos-feature is undefined in dicrc"; + + max_grouping_size_ = param.template get("max-grouping-size"); + if (max_grouping_size_ == 0) { + max_grouping_size_ = DEFAULT_MAX_GROUPING_SIZE; + } + + return true; +} + +namespace { +inline bool partial_match(const char *f1, const char *f2) { + if (std::strcmp(f1, "*") == 0) { + return true; + } + + scoped_fixed_array buf1; + scoped_fixed_array buf2; + scoped_fixed_array c1; + scoped_fixed_array c2; + + std::strncpy(buf1.get(), f1, buf1.size()); + std::strncpy(buf2.get(), f2, buf2.size()); + + const size_t n1 = tokenizeCSV(buf1.get(), c1.get(), c1.size()); + const size_t n2 = tokenizeCSV(buf2.get(), c2.get(), c2.size()); + const size_t n = std::min(n1, n2); + + for (size_t i = 0; i < n; ++i) { + if (std::strcmp(c1[i], "*") != 0 && + std::strcmp(c1[i], c2[i]) != 0) { + return false; + } + } + + return true; +} + +template +bool is_valid_node(const Lattice *lattice, N *node) { + const size_t end_pos = node->surface - lattice->sentence() + node->length; + if (lattice->boundary_constraint(end_pos) == MECAB_INSIDE_TOKEN) { + return false; + } + const size_t begin_pos = + node->surface - lattice->sentence() + node->length - node->rlength; + const char *feature = lattice->feature_constraint(begin_pos); + if (!feature) { + return true; + } + if (lattice->boundary_constraint(begin_pos) == MECAB_TOKEN_BOUNDARY && + lattice->boundary_constraint(end_pos) == MECAB_TOKEN_BOUNDARY && + partial_match(feature, node->feature)) { + return true; + } + return false; +} +} // namespace + +#define ADDUNKNWON do { \ + const Token *token = unk_tokens_[cinfo.default_type].first; \ + size_t size = unk_tokens_[cinfo.default_type].second; \ + for (size_t k = 0; k < size; ++k) { \ + N *new_node = allocator->newNode(); \ + read_node_info(unkdic_, *(token + k), &new_node); \ + new_node->char_type = cinfo.default_type; \ + new_node->surface = begin2; \ + new_node->length = begin3 - begin2; \ + new_node->rlength = begin3 - begin; \ + new_node->stat = MECAB_UNK_NODE; \ + new_node->bnext = result_node; \ + if (unk_feature_.get()) new_node->feature = unk_feature_.get(); \ + if (isPartial && !is_valid_node(lattice, new_node)) { continue; } \ + result_node = new_node; } } while (0) + +template +template +N *Tokenizer::lookup(const char *begin, const char *end, + Allocator *allocator, Lattice *lattice) const { + CharInfo cinfo; + N *result_node = 0; + size_t mblen = 0; + size_t clen = 0; + + end = static_cast(end - begin) >= 65535 ? begin + 65535 : end; + + if (isPartial) { + const size_t begin_pos = begin - lattice->sentence(); + for (size_t n = begin_pos + 1; n < lattice->size(); ++n) { + if (lattice->boundary_constraint(n) == MECAB_TOKEN_BOUNDARY) { + end = lattice->sentence() + n; + break; + } + } + } + + const char *begin2 = property_.seekToOtherType(begin, end, space_, + &cinfo, &mblen, &clen); + + Dictionary::result_type *daresults = allocator->mutable_results(); + const size_t results_size = allocator->results_size(); + + for (std::vector::const_iterator it = dic_.begin(); + it != dic_.end(); ++it) { + const size_t n = (*it)->commonPrefixSearch( + begin2, + static_cast(end - begin2), + daresults, results_size); + for (size_t i = 0; i < n; ++i) { + size_t size = (*it)->token_size(daresults[i]); + const Token *token = (*it)->token(daresults[i]); + for (size_t j = 0; j < size; ++j) { + N *new_node = allocator->newNode(); + read_node_info(**it, *(token + j), &new_node); + new_node->length = daresults[i].length; + new_node->rlength = begin2 - begin + new_node->length; + new_node->surface = begin2; + new_node->stat = MECAB_NOR_NODE; + new_node->char_type = cinfo.default_type; + if (isPartial && !is_valid_node(lattice, new_node)) { + continue; + } + new_node->bnext = result_node; + result_node = new_node; + } + } + } + + if (result_node && !cinfo.invoke) { + return result_node; + } + + const char *begin3 = begin2 + mblen; + const char *group_begin3 = 0; + + if (begin3 > end) { + ADDUNKNWON; + if (result_node) { + return result_node; + } + } + + if (cinfo.group) { + const char *tmp = begin3; + CharInfo fail; + begin3 = property_.seekToOtherType(begin3, end, cinfo, + &fail, &mblen, &clen); + if (clen <= max_grouping_size_) { + ADDUNKNWON; + } + group_begin3 = begin3; + begin3 = tmp; + } + + for (size_t i = 1; i <= cinfo.length; ++i) { + if (begin3 > end) { + break; + } + if (begin3 == group_begin3) { + continue; + } + clen = i; + ADDUNKNWON; + if (!cinfo.isKindOf(property_.getCharInfo(begin3, end, &mblen))) { + break; + } + begin3 += mblen; + } + + if (!result_node) { + ADDUNKNWON; + } + + if (isPartial && !result_node) { + begin3 = begin2; + while (true) { + cinfo = property_.getCharInfo(begin3, end, &mblen); + begin3 += mblen; + if (begin3 > end || + lattice->boundary_constraint(begin3 - lattice->sentence()) + != MECAB_INSIDE_TOKEN) { + break; + } + } + ADDUNKNWON; + + if (!result_node) { + N *new_node = allocator->newNode(); + new_node->char_type = cinfo.default_type; + new_node->surface = begin2; + new_node->length = begin3 - begin2; + new_node->rlength = begin3 - begin; + new_node->stat = MECAB_UNK_NODE; + new_node->bnext = result_node; + new_node->feature = + lattice->feature_constraint(begin - lattice->sentence()); + CHECK_DIE(new_node->feature); + result_node = new_node; + } + } + + return result_node; +} + +#undef ADDUNKNWON + +template +const DictionaryInfo *Tokenizer::dictionary_info() const { + return const_cast(dictionary_info_); +} + +template +void Tokenizer::close() { + for (std::vector::iterator it = dic_.begin(); + it != dic_.end(); ++it) { + delete *it; + } + dic_.clear(); + unk_tokens_.clear(); + property_.close(); +} +} From 1c9a7bedac527e65dce14fc21a94f3cb524fa1da Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:33 +0800 Subject: [PATCH 049/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/tokenizer.h | 134 ++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 third_party/mecab/src/tokenizer.h diff --git a/third_party/mecab/src/tokenizer.h b/third_party/mecab/src/tokenizer.h new file mode 100644 index 00000000..e1d6fb74 --- /dev/null +++ b/third_party/mecab/src/tokenizer.h @@ -0,0 +1,134 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_TOKENIZER_H_ +#define MECAB_TOKENIZER_H_ + +#include "mecab.h" +#include "freelist.h" +#include "dictionary.h" +#include "char_property.h" +#include "nbest_generator.h" +#include "scoped_ptr.h" + +namespace MeCab { + +class Param; +class NBestGenerator; + +template +class Allocator { + public: + N *newNode() { + N *node = node_freelist_->alloc(); + std::memset(node, 0, sizeof(N)); + node->id = id_++; + return node; + } + + P *newPath() { + if (!path_freelist_.get()) { + path_freelist_.reset(new FreeList

(PATH_FREELIST_SIZE)); + } + return path_freelist_->alloc(); + } + + Dictionary::result_type *mutable_results() { + return results_.get(); + } + + char *alloc(size_t size) { + if (!char_freelist_.get()) { + char_freelist_.reset(new ChunkFreeList(BUF_SIZE)); + } + return char_freelist_->alloc(size + 1); + } + + char *strdup(const char *str, size_t size) { + char *n = alloc(size + 1); + std::strncpy(n, str, size + 1); + return n; + } + + NBestGenerator *nbest_generator() { + if (!nbest_generator_.get()) { + nbest_generator_.reset(new NBestGenerator); + } + return nbest_generator_.get(); + } + + char *partial_buffer(size_t size) { + partial_buffer_.resize(size); + return &partial_buffer_[0]; + } + + size_t results_size() const { + return kResultsSize; + } + + void free() { + id_ = 0; + node_freelist_->free(); + if (path_freelist_.get()) { + path_freelist_->free(); + } + if (char_freelist_.get()) { + char_freelist_->free(); + } + } + + Allocator() + : id_(0), + node_freelist_(new FreeList(NODE_FREELIST_SIZE)), + path_freelist_(0), + char_freelist_(0), + nbest_generator_(0), + results_(new Dictionary::result_type[kResultsSize]) {} + virtual ~Allocator() {} + + private: + static const size_t kResultsSize = 512; + size_t id_; + scoped_ptr > node_freelist_; + scoped_ptr > path_freelist_; + scoped_ptr > char_freelist_; + scoped_ptr nbest_generator_; + std::vector partial_buffer_; + scoped_array results_; +}; + +template +class Tokenizer { + private: + std::vector dic_; + Dictionary unkdic_; + scoped_string bos_feature_; + scoped_string unk_feature_; + FreeList dictionary_info_freelist_; + std::vector > unk_tokens_; + DictionaryInfo *dictionary_info_; + CharInfo space_; + CharProperty property_; + size_t max_grouping_size_; + whatlog what_; + + public: + N *getBOSNode(Allocator *allocator) const; + N *getEOSNode(Allocator *allocator) const; + template N *lookup(const char *begin, const char *end, + Allocator *allocator, + Lattice *lattice) const; + bool open(const Param ¶m); + void close(); + + const DictionaryInfo *dictionary_info() const; + + const char *what() { return what_.str(); } + + explicit Tokenizer(); + virtual ~Tokenizer() { this->close(); } +}; +} +#endif // MECAB_TOKENIZER_H_ From 020f1feed064079ae1a89a4c70f51fb6c8a73cf2 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:35 +0800 Subject: [PATCH 050/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/ucs.h | 148 ++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 third_party/mecab/src/ucs.h diff --git a/third_party/mecab/src/ucs.h b/third_party/mecab/src/ucs.h new file mode 100644 index 00000000..8600161b --- /dev/null +++ b/third_party/mecab/src/ucs.h @@ -0,0 +1,148 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_UCS_H +#define MECAB_UCS_H + +#ifndef MECAB_USE_UTF8_ONLY +#include "ucstable.h" +#endif + +namespace MeCab { + +// All internal codes are represented in UCS2, +// if you want to use specific local codes, e.g, big5/euc-kr, +// make a function which maps the local code to the UCS code. + +inline unsigned short utf8_to_ucs2(const char *begin, const char *end, + size_t* mblen) { + const size_t len = end - begin; + + if (static_cast(begin[0]) < 0x80) { + *mblen = 1; + return static_cast(begin[0]); + + } else if (len >= 2 && (begin[0] & 0xe0) == 0xc0) { + *mblen = 2; + return((begin[0] & 0x1f) << 6) |(begin[1] & 0x3f); + + } else if (len >= 3 && (begin[0] & 0xf0) == 0xe0) { + *mblen = 3; + return ((begin[0] & 0x0f) << 12) | + ((begin[1] & 0x3f) << 6) |(begin[2] & 0x3f); + + /* belows are out of UCS2 */ + } else if (len >= 4 && (begin[0] & 0xf8) == 0xf0) { + *mblen = 4; + return 0; + + } else if (len >= 5 && (begin[0] & 0xfc) == 0xf8) { + *mblen = 5; + return 0; + + } else if (len >= 6 && (begin[0] & 0xfe) == 0xfc) { + *mblen = 6; + return 0; + + } else { + *mblen = 1; + return 0; + } +} + +inline unsigned short ascii_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + *mblen = 1; + return static_cast(begin[0]); +} + +inline unsigned short utf16be_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + if (len <= 1) { + *mblen = 1; + return 0; + } + *mblen = 2; +#if defined WORDS_BIGENDIAN + return (begin[0] << 8 | begin[1]); +#else + return (begin[1] << 8 | begin[0]); +#endif + return 0; +} + +inline unsigned short utf16le_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + if (len <= 1) { + *mblen = 1; + return 0; + } + *mblen = 2; +#if defined WORDS_BIGENDIAN + return (begin[1] << 8 | begin[0]); +#else + return (begin[0] << 8 | begin[1]); +#endif +} + +inline unsigned short utf16_to_ucs2(const char *begin, const char *end, + size_t *mblen) { +#if defined WORDS_BIGENDIAN + return utf16be_to_ucs2(begin, end, mblen); +#else + return utf16le_to_ucs2(begin, end, mblen); +#endif +} + + +#ifndef MECAB_USE_UTF8_ONLY +inline unsigned short euc_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + + // JISX 0212, 0213 + if (static_cast(begin[0]) == 0x8f && len >= 3) { + unsigned short key = (static_cast(begin[1]) << 8) + + static_cast(begin[2]); + if (key < 0xA0A0) { // offset violation + *mblen = 1; + return static_cast(begin[0]); + } + *mblen = 3; + return euc_hojo_tbl[ key - 0xA0A0 ]; + // JISX 0208 + 0201 + } else if ((static_cast(begin[0]) & 0x80) && len >= 2) { + *mblen = 2; + return euc_tbl[(static_cast(begin[0]) << 8) + + static_cast(begin[1]) ]; + } else { + *mblen = 1; + return static_cast(begin[0]); + } +} + +inline unsigned short cp932_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + + if ((static_cast(begin[0]) >= 0xA1 && + static_cast(begin[0]) <= 0xDF)) { + *mblen = 1; + return cp932_tbl[static_cast(begin[0]) ]; + } else if ((static_cast(begin[0]) & 0x80) && len >= 2) { + *mblen = 2; + return cp932_tbl[(static_cast(begin[0]) << 8) + + static_cast(begin[1]) ]; + } else { + *mblen = 1; + return static_cast(begin[0]); + } +} +#endif +} + +#endif From 54a3fa92422c9b7aacb363d992b80acfd0f094b1 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:36 +0800 Subject: [PATCH 051/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/utils.cpp | 564 ++++++++++++++++++++++++++++++++ 1 file changed, 564 insertions(+) create mode 100644 third_party/mecab/src/utils.cpp diff --git a/third_party/mecab/src/utils.cpp b/third_party/mecab/src/utils.cpp new file mode 100644 index 00000000..80fd613a --- /dev/null +++ b/third_party/mecab/src/utils.cpp @@ -0,0 +1,564 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#ifdef HAVE_DIRENT_H +#include +#endif + +#ifdef HAVE_WINDOWS_H +#define NOMINMAX +#include +#include +#endif + +#include + +#if defined(_WIN32) && !defined(__CYGWIN__) +extern HINSTANCE DllInstance; +#endif + +#include "common.h" +#include "mecab.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +#if defined(_WIN32) && !defined(__CYGWIN__) +std::wstring Utf8ToWide(const std::string &input) { + int output_length = ::MultiByteToWideChar(CP_UTF8, 0, + input.c_str(), -1, NULL, 0); + output_length = output_length <= 0 ? 0 : output_length - 1; + if (output_length == 0) { + return L""; + } + scoped_array input_wide(new wchar_t[output_length + 1]); + const int result = ::MultiByteToWideChar(CP_UTF8, 0, input.c_str(), -1, + input_wide.get(), output_length + 1); + std::wstring output; + if (result > 0) { + output.assign(input_wide.get()); + } + return output; +} + +std::string WideToUtf8(const std::wstring &input) { + const int output_length = ::WideCharToMultiByte(CP_UTF8, 0, + input.c_str(), -1, NULL, 0, + NULL, NULL); + if (output_length == 0) { + return ""; + } + + scoped_array input_encoded(new char[output_length + 1]); + const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1, + input_encoded.get(), + output_length + 1, NULL, NULL); + std::string output; + if (result > 0) { + output.assign(input_encoded.get()); + } + return output; +} +#endif + +int decode_charset(const char *charset) { + std::string tmp = charset; + toLower(&tmp); + if (tmp == "sjis" || tmp == "shift-jis" || + tmp == "shift_jis" || tmp == "cp932") + return CP932; + else if (tmp == "euc" || tmp == "euc_jp" || + tmp == "euc-jp") + return EUC_JP; + else if (tmp == "utf8" || tmp == "utf_8" || + tmp == "utf-8") + return UTF8; + else if (tmp == "utf16" || tmp == "utf_16" || + tmp == "utf-16") + return UTF16; + else if (tmp == "utf16be" || tmp == "utf_16be" || + tmp == "utf-16be") + return UTF16BE; + else if (tmp == "utf16le" || tmp == "utf_16le" || + tmp == "utf-16le") + return UTF16LE; + else if (tmp == "ascii") + return ASCII; + + return UTF8; // default is UTF8 +} + +std::string create_filename(const std::string &path, + const std::string &file) { + std::string s = path; +#if defined(_WIN32) && !defined(__CYGWIN__) + if (s.size() && s[s.size()-1] != '\\') s += '\\'; +#else + if (s.size() && s[s.size()-1] != '/') s += '/'; +#endif + s += file; + return s; +} + +void remove_filename(std::string *s) { + int len = static_cast(s->size()) - 1; + bool ok = false; + for (; len >= 0; --len) { +#if defined(_WIN32) && !defined(__CYGWIN__) + if ((*s)[len] == '\\') { + ok = true; + break; + } +#else + if ((*s)[len] == '/') { + ok = true; + break; + } +#endif + } + if (ok) + *s = s->substr(0, len); + else + *s = "."; +} + +void remove_pathname(std::string *s) { + int len = static_cast(s->size()) - 1; + bool ok = false; + for (; len >= 0; --len) { +#if defined(_WIN32) && !defined(__CYGWIN__) + if ((*s)[len] == '\\') { + ok = true; + break; + } +#else + if ((*s)[len] == '/') { + ok = true; + break; + } +#endif + } + if (ok) + *s = s->substr(len + 1, s->size() - len); + else + *s = "."; +} + +void replace_string(std::string *s, + const std::string &src, + const std::string &dst) { + const std::string::size_type pos = s->find(src); + if (pos != std::string::npos) { + s->replace(pos, src.size(), dst); + } +} + +void enum_csv_dictionaries(const char *path, + std::vector *dics) { + dics->clear(); + +#if defined(_WIN32) && !defined(__CYGWIN__) + WIN32_FIND_DATAW wfd; + HANDLE hFind; + const std::wstring pat = Utf8ToWide(create_filename(path, "*.csv")); + hFind = ::FindFirstFileW(pat.c_str(), &wfd); + CHECK_DIE(hFind != INVALID_HANDLE_VALUE) + << "Invalid File Handle. Get Last Error reports"; + do { + std::string tmp = create_filename(path, WideToUtf8(wfd.cFileName)); + dics->push_back(tmp); + } while (::FindNextFileW(hFind, &wfd)); + ::FindClose(hFind); +#else + DIR *dir = opendir(path); + CHECK_DIE(dir) << "no such directory: " << path; + + for (struct dirent *dp = readdir(dir); + dp; + dp = readdir(dir)) { + const std::string tmp = dp->d_name; + if (tmp.size() >= 5) { + std::string ext = tmp.substr(tmp.size() - 4, 4); + toLower(&ext); + if (ext == ".csv") { + dics->push_back(create_filename(path, tmp)); + } + } + } + closedir(dir); +#endif +} + +bool toLower(std::string *s) { + for (size_t i = 0; i < s->size(); ++i) { + char c = (*s)[i]; + if ((c >= 'A') && (c <= 'Z')) { + c += 'a' - 'A'; + (*s)[i] = c; + } + } + return true; +} + +bool escape_csv_element(std::string *w) { + if (w->find(',') != std::string::npos || + w->find('"') != std::string::npos) { + std::string tmp = "\""; + for (size_t j = 0; j < w->size(); j++) { + if ((*w)[j] == '"') tmp += '"'; + tmp += (*w)[j]; + } + tmp += '"'; + *w = tmp; + } + return true; +} + +int progress_bar(const char* message, size_t current, size_t total) { + static char bar[] = "###########################################"; + static int scale = sizeof(bar) - 1; + static int prev = 0; + + int cur_percentage = static_cast(100.0 * current/total); + int bar_len = static_cast(1.0 * current*scale/total); + + if (prev != cur_percentage) { + printf("%s: %3d%% |%.*s%*s| ", message, cur_percentage, + bar_len, bar, scale - bar_len, ""); + if (cur_percentage == 100) + printf("\n"); + else + printf("\r"); + fflush(stdout); + } + + prev = cur_percentage; + + return 1; +} + +int load_request_type(const Param ¶m) { + int request_type = MECAB_ONE_BEST; + + if (param.get("allocate-sentence")) { + request_type |= MECAB_ALLOCATE_SENTENCE; + } + + if (param.get("partial")) { + request_type |= MECAB_PARTIAL; + } + + if (param.get("all-morphs")) { + request_type |= MECAB_ALL_MORPHS; + } + + if (param.get("marginal")) { + request_type |= MECAB_MARGINAL_PROB; + } + + const int nbest = param.get("nbest"); + if (nbest >= 2) { + request_type |= MECAB_NBEST; + } + + // DEPRECATED: + const int lattice_level = param.get("lattice-level"); + if (lattice_level >= 1) { + request_type |= MECAB_NBEST; + } + + if (lattice_level >= 2) { + request_type |= MECAB_MARGINAL_PROB; + } + + return request_type; +} + +bool load_dictionary_resource(Param *param) { + std::string rcfile = param->get("rcfile"); + +#ifdef HAVE_GETENV + if (rcfile.empty()) { + const char *homedir = getenv("HOME"); + if (homedir) { + const std::string s = MeCab::create_filename(std::string(homedir), + ".mecabrc"); + std::ifstream ifs(WPATH(s.c_str())); + if (ifs) { + rcfile = s; + } + } + } + + if (rcfile.empty()) { + const char *rcenv = getenv("MECABRC"); + if (rcenv) { + rcfile = rcenv; + } + } +#endif + +#if defined (HAVE_GETENV) && defined(_WIN32) && !defined(__CYGWIN__) + if (rcfile.empty()) { + scoped_fixed_array buf; + const DWORD len = ::GetEnvironmentVariableW(L"MECABRC", + buf.get(), + buf.size()); + if (len < buf.size() && len > 0) { + rcfile = WideToUtf8(buf.get()); + } + } +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) + HKEY hKey; + scoped_fixed_array v; + DWORD vt; + DWORD size = v.size() * sizeof(v[0]); + DWORD qvres; + + if (rcfile.empty()) { + ::RegOpenKeyExW(HKEY_LOCAL_MACHINE, L"software\\mecab", 0, KEY_READ, &hKey); + qvres = ::RegQueryValueExW(hKey, L"mecabrc", 0, &vt, + reinterpret_cast(v.get()), &size); + ::RegCloseKey(hKey); + if (qvres == ERROR_SUCCESS && vt == REG_SZ) { + rcfile = WideToUtf8(v.get()); + } + } + + if (rcfile.empty()) { + ::RegOpenKeyExW(HKEY_CURRENT_USER, L"software\\mecab", 0, KEY_READ, &hKey); + qvres = ::RegQueryValueExW(hKey, L"mecabrc", 0, &vt, + reinterpret_cast(v.get()), &size); + ::RegCloseKey(hKey); + if (qvres == ERROR_SUCCESS && vt == REG_SZ) { + rcfile = WideToUtf8(v.get()); + } + } + + if (rcfile.empty()) { + vt = ::GetModuleFileNameW(DllInstance, v.get(), size); + if (vt != 0) { + scoped_fixed_array drive; + scoped_fixed_array dir; + _wsplitpath(v.get(), drive.get(), dir.get(), NULL, NULL); + const std::wstring path = + std::wstring(drive.get()) + std::wstring(dir.get()) + L"mecabrc"; + if (::GetFileAttributesW(path.c_str()) != -1) { + rcfile = WideToUtf8(path); + } + } + } +#endif + + if (rcfile.empty()) { + rcfile = MECAB_DEFAULT_RC; + } + + if (!param->load(rcfile.c_str())) { + return false; + } + + std::string dicdir = param->get("dicdir"); + if (dicdir.empty()) { + dicdir = "."; // current + } + remove_filename(&rcfile); + replace_string(&dicdir, "$(rcpath)", rcfile); + param->set("dicdir", dicdir, true); + dicdir = create_filename(dicdir, DICRC); + + if (!param->load(dicdir.c_str())) { + return false; + } + + return true; +} + +namespace { +// Copied from MurmurHash3.cpp +// http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp +//----------------------------------------------------------------------------- +// Platform-specific functions and macros +// Microsoft Visual Studio +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#define ROTL32(x,y) _rotl(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32 ( uint32_t x, uint8_t r ) { + return (x << r) | (x >> (32 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint32_t getblock ( const uint32_t * p, int i ) { + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint32_t fmix (uint32_t h) { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +void MurmurHash3_x86_128(const void * key, const int len, + uint32_t seed, char *out) { + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + uint32_t c1 = 0x239b961b; + uint32_t c2 = 0xab0e9789; + uint32_t c3 = 0x38b34ae5; + uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock(blocks,i*4+0); + uint32_t k2 = getblock(blocks,i*4+1); + uint32_t k3 = getblock(blocks,i*4+2); + uint32_t k4 = getblock(blocks,i*4+3); + + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + + h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; + + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; + + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; + + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch(len & 15) + { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = fmix(h1); + h2 = fmix(h2); + h3 = fmix(h3); + h4 = fmix(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + std::memcpy(out, reinterpret_cast(&h1), 4); + std::memcpy(out + 4, reinterpret_cast(&h2), 4); + std::memcpy(out + 8, reinterpret_cast(&h3), 4); + std::memcpy(out+ 12, reinterpret_cast(&h4), 4); +} +} + +uint64_t fingerprint(const char *str, size_t size) { + uint64_t result[2] = { 0 }; + const uint32_t kFingerPrint32Seed = 0xfd14deff; + MurmurHash3_x86_128(str, size, kFingerPrint32Seed, + reinterpret_cast(result)); + return result[0]; +} + +uint64_t fingerprint(const std::string &str) { + return fingerprint(str.data(), str.size()); +} + +bool file_exists(const char *filename) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + return false; + } + return true; +} +} // namespace MeCab From 7752955e618d8a4764a0566c0db3e1d5a93c0262 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:38 +0800 Subject: [PATCH 052/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/utils.h | 258 ++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 third_party/mecab/src/utils.h diff --git a/third_party/mecab/src/utils.h b/third_party/mecab/src/utils.h new file mode 100644 index 00000000..28ce2afd --- /dev/null +++ b/third_party/mecab/src/utils.h @@ -0,0 +1,258 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_UTILS_H +#define MECAB_UTILS_H + +#include +#include +#include +#include +#include +#include +#include "common.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_STDINT_H +#include +#else // HAVE_STDINT_H +#if defined(_WIN32) && !defined(__CYGWIN__) +#if defined(_MSC_VER) && (_MSC_VER <= 1500) +typedef unsigned char uint8_t; +typedef unsigned long uint32_t; +typedef unsigned long long uint64_t; +#else // _MSC_VER +#include +#endif // _MSC_VER +#else // _WIN32 +typedef unsigned char uint8_t; +typedef unsigned long uint32_t; +typedef unsigned __int64 uint64_t; +#endif // _WIN32 +#endif // HAVE_STDINT_H + +namespace MeCab { + +class Param; + +enum { EUC_JP, CP932, UTF8, UTF16, UTF16LE, UTF16BE, ASCII }; +int decode_charset(const char *charset); + +void inline dtoa(double val, char *s) { + std::sprintf(s, "%-16f", val); + char *p = s; + for (; *p != ' '; ++p) {} + *p = '\0'; + return; +} + +template +inline void itoa(T val, char *s) { + char *t; + T mod; + + if (val < 0) { + *s++ = '-'; + val = -val; + } + t = s; + + while (val) { + mod = val % 10; + *t++ = static_cast(mod) + '0'; + val /= 10; + } + + if (s == t) *t++ = '0'; + *t = '\0'; + std::reverse(s, t); + + return; +} + +template +inline void uitoa(T val, char *s) { + char *t; + T mod; + t = s; + while (val) { + mod = val % 10; + *t++ = static_cast(mod) + '0'; + val /= 10; + } + + if (s == t) *t++ = '0'; + *t = '\0'; + std::reverse(s, t); + return; +} + +inline const char *read_ptr(const char **ptr, size_t size) { + const char *r = *ptr; + *ptr += size; + return r; +} + +template +inline void read_static(const char **ptr, T& value) { + const char *r = read_ptr(ptr, sizeof(T)); + memcpy(&value, r, sizeof(T)); +} + +bool file_exists(const char *filename); + +int load_request_type(const Param ¶m); + +bool load_dictionary_resource(Param *); + +bool escape_csv_element(std::string *w); + +void enum_csv_dictionaries(const char *path, + std::vector *dics); + +int progress_bar(const char* message, size_t current, size_t total); + +bool toLower(std::string *); + +std::string create_filename(const std::string &path, + const std::string &file); +void remove_filename(std::string *s); +void remove_pathname(std::string *s); +void replace_string(std::string *s, + const std::string &src, + const std::string &dst); + +template +inline size_t tokenizeCSV(char *str, + Iterator out, size_t max) { + char *eos = str + std::strlen(str); + char *start = 0; + char *end = 0; + size_t n = 0; + + for (; str < eos; ++str) { + // skip white spaces + while (*str == ' ' || *str == '\t') ++str; + if (*str == '"') { + start = ++str; + end = start; + for (; str < eos; ++str) { + if (*str == '"') { + str++; + if (*str != '"') + break; + } + *end++ = *str; + } + str = std::find(str, eos, ','); + } else { + start = str; + str = std::find(str, eos, ','); + end = str; + } + if (max-- > 1) *end = '\0'; + *out++ = start; + ++n; + if (max == 0) break; + } + + return n; +} + +template +inline size_t tokenize(char *str, const char *del, + Iterator out, size_t max) { + char *stre = str + std::strlen(str); + const char *dele = del + std::strlen(del); + size_t size = 0; + + while (size < max) { + char *n = std::find_first_of(str, stre, del, dele); + *n = '\0'; + *out++ = str; + ++size; + if (n == stre) break; + str = n + 1; + } + + return size; +} + +// continus run of space is regarded as one space +template +inline size_t tokenize2(char *str, const char *del, + Iterator out, size_t max) { + char *stre = str + std::strlen(str); + const char *dele = del + std::strlen(del); + size_t size = 0; + + while (size < max) { + char *n = std::find_first_of(str, stre, del, dele); + *n = '\0'; + if (*str != '\0') { + *out++ = str; + ++size; + } + if (n == stre) break; + str = n + 1; + } + + return size; +} + +inline double logsumexp(double x, double y, bool flg) { +#define MINUS_LOG_EPSILON 50 + + if (flg) return y; // init mode + double vmin = std::min(x, y); + double vmax = std::max(x, y); + if (vmax > vmin + MINUS_LOG_EPSILON) { + return vmax; + } else { + return vmax + std::log(std::exp(vmin - vmax) + 1.0); + } +} + +inline short int tocost(double d, int n) { + static const short max = +32767; + static const short min = -32767; + return static_cast(std::max( + std::min( + -n * d, + static_cast(max)), + static_cast(min)) ); +} + +inline char getEscapedChar(const char p) { + switch (p) { + case '0': return '\0'; + case 'a': return '\a'; + case 'b': return '\b'; + case 't': return '\t'; + case 'n': return '\n'; + case 'v': return '\v'; + case 'f': return '\f'; + case 'r': return '\r'; + case 's': return ' '; + case '\\': return '\\'; + default: break; + } + + return '\0'; // never be here +} + +// return 64 bit hash +uint64_t fingerprint(const char *str, size_t size); +uint64_t fingerprint(const std::string &str); + +#if defined(_WIN32) && !defined(__CYGWIN__) +std::wstring Utf8ToWide(const std::string &input); +std::string WideToUtf8(const std::wstring &input); +#endif +} +#endif From e4d025431bd6d175b8c8023f5d1b886209b4695e Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:40 +0800 Subject: [PATCH 053/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/viterbi.cpp | 413 ++++++++++++++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 third_party/mecab/src/viterbi.cpp diff --git a/third_party/mecab/src/viterbi.cpp b/third_party/mecab/src/viterbi.cpp new file mode 100644 index 00000000..ca089a94 --- /dev/null +++ b/third_party/mecab/src/viterbi.cpp @@ -0,0 +1,413 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "connector.h" +#include "mecab.h" +#include "nbest_generator.h" +#include "param.h" +#include "viterbi.h" +#include "scoped_ptr.h" +#include "string_buffer.h" +#include "tokenizer.h" + +namespace MeCab { + +namespace { +void calc_alpha(Node *n, double beta) { + n->alpha = 0.0; + for (Path *path = n->lpath; path; path = path->lnext) { + n->alpha = logsumexp(n->alpha, + -beta * path->cost + path->lnode->alpha, + path == n->lpath); + } +} + +void calc_beta(Node *n, double beta) { + n->beta = 0.0; + for (Path *path = n->rpath; path; path = path->rnext) { + n->beta = logsumexp(n->beta, + -beta * path->cost + path->rnode->beta, + path == n->rpath); + } +} +} // namespace + +Viterbi::Viterbi() + : tokenizer_(0), connector_(0), + cost_factor_(0) {} + +Viterbi::~Viterbi() {} + +bool Viterbi::open(const Param ¶m) { + tokenizer_.reset(new Tokenizer); + CHECK_FALSE(tokenizer_->open(param)) << tokenizer_->what(); + CHECK_FALSE(tokenizer_->dictionary_info()) << "Dictionary is empty"; + + connector_.reset(new Connector); + CHECK_FALSE(connector_->open(param)) << connector_->what(); + + CHECK_FALSE(tokenizer_->dictionary_info()->lsize == + connector_->left_size() && + tokenizer_->dictionary_info()->rsize == + connector_->right_size()) + << "Transition table and dictionary are not compatible"; + + cost_factor_ = param.get("cost-factor"); + if (cost_factor_ == 0) { + cost_factor_ = 800; + } + + return true; +} + +bool Viterbi::analyze(Lattice *lattice) const { + if (!lattice || !lattice->sentence()) { + return false; + } + + if (!initPartial(lattice)) { + return false; + } + + bool result = false; + if (lattice->has_request_type(MECAB_NBEST) || + lattice->has_request_type(MECAB_MARGINAL_PROB)) { + // IsAllPath=true + if (lattice->has_constraint()) { + result = viterbi(lattice); + } else { + result = viterbi(lattice); + } + } else { + // IsAllPath=false + if (lattice->has_constraint()) { + result = viterbi(lattice); + } else { + result = viterbi(lattice); + } + } + + if (!result) { + return false; + } + + if (!forwardbackward(lattice)) { + return false; + } + + if (!buildBestLattice(lattice)) { + return false; + } + + if (!buildAllLattice(lattice)) { + return false; + } + + if (!initNBest(lattice)) { + return false; + } + + return true; +} + +const Tokenizer *Viterbi::tokenizer() const { + return tokenizer_.get(); +} + +const Connector *Viterbi::connector() const { + return connector_.get(); +} + +// static +bool Viterbi::forwardbackward(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_MARGINAL_PROB)) { + return true; + } + + Node **end_node_list = lattice->end_nodes(); + Node **begin_node_list = lattice->begin_nodes(); + + const size_t len = lattice->size(); + const double theta = lattice->theta(); + + end_node_list[0]->alpha = 0.0; + for (int pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + calc_alpha(node, theta); + } + } + + begin_node_list[len]->beta = 0.0; + for (int pos = static_cast(len); pos >= 0; --pos) { + for (Node *node = end_node_list[pos]; node; node = node->enext) { + calc_beta(node, theta); + } + } + + const double Z = begin_node_list[len]->alpha; + lattice->set_Z(Z); // alpha of EOS + + for (int pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + node->prob = std::exp(node->alpha + node->beta - Z); + for (Path *path = node->lpath; path; path = path->lnext) { + path->prob = std::exp(path->lnode->alpha + - theta * path->cost + + path->rnode->beta - Z); + } + } + } + + return true; +} + +// static +bool Viterbi::buildResultForNBest(Lattice *lattice) { + return buildAllLattice(lattice); +} + +// static +bool Viterbi::buildAllLattice(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_ALL_MORPHS)) { + return true; + } + + Node *prev = lattice->bos_node(); + const size_t len = lattice->size(); + Node **begin_node_list = lattice->begin_nodes(); + + for (long pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + prev->next = node; + node->prev = prev; + prev = node; + } + } + + return true; +} + +// static +bool Viterbi::buildAlternative(Lattice *lattice) { + Node **begin_node_list = lattice->begin_nodes(); + + const Node *bos_node = lattice->bos_node(); + for (const Node *node = bos_node; node; node = node->next) { + if (node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE) { + continue; + } + const size_t pos = node->surface - lattice->sentence() - + node->rlength + node->length; + std::cout.write(node->surface, node->length); + std::cout << "\t" << node->feature << std::endl; + for (const Node *anode = begin_node_list[pos]; + anode; anode = anode->bnext) { + if (anode->rlength == node->rlength && + anode->length == node->length) { + std::cout << "@ "; + std::cout.write(anode->surface, anode->length); + std::cout << "\t" << anode->feature << std::endl; + } + } + } + + std::cout << "EOS" << std::endl; + + return true; +} + +// static +bool Viterbi::buildBestLattice(Lattice *lattice) { + Node *node = lattice->eos_node(); + for (Node *prev_node; node->prev;) { + node->isbest = 1; + prev_node = node->prev; + prev_node->next = node; + node = prev_node; + } + + return true; +} + +// static +bool Viterbi::initNBest(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_NBEST)) { + return true; + } + lattice->allocator()->nbest_generator()->set(lattice); + return true; +} + +// static +bool Viterbi::initPartial(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_PARTIAL)) { + if (lattice->has_constraint()) { + lattice->set_boundary_constraint(0, MECAB_TOKEN_BOUNDARY); + lattice->set_boundary_constraint(lattice->size(), + MECAB_TOKEN_BOUNDARY); + } + return true; + } + + Allocator *allocator = lattice->allocator(); + char *str = allocator->partial_buffer(lattice->size() + 1); + strncpy(str, lattice->sentence(), lattice->size() + 1); + + std::vector lines; + const size_t lsize = tokenize(str, "\n", + std::back_inserter(lines), + lattice->size() + 1); + char* column[2]; + scoped_array buf(new char[lattice->size() + 1]); + StringBuffer os(buf.get(), lattice->size() + 1); + + std::vector > tokens; + tokens.reserve(lsize); + + size_t pos = 0; + for (size_t i = 0; i < lsize; ++i) { + const size_t size = tokenize(lines[i], "\t", column, 2); + if (size == 1 && std::strcmp(column[0], "EOS") == 0) { + break; + } + const size_t len = std::strlen(column[0]); + if (size == 2) { + tokens.push_back(std::make_pair(column[0], column[1])); + } else { + tokens.push_back(std::make_pair(column[0], reinterpret_cast(0))); + } + os << column[0]; + pos += len; + } + + os << '\0'; + + lattice->set_sentence(os.str()); + + pos = 0; + for (size_t i = 0; i < tokens.size(); ++i) { + const char *surface = tokens[i].first; + const char *feature = tokens[i].second; + const size_t len = std::strlen(surface); + lattice->set_boundary_constraint(pos, MECAB_TOKEN_BOUNDARY); + lattice->set_boundary_constraint(pos + len, MECAB_TOKEN_BOUNDARY); + if (feature) { + lattice->set_feature_constraint(pos, pos + len, feature); + for (size_t n = 1; n < len; ++n) { + lattice->set_boundary_constraint(pos + n, + MECAB_INSIDE_TOKEN); + } + } + pos += len; + } + + return true; +} + +namespace { +template bool connect(size_t pos, Node *rnode, + Node **begin_node_list, + Node **end_node_list, + const Connector *connector, + Allocator *allocator) { + for (;rnode; rnode = rnode->bnext) { + long best_cost = 2147483647; + Node* best_node = 0; + for (Node *lnode = end_node_list[pos]; lnode; lnode = lnode->enext) { + int lcost = connector->cost(lnode, rnode); // local cost + long cost = lnode->cost + lcost; + + if (cost < best_cost) { + best_node = lnode; + best_cost = cost; + } + + if (IsAllPath) { + Path *path = allocator->newPath(); + path->cost = lcost; + path->rnode = rnode; + path->lnode = lnode; + path->lnext = rnode->lpath; + rnode->lpath = path; + path->rnext = lnode->rpath; + lnode->rpath = path; + } + } + + // overflow check 2003/03/09 + if (!best_node) { + return false; + } + + rnode->prev = best_node; + rnode->next = 0; + rnode->cost = best_cost; + const size_t x = rnode->rlength + pos; + rnode->enext = end_node_list[x]; + end_node_list[x] = rnode; + } + + return true; +} +} // namespace + +template +bool Viterbi::viterbi(Lattice *lattice) const { + Node **end_node_list = lattice->end_nodes(); + Node **begin_node_list = lattice->begin_nodes(); + Allocator *allocator = lattice->allocator(); + const size_t len = lattice->size(); + const char *begin = lattice->sentence(); + const char *end = begin + len; + + Node *bos_node = tokenizer_->getBOSNode(lattice->allocator()); + bos_node->surface = lattice->sentence(); + end_node_list[0] = bos_node; + + for (size_t pos = 0; pos < len; ++pos) { + if (end_node_list[pos]) { + Node *right_node = tokenizer_->lookup(begin + pos, end, + allocator, lattice); + begin_node_list[pos] = right_node; + if (!connect(pos, right_node, + begin_node_list, + end_node_list, + connector_.get(), + allocator)) { + lattice->set_what("too long sentence."); + return false; + } + } + } + + Node *eos_node = tokenizer_->getEOSNode(lattice->allocator()); + eos_node->surface = lattice->sentence() + lattice->size(); + begin_node_list[lattice->size()] = eos_node; + + for (long pos = len; static_cast(pos) >= 0; --pos) { + if (end_node_list[pos]) { + if (!connect(pos, eos_node, + begin_node_list, + end_node_list, + connector_.get(), + allocator)) { + lattice->set_what("too long sentence."); + return false; + } + break; + } + } + + end_node_list[0] = bos_node; + begin_node_list[lattice->size()] = eos_node; + + return true; +} +} // Mecab From 938d03770fe1c2d5fdf1af77a2fbad3ff1e6ba67 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:41 +0800 Subject: [PATCH 054/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/viterbi.h | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 third_party/mecab/src/viterbi.h diff --git a/third_party/mecab/src/viterbi.h b/third_party/mecab/src/viterbi.h new file mode 100644 index 00000000..b40667f2 --- /dev/null +++ b/third_party/mecab/src/viterbi.h @@ -0,0 +1,53 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_VITERBI_H_ +#define MECAB_VITERBI_H_ + +#include +#include "mecab.h" +#include "thread.h" + +namespace MeCab { + +class Lattice; +class Param; +class Connector; +template class Tokenizer; + +class Viterbi { + public: + bool open(const Param ¶m); + + bool analyze(Lattice *lattice) const; + + const Tokenizer *tokenizer() const; + + const Connector *connector() const; + + const char *what() { return what_.str(); } + + static bool buildResultForNBest(Lattice *lattice); + + Viterbi(); + virtual ~Viterbi(); + + private: + template bool viterbi(Lattice *lattice) const; + + static bool forwardbackward(Lattice *lattice); + static bool initPartial(Lattice *lattice); + static bool initNBest(Lattice *lattice); + static bool buildBestLattice(Lattice *lattice); + static bool buildAllLattice(Lattice *lattice); + static bool buildAlternative(Lattice *lattice); + + scoped_ptr > tokenizer_; + scoped_ptr connector_; + int cost_factor_; + whatlog what_; +}; +} +#endif // MECAB_VITERBI_H_ From 1a26ae8226d3cd967854597fb03548433ff21332 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:43 +0800 Subject: [PATCH 055/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/winmain.h | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 third_party/mecab/src/winmain.h diff --git a/third_party/mecab/src/winmain.h b/third_party/mecab/src/winmain.h new file mode 100644 index 00000000..64ca02bc --- /dev/null +++ b/third_party/mecab/src/winmain.h @@ -0,0 +1,69 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#if defined(_WIN32) || defined(__CYGWIN__) + +#include +#include + +namespace { +class CommandLine { + public: + CommandLine(int argc, wchar_t **argv) : argc_(argc), argv_(0) { + argv_ = new char * [argc_]; + for (int i = 0; i < argc_; ++i) { + const std::string arg = WideToUtf8(argv[i]); + argv_[i] = new char[arg.size() + 1]; + ::memcpy(argv_[i], arg.data(), arg.size()); + argv_[i][arg.size()] = '\0'; + } + } + ~CommandLine() { + for (int i = 0; i < argc_; ++i) { + delete [] argv_[i]; + } + delete [] argv_; + } + + int argc() const { return argc_; } + char **argv() const { return argv_; } + + private: + static std::string WideToUtf8(const std::wstring &input) { + const int output_length = ::WideCharToMultiByte(CP_UTF8, 0, + input.c_str(), -1, NULL, 0, + NULL, NULL); + if (output_length == 0) { + return ""; + } + + char *input_encoded = new char[output_length + 1]; + const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1, + input_encoded, + output_length + 1, NULL, NULL); + std::string output; + if (result > 0) { + output.assign(input_encoded); + } + delete [] input_encoded; + return output; + } + + int argc_; + char **argv_; +}; +} // namespace + +#define main(argc, argv) wmain_to_main_wrapper(argc, argv) + +int wmain_to_main_wrapper(int argc, char **argv); + +#if defined(__MINGW32__) +extern "C" +#endif +int wmain(int argc, wchar_t **argv) { + CommandLine cmd(argc, argv); + return wmain_to_main_wrapper(cmd.argc(), cmd.argv()); +} +#endif From ea3546b173ea7b8af75f2550c72f6f2ee0a72344 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:44 +0800 Subject: [PATCH 056/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/writer.cpp | 412 +++++++++++++++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 third_party/mecab/src/writer.cpp diff --git a/third_party/mecab/src/writer.cpp b/third_party/mecab/src/writer.cpp new file mode 100644 index 00000000..14ea0178 --- /dev/null +++ b/third_party/mecab/src/writer.cpp @@ -0,0 +1,412 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "param.h" +#include "string_buffer.h" +#include "utils.h" +#include "writer.h" + +namespace MeCab { + +Writer::Writer() : write_(&Writer::writeLattice) {} +Writer::~Writer() {} + +void Writer::close() { + write_ = &Writer::writeLattice; +} + +bool Writer::open(const Param ¶m) { + const std::string ostyle = param.get("output-format-type"); + write_ = &Writer::writeLattice; + + if (ostyle == "wakati") { + write_ = &Writer::writeWakati; + } else if (ostyle == "none") { + write_ = &Writer::writeNone; + } else if (ostyle == "dump") { + write_ = &Writer::writeDump; + } else if (ostyle == "em") { + write_ = &Writer::writeEM; + } else { + // default values + std::string node_format = "%m\\t%H\\n"; + std::string unk_format = "%m\\t%H\\n"; + std::string bos_format = ""; + std::string eos_format = "EOS\\n"; + std::string eon_format = ""; + + std::string node_format_key = "node-format"; + std::string bos_format_key = "bos-format"; + std::string eos_format_key = "eos-format"; + std::string unk_format_key = "unk-format"; + std::string eon_format_key = "eon-format"; + + if (!ostyle.empty()) { + node_format_key += "-"; + node_format_key += ostyle; + bos_format_key += "-"; + bos_format_key += ostyle; + eos_format_key += "-"; + eos_format_key += ostyle; + unk_format_key += "-"; + unk_format_key += ostyle; + eon_format_key += "-"; + eon_format_key += ostyle; + const std::string tmp = param.get(node_format_key.c_str()); + CHECK_FALSE(!tmp.empty()) << "unknown format type [" << ostyle << "]"; + } + + const std::string node_format2 = + param.get(node_format_key.c_str()); + const std::string bos_format2 = + param.get(bos_format_key.c_str()); + const std::string eos_format2 = + param.get(eos_format_key.c_str()); + const std::string unk_format2 = + param.get(unk_format_key.c_str()); + const std::string eon_format2 = + param.get(eon_format_key.c_str()); + + if (node_format != node_format2 || bos_format != bos_format2 || + eos_format != eos_format2 || unk_format != unk_format2) { + write_ = &Writer::writeUser; + if (node_format != node_format2) { + node_format = node_format2; + } + if (bos_format != bos_format2) { + bos_format = bos_format2; + } + if (eos_format != eos_format2) { + eos_format = eos_format2; + } + if (unk_format != unk_format2) { + unk_format = unk_format2; + } else if (node_format != node_format2) { + unk_format = node_format2; + } else { + unk_format = node_format; + } + if (eon_format != eon_format2) { + eon_format = eon_format2; + } + node_format_.reset_string(node_format.c_str()); + bos_format_.reset_string(bos_format.c_str()); + eos_format_.reset_string(eos_format.c_str()); + unk_format_.reset_string(unk_format.c_str()); + eon_format_.reset_string(eon_format.c_str()); + } + } + + return true; +} + +bool Writer::write(Lattice *lattice, StringBuffer *os) const { + if (!lattice || !lattice->is_available()) { + return false; + } + return (this->*write_)(lattice, os); +} + +bool Writer::writeLattice(Lattice *lattice, StringBuffer *os) const { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature; // << '\t'; + *os << '\n'; + } + *os << "EOS\n"; + return true; +} + +bool Writer::writeWakati(Lattice *lattice, StringBuffer *os) const { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << ' '; + } + *os << '\n'; + return true; +} + +bool Writer::writeNone(Lattice *lattice, StringBuffer *os) const { + return true; // do nothing +} + +bool Writer::writeEM(Lattice *lattice, StringBuffer *os) const { + static const float min_prob = 0.0001; + for (const Node *node = lattice->bos_node(); node; node = node->next) { + if (node->prob >= min_prob) { + *os << "U\t"; + if (node->stat == MECAB_BOS_NODE) { + *os << "BOS"; + } else if (node->stat == MECAB_EOS_NODE) { + *os << "EOS"; + } else { + os->write(node->surface, node->length); + } + *os << '\t' << node->feature << '\t' << node->prob << '\n'; + } + for (const Path *path = node->lpath; path; path = path->lnext) { + if (path->prob >= min_prob) { + *os << "B\t" << path->lnode->feature << '\t' + << node->feature << '\t' << path->prob << '\n'; + } + } + } + *os << "EOS\n"; + return true; +} + +bool Writer::writeDump(Lattice *lattice, StringBuffer *os) const { + const char *str = lattice->sentence(); + for (const Node *node = lattice->bos_node(); node; node = node->next) { + *os << node->id << ' '; + if (node->stat == MECAB_BOS_NODE) { + *os << "BOS"; + } else if (node->stat == MECAB_EOS_NODE) { + *os << "EOS"; + } else { + os->write(node->surface, node->length); + } + + *os << ' ' << node->feature + << ' ' << static_cast(node->surface - str) + << ' ' << static_cast(node->surface - str + node->length) + << ' ' << node->rcAttr + << ' ' << node->lcAttr + << ' ' << node->posid + << ' ' << static_cast(node->char_type) + << ' ' << static_cast(node->stat) + << ' ' << static_cast(node->isbest) + << ' ' << node->alpha + << ' ' << node->beta + << ' ' << node->prob + << ' ' << node->cost; + + for (const Path *path = node->lpath; path; path = path->lnext) { + *os << ' ' << path->lnode->id << ':' << path->cost << ':' << path->prob; + } + *os << '\n'; + } + return true; +} + +bool Writer::writeUser(Lattice *lattice, StringBuffer *os) const { + if (!writeNode(lattice, bos_format_.get(), lattice->bos_node(), os)) { + return false; + } + const Node *node = 0; + for (node = lattice->bos_node()->next; node->next; node = node->next) { + const char *fmt = (node->stat == MECAB_UNK_NODE ? unk_format_.get() : + node_format_.get()); + if (!writeNode(lattice, fmt, node, os)) { + return false; + } + } + if (!writeNode(lattice, eos_format_.get(), node, os)) { + return false; + } + return true; +} + +bool Writer::writeNode(Lattice *lattice, const Node *node, + StringBuffer *os) const { + switch (node->stat) { + case MECAB_BOS_NODE: + return writeNode(lattice, bos_format_.get(), node, os); + case MECAB_EOS_NODE: + return writeNode(lattice, eos_format_.get(), node, os); + case MECAB_UNK_NODE: + return writeNode(lattice, unk_format_.get(), node, os); + case MECAB_NOR_NODE: + return writeNode(lattice, node_format_.get(), node, os); + case MECAB_EON_NODE: + return writeNode(lattice, eon_format_.get(), node, os); + } + return true; +} + +bool Writer::writeNode(Lattice *lattice, + const char *p, + const Node *node, + StringBuffer *os) const { + scoped_fixed_array buf; + scoped_fixed_array ptr; + size_t psize = 0; + + for (; *p; p++) { + switch (*p) { + default: *os << *p; break; + + case '\\': *os << getEscapedChar(*++p); break; + + case '%': { // macros + switch (*++p) { + default: { + const std::string error = "unknown meta char: " + *p; + lattice->set_what(error.c_str()); + return false; + } + // input sentence + case 'S': os->write(lattice->sentence(), lattice->size()); break; + // sentence length + case 'L': *os << lattice->size(); break; + // morph + case 'm': os->write(node->surface, node->length); break; + case 'M': os->write(reinterpret_cast + (node->surface - node->rlength + node->length), + node->rlength); + break; + case 'h': *os << node->posid; break; // Part-Of-Speech ID + case '%': *os << '%'; break; // % + case 'c': *os << static_cast(node->wcost); break; // word cost + case 'H': *os << node->feature; break; + case 't': *os << static_cast(node->char_type); break; + case 's': *os << static_cast(node->stat); break; + case 'P': *os << node->prob; break; + case 'p': { + switch (*++p) { + default: + lattice->set_what("[iseSCwcnblLh] is required after %p"); + return false; + case 'i': *os << node->id; break; // node id + case 'S': os->write(reinterpret_cast + (node->surface - + node->rlength + node->length), + node->rlength - node->length); + break; // space + // start position + case 's': *os << static_cast( + node->surface - lattice->sentence()); + break; + // end position + case 'e': *os << static_cast + (node->surface - lattice->sentence() + node->length); + break; + // connection cost + case 'C': *os << node->cost - + node->prev->cost - node->wcost; + break; + case 'w': *os << node->wcost; break; // word cost + case 'c': *os << node->cost; break; // best cost + case 'n': *os << (node->cost - node->prev->cost); break; + // node cost + // * if best path, otherwise ' ' + case 'b': *os << (node->isbest ? '*' : ' '); break; + case 'P': *os << node->prob; break; + case 'A': *os << node->alpha; break; + case 'B': *os << node->beta; break; + case 'l': *os << node->length; break; // length of morph + // length of morph including the spaces + case 'L': *os << node->rlength; break; + case 'h': { // Hidden Layer ID + switch (*++p) { + default: + lattice->set_what("lr is required after %ph"); + return false; + case 'l': *os << node->lcAttr; break; // current + case 'r': *os << node->rcAttr; break; // prev + } + } break; + + case 'p': { + char mode = *++p; + char sep = *++p; + if (sep == '\\') { + sep = getEscapedChar(*++p); + } + if (!node->lpath) { + lattice->set_what("no path information is available"); + return false; + } + for (Path *path = node->lpath; path; path = path->lnext) { + if (path != node->lpath) *os << sep; + switch (mode) { + case 'i': *os << path->lnode->id; break; + case 'c': *os << path->cost; break; + case 'P': *os << path->prob; break; + default: + lattice->set_what("[icP] is required after %pp"); + return false; + } + } + } break; + + } + } break; + + case 'F': + case 'f': { + if (node->feature[0] == '\0') { + lattice->set_what("no feature information available"); + return false; + } + if (!psize) { + std::strncpy(buf.get(), node->feature, buf.size()); + psize = tokenizeCSV(buf.get(), ptr.get(), ptr.size()); + } + + // separator + char separator = '\t'; // default separator + if (*p == 'F') { // change separator + if (*++p == '\\') { + separator = getEscapedChar(*++p); + } else { + separator = *p; + } + } + + if (*++p !='[') { + lattice->set_what("cannot find '['"); + return false; + } + size_t n = 0; + bool sep = false; + bool isfil = false; + p++; + + for (;; ++p) { + switch (*p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n +(*p - '0'); + break; + case ',': case ']': + if (n >= psize) { + lattice->set_what("given index is out of range"); + return false; + } + isfil = (ptr[n][0] != '*'); + if (isfil) { + if (sep) { + *os << separator; + } + *os << ptr[n]; + } + if (*p == ']') { + goto last; + } + sep = isfil; + n = 0; + break; + default: + lattice->set_what("cannot find ']'"); + return false; + } + } + } last: break; + } // end switch + } break; // end case '%' + } // end switch + } + + return true; +} +} From 36ba02a7b061554681995cf56eb020a43ac3eca8 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:46 +0800 Subject: [PATCH 057/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/mecab/src/writer.h | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 third_party/mecab/src/writer.h diff --git a/third_party/mecab/src/writer.h b/third_party/mecab/src/writer.h new file mode 100644 index 00000000..defde406 --- /dev/null +++ b/third_party/mecab/src/writer.h @@ -0,0 +1,57 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_WRITER_H_ +#define MECAB_WRITER_H_ + +#include +#include "common.h" +#include "mecab.h" +#include "utils.h" +#include "scoped_ptr.h" +#include "string_buffer.h" + +namespace MeCab { + +class Param; + +class Writer { + public: + Writer(); + virtual ~Writer(); + bool open(const Param ¶m); + void close(); + + bool writeNode(Lattice *lattice, + const char *format, + const Node *node, StringBuffer *s) const; + bool writeNode(Lattice *lattice, + const Node *node, + StringBuffer *s) const; + + bool write(Lattice *lattice, StringBuffer *node) const; + + const char *what() { return what_.str(); } + + private: + scoped_string node_format_; + scoped_string bos_format_; + scoped_string eos_format_; + scoped_string unk_format_; + scoped_string eon_format_; + whatlog what_; + + bool writeLattice(Lattice *lattice, StringBuffer *s) const; + bool writeWakati(Lattice *lattice, StringBuffer *s) const; + bool writeNone(Lattice *lattice, StringBuffer *s) const; + bool writeUser(Lattice *lattice, StringBuffer *s) const; + bool writeDump(Lattice *lattice, StringBuffer *s) const; + bool writeEM(Lattice *lattice, StringBuffer *s) const; + + bool (Writer::*write_)(Lattice *lattice, StringBuffer *s) const; +}; +} + +#endif // WRITER_H_ From 07fe5a8c781504033411abbd3dd15920084476e3 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:47 +0800 Subject: [PATCH 058/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/.gitignore | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 third_party/cppjieba/.gitignore diff --git a/third_party/cppjieba/.gitignore b/third_party/cppjieba/.gitignore new file mode 100644 index 00000000..4a1921d3 --- /dev/null +++ b/third_party/cppjieba/.gitignore @@ -0,0 +1,19 @@ +tags +*.demo +*swp +*.out +*.o +*.d +*.ut +log +main +lib*.a +*_demo +segdict* +prior.gbk +tmp +t.* +*.pid +build +Testing/Temporary/CTestCostData.txt +Testing/Temporary/LastTest.log From a8fa1d5eb6d3ee12c26bb0b8ff4d704ffbe094a8 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:49 +0800 Subject: [PATCH 059/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/CMakeLists.txt | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 third_party/cppjieba/CMakeLists.txt diff --git a/third_party/cppjieba/CMakeLists.txt b/third_party/cppjieba/CMakeLists.txt new file mode 100644 index 00000000..f7fbd231 --- /dev/null +++ b/third_party/cppjieba/CMakeLists.txt @@ -0,0 +1,47 @@ +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +PROJECT(CPPJIEBA) + +# Use vendored limonp +set(LIMONP_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/deps/limonp/include") +INCLUDE_DIRECTORIES("${LIMONP_INCLUDE_DIR}" + ${PROJECT_SOURCE_DIR}/include) + +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) +endif() +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +ADD_DEFINITIONS(-O3 -g) + +# Define a variable to check if this is the top-level project +if(NOT DEFINED CPPJIEBA_TOP_LEVEL_PROJECT) + if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(CPPJIEBA_TOP_LEVEL_PROJECT ON) + else() + set(CPPJIEBA_TOP_LEVEL_PROJECT OFF) + endif() +endif() + +if(NOT TARGET cppjieba) + add_library(cppjieba INTERFACE) + target_include_directories(cppjieba INTERFACE + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/deps/limonp/include + ) +endif() + +include(GNUInstallDirs) +install(DIRECTORY include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(DIRECTORY dict/ + DESTINATION ${CMAKE_INSTALL_DATADIR}/cppjieba/dict) + +if(CPPJIEBA_TOP_LEVEL_PROJECT) + ENABLE_TESTING() + + message(STATUS "MSVC value: ${MSVC}") + ADD_SUBDIRECTORY(test) + ADD_TEST(NAME ./test/test.run COMMAND ./test/test.run) + ADD_TEST(NAME ./load_test COMMAND ./load_test) +endif() From 1d234d8c572216b6fdd924b0eca49d138a973d64 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:51 +0800 Subject: [PATCH 060/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/LICENSE | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 third_party/cppjieba/LICENSE diff --git a/third_party/cppjieba/LICENSE b/third_party/cppjieba/LICENSE new file mode 100644 index 00000000..6308e939 --- /dev/null +++ b/third_party/cppjieba/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From c1fb15fd48f757170944fa362160d781dc28b499 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:52 +0800 Subject: [PATCH 061/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/VERSION | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 third_party/cppjieba/VERSION diff --git a/third_party/cppjieba/VERSION b/third_party/cppjieba/VERSION new file mode 100644 index 00000000..55a34b06 --- /dev/null +++ b/third_party/cppjieba/VERSION @@ -0,0 +1,2 @@ +v5.6.0 +git@github.com:yanyiwu/cppjieba.git From 13848748c0cdc80f88baff1b3b0c932869759c3c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:54 +0800 Subject: [PATCH 062/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/deps/limonp/.gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/.gitignore diff --git a/third_party/cppjieba/deps/limonp/.gitignore b/third_party/cppjieba/deps/limonp/.gitignore new file mode 100644 index 00000000..ad7223aa --- /dev/null +++ b/third_party/cppjieba/deps/limonp/.gitignore @@ -0,0 +1,9 @@ +*.o +*.ut +libcm.a +tags +*.d +build +t.cpp +a.out +*.swp From f26ca773acf1bb88ca0b732f3734b3eea9c2ff4d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:56 +0800 Subject: [PATCH 063/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/deps/limonp/.gitmodules | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 third_party/cppjieba/deps/limonp/.gitmodules diff --git a/third_party/cppjieba/deps/limonp/.gitmodules b/third_party/cppjieba/deps/limonp/.gitmodules new file mode 100644 index 00000000..e69de29b From b55aace7117b86299adc18e7d82af870d89cff75 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:57 +0800 Subject: [PATCH 064/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/deps/limonp/CHANGELOG.md | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/CHANGELOG.md diff --git a/third_party/cppjieba/deps/limonp/CHANGELOG.md b/third_party/cppjieba/deps/limonp/CHANGELOG.md new file mode 100644 index 00000000..17427cef --- /dev/null +++ b/third_party/cppjieba/deps/limonp/CHANGELOG.md @@ -0,0 +1,169 @@ +# CHANGELOG + +## v1.0.1 + ++ [CI] Update GitHub Actions configurations + - Add stale issues workflow + - Update checkout action from v2 to v4 + - Update macOS test environments (remove macOS-12, add macOS-15) ++ [dep] Update googletest to release-1.12.1 ++ [doc] Add build instructions to README.md + +## v1.0.0 + ++ rm thread pool demo ++ deleted: include/limonp/BlockingQueue.hpp ++ deleted: include/limonp/BoundedBlockingQueue.hpp ++ deleted: include/limonp/BoundedQueue.hpp ++ deleted: include/limonp/MutexLock.hpp ++ deleted: include/limonp/Thread.hpp ++ deleted: include/limonp/ThreadPool.hpp ++ deleted: test/unittest/TBlockingQueue.cpp ++ deleted: test/unittest/TBoundedQueue.cpp ++ deleted: test/unittest/TMutexLock.cpp ++ deleted: test/unittest/TThread.cpp ++ deleted: test/unittest/TThreadPool.cpp ++ rm FileLock ++ rm Md5.hpp + +## v0.9.0 + ++ [c++20] compatibility ++ [c++17] compatibility + +## v0.8.1 + ++ [CI] fix windows gtest thread link error ++ [submodule] rm test/googletest ++ [CMake] FetchContent googletest ++ [CMake] required 3.5 -> 3.14 + +## v0.8.0 + ++ [StringUtil] Fix windows assert typo ++ [CMake] find_package(Threads REQUIRED); target_link_libraries(... Threads::Threads) ++ [CMAKE][CI] windows: 2019,2022 ++ [CMAKE][CI] matrix.build_type[Release, Debug] ++ [unittest] disable #TMd5.cpp ++ [unittest] disable #TFileLock.cpp ++ [unittest] disable #TBoundedQueue.cpp ++ [unittest] disable #TMutexLock.cpp ++ [unittest] disable #TBlockingQueue.cpp ++ [unittest] disable #TThread.cpp ++ [unittest] disable #TThreadPool.cpp + +## v0.7.2 + ++ [CI] ubuntu version from 20 to 22, macos version from 12 to 14 ++ [test/unittes] uint->size_t ++ [googletest] v1.6.0->v1.10.0 ++ [CMake] version required 3.0 -> 3.5 + +## v0.7.1 + ++ [CMake] fix CMAKE_CXX_STANDARD passed from github/actions and [c++11, c++14] only + +## v0.7.0 + ++ [CI] Added os.macos and cpp_version=[c++98, c++03, c++11, c++14, c++17, c++20] ++ [git submodule] Added googletest-release-v1.6.0 + +## v0.6.7 + ++ Merged [pr35](https://github.com/yanyiwu/limonp/pull/35) ++ Merged [pr33](https://github.com/yanyiwu/limonp/pull/33) ++ Merged [pr32](https://github.com/yanyiwu/limonp/pull/32) + +## v0.6.6 + ++ Merged [pr-31 To be compatible with cpp17 and later, use lambda instead of std::not1 & std::bind2nd #31](https://github.com/yanyiwu/limonp/pull/31) + +## v0.6.5 + ++ Merged [pr-25 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/25) ++ Merged [pr-26 fix license for end of line sequence and remove useless signature](https://github.com/yanyiwu/limonp/pull/26) ++ Merged [pr-27 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/27) ++ Merged [pr-28 add a target to be ready to support installation](https://github.com/yanyiwu/limonp/pull/28) ++ Merged [pr-29 Installable by cmake](https://github.com/yanyiwu/limonp/pull/29) ++ Merged [pr-30 Replace localtime with localtime_s on Windows and localtime_r on Linux](https://github.com/yanyiwu/limonp/pull/30) + +## v0.6.4 + ++ merge [fixup gcc8 warnings](https://github.com/yanyiwu/gojieba/pull/70) + +## v0.6.3 + ++ remove compiler conplained macro + +## v0.6.2 + ++ merge [pr-18](https://github.com/yanyiwu/limonp/pull/18/files) + +## v0.6.1 + +add Specialized template for vector + +when it is `vector`, print like this: ["hello", "world"]; (special case) +when it is `vector`, print like this: [1, 10, 1000]; (common cases) + +## v0.6.0 + ++ remove Trim out of Split. + +## v0.5.6 + ++ fix hidden trouble. + +## v0.5.5 + ++ macro name LOG and CHECK in Logging.hpp is so easy to confict with other lib, so I have to rename them to XLOG and XCHECK for avoiding those macro name conflicts. + +## v0.5.4 + ++ add ForcePublic.hpp ++ Add Utf8ToUnicode32 and Unicode32ToUtf8 in StringUtil.hpp + +## v0.5.3 + ++ Fix incompatibility problem about 'time.h' in Windows. + +## v0.5.2 + ++ Fix incompatibility problem about `enum {INFO ...}` name conflicts in Windows . ++ So from this version begin: the compile flags: `-DLOGGING_LEVEL=WARNING` must be changed to `-DLOGGING_LEVEL=LL_WARNING` + +## v0.5.1 + ++ add `ThreadPool::Stop()` to wait util all the threads finished. +If Stop() has not been called, it will be called when the ThreadPool destructing. + +## v0.5.0 + ++ Reorganized directories: include/ -> include/limonp/ ... and so on. ++ Add `NewClosure` in Closure.hpp, 0~3 arguments have been supported. ++ Update ThreadPool, use `NewClosure` instead of `CreateTask` + +## v0.4.1 + ++ `CHECK(exp) << "log message"` supported; + +## v0.4.0 + ++ add test/demo.cc as example. ++ move `print` macro to StdExtension.hpp ++ BigChange: rewrite `log` module, use `LOG(INFO) << "xxx" ` instead `LogInfo` . ++ remove HandyMacro.hpp, add CHECK in Logging.hpp instead. + +## v0.3.0 + ++ remove 'MysqlClient.hpp', 'InitOnOff.hpp', 'CastFloat.hpp' ++ add 'Closure.hpp' ++ uniform code style + +## v0.2.0 + ++ `namespace limonp`, not `Limonp` . + +## v0.1.0 + ++ Basic functions From d467504862822c46ffb19a3b0fa963270918e439 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:40:58 +0800 Subject: [PATCH 065/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/deps/limonp/CMakeLists.txt | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/CMakeLists.txt diff --git a/third_party/cppjieba/deps/limonp/CMakeLists.txt b/third_party/cppjieba/deps/limonp/CMakeLists.txt new file mode 100644 index 00000000..77c4210d --- /dev/null +++ b/third_party/cppjieba/deps/limonp/CMakeLists.txt @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.14) + +PROJECT(limonp + LANGUAGES CXX) + +################ +# cmake config # +################ + +if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11) +endif() +message(STATUS "CMAKE_CXX_STANDARD is ${CMAKE_CXX_STANDARD}") + +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + + +############## +# dependency # +############## +include(GNUInstallDirs) + +########## +# target # +########## + +add_library(${PROJECT_NAME} INTERFACE) +add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) + +target_include_directories(${PROJECT_NAME} + INTERFACE + $ + $) + +######## +# test # +######## + +ENABLE_TESTING() + +ADD_SUBDIRECTORY(test) +ADD_TEST(NAME ./test/demo COMMAND ./test/demo) +ADD_TEST(NAME ./test/test.run COMMAND ./test/test.run) + +########### +# install # +########### + +include(GNUInstallDirs) + +install(TARGETS ${PROJECT_NAME} + EXPORT ${PROJECT_NAME}) + +install(EXPORT ${PROJECT_NAME} + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}/ + NAMESPACE ${PROJECT_NAME}:: + FILE ${PROJECT_NAME}-config.cmake) + +install(DIRECTORY include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) From 827d7f49a1c59431e5a79fef16ece59cc3f72022 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:00 +0800 Subject: [PATCH 066/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/deps/limonp/LICENSE | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/LICENSE diff --git a/third_party/cppjieba/deps/limonp/LICENSE b/third_party/cppjieba/deps/limonp/LICENSE new file mode 100644 index 00000000..6308e939 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 5624444f8e32862fd64b8e432b3605bea612893a Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:02 +0800 Subject: [PATCH 067/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../limonp/include/limonp/ArgvContext.hpp | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp b/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp new file mode 100644 index 00000000..ba3abe06 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp @@ -0,0 +1,70 @@ +/************************************ + * file enc : ascii + * author : wuyanyi09@gmail.com + ************************************/ + +#ifndef LIMONP_ARGV_FUNCTS_H +#define LIMONP_ARGV_FUNCTS_H + +#include +#include +#include "StringUtil.hpp" + +namespace limonp { + +using namespace std; + +class ArgvContext { + public : + ArgvContext(int argc, const char* const * argv) { + for(int i = 0; i < argc; i++) { + if(StartsWith(argv[i], "-")) { + if(i + 1 < argc && !StartsWith(argv[i + 1], "-")) { + mpss_[argv[i]] = argv[i+1]; + i++; + } else { + sset_.insert(argv[i]); + } + } else { + args_.push_back(argv[i]); + } + } + } + ~ArgvContext() { + } + + friend ostream& operator << (ostream& os, const ArgvContext& args); + string operator [](size_t i) const { + if(i < args_.size()) { + return args_[i]; + } + return ""; + } + string operator [](const string& key) const { + map::const_iterator it = mpss_.find(key); + if(it != mpss_.end()) { + return it->second; + } + return ""; + } + + bool HasKey(const string& key) const { + if(mpss_.find(key) != mpss_.end() || sset_.find(key) != sset_.end()) { + return true; + } + return false; + } + + private: + vector args_; + map mpss_; + set sset_; +}; // class ArgvContext + +inline ostream& operator << (ostream& os, const ArgvContext& args) { + return os< Date: Fri, 14 Aug 2026 03:41:03 +0800 Subject: [PATCH 068/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/Closure.hpp | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Closure.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Closure.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Closure.hpp new file mode 100644 index 00000000..c9d9dd49 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/Closure.hpp @@ -0,0 +1,206 @@ +#ifndef LIMONP_CLOSURE_HPP +#define LIMONP_CLOSURE_HPP + +namespace limonp { + +class ClosureInterface { + public: + virtual ~ClosureInterface() { + } + virtual void Run() = 0; +}; + +template +class Closure0: public ClosureInterface { + public: + Closure0(Funct fun) { + fun_ = fun; + } + virtual ~Closure0() { + } + virtual void Run() { + (*fun_)(); + } + private: + Funct fun_; +}; + +template +class Closure1: public ClosureInterface { + public: + Closure1(Funct fun, Arg1 arg1) { + fun_ = fun; + arg1_ = arg1; + } + virtual ~Closure1() { + } + virtual void Run() { + (*fun_)(arg1_); + } + private: + Funct fun_; + Arg1 arg1_; +}; + +template +class Closure2: public ClosureInterface { + public: + Closure2(Funct fun, Arg1 arg1, Arg2 arg2) { + fun_ = fun; + arg1_ = arg1; + arg2_ = arg2; + } + virtual ~Closure2() { + } + virtual void Run() { + (*fun_)(arg1_, arg2_); + } + private: + Funct fun_; + Arg1 arg1_; + Arg2 arg2_; +}; + +template +class Closure3: public ClosureInterface { + public: + Closure3(Funct fun, Arg1 arg1, Arg2 arg2, Arg3 arg3) { + fun_ = fun; + arg1_ = arg1; + arg2_ = arg2; + arg3_ = arg3; + } + virtual ~Closure3() { + } + virtual void Run() { + (*fun_)(arg1_, arg2_, arg3_); + } + private: + Funct fun_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; +}; + +template +class ObjClosure0: public ClosureInterface { + public: + ObjClosure0(Obj* p, Funct fun) { + p_ = p; + fun_ = fun; + } + virtual ~ObjClosure0() { + } + virtual void Run() { + (p_->*fun_)(); + } + private: + Obj* p_; + Funct fun_; +}; + +template +class ObjClosure1: public ClosureInterface { + public: + ObjClosure1(Obj* p, Funct fun, Arg1 arg1) { + p_ = p; + fun_ = fun; + arg1_ = arg1; + } + virtual ~ObjClosure1() { + } + virtual void Run() { + (p_->*fun_)(arg1_); + } + private: + Obj* p_; + Funct fun_; + Arg1 arg1_; +}; + +template +class ObjClosure2: public ClosureInterface { + public: + ObjClosure2(Obj* p, Funct fun, Arg1 arg1, Arg2 arg2) { + p_ = p; + fun_ = fun; + arg1_ = arg1; + arg2_ = arg2; + } + virtual ~ObjClosure2() { + } + virtual void Run() { + (p_->*fun_)(arg1_, arg2_); + } + private: + Obj* p_; + Funct fun_; + Arg1 arg1_; + Arg2 arg2_; +}; +template +class ObjClosure3: public ClosureInterface { + public: + ObjClosure3(Obj* p, Funct fun, Arg1 arg1, Arg2 arg2, Arg3 arg3) { + p_ = p; + fun_ = fun; + arg1_ = arg1; + arg2_ = arg2; + arg3_ = arg3; + } + virtual ~ObjClosure3() { + } + virtual void Run() { + (p_->*fun_)(arg1_, arg2_, arg3_); + } + private: + Obj* p_; + Funct fun_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; +}; + +template +ClosureInterface* NewClosure(R (*fun)()) { + return new Closure0(fun); +} + +template +ClosureInterface* NewClosure(R (*fun)(Arg1), Arg1 arg1) { + return new Closure1(fun, arg1); +} + +template +ClosureInterface* NewClosure(R (*fun)(Arg1, Arg2), Arg1 arg1, Arg2 arg2) { + return new Closure2(fun, arg1, arg2); +} + +template +ClosureInterface* NewClosure(R (*fun)(Arg1, Arg2, Arg3), Arg1 arg1, Arg2 arg2, Arg3 arg3) { + return new Closure3(fun, arg1, arg2, arg3); +} + +template +ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)()) { + return new ObjClosure0(obj, fun); +} + +template +ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1), Arg1 arg1) { + return new ObjClosure1(obj, fun, arg1); +} + +template +ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1, Arg2), Arg1 arg1, Arg2 arg2) { + return new ObjClosure2(obj, fun, arg1, arg2); +} + +template +ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1, Arg2, Arg3), Arg1 arg1, Arg2 arg2, Arg3 arg3) { + return new ObjClosure3(obj, fun, arg1, arg2, arg3); +} + +} // namespace limonp + +#endif // LIMONP_CLOSURE_HPP From 8af99fe931c4c1603e0cb113ceacbb44f0cc9cc3 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:05 +0800 Subject: [PATCH 069/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/Colors.hpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp new file mode 100644 index 00000000..04edd7eb --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp @@ -0,0 +1,31 @@ +#ifndef LIMONP_COLOR_PRINT_HPP +#define LIMONP_COLOR_PRINT_HPP + +#include +#include + +namespace limonp { + +using std::string; + +enum Color { + BLACK = 30, + RED, + GREEN, + YELLOW, + BLUE, + PURPLE +}; // enum Color + +static void ColorPrintln(enum Color color, const char * fmt, ...) { + va_list ap; + printf("\033[0;%dm", color); + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); + printf("\033[0m\n"); // if not \n , in some situation , the next lines will be set the same color unexpectedly +} + +} // namespace limonp + +#endif // LIMONP_COLOR_PRINT_HPP From 7c6e0e437ce28cf92a24b1e0109413a4d68e9918 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:07 +0800 Subject: [PATCH 070/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/Condition.hpp | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp new file mode 100644 index 00000000..656a61d7 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp @@ -0,0 +1,38 @@ +#ifndef LIMONP_CONDITION_HPP +#define LIMONP_CONDITION_HPP + +#include "MutexLock.hpp" + +namespace limonp { + +class Condition : NonCopyable { + public: + explicit Condition(MutexLock& mutex) + : mutex_(mutex) { + XCHECK(!pthread_cond_init(&pcond_, NULL)); + } + + ~Condition() { + XCHECK(!pthread_cond_destroy(&pcond_)); + } + + void Wait() { + XCHECK(!pthread_cond_wait(&pcond_, mutex_.GetPthreadMutex())); + } + + void Notify() { + XCHECK(!pthread_cond_signal(&pcond_)); + } + + void NotifyAll() { + XCHECK(!pthread_cond_broadcast(&pcond_)); + } + + private: + MutexLock& mutex_; + pthread_cond_t pcond_; +}; // class Condition + +} // namespace limonp + +#endif // LIMONP_CONDITION_HPP From 3c20751f26ad575712bef82db09b1a8e4320be23 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:08 +0800 Subject: [PATCH 071/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/Config.hpp | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Config.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp new file mode 100644 index 00000000..c98f2227 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp @@ -0,0 +1,103 @@ +/************************************ + * file enc : utf8 + * author : wuyanyi09@gmail.com + ************************************/ +#ifndef LIMONP_CONFIG_H +#define LIMONP_CONFIG_H + +#include +#include +#include +#include +#include "StringUtil.hpp" + +namespace limonp { + +using namespace std; + +class Config { + public: + explicit Config(const string& filePath) { + LoadFile(filePath); + } + + operator bool () { + return !map_.empty(); + } + + string Get(const string& key, const string& defaultvalue) const { + map::const_iterator it = map_.find(key); + if(map_.end() != it) { + return it->second; + } + return defaultvalue; + } + int Get(const string& key, int defaultvalue) const { + string str = Get(key, ""); + if("" == str) { + return defaultvalue; + } + return atoi(str.c_str()); + } + const char* operator [] (const char* key) const { + if(NULL == key) { + return NULL; + } + map::const_iterator it = map_.find(key); + if(map_.end() != it) { + return it->second.c_str(); + } + return NULL; + } + + string GetConfigInfo() const { + string res; + res << *this; + return res; + } + + private: + void LoadFile(const string& filePath) { + ifstream ifs(filePath.c_str()); + assert(ifs); + string line; + vector vecBuf; + size_t lineno = 0; + while(getline(ifs, line)) { + lineno ++; + Trim(line); + if(line.empty() || StartsWith(line, "#")) { + continue; + } + vecBuf.clear(); + Split(line, vecBuf, "="); + if(2 != vecBuf.size()) { + fprintf(stderr, "line[%s] illegal.\n", line.c_str()); + assert(false); + continue; + } + string& key = vecBuf[0]; + string& value = vecBuf[1]; + Trim(key); + Trim(value); + if(!map_.insert(make_pair(key, value)).second) { + fprintf(stderr, "key[%s] already exits.\n", key.c_str()); + assert(false); + continue; + } + } + ifs.close(); + } + + friend ostream& operator << (ostream& os, const Config& config); + + map map_; +}; // class Config + +inline ostream& operator << (ostream& os, const Config& config) { + return os << config.map_; +} + +} // namespace limonp + +#endif // LIMONP_CONFIG_H From 62f7e317a3afe52380f6e3c7359b92bd1e2e397e Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:11 +0800 Subject: [PATCH 072/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/deps/limonp/include/limonp/ForcePublic.hpp | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp b/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp new file mode 100644 index 00000000..20766820 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp @@ -0,0 +1,7 @@ +#ifndef LIMONP_FORCE_PUBLIC_H +#define LIMONP_FORCE_PUBLIC_H + +#define private public +#define protected public + +#endif // LIMONP_FORCE_PUBLIC_H From c621decdaf94b90d3e4728c33b8c83aa0e0351d1 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:13 +0800 Subject: [PATCH 073/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../limonp/include/limonp/LocalVector.hpp | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp b/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp new file mode 100644 index 00000000..11339cc8 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp @@ -0,0 +1,139 @@ +#ifndef LIMONP_LOCAL_VECTOR_HPP +#define LIMONP_LOCAL_VECTOR_HPP + +#include +#include +#include +#include + +namespace limonp { +using namespace std; +/* + * LocalVector : T must be primitive type (char , int, size_t), if T is struct or class, LocalVector may be dangerous.. + * LocalVector is simple and not well-tested. + */ +const size_t LOCAL_VECTOR_BUFFER_SIZE = 16; +template +class LocalVector { + public: + typedef const T* const_iterator ; + typedef T value_type; + typedef size_t size_type; + private: + T buffer_[LOCAL_VECTOR_BUFFER_SIZE]; + T * ptr_; + size_t size_; + size_t capacity_; + public: + LocalVector() { + init_(); + }; + LocalVector(const LocalVector& vec) { + init_(); + *this = vec; + } + LocalVector(const_iterator begin, const_iterator end) { // TODO: make it faster + init_(); + while(begin != end) { + push_back(*begin++); + } + } + LocalVector(size_t size, const T& t) { // TODO: make it faster + init_(); + while(size--) { + push_back(t); + } + } + ~LocalVector() { + if(ptr_ != buffer_) { + free(ptr_); + } + }; + public: + LocalVector& operator = (const LocalVector& vec) { + clear(); + size_ = vec.size(); + capacity_ = vec.capacity(); + if(vec.buffer_ == vec.ptr_) { + memcpy(static_cast(buffer_), vec.buffer_, sizeof(T) * size_); + ptr_ = buffer_; + } else { + ptr_ = (T*) malloc(vec.capacity() * sizeof(T)); + assert(ptr_); + memcpy(static_cast(ptr_), vec.ptr_, vec.size() * sizeof(T)); + } + return *this; + } + private: + void init_() { + ptr_ = buffer_; + size_ = 0; + capacity_ = LOCAL_VECTOR_BUFFER_SIZE; + } + public: + T& operator [] (size_t i) { + return ptr_[i]; + } + const T& operator [] (size_t i) const { + return ptr_[i]; + } + void push_back(const T& t) { + if(size_ == capacity_) { + assert(capacity_); + reserve(capacity_ * 2); + } + ptr_[size_ ++ ] = t; + } + void reserve(size_t size) { + if(size <= capacity_) { + return; + } + T * next = (T*)malloc(sizeof(T) * size); + assert(next); + T * old = ptr_; + ptr_ = next; + memcpy(static_cast(ptr_), old, sizeof(T) * capacity_); + capacity_ = size; + if(old != buffer_) { + free(old); + } + } + bool empty() const { + return 0 == size(); + } + size_t size() const { + return size_; + } + size_t capacity() const { + return capacity_; + } + const_iterator begin() const { + return ptr_; + } + const_iterator end() const { + return ptr_ + size_; + } + void clear() { + if(ptr_ != buffer_) { + free(ptr_); + } + init_(); + } +}; + +template +ostream & operator << (ostream& os, const LocalVector& vec) { + if(vec.empty()) { + return os << "[]"; + } + os<<"[\""< Date: Fri, 14 Aug 2026 03:41:16 +0800 Subject: [PATCH 074/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/Logging.hpp | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Logging.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Logging.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Logging.hpp new file mode 100644 index 00000000..beb5a7e2 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/Logging.hpp @@ -0,0 +1,91 @@ +#ifndef LIMONP_LOGGING_HPP +#define LIMONP_LOGGING_HPP + +#include +#include +#include +#include +#include + +#ifdef XLOG +#error "XLOG has been defined already" +#endif // XLOG +#ifdef XCHECK +#error "XCHECK has been defined already" +#endif // XCHECK + +#define XLOG(level) limonp::Logger(limonp::LL_##level, __FILE__, __LINE__).Stream() +#define XCHECK(exp) if(!(exp)) XLOG(FATAL) << "exp: ["#exp << "] false. " + +namespace limonp { + +enum { + LL_DEBUG = 0, + LL_INFO = 1, + LL_WARNING = 2, + LL_ERROR = 3, + LL_FATAL = 4, +}; // enum + +static const char * LOG_LEVEL_ARRAY[] = {"DEBUG","INFO","WARN","ERROR","FATAL"}; +static const char * LOG_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"; + +class Logger { + public: + Logger(size_t level, const char* filename, int lineno) + : level_(level) { +#ifdef LOGGING_LEVEL + if (level_ < LOGGING_LEVEL) { + return; + } +#endif + assert(level_ <= sizeof(LOG_LEVEL_ARRAY)/sizeof(*LOG_LEVEL_ARRAY)); + + char buf[32]; + + time_t timeNow; + time(&timeNow); + + struct tm tmNow; + + #if defined(_WIN32) || defined(_WIN64) + errno_t e = localtime_s(&tmNow, &timeNow); + assert(e == 0); + #else + struct tm * tm_tmp = localtime_r(&timeNow, &tmNow); + (void)tm_tmp; + assert(tm_tmp != nullptr); + #endif + + strftime(buf, sizeof(buf), LOG_TIME_FORMAT, &tmNow); + + stream_ << buf + << " " << filename + << ":" << lineno + << " " << LOG_LEVEL_ARRAY[level_] + << " "; + } + ~Logger() { +#ifdef LOGGING_LEVEL + if (level_ < LOGGING_LEVEL) { + return; + } +#endif + std::cerr << stream_.str() << std::endl; + if (level_ == LL_FATAL) { + abort(); + } + } + + std::ostream& Stream() { + return stream_; + } + + private: + std::ostringstream stream_; + size_t level_; +}; // class Logger + +} // namespace limonp + +#endif // LIMONP_LOGGING_HPP From 0a76d638cbc204e92c84db61296f7f521369b67f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:18 +0800 Subject: [PATCH 075/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../limonp/include/limonp/NonCopyable.hpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp b/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp new file mode 100644 index 00000000..145400f4 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp @@ -0,0 +1,21 @@ +/************************************ + ************************************/ +#ifndef LIMONP_NONCOPYABLE_H +#define LIMONP_NONCOPYABLE_H + +namespace limonp { + +class NonCopyable { + protected: + NonCopyable() { + } + ~NonCopyable() { + } + private: + NonCopyable(const NonCopyable& ); + const NonCopyable& operator=(const NonCopyable& ); +}; // class NonCopyable + +} // namespace limonp + +#endif // LIMONP_NONCOPYABLE_H From cd0fcf5bb2a1559829e852aafaf8cea2817535e3 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:20 +0800 Subject: [PATCH 076/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../limonp/include/limonp/StdExtension.hpp | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp b/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp new file mode 100644 index 00000000..cf00e941 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp @@ -0,0 +1,157 @@ +#ifndef LIMONP_STD_EXTEMSION_HPP +#define LIMONP_STD_EXTEMSION_HPP + +#include + +#ifdef __APPLE__ +#include +#include +#elif(__cplusplus >= 201103L) +#include +#include +#elif defined _MSC_VER +#include +#include +#else +#include +#include +namespace std { +using std::tr1::unordered_map; +using std::tr1::unordered_set; +} + +#endif + +#include +#include +#include +#include +#include +#include + +namespace std { + +template +ostream& operator << (ostream& os, const vector& v) { + if(v.empty()) { + return os << "[]"; + } + os<<"["< +inline ostream& operator << (ostream& os, const vector& v) { + if(v.empty()) { + return os << "[]"; + } + os<<"[\""< +ostream& operator << (ostream& os, const deque& dq) { + if(dq.empty()) { + return os << "[]"; + } + os<<"[\""< +ostream& operator << (ostream& os, const pair& pr) { + os << pr.first << ":" << pr.second ; + return os; +} + + +template +string& operator << (string& str, const T& obj) { + stringstream ss; + ss << obj; // call ostream& operator << (ostream& os, + return str = ss.str(); +} + +template +ostream& operator << (ostream& os, const map& mp) { + if(mp.empty()) { + os<<"{}"; + return os; + } + os<<'{'; + typename map::const_iterator it = mp.begin(); + os<<*it; + it++; + while(it != mp.end()) { + os<<", "<<*it; + it++; + } + os<<'}'; + return os; +} +template +ostream& operator << (ostream& os, const std::unordered_map& mp) { + if(mp.empty()) { + return os << "{}"; + } + os<<'{'; + typename std::unordered_map::const_iterator it = mp.begin(); + os<<*it; + it++; + while(it != mp.end()) { + os<<", "<<*it++; + } + return os<<'}'; +} + +template +ostream& operator << (ostream& os, const set& st) { + if(st.empty()) { + os << "{}"; + return os; + } + os<<'{'; + typename set::const_iterator it = st.begin(); + os<<*it; + it++; + while(it != st.end()) { + os<<", "<<*it; + it++; + } + os<<'}'; + return os; +} + +template +bool IsIn(const ContainType& contain, const KeyType& key) { + return contain.end() != contain.find(key); +} + +template +basic_string & operator << (basic_string & s, ifstream & ifs) { + return s.assign((istreambuf_iterator(ifs)), istreambuf_iterator()); +} + +template +ofstream & operator << (ofstream & ofs, const basic_string& s) { + ostreambuf_iterator itr (ofs); + copy(s.begin(), s.end(), itr); + return ofs; +} + +} // namespace std + +#endif From 4a552d68ce70cf2c5eec125ef5de7dd4d1095855 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:22 +0800 Subject: [PATCH 077/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../deps/limonp/include/limonp/StringUtil.hpp | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp diff --git a/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp b/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp new file mode 100644 index 00000000..11869b42 --- /dev/null +++ b/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp @@ -0,0 +1,367 @@ +/************************************ + * file enc : ascii + * author : wuyanyi09@gmail.com + ************************************/ +#ifndef LIMONP_STR_FUNCTS_H +#define LIMONP_STR_FUNCTS_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "StdExtension.hpp" + +namespace limonp { +using namespace std; + +template +void Join(T begin, T end, string& res, const string& connector) { + if(begin == end) { + return; + } + stringstream ss; + ss<<*begin; + begin++; + while(begin != end) { + ss << connector << *begin; + begin ++; + } + res = ss.str(); +} + +template +string Join(T begin, T end, const string& connector) { + string res; + Join(begin ,end, res, connector); + return res; +} + +inline string& Upper(string& str) { + transform(str.begin(), str.end(), str.begin(), (int (*)(int))toupper); + return str; +} + +inline string& Lower(string& str) { + transform(str.begin(), str.end(), str.begin(), (int (*)(int))tolower); + return str; +} + +inline bool IsSpace(unsigned c) { + // when passing large int as the argument of isspace, it core dump, so here need a type cast. + return c > 0xff ? false : std::isspace(c & 0xff) != 0; +} + +inline std::string& LTrim(std::string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { + return !std::isspace(ch); + })); + return s; +} + +inline std::string& RTrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { + return !std::isspace(ch); + }).base(), s.end()); + return s; +} + +inline std::string& Trim(std::string &s) { + return LTrim(RTrim(s)); +} + +inline std::string& LTrim(std::string& s, char x) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), + [x](unsigned char c) { return !std::isspace(c) && c != x; })); + return s; +} + +inline std::string& RTrim(std::string& s, char x) { + s.erase(std::find_if(s.rbegin(), s.rend(), + [x](unsigned char c) { return !std::isspace(c) && c != x; }).base(), s.end()); + return s; +} + +inline std::string& Trim(std::string &s, char x) { + return LTrim(RTrim(s, x), x); +} + +inline void Split(const string& src, vector& res, const string& pattern, size_t maxsplit = string::npos) { + res.clear(); + size_t Start = 0; + size_t end = 0; + string sub; + while(Start < src.size()) { + end = src.find_first_of(pattern, Start); + if(string::npos == end || res.size() >= maxsplit) { + sub = src.substr(Start); + res.push_back(sub); + return; + } + sub = src.substr(Start, end - Start); + res.push_back(sub); + Start = end + 1; + } + return; +} + +inline vector Split(const string& src, const string& pattern, size_t maxsplit = string::npos) { + vector res; + Split(src, res, pattern, maxsplit); + return res; +} + +inline bool StartsWith(const string& str, const string& prefix) { + if(prefix.length() > str.length()) { + return false; + } + return 0 == str.compare(0, prefix.length(), prefix); +} + +inline bool EndsWith(const string& str, const string& suffix) { + if(suffix.length() > str.length()) { + return false; + } + return 0 == str.compare(str.length() - suffix.length(), suffix.length(), suffix); +} + +inline bool IsInStr(const string& str, char ch) { + return str.find(ch) != string::npos; +} + +inline uint16_t TwocharToUint16(char high, char low) { + return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff)); +} + +template +bool Utf8ToUnicode(const char * const str, size_t len, Uint16Container& vec) { + if(!str) { + return false; + } + char ch1, ch2; + uint16_t tmp; + vec.clear(); + for(size_t i = 0; i < len;) { + if(!(str[i] & 0x80)) { // 0xxxxxxx + vec.push_back(str[i]); + i++; + } else if ((uint8_t)str[i] <= 0xdf && i + 1 < len) { // 110xxxxxx + ch1 = (str[i] >> 2) & 0x07; + ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 ); + tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff)); + vec.push_back(tmp); + i += 2; + } else if((uint8_t)str[i] <= 0xef && i + 2 < len) { + ch1 = ((uint8_t)str[i] << 4) | ((str[i+1] >> 2) & 0x0f ); + ch2 = (((uint8_t)str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f); + tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff)); + vec.push_back(tmp); + i += 3; + } else { + return false; + } + } + return true; +} + +template +bool Utf8ToUnicode(const string& str, Uint16Container& vec) { + return Utf8ToUnicode(str.c_str(), str.size(), vec); +} + +template +bool Utf8ToUnicode32(const string& str, Uint32Container& vec) { + uint32_t tmp; + vec.clear(); + for(size_t i = 0; i < str.size();) { + if(!(str[i] & 0x80)) { // 0xxxxxxx + // 7bit, total 7bit + tmp = (uint8_t)(str[i]) & 0x7f; + i++; + } else if ((uint8_t)str[i] <= 0xdf && i + 1 < str.size()) { // 110xxxxxx + // 5bit, total 5bit + tmp = (uint8_t)(str[i]) & 0x1f; + + // 6bit, total 11bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+1]) & 0x3f; + i += 2; + } else if((uint8_t)str[i] <= 0xef && i + 2 < str.size()) { // 1110xxxxxx + // 4bit, total 4bit + tmp = (uint8_t)(str[i]) & 0x0f; + + // 6bit, total 10bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+1]) & 0x3f; + + // 6bit, total 16bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+2]) & 0x3f; + + i += 3; + } else if((uint8_t)str[i] <= 0xf7 && i + 3 < str.size()) { // 11110xxxx + // 3bit, total 3bit + tmp = (uint8_t)(str[i]) & 0x07; + + // 6bit, total 9bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+1]) & 0x3f; + + // 6bit, total 15bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+2]) & 0x3f; + + // 6bit, total 21bit + tmp <<= 6; + tmp |= (uint8_t)(str[i+3]) & 0x3f; + + i += 4; + } else { + return false; + } + vec.push_back(tmp); + } + return true; +} + +template +void Unicode32ToUtf8(Uint32ContainerConIter begin, Uint32ContainerConIter end, string& res) { + res.clear(); + uint32_t ui; + while(begin != end) { + ui = *begin; + if(ui <= 0x7f) { + res += char(ui); + } else if(ui <= 0x7ff) { + res += char(((ui >> 6) & 0x1f) | 0xc0); + res += char((ui & 0x3f) | 0x80); + } else if(ui <= 0xffff) { + res += char(((ui >> 12) & 0x0f) | 0xe0); + res += char(((ui >> 6) & 0x3f) | 0x80); + res += char((ui & 0x3f) | 0x80); + } else { + res += char(((ui >> 18) & 0x03) | 0xf0); + res += char(((ui >> 12) & 0x3f) | 0x80); + res += char(((ui >> 6) & 0x3f) | 0x80); + res += char((ui & 0x3f) | 0x80); + } + begin ++; + } +} + +template +void UnicodeToUtf8(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) { + res.clear(); + uint16_t ui; + while(begin != end) { + ui = *begin; + if(ui <= 0x7f) { + res += char(ui); + } else if(ui <= 0x7ff) { + res += char(((ui>>6) & 0x1f) | 0xc0); + res += char((ui & 0x3f) | 0x80); + } else { + res += char(((ui >> 12) & 0x0f )| 0xe0); + res += char(((ui>>6) & 0x3f )| 0x80 ); + res += char((ui & 0x3f) | 0x80); + } + begin ++; + } +} + + +template +bool GBKTrans(const char* const str, size_t len, Uint16Container& vec) { + vec.clear(); + if(!str) { + return true; + } + size_t i = 0; + while(i < len) { + if(0 == (str[i] & 0x80)) { + vec.push_back(uint16_t(str[i])); + i++; + } else { + if(i + 1 < len) { //&& (str[i+1] & 0x80)) + uint16_t tmp = (((uint16_t(str[i]) & 0x00ff ) << 8) | (uint16_t(str[i+1]) & 0x00ff)); + vec.push_back(tmp); + i += 2; + } else { + return false; + } + } + } + return true; +} + +template +bool GBKTrans(const string& str, Uint16Container& vec) { + return GBKTrans(str.c_str(), str.size(), vec); +} + +template +void GBKTrans(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) { + res.clear(); + //pair pa; + char first, second; + while(begin != end) { + //pa = uint16ToChar2(*begin); + first = ((*begin)>>8) & 0x00ff; + second = (*begin) & 0x00ff; + if(first & 0x80) { + res += first; + res += second; + } else { + res += second; + } + begin++; + } +} + +/* + * format example: "%Y-%m-%d %H:%M:%S" + */ +inline void GetTime(const string& format, string& timeStr) { + time_t timeNow; + time(&timeNow); + + struct tm tmNow; + + #if defined(_WIN32) || defined(_WIN64) + errno_t e = localtime_s(&tmNow, &timeNow); + assert(e == 0); + #else + struct tm * tm_tmp = localtime_r(&timeNow, &tmNow); + (void)tm_tmp; + assert(tm_tmp != nullptr); + #endif + + timeStr.resize(64); + + size_t len = strftime((char*)timeStr.c_str(), timeStr.size(), format.c_str(), &tmNow); + + timeStr.resize(len); +} + +inline string PathJoin(const string& path1, const string& path2) { + if(EndsWith(path1, "/")) { + return path1 + path2; + } + return path1 + "/" + path2; +} + +} +#endif From ab33310378c85a8a1238d285314916a7ff9f12e6 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:24 +0800 Subject: [PATCH 078/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/dict/README.md | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 third_party/cppjieba/dict/README.md diff --git a/third_party/cppjieba/dict/README.md b/third_party/cppjieba/dict/README.md new file mode 100644 index 00000000..88791189 --- /dev/null +++ b/third_party/cppjieba/dict/README.md @@ -0,0 +1,31 @@ +# CppJieba字典 + +文件后缀名代表的是词典的编码方式。 +比如filename.utf8 是 utf8编码，filename.gbk 是 gbk编码方式。 + + +## 分词 + +### jieba.dict.utf8/gbk + +作为最大概率法(MPSegment: Max Probability)分词所使用的词典。 + +### hmm_model.utf8/gbk + +作为隐式马尔科夫模型(HMMSegment: Hidden Markov Model)分词所使用的词典。 + +__对于MixSegment(混合MPSegment和HMMSegment两者)则同时使用以上两个词典__ + + +## 关键词抽取 + +### idf.utf8 + +IDF(Inverse Document Frequency) +在KeywordExtractor中，使用的是经典的TF-IDF算法，所以需要这么一个词典提供IDF信息。 + +### stop_words.utf8 + +停用词词典 + + From 4b1a61d19304ec1c9dfa04cbd3a3aee74d62958c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:25 +0800 Subject: [PATCH 079/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/dict/stop_words.utf8 | 1534 +++++++++++++++++++++ 1 file changed, 1534 insertions(+) create mode 100644 third_party/cppjieba/dict/stop_words.utf8 diff --git a/third_party/cppjieba/dict/stop_words.utf8 b/third_party/cppjieba/dict/stop_words.utf8 new file mode 100644 index 00000000..32ac9e67 --- /dev/null +++ b/third_party/cppjieba/dict/stop_words.utf8 @@ -0,0 +1,1534 @@ +" +. +。 +, +、 +！ +？ +： +； +` +﹑ +• +＂ +^ +… +‘ +’ +“ +” +〝 +〞 +~ +\ +∕ +| +¦ +‖ +—　 +( +) +〈 +〉 +﹞ +﹝ +「 +」 +‹ +› +〖 +〗 +】 +【 +» +« +』 +『 +〕 +〔 +》 +《 +} +{ +] +[ +﹐ +¸ +﹕ +︰ +﹔ +; +！ +¡ +？ +¿ +﹖ +﹌ +﹏ +﹋ +＇ +´ +ˊ +ˋ +- +― +﹫ +@ +︳ +︴ +_ +¯ +＿ +￣ +﹢ ++ +﹦ += +﹤ +‐ +< +­ +˜ +~ +﹟ +# +﹩ +$ +﹠ +& +﹪ +% +﹡ +* +﹨ +\ +﹍ +﹉ +﹎ +﹊ +ˇ +︵ +︶ +︷ +︸ +︹ +︿ +﹀ +︺ +︽ +︾ +_ +ˉ +﹁ +﹂ +﹃ +﹄ +︻ +︼ +的 +了 +the +a +an +that +those +this +that +$ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +? +_ +“ +” +、 +。 +《 +》 +一 +一些 +一何 +一切 +一则 +一方面 +一旦 +一来 +一样 +一般 +一转眼 +万一 +上 +上下 +下 +不 +不仅 +不但 +不光 +不单 +不只 +不外乎 +不如 +不妨 +不尽 +不尽然 +不得 +不怕 +不惟 +不成 +不拘 +不料 +不是 +不比 +不然 +不特 +不独 +不管 +不至于 +不若 +不论 +不过 +不问 +与 +与其 +与其说 +与否 +与此同时 +且 +且不说 +且说 +两者 +个 +个别 +临 +为 +为了 +为什么 +为何 +为止 +为此 +为着 +乃 +乃至 +乃至于 +么 +之 +之一 +之所以 +之类 +乌乎 +乎 +乘 +也 +也好 +也罢 +了 +二来 +于 +于是 +于是乎 +云云 +云尔 +些 +亦 +人 +人们 +人家 +什么 +什么样 +今 +介于 +仍 +仍旧 +从 +从此 +从而 +他 +他人 +他们 +以 +以上 +以为 +以便 +以免 +以及 +以故 +以期 +以来 +以至 +以至于 +以致 +们 +任 +任何 +任凭 +似的 +但 +但凡 +但是 +何 +何以 +何况 +何处 +何时 +余外 +作为 +你 +你们 +使 +使得 +例如 +依 +依据 +依照 +便于 +俺 +俺们 +倘 +倘使 +倘或 +倘然 +倘若 +借 +假使 +假如 +假若 +傥然 +像 +儿 +先不先 +光是 +全体 +全部 +兮 +关于 +其 +其一 +其中 +其二 +其他 +其余 +其它 +其次 +具体地说 +具体说来 +兼之 +内 +再 +再其次 +再则 +再有 +再者 +再者说 +再说 +冒 +冲 +况且 +几 +几时 +凡 +凡是 +凭 +凭借 +出于 +出来 +分别 +则 +则甚 +别 +别人 +别处 +别是 +别的 +别管 +别说 +到 +前后 +前此 +前者 +加之 +加以 +即 +即令 +即使 +即便 +即如 +即或 +即若 +却 +去 +又 +又及 +及 +及其 +及至 +反之 +反而 +反过来 +反过来说 +受到 +另 +另一方面 +另外 +另悉 +只 +只当 +只怕 +只是 +只有 +只消 +只要 +只限 +叫 +叮咚 +可 +可以 +可是 +可见 +各 +各个 +各位 +各种 +各自 +同 +同时 +后 +后者 +向 +向使 +向着 +吓 +吗 +否则 +吧 +吧哒 +吱 +呀 +呃 +呕 +呗 +呜 +呜呼 +呢 +呵 +呵呵 +呸 +呼哧 +咋 +和 +咚 +咦 +咧 +咱 +咱们 +咳 +哇 +哈 +哈哈 +哉 +哎 +哎呀 +哎哟 +哗 +哟 +哦 +哩 +哪 +哪个 +哪些 +哪儿 +哪天 +哪年 +哪怕 +哪样 +哪边 +哪里 +哼 +哼唷 +唉 +唯有 +啊 +啐 +啥 +啦 +啪达 +啷当 +喂 +喏 +喔唷 +喽 +嗡 +嗡嗡 +嗬 +嗯 +嗳 +嘎 +嘎登 +嘘 +嘛 +嘻 +嘿 +嘿嘿 +因 +因为 +因了 +因此 +因着 +因而 +固然 +在 +在下 +在于 +地 +基于 +处在 +多 +多么 +多少 +大 +大家 +她 +她们 +好 +如 +如上 +如上所述 +如下 +如何 +如其 +如同 +如是 +如果 +如此 +如若 +始而 +孰料 +孰知 +宁 +宁可 +宁愿 +宁肯 +它 +它们 +对 +对于 +对待 +对方 +对比 +将 +小 +尔 +尔后 +尔尔 +尚且 +就 +就是 +就是了 +就是说 +就算 +就要 +尽 +尽管 +尽管如此 +岂但 +己 +已 +已矣 +巴 +巴巴 +并 +并且 +并非 +庶乎 +庶几 +开外 +开始 +归 +归齐 +当 +当地 +当然 +当着 +彼 +彼时 +彼此 +往 +待 +很 +得 +得了 +怎 +怎么 +怎么办 +怎么样 +怎奈 +怎样 +总之 +总的来看 +总的来说 +总的说来 +总而言之 +恰恰相反 +您 +惟其 +慢说 +我 +我们 +或 +或则 +或是 +或曰 +或者 +截至 +所 +所以 +所在 +所幸 +所有 +才 +才能 +打 +打从 +把 +抑或 +拿 +按 +按照 +换句话说 +换言之 +据 +据此 +接着 +故 +故此 +故而 +旁人 +无 +无宁 +无论 +既 +既往 +既是 +既然 +时候 +是 +是以 +是的 +曾 +替 +替代 +最 +有 +有些 +有关 +有及 +有时 +有的 +望 +朝 +朝着 +本 +本人 +本地 +本着 +本身 +来 +来着 +来自 +来说 +极了 +果然 +果真 +某 +某个 +某些 +某某 +根据 +欤 +正值 +正如 +正巧 +正是 +此 +此地 +此处 +此外 +此时 +此次 +此间 +毋宁 +每 +每当 +比 +比及 +比如 +比方 +没奈何 +沿 +沿着 +漫说 +焉 +然则 +然后 +然而 +照 +照着 +犹且 +犹自 +甚且 +甚么 +甚或 +甚而 +甚至 +甚至于 +用 +用来 +由 +由于 +由是 +由此 +由此可见 +的 +的确 +的话 +直到 +相对而言 +省得 +看 +眨眼 +着 +着呢 +矣 +矣乎 +矣哉 +离 +竟而 +第 +等 +等到 +等等 +简言之 +管 +类如 +紧接着 +纵 +纵令 +纵使 +纵然 +经 +经过 +结果 +给 +继之 +继后 +继而 +综上所述 +罢了 +者 +而 +而且 +而况 +而后 +而外 +而已 +而是 +而言 +能 +能否 +腾 +自 +自个儿 +自从 +自各儿 +自后 +自家 +自己 +自打 +自身 +至 +至于 +至今 +至若 +致 +般的 +若 +若夫 +若是 +若果 +若非 +莫不然 +莫如 +莫若 +虽 +虽则 +虽然 +虽说 +被 +要 +要不 +要不是 +要不然 +要么 +要是 +譬喻 +譬如 +让 +许多 +论 +设使 +设或 +设若 +诚如 +诚然 +该 +说来 +诸 +诸位 +诸如 +谁 +谁人 +谁料 +谁知 +贼死 +赖以 +赶 +起 +起见 +趁 +趁着 +越是 +距 +跟 +较 +较之 +边 +过 +还 +还是 +还有 +还要 +这 +这一来 +这个 +这么 +这么些 +这么样 +这么点儿 +这些 +这会儿 +这儿 +这就是说 +这时 +这样 +这次 +这般 +这边 +这里 +进而 +连 +连同 +逐步 +通过 +遵循 +遵照 +那 +那个 +那么 +那么些 +那么样 +那些 +那会儿 +那儿 +那时 +那样 +那般 +那边 +那里 +都 +鄙人 +鉴于 +针对 +阿 +除 +除了 +除外 +除开 +除此之外 +除非 +随 +随后 +随时 +随着 +难道说 +非但 +非徒 +非特 +非独 +靠 +顺 +顺着 +首先 +！ +， +： +； +？ +to +can +could +dare +do +did +does +may +might +would +should +must +will +ought +shall +need +is +a +am +are +about +according +after +against +all +almost +also +although +among +an +and +another +any +anything +approximately +as +asked +at +back +because +before +besides +between +both +but +by +call +called +currently +despite +did +do +dr +during +each +earlier +eight +even +eventually +every +everything +five +for +four +from +he +her +here +his +how +however +i +if +in +indeed +instead +it +its +just +last +like +major +many +may +maybe +meanwhile +more +moreover +most +mr +mrs +ms +much +my +neither +net +never +nevertheless +nine +no +none +not +nothing +now +of +on +once +one +only +or +other +our +over +partly +perhaps +prior +regarding +separately +seven +several +she +should +similarly +since +six +so +some +somehow +still +such +ten +that +the +their +then +there +therefore +these +they +this +those +though +three +to +two +under +unless +unlike +until +volume +we +what +whatever +whats +when +where +which +while +why +with +without +yesterday +yet +you +your +aboard +about +above +according to +across +afore +after +against +agin +along +alongside +amid +amidst +among +amongst +anent +around +as +aslant +astride +at +athwart +bar +because of +before +behind +below +beneath +beside +besides +between +betwixt +beyond +but +by +circa +despite +down +during +due to +ere +except +for +from +in +inside +into +less +like +mid +midst +minus +near +next +nigh +nigher +nighest +notwithstanding +of +off +on +on to +onto +out +out of +outside +over +past +pending +per +plus +qua +re +round +sans +save +since +through +throughout +thru +till +to +toward +towards +under +underneath +unlike +until +unto +up +upon +versus +via +vice +with +within +without +he +her +herself +hers +him +himself +his +I +it +its +itself +me +mine +my +myself +ours +she +their +theirs +them +themselves +they +us +we +our +ourselves +you +your +yours +yourselves +yourself +this +that +these +those +" +' +'' +( +) +*LRB* +*RRB* + + + + + +@ +& +[ +] +` +`` +e.g., +{ +} +" +“ +” +-RRB- +-LRB- +-- +a +about +above +across +after +afterwards +again +against +all +almost +alone +along +already +also +although +always +am +among +amongst +amoungst +amount +an +and +another +any +anyhow +anyone +anything +anyway +anywhere +are +around +as +at +back +be +became +because +become +becomes +becoming +been +before +beforehand +behind +being +below +beside +besides +between +beyond +bill +both +bottom +but +by +call +can +cannot +cant +co +computer +con +could +couldnt +cry +de +describe +detail +do +done +down +due +during +each +eg +eight +either +eleven +else +elsewhere +empty +enough +etc +even +ever +every +everyone +everything +everywhere +except +few +fifteen +fify +fill +find +fire +first +five +for +former +formerly +forty +found +four +from +front +full +further +get +give +go +had +has +hasnt +have +he +hence +her +here +hereafter +hereby +herein +hereupon +hers +herself +him +himself +his +how +however +hundred +i +ie +if +in +inc +indeed +interest +into +is +it +its +itself +keep +last +latter +latterly +least +less +ltd +made +many +may +me +meanwhile +might +mill +mine +more +moreover +most +mostly +move +much +must +my +myself +name +namely +neither +never +nevertheless +next +nine +no +nobody +none +noone +nor +not +nothing +now +nowhere +of +off +often +on +once +one +only +onto +or +other +others +otherwise +our +ours +ourselves +out +over +own +part +per +perhaps +please +put +rather +re +same +see +seem +seemed +seeming +seems +serious +several +she +should +show +side +since +sincere +six +sixty +so +some +somehow +someone +something +sometime +sometimes +somewhere +still +such +system +take +ten +than +that +the +their +them +themselves +then +thence +there +thereafter +thereby +therefore +therein +thereupon +these +they +thick +thin +third +this +those +though +three +through +throughout +thru +thus +to +together +too +top +toward +towards +twelve +twenty +two +un +under +until +up +upon +us +very +via +was +we +well +were +what +whatever +when +whence +whenever +where +whereafter +whereas +whereby +wherein +whereupon +wherever +whether +which +while +whither +who +whoever +whole +whom +whose +why +will +with +within +without +would +yet +you +your +yours +yourself +yourselves + + +: +/ +（ +> +） +< +! From 71f01f5825a0cc03fdedcb898f538007e95c884f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:27 +0800 Subject: [PATCH 080/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- third_party/cppjieba/dict/user.dict.utf8 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 third_party/cppjieba/dict/user.dict.utf8 diff --git a/third_party/cppjieba/dict/user.dict.utf8 b/third_party/cppjieba/dict/user.dict.utf8 new file mode 100644 index 00000000..f4a1a4b8 --- /dev/null +++ b/third_party/cppjieba/dict/user.dict.utf8 @@ -0,0 +1,10 @@ +云计算 +区块链 10 nz +机器学习 10 n +深度学习 10 n +神经网络 10 n +人工智能 10 n +数据挖掘 10 n +自然语言处理 10 n +大模型 10 n +大语言模型 10 n \ No newline at end of file From 5437cf5a80e3d496d25bd324bd98b3b345303c80 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:28 +0800 Subject: [PATCH 081/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/dict/pos_dict/prob_start.utf8 | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 third_party/cppjieba/dict/pos_dict/prob_start.utf8 diff --git a/third_party/cppjieba/dict/pos_dict/prob_start.utf8 b/third_party/cppjieba/dict/pos_dict/prob_start.utf8 new file mode 100644 index 00000000..433750d5 --- /dev/null +++ b/third_party/cppjieba/dict/pos_dict/prob_start.utf8 @@ -0,0 +1,259 @@ +#初始状态的概率 +#格式 +#状态:概率 +B,a:-4.7623052146 +B,ad:-6.68006603678 +B,ag:-3.14e+100 +B,an:-8.69708322302 +B,b:-5.01837436211 +B,bg:-3.14e+100 +B,c:-3.42388018495 +B,d:-3.97504752976 +B,df:-8.88897423083 +B,dg:-3.14e+100 +B,e:-8.56355183039 +B,en:-3.14e+100 +B,f:-5.49163041848 +B,g:-3.14e+100 +B,h:-13.53336513 +B,i:-6.11578472756 +B,in:-3.14e+100 +B,j:-5.05761912847 +B,jn:-3.14e+100 +B,k:-3.14e+100 +B,l:-4.90588358466 +B,ln:-3.14e+100 +B,m:-3.6524299819 +B,mg:-3.14e+100 +B,mq:-6.7869530014 +B,n:-1.69662577975 +B,ng:-3.14e+100 +B,nr:-2.23104959138 +B,nrfg:-5.87372217541 +B,nrt:-4.98564273352 +B,ns:-2.8228438315 +B,nt:-4.84609166818 +B,nz:-3.94698846058 +B,o:-8.43349870215 +B,p:-4.20098413209 +B,q:-6.99812385896 +B,qe:-3.14e+100 +B,qg:-3.14e+100 +B,r:-3.40981877908 +B,rg:-3.14e+100 +B,rr:-12.4347528413 +B,rz:-7.94611647157 +B,s:-5.52267359084 +B,t:-3.36474790945 +B,tg:-3.14e+100 +B,u:-9.1639172775 +B,ud:-3.14e+100 +B,ug:-3.14e+100 +B,uj:-3.14e+100 +B,ul:-3.14e+100 +B,uv:-3.14e+100 +B,uz:-3.14e+100 +B,v:-2.67405848743 +B,vd:-9.04472876024 +B,vg:-3.14e+100 +B,vi:-12.4347528413 +B,vn:-4.33156108902 +B,vq:-12.1470707689 +B,w:-3.14e+100 +B,x:-3.14e+100 +B,y:-9.84448567586 +B,yg:-3.14e+100 +B,z:-7.04568111149 +B,zg:-3.14e+100 +E,a:-3.14e+100 +E,ad:-3.14e+100 +E,ag:-3.14e+100 +E,an:-3.14e+100 +E,b:-3.14e+100 +E,bg:-3.14e+100 +E,c:-3.14e+100 +E,d:-3.14e+100 +E,df:-3.14e+100 +E,dg:-3.14e+100 +E,e:-3.14e+100 +E,en:-3.14e+100 +E,f:-3.14e+100 +E,g:-3.14e+100 +E,h:-3.14e+100 +E,i:-3.14e+100 +E,in:-3.14e+100 +E,j:-3.14e+100 +E,jn:-3.14e+100 +E,k:-3.14e+100 +E,l:-3.14e+100 +E,ln:-3.14e+100 +E,m:-3.14e+100 +E,mg:-3.14e+100 +E,mq:-3.14e+100 +E,n:-3.14e+100 +E,ng:-3.14e+100 +E,nr:-3.14e+100 +E,nrfg:-3.14e+100 +E,nrt:-3.14e+100 +E,ns:-3.14e+100 +E,nt:-3.14e+100 +E,nz:-3.14e+100 +E,o:-3.14e+100 +E,p:-3.14e+100 +E,q:-3.14e+100 +E,qe:-3.14e+100 +E,qg:-3.14e+100 +E,r:-3.14e+100 +E,rg:-3.14e+100 +E,rr:-3.14e+100 +E,rz:-3.14e+100 +E,s:-3.14e+100 +E,t:-3.14e+100 +E,tg:-3.14e+100 +E,u:-3.14e+100 +E,ud:-3.14e+100 +E,ug:-3.14e+100 +E,uj:-3.14e+100 +E,ul:-3.14e+100 +E,uv:-3.14e+100 +E,uz:-3.14e+100 +E,v:-3.14e+100 +E,vd:-3.14e+100 +E,vg:-3.14e+100 +E,vi:-3.14e+100 +E,vn:-3.14e+100 +E,vq:-3.14e+100 +E,w:-3.14e+100 +E,x:-3.14e+100 +E,y:-3.14e+100 +E,yg:-3.14e+100 +E,z:-3.14e+100 +E,zg:-3.14e+100 +M,a:-3.14e+100 +M,ad:-3.14e+100 +M,ag:-3.14e+100 +M,an:-3.14e+100 +M,b:-3.14e+100 +M,bg:-3.14e+100 +M,c:-3.14e+100 +M,d:-3.14e+100 +M,df:-3.14e+100 +M,dg:-3.14e+100 +M,e:-3.14e+100 +M,en:-3.14e+100 +M,f:-3.14e+100 +M,g:-3.14e+100 +M,h:-3.14e+100 +M,i:-3.14e+100 +M,in:-3.14e+100 +M,j:-3.14e+100 +M,jn:-3.14e+100 +M,k:-3.14e+100 +M,l:-3.14e+100 +M,ln:-3.14e+100 +M,m:-3.14e+100 +M,mg:-3.14e+100 +M,mq:-3.14e+100 +M,n:-3.14e+100 +M,ng:-3.14e+100 +M,nr:-3.14e+100 +M,nrfg:-3.14e+100 +M,nrt:-3.14e+100 +M,ns:-3.14e+100 +M,nt:-3.14e+100 +M,nz:-3.14e+100 +M,o:-3.14e+100 +M,p:-3.14e+100 +M,q:-3.14e+100 +M,qe:-3.14e+100 +M,qg:-3.14e+100 +M,r:-3.14e+100 +M,rg:-3.14e+100 +M,rr:-3.14e+100 +M,rz:-3.14e+100 +M,s:-3.14e+100 +M,t:-3.14e+100 +M,tg:-3.14e+100 +M,u:-3.14e+100 +M,ud:-3.14e+100 +M,ug:-3.14e+100 +M,uj:-3.14e+100 +M,ul:-3.14e+100 +M,uv:-3.14e+100 +M,uz:-3.14e+100 +M,v:-3.14e+100 +M,vd:-3.14e+100 +M,vg:-3.14e+100 +M,vi:-3.14e+100 +M,vn:-3.14e+100 +M,vq:-3.14e+100 +M,w:-3.14e+100 +M,x:-3.14e+100 +M,y:-3.14e+100 +M,yg:-3.14e+100 +M,z:-3.14e+100 +M,zg:-3.14e+100 +S,a:-3.90253968313 +S,ad:-11.0484584802 +S,ag:-6.95411391796 +S,an:-12.8402179494 +S,b:-6.47288876397 +S,bg:-3.14e+100 +S,c:-4.78696679586 +S,d:-3.90391976418 +S,df:-3.14e+100 +S,dg:-8.9483976513 +S,e:-5.94251300628 +S,en:-3.14e+100 +S,f:-5.19482024998 +S,g:-6.50782681533 +S,h:-8.65056320738 +S,i:-3.14e+100 +S,in:-3.14e+100 +S,j:-4.91199211964 +S,jn:-3.14e+100 +S,k:-6.94032059583 +S,l:-3.14e+100 +S,ln:-3.14e+100 +S,m:-3.26920065212 +S,mg:-10.8253149289 +S,mq:-3.14e+100 +S,n:-3.85514838976 +S,ng:-4.9134348611 +S,nr:-4.48366310396 +S,nrfg:-3.14e+100 +S,nrt:-3.14e+100 +S,ns:-3.14e+100 +S,nt:-12.1470707689 +S,nz:-3.14e+100 +S,o:-8.46446092775 +S,p:-2.98684018136 +S,q:-4.88865861826 +S,qe:-3.14e+100 +S,qg:-3.14e+100 +S,r:-2.76353367841 +S,rg:-10.2752685919 +S,rr:-3.14e+100 +S,rz:-3.14e+100 +S,s:-3.14e+100 +S,t:-3.14e+100 +S,tg:-6.27284253188 +S,u:-6.94032059583 +S,ud:-7.72823016105 +S,ug:-7.53940370266 +S,uj:-6.85251045118 +S,ul:-8.41537131755 +S,uv:-8.15808672229 +S,uz:-9.29925862537 +S,v:-3.05329230341 +S,vd:-3.14e+100 +S,vg:-5.94301818437 +S,vi:-3.14e+100 +S,vn:-11.4539235883 +S,vq:-3.14e+100 +S,w:-3.14e+100 +S,x:-8.42741965607 +S,y:-6.19707946995 +S,yg:-13.53336513 +S,z:-3.14e+100 +S,zg:-3.14e+100 From 15af2cc230800a846ac1475958355bacead9e72d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:30 +0800 Subject: [PATCH 082/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/DictTrie.hpp | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/DictTrie.hpp diff --git a/third_party/cppjieba/include/cppjieba/DictTrie.hpp b/third_party/cppjieba/include/cppjieba/DictTrie.hpp new file mode 100644 index 00000000..ea8916c1 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/DictTrie.hpp @@ -0,0 +1,280 @@ +#ifndef CPPJIEBA_DICT_TRIE_HPP +#define CPPJIEBA_DICT_TRIE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "limonp/StringUtil.hpp" +#include "limonp/Logging.hpp" +#include "Unicode.hpp" +#include "Trie.hpp" + +namespace cppjieba { + +const double MIN_DOUBLE = -3.14e+100; +const double MAX_DOUBLE = 3.14e+100; +const size_t DICT_COLUMN_NUM = 3; +const char* const UNKNOWN_TAG = ""; + +class DictTrie { + public: + enum UserWordWeightOption { + WordWeightMin, + WordWeightMedian, + WordWeightMax, + }; // enum UserWordWeightOption + + DictTrie(const std::string& dict_path, const std::string& user_dict_paths = "", UserWordWeightOption user_word_weight_opt = WordWeightMedian) { + Init(dict_path, user_dict_paths, user_word_weight_opt); + } + + ~DictTrie() { + delete trie_; + } + + bool InsertUserWord(const std::string& word, const std::string& tag = UNKNOWN_TAG) { + DictUnit node_info; + if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) { + return false; + } + active_node_infos_.push_back(node_info); + trie_->InsertNode(node_info.word, &active_node_infos_.back()); + return true; + } + + bool InsertUserWord(const std::string& word,int freq, const std::string& tag = UNKNOWN_TAG) { + DictUnit node_info; + double weight = freq ? log(1.0 * freq / freq_sum_) : user_word_default_weight_ ; + if (!MakeNodeInfo(node_info, word, weight , tag)) { + return false; + } + active_node_infos_.push_back(node_info); + trie_->InsertNode(node_info.word, &active_node_infos_.back()); + return true; + } + + bool DeleteUserWord(const std::string& word, const std::string& tag = UNKNOWN_TAG) { + DictUnit node_info; + if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) { + return false; + } + trie_->DeleteNode(node_info.word, &node_info); + return true; + } + + const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { + return trie_->Find(begin, end); + } + + void Find(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator end, + std::vector&res, + size_t max_word_len = MAX_WORD_LENGTH) const { + trie_->Find(begin, end, res, max_word_len); + } + + bool Find(const std::string& word) + { + const DictUnit *tmp = NULL; + RuneStrArray runes; + if (!DecodeUTF8RunesInString(word, runes)) + { + XLOG(ERROR) << "Decode failed."; + } + tmp = Find(runes.begin(), runes.end()); + if (tmp == NULL) + { + return false; + } + else + { + return true; + } + } + + bool IsUserDictSingleChineseWord(const Rune& word) const { + return IsIn(user_dict_single_chinese_word_, word); + } + + double GetMinWeight() const { + return min_weight_; + } + + void InserUserDictNode(const std::string& line) { + std::vector buf; + DictUnit node_info; + limonp::Split(line, buf, " "); + if(buf.size() == 1){ + MakeNodeInfo(node_info, + buf[0], + user_word_default_weight_, + UNKNOWN_TAG); + } else if (buf.size() == 2) { + MakeNodeInfo(node_info, + buf[0], + user_word_default_weight_, + buf[1]); + } else if (buf.size() == 3) { + int freq = atoi(buf[1].c_str()); + assert(freq_sum_ > 0.0); + double weight = log(1.0 * freq / freq_sum_); + MakeNodeInfo(node_info, buf[0], weight, buf[2]); + } + static_node_infos_.push_back(node_info); + if (node_info.word.size() == 1) { + user_dict_single_chinese_word_.insert(node_info.word[0]); + } + } + + void LoadUserDict(const std::vector& buf) { + for (size_t i = 0; i < buf.size(); i++) { + InserUserDictNode(buf[i]); + } + } + + void LoadUserDict(const std::set& buf) { + std::set::const_iterator iter; + for (iter = buf.begin(); iter != buf.end(); iter++){ + InserUserDictNode(*iter); + } + } + + void LoadUserDict(const std::string& filePaths) { + std::vector files = limonp::Split(filePaths, "|;"); + for (size_t i = 0; i < files.size(); i++) { + std::ifstream ifs(files[i].c_str()); + XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; + std::string line; + + while(getline(ifs, line)) { + if (line.size() == 0) { + continue; + } + InserUserDictNode(line); + } + } + } + + + private: + void Init(const std::string& dict_path, const std::string& user_dict_paths, UserWordWeightOption user_word_weight_opt) { + LoadDict(dict_path); + freq_sum_ = CalcFreqSum(static_node_infos_); + CalculateWeight(static_node_infos_, freq_sum_); + SetStaticWordWeights(user_word_weight_opt); + + if (user_dict_paths.size()) { + LoadUserDict(user_dict_paths); + } + Shrink(static_node_infos_); + CreateTrie(static_node_infos_); + } + + void CreateTrie(const std::vector& dictUnits) { + assert(dictUnits.size()); + std::vector words; + std::vector valuePointers; + for (size_t i = 0 ; i < dictUnits.size(); i ++) { + words.push_back(dictUnits[i].word); + valuePointers.push_back(&dictUnits[i]); + } + + trie_ = new Trie(words, valuePointers); + } + + bool MakeNodeInfo(DictUnit& node_info, + const std::string& word, + double weight, + const std::string& tag) { + if (!DecodeUTF8RunesInString(word, node_info.word)) { + XLOG(ERROR) << "UTF-8 decode failed for dict word: " << word; + return false; + } + node_info.weight = weight; + node_info.tag = tag; + return true; + } + + void LoadDict(const std::string& filePath) { + std::ifstream ifs(filePath.c_str()); + XCHECK(ifs.is_open()) << "open " << filePath << " failed."; + std::string line; + std::vector buf; + + DictUnit node_info; + while (getline(ifs, line)) { + limonp::Split(line, buf, " "); + XCHECK(buf.size() == DICT_COLUMN_NUM) << "split result illegal, line:" << line; + MakeNodeInfo(node_info, + buf[0], + atof(buf[1].c_str()), + buf[2]); + static_node_infos_.push_back(node_info); + } + } + + static bool WeightCompare(const DictUnit& lhs, const DictUnit& rhs) { + return lhs.weight < rhs.weight; + } + + void SetStaticWordWeights(UserWordWeightOption option) { + XCHECK(!static_node_infos_.empty()); + std::vector x = static_node_infos_; + std::sort(x.begin(), x.end(), WeightCompare); + min_weight_ = x[0].weight; + max_weight_ = x[x.size() - 1].weight; + median_weight_ = x[x.size() / 2].weight; + switch (option) { + case WordWeightMin: + user_word_default_weight_ = min_weight_; + break; + case WordWeightMedian: + user_word_default_weight_ = median_weight_; + break; + default: + user_word_default_weight_ = max_weight_; + break; + } + } + + double CalcFreqSum(const std::vector& node_infos) const { + double sum = 0.0; + for (size_t i = 0; i < node_infos.size(); i++) { + sum += node_infos[i].weight; + } + return sum; + } + + void CalculateWeight(std::vector& node_infos, double sum) const { + assert(sum > 0.0); + for (size_t i = 0; i < node_infos.size(); i++) { + DictUnit& node_info = node_infos[i]; + assert(node_info.weight > 0.0); + node_info.weight = log(double(node_info.weight)/sum); + } + } + + void Shrink(std::vector& units) const { + std::vector(units.begin(), units.end()).swap(units); + } + + std::vector static_node_infos_; + std::deque active_node_infos_; // must not be std::vector + Trie * trie_; + + double freq_sum_; + double min_weight_; + double max_weight_; + double median_weight_; + double user_word_default_weight_; + std::unordered_set user_dict_single_chinese_word_; +}; +} + +#endif From 6ce6173ba3233d51a002556f5ae503cc59fafed4 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:31 +0800 Subject: [PATCH 083/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/FullSegment.hpp | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/FullSegment.hpp diff --git a/third_party/cppjieba/include/cppjieba/FullSegment.hpp b/third_party/cppjieba/include/cppjieba/FullSegment.hpp new file mode 100644 index 00000000..79d5211e --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/FullSegment.hpp @@ -0,0 +1,93 @@ +#ifndef CPPJIEBA_FULLSEGMENT_H +#define CPPJIEBA_FULLSEGMENT_H + +#include +#include +#include +#include "limonp/Logging.hpp" +#include "DictTrie.hpp" +#include "SegmentBase.hpp" +#include "Unicode.hpp" + +namespace cppjieba { +class FullSegment: public SegmentBase { + public: + FullSegment(const string& dictPath) { + dictTrie_ = new DictTrie(dictPath); + isNeedDestroy_ = true; + } + FullSegment(const DictTrie* dictTrie) + : dictTrie_(dictTrie), isNeedDestroy_(false) { + assert(dictTrie_); + } + ~FullSegment() { + if (isNeedDestroy_) { + delete dictTrie_; + } + } + void Cut(const string& sentence, + vector& words) const { + vector tmp; + Cut(sentence, tmp); + GetStringsFromWords(tmp, words); + } + void Cut(const string& sentence, + vector& words) const { + PreFilter pre_filter(symbols_, sentence); + PreFilter::Range range; + vector wrs; + wrs.reserve(sentence.size()/2); + while (pre_filter.HasNext()) { + range = pre_filter.Next(); + Cut(range.begin, range.end, wrs); + } + words.clear(); + words.reserve(wrs.size()); + GetWordsFromWordRanges(sentence, wrs, words); + } + void Cut(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator end, + vector& res) const { + // result of searching in trie tree + LocalVector > tRes; + + // max index of res's words + size_t maxIdx = 0; + + // always equals to (uItr - begin) + size_t uIdx = 0; + + // tmp variables + size_t wordLen = 0; + assert(dictTrie_); + vector dags; + dictTrie_->Find(begin, end, dags); + for (size_t i = 0; i < dags.size(); i++) { + for (size_t j = 0; j < dags[i].nexts.size(); j++) { + size_t nextoffset = dags[i].nexts[j].first; + assert(nextoffset < dags.size()); + const DictUnit* du = dags[i].nexts[j].second; + if (du == NULL) { + if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) { + WordRange wr(begin + i, begin + nextoffset); + res.push_back(wr); + } + } else { + wordLen = du->word.size(); + if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) { + WordRange wr(begin + i, begin + nextoffset); + res.push_back(wr); + } + } + maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx; + } + uIdx++; + } + } + private: + const DictTrie* dictTrie_; + bool isNeedDestroy_; +}; +} + +#endif From 4ea87dfdaa519f209d053c5ea594513bfffd78df Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:33 +0800 Subject: [PATCH 084/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/HMMModel.hpp | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/HMMModel.hpp diff --git a/third_party/cppjieba/include/cppjieba/HMMModel.hpp b/third_party/cppjieba/include/cppjieba/HMMModel.hpp new file mode 100644 index 00000000..3921faaf --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/HMMModel.hpp @@ -0,0 +1,129 @@ +#ifndef CPPJIEBA_HMMMODEL_H +#define CPPJIEBA_HMMMODEL_H + +#include "limonp/StringUtil.hpp" +#include "Trie.hpp" + +namespace cppjieba { + +using namespace limonp; +typedef unordered_map EmitProbMap; + +struct HMMModel { + /* + * STATUS: + * 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S + * */ + enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4}; + + HMMModel(const string& modelPath) { + memset(startProb, 0, sizeof(startProb)); + memset(transProb, 0, sizeof(transProb)); + statMap[0] = 'B'; + statMap[1] = 'E'; + statMap[2] = 'M'; + statMap[3] = 'S'; + emitProbVec.push_back(&emitProbB); + emitProbVec.push_back(&emitProbE); + emitProbVec.push_back(&emitProbM); + emitProbVec.push_back(&emitProbS); + LoadModel(modelPath); + } + ~HMMModel() { + } + void LoadModel(const string& filePath) { + ifstream ifile(filePath.c_str()); + XCHECK(ifile.is_open()) << "open " << filePath << " failed"; + string line; + vector tmp; + vector tmp2; + //Load startProb + XCHECK(GetLine(ifile, line)); + Split(line, tmp, " "); + XCHECK(tmp.size() == STATUS_SUM); + for (size_t j = 0; j< tmp.size(); j++) { + startProb[j] = atof(tmp[j].c_str()); + } + + //Load transProb + for (size_t i = 0; i < STATUS_SUM; i++) { + XCHECK(GetLine(ifile, line)); + Split(line, tmp, " "); + XCHECK(tmp.size() == STATUS_SUM); + for (size_t j =0; j < STATUS_SUM; j++) { + transProb[i][j] = atof(tmp[j].c_str()); + } + } + + //Load emitProbB + XCHECK(GetLine(ifile, line)); + XCHECK(LoadEmitProb(line, emitProbB)); + + //Load emitProbE + XCHECK(GetLine(ifile, line)); + XCHECK(LoadEmitProb(line, emitProbE)); + + //Load emitProbM + XCHECK(GetLine(ifile, line)); + XCHECK(LoadEmitProb(line, emitProbM)); + + //Load emitProbS + XCHECK(GetLine(ifile, line)); + XCHECK(LoadEmitProb(line, emitProbS)); + } + double GetEmitProb(const EmitProbMap* ptMp, Rune key, + double defVal)const { + EmitProbMap::const_iterator cit = ptMp->find(key); + if (cit == ptMp->end()) { + return defVal; + } + return cit->second; + } + bool GetLine(ifstream& ifile, string& line) { + while (getline(ifile, line)) { + Trim(line); + if (line.empty()) { + continue; + } + if (StartsWith(line, "#")) { + continue; + } + return true; + } + return false; + } + bool LoadEmitProb(const string& line, EmitProbMap& mp) { + if (line.empty()) { + return false; + } + vector tmp, tmp2; + Unicode unicode; + Split(line, tmp, ","); + for (size_t i = 0; i < tmp.size(); i++) { + Split(tmp[i], tmp2, ":"); + if (2 != tmp2.size()) { + XLOG(ERROR) << "emitProb illegal."; + return false; + } + if (!DecodeUTF8RunesInString(tmp2[0], unicode) || unicode.size() != 1) { + XLOG(ERROR) << "TransCode failed."; + return false; + } + mp[unicode[0]] = atof(tmp2[1].c_str()); + } + return true; + } + + char statMap[STATUS_SUM]; + double startProb[STATUS_SUM]; + double transProb[STATUS_SUM][STATUS_SUM]; + EmitProbMap emitProbB; + EmitProbMap emitProbE; + EmitProbMap emitProbM; + EmitProbMap emitProbS; + vector emitProbVec; +}; // struct HMMModel + +} // namespace cppjieba + +#endif From eb990f2533cddfcb3ff0ccb1561d4b4b6444c5b0 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:35 +0800 Subject: [PATCH 085/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/HMMSegment.hpp | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/HMMSegment.hpp diff --git a/third_party/cppjieba/include/cppjieba/HMMSegment.hpp b/third_party/cppjieba/include/cppjieba/HMMSegment.hpp new file mode 100644 index 00000000..d515c049 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/HMMSegment.hpp @@ -0,0 +1,190 @@ +#ifndef CPPJIBEA_HMMSEGMENT_H +#define CPPJIBEA_HMMSEGMENT_H + +#include +#include +#include +#include +#include "HMMModel.hpp" +#include "SegmentBase.hpp" + +namespace cppjieba { +class HMMSegment: public SegmentBase { + public: + HMMSegment(const string& filePath) + : model_(new HMMModel(filePath)), isNeedDestroy_(true) { + } + HMMSegment(const HMMModel* model) + : model_(model), isNeedDestroy_(false) { + } + ~HMMSegment() { + if (isNeedDestroy_) { + delete model_; + } + } + + void Cut(const string& sentence, + vector& words) const { + vector tmp; + Cut(sentence, tmp); + GetStringsFromWords(tmp, words); + } + void Cut(const string& sentence, + vector& words) const { + PreFilter pre_filter(symbols_, sentence); + PreFilter::Range range; + vector wrs; + wrs.reserve(sentence.size()/2); + while (pre_filter.HasNext()) { + range = pre_filter.Next(); + Cut(range.begin, range.end, wrs); + } + words.clear(); + words.reserve(wrs.size()); + GetWordsFromWordRanges(sentence, wrs, words); + } + void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res) const { + RuneStrArray::const_iterator left = begin; + RuneStrArray::const_iterator right = begin; + while (right != end) { + if (right->rune < 0x80) { + if (left != right) { + InternalCut(left, right, res); + } + left = right; + do { + right = SequentialLetterRule(left, end); + if (right != left) { + break; + } + right = NumbersRule(left, end); + if (right != left) { + break; + } + right ++; + } while (false); + WordRange wr(left, right - 1); + res.push_back(wr); + left = right; + } else { + right++; + } + } + if (left != right) { + InternalCut(left, right, res); + } + } + private: + // sequential letters rule + RuneStrArray::const_iterator SequentialLetterRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { + Rune x = begin->rune; + if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) { + begin ++; + } else { + return begin; + } + while (begin != end) { + x = begin->rune; + if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) { + begin ++; + } else { + break; + } + } + return begin; + } + // + RuneStrArray::const_iterator NumbersRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { + Rune x = begin->rune; + if ('0' <= x && x <= '9') { + begin ++; + } else { + return begin; + } + while (begin != end) { + x = begin->rune; + if ( ('0' <= x && x <= '9') || x == '.') { + begin++; + } else { + break; + } + } + return begin; + } + void InternalCut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res) const { + vector status; + Viterbi(begin, end, status); + + RuneStrArray::const_iterator left = begin; + RuneStrArray::const_iterator right; + for (size_t i = 0; i < status.size(); i++) { + if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i]) + right = begin + i + 1; + WordRange wr(left, right - 1); + res.push_back(wr); + left = right; + } + } + } + + void Viterbi(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator end, + vector& status) const { + size_t Y = HMMModel::STATUS_SUM; + size_t X = end - begin; + + size_t XYSize = X * Y; + size_t now, old, stat; + double tmp, endE, endS; + + vector path(XYSize); + vector weight(XYSize); + + //start + for (size_t y = 0; y < Y; y++) { + weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE); + path[0 + y * X] = -1; + } + + double emitProb; + + for (size_t x = 1; x < X; x++) { + for (size_t y = 0; y < Y; y++) { + now = x + y*X; + weight[now] = MIN_DOUBLE; + path[now] = HMMModel::E; // warning + emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE); + for (size_t preY = 0; preY < Y; preY++) { + old = x - 1 + preY * X; + tmp = weight[old] + model_->transProb[preY][y] + emitProb; + if (tmp > weight[now]) { + weight[now] = tmp; + path[now] = preY; + } + } + } + } + + endE = weight[X-1+HMMModel::E*X]; + endS = weight[X-1+HMMModel::S*X]; + stat = 0; + if (endE >= endS) { + stat = HMMModel::E; + } else { + stat = HMMModel::S; + } + + status.resize(X); + for (int x = X -1 ; x >= 0; x--) { + status[x] = stat; + stat = path[x + stat*X]; + } + } + + const HMMModel* model_; + bool isNeedDestroy_; +}; // class HMMSegment + +} // namespace cppjieba + +#endif From 63b54a69a602302a9a0400d2ecae44b1b0714dfd Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:37 +0800 Subject: [PATCH 086/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/Jieba.hpp | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/Jieba.hpp diff --git a/third_party/cppjieba/include/cppjieba/Jieba.hpp b/third_party/cppjieba/include/cppjieba/Jieba.hpp new file mode 100644 index 00000000..01fea361 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/Jieba.hpp @@ -0,0 +1,169 @@ +#ifndef CPPJIEAB_JIEBA_H +#define CPPJIEAB_JIEBA_H + +#include "QuerySegment.hpp" +#include "KeywordExtractor.hpp" + +namespace cppjieba { + +class Jieba { + public: + Jieba(const string& dict_path = "", + const string& model_path = "", + const string& user_dict_path = "", + const string& idf_path = "", + const string& stop_word_path = "") + : dict_trie_(getPath(dict_path, "jieba.dict.utf8"), getPath(user_dict_path, "user.dict.utf8")), + model_(getPath(model_path, "hmm_model.utf8")), + mp_seg_(&dict_trie_), + hmm_seg_(&model_), + mix_seg_(&dict_trie_, &model_), + full_seg_(&dict_trie_), + query_seg_(&dict_trie_, &model_), + extractor(&dict_trie_, &model_, + getPath(idf_path, "idf.utf8"), + getPath(stop_word_path, "stop_words.utf8")) { + } + ~Jieba() { + } + + struct LocWord { + string word; + size_t begin; + size_t end; + }; // struct LocWord + + void Cut(const string& sentence, vector& words, bool hmm = true) const { + mix_seg_.Cut(sentence, words, hmm); + } + void Cut(const string& sentence, vector& words, bool hmm = true) const { + mix_seg_.Cut(sentence, words, hmm); + } + void CutAll(const string& sentence, vector& words) const { + full_seg_.Cut(sentence, words); + } + void CutAll(const string& sentence, vector& words) const { + full_seg_.Cut(sentence, words); + } + void CutForSearch(const string& sentence, vector& words, bool hmm = true) const { + query_seg_.Cut(sentence, words, hmm); + } + void CutForSearch(const string& sentence, vector& words, bool hmm = true) const { + query_seg_.Cut(sentence, words, hmm); + } + void CutHMM(const string& sentence, vector& words) const { + hmm_seg_.Cut(sentence, words); + } + void CutHMM(const string& sentence, vector& words) const { + hmm_seg_.Cut(sentence, words); + } + void CutSmall(const string& sentence, vector& words, size_t max_word_len) const { + mp_seg_.Cut(sentence, words, max_word_len); + } + void CutSmall(const string& sentence, vector& words, size_t max_word_len) const { + mp_seg_.Cut(sentence, words, max_word_len); + } + + void Tag(const string& sentence, vector >& words) const { + mix_seg_.Tag(sentence, words); + } + string LookupTag(const string &str) const { + return mix_seg_.LookupTag(str); + } + bool InsertUserWord(const string& word, const string& tag = UNKNOWN_TAG) { + return dict_trie_.InsertUserWord(word, tag); + } + + bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) { + return dict_trie_.InsertUserWord(word,freq, tag); + } + + bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) { + return dict_trie_.DeleteUserWord(word, tag); + } + + bool Find(const string& word) + { + return dict_trie_.Find(word); + } + + void ResetSeparators(const string& s) { + //TODO + mp_seg_.ResetSeparators(s); + hmm_seg_.ResetSeparators(s); + mix_seg_.ResetSeparators(s); + full_seg_.ResetSeparators(s); + query_seg_.ResetSeparators(s); + } + + const DictTrie* GetDictTrie() const { + return &dict_trie_; + } + + const HMMModel* GetHMMModel() const { + return &model_; + } + + void LoadUserDict(const vector& buf) { + dict_trie_.LoadUserDict(buf); + } + + void LoadUserDict(const set& buf) { + dict_trie_.LoadUserDict(buf); + } + + void LoadUserDict(const string& path) { + dict_trie_.LoadUserDict(path); + } + + private: + static string pathJoin(const string& dir, const string& filename) { + if (dir.empty()) { + return filename; + } + + char last_char = dir[dir.length() - 1]; + if (last_char == '/' || last_char == '\\') { + return dir + filename; + } else { + #ifdef _WIN32 + return dir + '\\' + filename; + #else + return dir + '/' + filename; + #endif + } + } + + static string getCurrentDirectory() { + string path(__FILE__); + size_t pos = path.find_last_of("/\\"); + return (pos == string::npos) ? "" : path.substr(0, pos); + } + + static string getPath(const string& path, const string& default_file) { + if (path.empty()) { + string current_dir = getCurrentDirectory(); + string parent_dir = current_dir.substr(0, current_dir.find_last_of("/\\")); + string grandparent_dir = parent_dir.substr(0, parent_dir.find_last_of("/\\")); + return pathJoin(pathJoin(grandparent_dir, "dict"), default_file); + } + return path; + } + + DictTrie dict_trie_; + HMMModel model_; + + // They share the same dict trie and model + MPSegment mp_seg_; + HMMSegment hmm_seg_; + MixSegment mix_seg_; + FullSegment full_seg_; + QuerySegment query_seg_; + + public: + KeywordExtractor extractor; +}; // class Jieba + +} // namespace cppjieba + +#endif // CPPJIEAB_JIEBA_H From 311ba3cc70913698e4ad0917459b424bb5c2875f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:38 +0800 Subject: [PATCH 087/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../include/cppjieba/KeywordExtractor.hpp | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp diff --git a/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp b/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp new file mode 100644 index 00000000..24b2c409 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp @@ -0,0 +1,149 @@ +#ifndef CPPJIEBA_KEYWORD_EXTRACTOR_H +#define CPPJIEBA_KEYWORD_EXTRACTOR_H + +#include +#include +#include +#include "MixSegment.hpp" + +namespace cppjieba { + +/*utf8*/ +class KeywordExtractor { + public: + struct Word { + std::string word; + std::vector offsets; + double weight; + }; // struct Word + + KeywordExtractor(const std::string& dictPath, + const std::string& hmmFilePath, + const std::string& idfPath, + const std::string& stopWordPath, + const std::string& userDict = "") + : segment_(dictPath, hmmFilePath, userDict) { + LoadIdfDict(idfPath); + LoadStopWordDict(stopWordPath); + } + KeywordExtractor(const DictTrie* dictTrie, + const HMMModel* model, + const std::string& idfPath, + const std::string& stopWordPath) + : segment_(dictTrie, model) { + LoadIdfDict(idfPath); + LoadStopWordDict(stopWordPath); + } + ~KeywordExtractor() { + } + + void Extract(const std::string& sentence, std::vector& keywords, size_t topN) const { + std::vector topWords; + Extract(sentence, topWords, topN); + for (size_t i = 0; i < topWords.size(); i++) { + keywords.push_back(topWords[i].word); + } + } + + void Extract(const std::string& sentence, std::vector >& keywords, size_t topN) const { + std::vector topWords; + Extract(sentence, topWords, topN); + for (size_t i = 0; i < topWords.size(); i++) { + keywords.push_back(pair(topWords[i].word, topWords[i].weight)); + } + } + + void Extract(const std::string& sentence, std::vector& keywords, size_t topN) const { + std::vector words; + segment_.Cut(sentence, words); + + std::map wordmap; + size_t offset = 0; + for (size_t i = 0; i < words.size(); ++i) { + size_t t = offset; + offset += words[i].size(); + if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) { + continue; + } + wordmap[words[i]].offsets.push_back(t); + wordmap[words[i]].weight += 1.0; + } + if (offset != sentence.size()) { + XLOG(ERROR) << "words illegal"; + return; + } + + keywords.clear(); + keywords.reserve(wordmap.size()); + for (std::map::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) { + std::unordered_map::const_iterator cit = idfMap_.find(itr->first); + if (cit != idfMap_.end()) { + itr->second.weight *= cit->second; + } else { + itr->second.weight *= idfAverage_; + } + itr->second.word = itr->first; + keywords.push_back(itr->second); + } + topN = min(topN, keywords.size()); + std::partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare); + keywords.resize(topN); + } + private: + void LoadIdfDict(const std::string& idfPath) { + std::ifstream ifs(idfPath.c_str()); + XCHECK(ifs.is_open()) << "open " << idfPath << " failed"; + std::string line ; + std::vector buf; + double idf = 0.0; + double idfSum = 0.0; + size_t lineno = 0; + for (; getline(ifs, line); lineno++) { + buf.clear(); + if (line.empty()) { + XLOG(ERROR) << "lineno: " << lineno << " empty. skipped."; + continue; + } + limonp::Split(line, buf, " "); + if (buf.size() != 2) { + XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " empty. skipped."; + continue; + } + idf = atof(buf[1].c_str()); + idfMap_[buf[0]] = idf; + idfSum += idf; + + } + + assert(lineno); + idfAverage_ = idfSum / lineno; + assert(idfAverage_ > 0.0); + } + void LoadStopWordDict(const std::string& filePath) { + std::ifstream ifs(filePath.c_str()); + XCHECK(ifs.is_open()) << "open " << filePath << " failed"; + std::string line ; + while (getline(ifs, line)) { + stopWords_.insert(line); + } + assert(stopWords_.size()); + } + + static bool Compare(const Word& lhs, const Word& rhs) { + return lhs.weight > rhs.weight; + } + + MixSegment segment_; + std::unordered_map idfMap_; + double idfAverage_; + + std::unordered_set stopWords_; +}; // class KeywordExtractor + +inline std::ostream& operator << (std::ostream& os, const KeywordExtractor::Word& word) { + return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; +} + +} // namespace cppjieba + +#endif From 3e3d6429d13dfe643bafca970b8bb12965ef72ef Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:40 +0800 Subject: [PATCH 088/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/MixSegment.hpp | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/MixSegment.hpp diff --git a/third_party/cppjieba/include/cppjieba/MixSegment.hpp b/third_party/cppjieba/include/cppjieba/MixSegment.hpp new file mode 100644 index 00000000..8fd24e90 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/MixSegment.hpp @@ -0,0 +1,109 @@ +#ifndef CPPJIEBA_MIXSEGMENT_H +#define CPPJIEBA_MIXSEGMENT_H + +#include +#include "MPSegment.hpp" +#include "HMMSegment.hpp" +#include "limonp/StringUtil.hpp" +#include "PosTagger.hpp" + +namespace cppjieba { +class MixSegment: public SegmentTagged { + public: + MixSegment(const string& mpSegDict, const string& hmmSegDict, + const string& userDict = "") + : mpSeg_(mpSegDict, userDict), + hmmSeg_(hmmSegDict) { + } + MixSegment(const DictTrie* dictTrie, const HMMModel* model) + : mpSeg_(dictTrie), hmmSeg_(model) { + } + ~MixSegment() { + } + + void Cut(const string& sentence, vector& words) const { + Cut(sentence, words, true); + } + void Cut(const string& sentence, vector& words, bool hmm) const { + vector tmp; + Cut(sentence, tmp, hmm); + GetStringsFromWords(tmp, words); + } + void Cut(const string& sentence, vector& words, bool hmm = true) const { + PreFilter pre_filter(symbols_, sentence); + PreFilter::Range range; + vector wrs; + wrs.reserve(sentence.size() / 2); + while (pre_filter.HasNext()) { + range = pre_filter.Next(); + Cut(range.begin, range.end, wrs, hmm); + } + words.clear(); + words.reserve(wrs.size()); + GetWordsFromWordRanges(sentence, wrs, words); + } + + void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res, bool hmm) const { + if (!hmm) { + mpSeg_.Cut(begin, end, res); + return; + } + vector words; + assert(end >= begin); + words.reserve(end - begin); + mpSeg_.Cut(begin, end, words); + + vector hmmRes; + hmmRes.reserve(end - begin); + for (size_t i = 0; i < words.size(); i++) { + //if mp Get a word, it's ok, put it into result + if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) { + res.push_back(words[i]); + continue; + } + + // if mp Get a single one and it is not in userdict, collect it in sequence + size_t j = i; + while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) { + j++; + } + + // Cut the sequence with hmm + assert(j - 1 >= i); + // TODO + hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes); + //put hmm result to result + for (size_t k = 0; k < hmmRes.size(); k++) { + res.push_back(hmmRes[k]); + } + + //clear tmp vars + hmmRes.clear(); + + //let i jump over this piece + i = j - 1; + } + } + + const DictTrie* GetDictTrie() const { + return mpSeg_.GetDictTrie(); + } + + bool Tag(const string& src, vector >& res) const { + return tagger_.Tag(src, res, *this); + } + + string LookupTag(const string &str) const { + return tagger_.LookupTag(str, *this); + } + + private: + MPSegment mpSeg_; + HMMSegment hmmSeg_; + PosTagger tagger_; + +}; // class MixSegment + +} // namespace cppjieba + +#endif From 65a5f2abaf12e5ef62120d8d77a417314d66b392 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:42 +0800 Subject: [PATCH 089/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/MPSegment.hpp | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/MPSegment.hpp diff --git a/third_party/cppjieba/include/cppjieba/MPSegment.hpp b/third_party/cppjieba/include/cppjieba/MPSegment.hpp new file mode 100644 index 00000000..bcbfaba6 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/MPSegment.hpp @@ -0,0 +1,137 @@ +#ifndef CPPJIEBA_MPSEGMENT_H +#define CPPJIEBA_MPSEGMENT_H + +#include +#include +#include +#include "limonp/Logging.hpp" +#include "DictTrie.hpp" +#include "SegmentTagged.hpp" +#include "PosTagger.hpp" + +namespace cppjieba { + +class MPSegment: public SegmentTagged { + public: + MPSegment(const string& dictPath, const string& userDictPath = "") + : dictTrie_(new DictTrie(dictPath, userDictPath)), isNeedDestroy_(true) { + } + MPSegment(const DictTrie* dictTrie) + : dictTrie_(dictTrie), isNeedDestroy_(false) { + assert(dictTrie_); + } + ~MPSegment() { + if (isNeedDestroy_) { + delete dictTrie_; + } + } + + void Cut(const string& sentence, vector& words) const { + Cut(sentence, words, MAX_WORD_LENGTH); + } + + void Cut(const string& sentence, + vector& words, + size_t max_word_len) const { + vector tmp; + Cut(sentence, tmp, max_word_len); + GetStringsFromWords(tmp, words); + } + void Cut(const string& sentence, + vector& words, + size_t max_word_len = MAX_WORD_LENGTH) const { + PreFilter pre_filter(symbols_, sentence); + PreFilter::Range range; + vector wrs; + wrs.reserve(sentence.size()/2); + while (pre_filter.HasNext()) { + range = pre_filter.Next(); + Cut(range.begin, range.end, wrs, max_word_len); + } + words.clear(); + words.reserve(wrs.size()); + GetWordsFromWordRanges(sentence, wrs, words); + } + void Cut(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator end, + vector& words, + size_t max_word_len = MAX_WORD_LENGTH) const { + vector dags; + dictTrie_->Find(begin, + end, + dags, + max_word_len); + CalcDP(dags); + CutByDag(begin, end, dags, words); + } + + const DictTrie* GetDictTrie() const { + return dictTrie_; + } + + bool Tag(const string& src, vector >& res) const { + return tagger_.Tag(src, res, *this); + } + + bool IsUserDictSingleChineseWord(const Rune& value) const { + return dictTrie_->IsUserDictSingleChineseWord(value); + } + private: + void CalcDP(vector& dags) const { + size_t nextPos; + const DictUnit* p; + double val; + + for (vector::reverse_iterator rit = dags.rbegin(); rit != dags.rend(); rit++) { + rit->pInfo = NULL; + rit->weight = MIN_DOUBLE; + assert(!rit->nexts.empty()); + for (LocalVector >::const_iterator it = rit->nexts.begin(); it != rit->nexts.end(); it++) { + nextPos = it->first; + p = it->second; + val = 0.0; + if (nextPos + 1 < dags.size()) { + val += dags[nextPos + 1].weight; + } + + if (p) { + val += p->weight; + } else { + val += dictTrie_->GetMinWeight(); + } + if (val > rit->weight) { + rit->pInfo = p; + rit->weight = val; + } + } + } + } + void CutByDag(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator /*end*/, + const vector& dags, + vector& words) const { + size_t i = 0; + while (i < dags.size()) { + const DictUnit* p = dags[i].pInfo; + if (p) { + assert(p->word.size() >= 1); + WordRange wr(begin + i, begin + i + p->word.size() - 1); + words.push_back(wr); + i += p->word.size(); + } else { //single chinese word + WordRange wr(begin + i, begin + i); + words.push_back(wr); + i++; + } + } + } + + const DictTrie* dictTrie_; + bool isNeedDestroy_; + PosTagger tagger_; + +}; // class MPSegment + +} // namespace cppjieba + +#endif From b455dbc1de0fe534bf83f354bffa8214b2fdd787 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:43 +0800 Subject: [PATCH 090/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/PosTagger.hpp | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/PosTagger.hpp diff --git a/third_party/cppjieba/include/cppjieba/PosTagger.hpp b/third_party/cppjieba/include/cppjieba/PosTagger.hpp new file mode 100644 index 00000000..15863306 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/PosTagger.hpp @@ -0,0 +1,77 @@ +#ifndef CPPJIEBA_POS_TAGGING_H +#define CPPJIEBA_POS_TAGGING_H + +#include "limonp/StringUtil.hpp" +#include "SegmentTagged.hpp" +#include "DictTrie.hpp" + +namespace cppjieba { +using namespace limonp; + +static const char* const POS_M = "m"; +static const char* const POS_ENG = "eng"; +static const char* const POS_X = "x"; + +class PosTagger { + public: + PosTagger() { + } + ~PosTagger() { + } + + bool Tag(const string& src, vector >& res, const SegmentTagged& segment) const { + vector CutRes; + segment.Cut(src, CutRes); + + for (vector::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) { + res.push_back(make_pair(*itr, LookupTag(*itr, segment))); + } + return !res.empty(); + } + + string LookupTag(const string &str, const SegmentTagged& segment) const { + const DictUnit *tmp = NULL; + RuneStrArray runes; + const DictTrie * dict = segment.GetDictTrie(); + assert(dict != NULL); + if (!DecodeUTF8RunesInString(str, runes)) { + XLOG(ERROR) << "UTF-8 decode failed for word: " << str; + return POS_X; + } + tmp = dict->Find(runes.begin(), runes.end()); + if (tmp == NULL || tmp->tag.empty()) { + return SpecialRule(runes); + } else { + return tmp->tag; + } + } + + private: + const char* SpecialRule(const RuneStrArray& unicode) const { + size_t m = 0; + size_t eng = 0; + for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) { + if (unicode[i].rune < 0x80) { + eng ++; + if ('0' <= unicode[i].rune && unicode[i].rune <= '9') { + m++; + } + } + } + // ascii char is not found + if (eng == 0) { + return POS_X; + } + // all the ascii is number char + if (m == eng) { + return POS_M; + } + // the ascii chars contain english letter + return POS_ENG; + } + +}; // class PosTagger + +} // namespace cppjieba + +#endif From 240c91de76e2af54acfae77967460f294a1cbd29 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:45 +0800 Subject: [PATCH 091/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/PreFilter.hpp | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/PreFilter.hpp diff --git a/third_party/cppjieba/include/cppjieba/PreFilter.hpp b/third_party/cppjieba/include/cppjieba/PreFilter.hpp new file mode 100644 index 00000000..deb750b5 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/PreFilter.hpp @@ -0,0 +1,54 @@ +#ifndef CPPJIEBA_PRE_FILTER_H +#define CPPJIEBA_PRE_FILTER_H + +#include "Trie.hpp" +#include "limonp/Logging.hpp" + +namespace cppjieba { + +class PreFilter { + public: + //TODO use WordRange instead of Range + struct Range { + RuneStrArray::const_iterator begin; + RuneStrArray::const_iterator end; + }; // struct Range + + PreFilter(const unordered_set& symbols, + const string& sentence) + : symbols_(symbols) { + if (!DecodeUTF8RunesInString(sentence, sentence_)) { + XLOG(ERROR) << "UTF-8 decode failed for input sentence"; + } + cursor_ = sentence_.begin(); + } + ~PreFilter() { + } + bool HasNext() const { + return cursor_ != sentence_.end(); + } + Range Next() { + Range range; + range.begin = cursor_; + while (cursor_ != sentence_.end()) { + if (IsIn(symbols_, cursor_->rune)) { + if (range.begin == cursor_) { + cursor_ ++; + } + range.end = cursor_; + return range; + } + cursor_ ++; + } + range.end = sentence_.end(); + return range; + } + private: + RuneStrArray::const_iterator cursor_; + RuneStrArray sentence_; + const unordered_set& symbols_; +}; // class PreFilter + +} // namespace cppjieba + +#endif // CPPJIEBA_PRE_FILTER_H From c0590f92c3a80c1d5a95429a0eaa43e529defec2 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:46 +0800 Subject: [PATCH 092/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../include/cppjieba/QuerySegment.hpp | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/QuerySegment.hpp diff --git a/third_party/cppjieba/include/cppjieba/QuerySegment.hpp b/third_party/cppjieba/include/cppjieba/QuerySegment.hpp new file mode 100644 index 00000000..6be886ab --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/QuerySegment.hpp @@ -0,0 +1,89 @@ +#ifndef CPPJIEBA_QUERYSEGMENT_H +#define CPPJIEBA_QUERYSEGMENT_H + +#include +#include +#include +#include "limonp/Logging.hpp" +#include "DictTrie.hpp" +#include "SegmentBase.hpp" +#include "FullSegment.hpp" +#include "MixSegment.hpp" +#include "Unicode.hpp" + +namespace cppjieba { +class QuerySegment: public SegmentBase { + public: + QuerySegment(const string& dict, const string& model, const string& userDict = "") + : mixSeg_(dict, model, userDict), + trie_(mixSeg_.GetDictTrie()) { + } + QuerySegment(const DictTrie* dictTrie, const HMMModel* model) + : mixSeg_(dictTrie, model), trie_(dictTrie) { + } + ~QuerySegment() { + } + + void Cut(const string& sentence, vector& words) const { + Cut(sentence, words, true); + } + void Cut(const string& sentence, vector& words, bool hmm) const { + vector tmp; + Cut(sentence, tmp, hmm); + GetStringsFromWords(tmp, words); + } + void Cut(const string& sentence, vector& words, bool hmm = true) const { + PreFilter pre_filter(symbols_, sentence); + PreFilter::Range range; + vector wrs; + wrs.reserve(sentence.size()/2); + while (pre_filter.HasNext()) { + range = pre_filter.Next(); + Cut(range.begin, range.end, wrs, hmm); + } + words.clear(); + words.reserve(wrs.size()); + GetWordsFromWordRanges(sentence, wrs, words); + } + void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res, bool hmm) const { + //use mix Cut first + vector mixRes; + mixSeg_.Cut(begin, end, mixRes, hmm); + + vector fullRes; + for (vector::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) { + if (mixResItr->Length() > 2) { + for (size_t i = 0; i + 1 < mixResItr->Length(); i++) { + WordRange wr(mixResItr->left + i, mixResItr->left + i + 1); + if (trie_->Find(wr.left, wr.right + 1) != NULL) { + res.push_back(wr); + } + } + } + if (mixResItr->Length() > 3) { + for (size_t i = 0; i + 2 < mixResItr->Length(); i++) { + WordRange wr(mixResItr->left + i, mixResItr->left + i + 2); + if (trie_->Find(wr.left, wr.right + 1) != NULL) { + res.push_back(wr); + } + } + } + res.push_back(*mixResItr); + } + } + private: + bool IsAllAscii(const Unicode& s) const { + for(size_t i = 0; i < s.size(); i++) { + if (s[i] >= 0x80) { + return false; + } + } + return true; + } + MixSegment mixSeg_; + const DictTrie* trie_; +}; // QuerySegment + +} // namespace cppjieba + +#endif From 1cc19e4d48ba82bc39827a9358af1f54025153b4 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:48 +0800 Subject: [PATCH 093/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/SegmentBase.hpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/SegmentBase.hpp diff --git a/third_party/cppjieba/include/cppjieba/SegmentBase.hpp b/third_party/cppjieba/include/cppjieba/SegmentBase.hpp new file mode 100644 index 00000000..130b2128 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/SegmentBase.hpp @@ -0,0 +1,46 @@ +#ifndef CPPJIEBA_SEGMENTBASE_H +#define CPPJIEBA_SEGMENTBASE_H + +#include "limonp/Logging.hpp" +#include "PreFilter.hpp" +#include + + +namespace cppjieba { + +const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82"; + +using namespace limonp; + +class SegmentBase { + public: + SegmentBase() { + XCHECK(ResetSeparators(SPECIAL_SEPARATORS)); + } + virtual ~SegmentBase() { + } + + virtual void Cut(const string& sentence, vector& words) const = 0; + + bool ResetSeparators(const string& s) { + symbols_.clear(); + RuneStrArray runes; + if (!DecodeUTF8RunesInString(s, runes)) { + XLOG(ERROR) << "UTF-8 decode failed for separators: " << s; + return false; + } + for (size_t i = 0; i < runes.size(); i++) { + if (!symbols_.insert(runes[i].rune).second) { + XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists"; + return false; + } + } + return true; + } + protected: + unordered_set symbols_; +}; // class SegmentBase + +} // cppjieba + +#endif From cd3aea115b32b935053f949551306b0f5671285b Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:49 +0800 Subject: [PATCH 094/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../include/cppjieba/SegmentTagged.hpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/SegmentTagged.hpp diff --git a/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp b/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp new file mode 100644 index 00000000..4d99a31a --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp @@ -0,0 +1,23 @@ +#ifndef CPPJIEBA_SEGMENTTAGGED_H +#define CPPJIEBA_SEGMENTTAGGED_H + +#include "SegmentBase.hpp" + +namespace cppjieba { + +class SegmentTagged : public SegmentBase{ + public: + SegmentTagged() { + } + virtual ~SegmentTagged() { + } + + virtual bool Tag(const string& src, vector >& res) const = 0; + + virtual const DictTrie* GetDictTrie() const = 0; + +}; // class SegmentTagged + +} // cppjieba + +#endif From 4872f50c215dc7552eae4bfa5c7bbed831596e39 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:51 +0800 Subject: [PATCH 095/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../include/cppjieba/TextRankExtractor.hpp | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp diff --git a/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp b/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp new file mode 100644 index 00000000..292d0a8f --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp @@ -0,0 +1,190 @@ +#ifndef CPPJIEBA_TEXTRANK_EXTRACTOR_H +#define CPPJIEBA_TEXTRANK_EXTRACTOR_H + +#include +#include "Jieba.hpp" + +namespace cppjieba { + using namespace limonp; + using namespace std; + + class TextRankExtractor { + public: + typedef struct _Word {string word;vector offsets;double weight;} Word; // struct Word + private: + typedef std::map WordMap; + + class WordGraph{ + private: + typedef double Score; + typedef string Node; + typedef std::set NodeSet; + + typedef std::map Edges; + typedef std::map Graph; + //typedef std::unordered_map Edges; + //typedef std::unordered_map Graph; + + double d; + Graph graph; + NodeSet nodeSet; + public: + WordGraph(): d(0.85) {}; + WordGraph(double in_d): d(in_d) {}; + + void addEdge(Node start,Node end,double weight){ + Edges temp; + Edges::iterator gotEdges; + nodeSet.insert(start); + nodeSet.insert(end); + graph[start][end]+=weight; + graph[end][start]+=weight; + } + + void rank(WordMap &ws,size_t rankTime=10){ + WordMap outSum; + Score wsdef, min_rank, max_rank; + + if( graph.size() == 0) + return; + + wsdef = 1.0 / graph.size(); + + for(Graph::iterator edges=graph.begin();edges!=graph.end();++edges){ + // edges->first start节点；edge->first end节点；edge->second 权重 + ws[edges->first].word=edges->first; + ws[edges->first].weight=wsdef; + outSum[edges->first].weight=0; + for(Edges::iterator edge=edges->second.begin();edge!=edges->second.end();++edge){ + outSum[edges->first].weight+=edge->second; + } + } + //sort(nodeSet.begin(),nodeSet.end()); 是否需要排序? + for( size_t i=0; ifirst end节点；edge->second 权重 + s += edge->second / outSum[edge->first].weight * ws[edge->first].weight; + ws[*node].weight = (1 - d) + d * s; + } + } + + min_rank=max_rank=ws.begin()->second.weight; + for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){ + if( i->second.weight < min_rank ){ + min_rank = i->second.weight; + } + if( i->second.weight > max_rank ){ + max_rank = i->second.weight; + } + } + for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){ + ws[i->first].weight = (i->second.weight - min_rank / 10.0) / (max_rank - min_rank / 10.0); + } + } + }; + + public: + TextRankExtractor(const string& dictPath, + const string& hmmFilePath, + const string& stopWordPath, + const string& userDict = "") + : segment_(dictPath, hmmFilePath, userDict) { + LoadStopWordDict(stopWordPath); + } + TextRankExtractor(const DictTrie* dictTrie, + const HMMModel* model, + const string& stopWordPath) + : segment_(dictTrie, model) { + LoadStopWordDict(stopWordPath); + } + TextRankExtractor(const Jieba& jieba, const string& stopWordPath) : segment_(jieba.GetDictTrie(), jieba.GetHMMModel()) { + LoadStopWordDict(stopWordPath); + } + ~TextRankExtractor() { + } + + void Extract(const string& sentence, vector& keywords, size_t topN) const { + vector topWords; + Extract(sentence, topWords, topN); + for (size_t i = 0; i < topWords.size(); i++) { + keywords.push_back(topWords[i].word); + } + } + + void Extract(const string& sentence, vector >& keywords, size_t topN) const { + vector topWords; + Extract(sentence, topWords, topN); + for (size_t i = 0; i < topWords.size(); i++) { + keywords.push_back(pair(topWords[i].word, topWords[i].weight)); + } + } + + void Extract(const string& sentence, vector& keywords, size_t topN, size_t span=5,size_t rankTime=10) const { + vector words; + segment_.Cut(sentence, words); + + TextRankExtractor::WordGraph graph; + WordMap wordmap; + size_t offset = 0; + + for(size_t i=0; i < words.size(); i++){ + size_t t = offset; + offset += words[i].size(); + if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) { + continue; + } + for(size_t j=i+1,skip=0;jsecond); + } + + topN = min(topN, keywords.size()); + partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare); + keywords.resize(topN); + } + private: + void LoadStopWordDict(const string& filePath) { + ifstream ifs(filePath.c_str()); + XCHECK(ifs.is_open()) << "open " << filePath << " failed"; + string line ; + while (getline(ifs, line)) { + stopWords_.insert(line); + } + assert(stopWords_.size()); + } + + static bool Compare(const Word &x,const Word &y){ + return x.weight > y.weight; + } + + MixSegment segment_; + unordered_set stopWords_; + }; // class TextRankExtractor + + inline ostream& operator << (ostream& os, const TextRankExtractor::Word& word) { + return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; + } +} // namespace cppjieba + +#endif + + From 41a5c381b5b1371a553a7541bdc9be22388dd0ed Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:53 +0800 Subject: [PATCH 096/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/Trie.hpp | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/Trie.hpp diff --git a/third_party/cppjieba/include/cppjieba/Trie.hpp b/third_party/cppjieba/include/cppjieba/Trie.hpp new file mode 100644 index 00000000..dc3c78a2 --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/Trie.hpp @@ -0,0 +1,193 @@ +#ifndef CPPJIEBA_TRIE_HPP +#define CPPJIEBA_TRIE_HPP + +#include +#include +#include "limonp/StdExtension.hpp" +#include "Unicode.hpp" + +namespace cppjieba { + +using namespace std; + +const size_t MAX_WORD_LENGTH = 512; + +struct DictUnit { + Unicode word; + double weight; + string tag; +}; // struct DictUnit + +struct Dag { + RuneStr runestr; + // [offset, nexts.first] + limonp::LocalVector > nexts; + const DictUnit * pInfo; + double weight; + size_t nextPos; // TODO + Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) { + } +}; // struct Dag + +typedef Rune TrieKey; + +class TrieNode { + public : + TrieNode(): next(NULL), ptValue(NULL) { + } + public: + typedef unordered_map NextMap; + NextMap *next; + const DictUnit *ptValue; +}; + +class Trie { + public: + Trie(const vector& keys, const vector& valuePointers) + : root_(new TrieNode) { + CreateTrie(keys, valuePointers); + } + ~Trie() { + DeleteNode(root_); + } + + const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { + if (begin == end) { + return NULL; + } + + const TrieNode* ptNode = root_; + TrieNode::NextMap::const_iterator citer; + for (RuneStrArray::const_iterator it = begin; it != end; it++) { + if (NULL == ptNode->next) { + return NULL; + } + citer = ptNode->next->find(it->rune); + if (ptNode->next->end() == citer) { + return NULL; + } + ptNode = citer->second; + } + return ptNode->ptValue; + } + + void Find(RuneStrArray::const_iterator begin, + RuneStrArray::const_iterator end, + vector&res, + size_t max_word_len = MAX_WORD_LENGTH) const { + assert(root_ != NULL); + res.resize(end - begin); + + const TrieNode *ptNode = NULL; + TrieNode::NextMap::const_iterator citer; + for (size_t i = 0; i < size_t(end - begin); i++) { + res[i].runestr = *(begin + i); + + if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) { + ptNode = citer->second; + } else { + ptNode = NULL; + } + if (ptNode != NULL) { + res[i].nexts.push_back(pair(i, ptNode->ptValue)); + } else { + res[i].nexts.push_back(pair(i, static_cast(NULL))); + } + + for (size_t j = i + 1; j < size_t(end - begin) && (j - i + 1) <= max_word_len; j++) { + if (ptNode == NULL || ptNode->next == NULL) { + break; + } + citer = ptNode->next->find((begin + j)->rune); + if (ptNode->next->end() == citer) { + break; + } + ptNode = citer->second; + if (NULL != ptNode->ptValue) { + res[i].nexts.push_back(pair(j, ptNode->ptValue)); + } + } + } + } + + void InsertNode(const Unicode& key, const DictUnit* ptValue) { + if (key.begin() == key.end()) { + return; + } + + TrieNode::NextMap::const_iterator kmIter; + TrieNode *ptNode = root_; + for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) { + if (NULL == ptNode->next) { + ptNode->next = new TrieNode::NextMap; + } + kmIter = ptNode->next->find(*citer); + if (ptNode->next->end() == kmIter) { + TrieNode *nextNode = new TrieNode; + + ptNode->next->insert(make_pair(*citer, nextNode)); + ptNode = nextNode; + } else { + ptNode = kmIter->second; + } + } + assert(ptNode != NULL); + ptNode->ptValue = ptValue; + } + void DeleteNode(const Unicode& key, const DictUnit* /*ptValue*/) { + if (key.begin() == key.end()) { + return; + } + //定义一个NextMap迭代器 + TrieNode::NextMap::const_iterator kmIter; + //定义一个指向root的TrieNode指针 + TrieNode *ptNode = root_; + for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) { + //链表不存在元素 + if (NULL == ptNode->next) { + return; + } + kmIter = ptNode->next->find(*citer); + //如果map中不存在,跳出循环 + if (ptNode->next->end() == kmIter) { + break; + } + //从unordered_map中擦除该项 + ptNode->next->erase(*citer); + //删除该node + ptNode = kmIter->second; + delete ptNode; + break; + } + return; + } + private: + void CreateTrie(const vector& keys, const vector& valuePointers) { + if (valuePointers.empty() || keys.empty()) { + return; + } + assert(keys.size() == valuePointers.size()); + + for (size_t i = 0; i < keys.size(); i++) { + InsertNode(keys[i], valuePointers[i]); + } + } + + void DeleteNode(TrieNode* node) { + if (NULL == node) { + return; + } + if (NULL != node->next) { + for (TrieNode::NextMap::iterator it = node->next->begin(); it != node->next->end(); ++it) { + DeleteNode(it->second); + } + delete node->next; + } + delete node; + } + + TrieNode* root_; +}; // class Trie +} // namespace cppjieba + +#endif // CPPJIEBA_TRIE_HPP From cf18848086739fd6a3cb55f58b19c7950294eb5f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:54 +0800 Subject: [PATCH 097/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- .../cppjieba/include/cppjieba/Unicode.hpp | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 third_party/cppjieba/include/cppjieba/Unicode.hpp diff --git a/third_party/cppjieba/include/cppjieba/Unicode.hpp b/third_party/cppjieba/include/cppjieba/Unicode.hpp new file mode 100644 index 00000000..9adec2ca --- /dev/null +++ b/third_party/cppjieba/include/cppjieba/Unicode.hpp @@ -0,0 +1,227 @@ +#ifndef CPPJIEBA_UNICODE_H +#define CPPJIEBA_UNICODE_H + +#include +#include +#include +#include +#include +#include "limonp/LocalVector.hpp" + +namespace cppjieba { + +using std::string; +using std::vector; + +typedef uint32_t Rune; + +struct Word { + string word; + uint32_t offset; + uint32_t unicode_offset; + uint32_t unicode_length; + Word(const string& w, uint32_t o) + : word(w), offset(o) { + } + Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length) + : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) { + } +}; // struct Word + +inline std::ostream& operator << (std::ostream& os, const Word& w) { + return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}"; +} + +struct RuneStr { + Rune rune; + uint32_t offset; + uint32_t len; + uint32_t unicode_offset; + uint32_t unicode_length; + RuneStr(): rune(0), offset(0), len(0), unicode_offset(0), unicode_length(0) { + } + RuneStr(Rune r, uint32_t o, uint32_t l) + : rune(r), offset(o), len(l), unicode_offset(0), unicode_length(0) { + } + RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length) + : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) { + } +}; // struct RuneStr + +inline std::ostream& operator << (std::ostream& os, const RuneStr& r) { + return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}"; +} + +typedef limonp::LocalVector Unicode; +typedef limonp::LocalVector RuneStrArray; + +// [left, right] +struct WordRange { + RuneStrArray::const_iterator left; + RuneStrArray::const_iterator right; + WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r) + : left(l), right(r) { + } + size_t Length() const { + return right - left + 1; + } + bool IsAllAscii() const { + for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) { + if (iter->rune >= 0x80) { + return false; + } + } + return true; + } +}; // struct WordRange + +struct RuneStrLite { + uint32_t rune; + uint32_t len; + RuneStrLite(): rune(0), len(0) { + } + RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) { + } +}; // struct RuneStrLite + +inline RuneStrLite DecodeUTF8ToRune(const char* str, size_t len) { + RuneStrLite rp(0, 0); + if (str == NULL || len == 0) { + return rp; + } + if (!(str[0] & 0x80)) { // 0xxxxxxx + // 7bit, total 7bit + rp.rune = (uint8_t)(str[0]) & 0x7f; + rp.len = 1; + } else if ((uint8_t)str[0] <= 0xdf && 1 < len) { + // 110xxxxxx + // 5bit, total 5bit + rp.rune = (uint8_t)(str[0]) & 0x1f; + + // 6bit, total 11bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[1]) & 0x3f; + rp.len = 2; + } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx + // 4bit, total 4bit + rp.rune = (uint8_t)(str[0]) & 0x0f; + + // 6bit, total 10bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[1]) & 0x3f; + + // 6bit, total 16bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[2]) & 0x3f; + + rp.len = 3; + } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx + // 3bit, total 3bit + rp.rune = (uint8_t)(str[0]) & 0x07; + + // 6bit, total 9bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[1]) & 0x3f; + + // 6bit, total 15bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[2]) & 0x3f; + + // 6bit, total 21bit + rp.rune <<= 6; + rp.rune |= (uint8_t)(str[3]) & 0x3f; + + rp.len = 4; + } else { + rp.rune = 0; + rp.len = 0; + } + return rp; +} + +inline bool DecodeUTF8RunesInString(const char* s, size_t len, RuneStrArray& runes) { + runes.clear(); + runes.reserve(len / 2); + for (uint32_t i = 0, j = 0; i < len;) { + RuneStrLite rp = DecodeUTF8ToRune(s + i, len - i); + if (rp.len == 0) { + runes.clear(); + return false; + } + RuneStr x(rp.rune, i, rp.len, j, 1); + runes.push_back(x); + i += rp.len; + ++j; + } + return true; +} + +inline bool DecodeUTF8RunesInString(const string& s, RuneStrArray& runes) { + return DecodeUTF8RunesInString(s.c_str(), s.size(), runes); +} + +inline bool DecodeUTF8RunesInString(const char* s, size_t len, Unicode& unicode) { + unicode.clear(); + RuneStrArray runes; + if (!DecodeUTF8RunesInString(s, len, runes)) { + return false; + } + unicode.reserve(runes.size()); + for (size_t i = 0; i < runes.size(); i++) { + unicode.push_back(runes[i].rune); + } + return true; +} + +inline bool IsSingleWord(const string& str) { + RuneStrLite rp = DecodeUTF8ToRune(str.c_str(), str.size()); + return rp.len == str.size(); +} + +inline bool DecodeUTF8RunesInString(const string& s, Unicode& unicode) { + return DecodeUTF8RunesInString(s.c_str(), s.size(), unicode); +} + +inline Unicode DecodeUTF8RunesInString(const string& s) { + Unicode result; + DecodeUTF8RunesInString(s, result); + return result; +} + + +// [left, right] +inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) { + assert(right->offset >= left->offset); + uint32_t len = right->offset - left->offset + right->len; + uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length; + return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length); +} + +inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) { + assert(right->offset >= left->offset); + uint32_t len = right->offset - left->offset + right->len; + return s.substr(left->offset, len); +} + +inline void GetWordsFromWordRanges(const string& s, const vector& wrs, vector& words) { + for (size_t i = 0; i < wrs.size(); i++) { + words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right)); + } +} + +inline vector GetWordsFromWordRanges(const string& s, const vector& wrs) { + vector result; + GetWordsFromWordRanges(s, wrs, result); + return result; +} + +inline void GetStringsFromWords(const vector& words, vector& strs) { + strs.resize(words.size()); + for (size_t i = 0; i < words.size(); ++i) { + strs[i] = words[i].word; + } +} + +} // namespace cppjieba + +#endif // CPPJIEBA_UNICODE_H From 3c0d78d413ff1caf957df4d145cd4a9d23318d4d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:55 +0800 Subject: [PATCH 098/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/CMakeLists.txt | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/fts/CMakeLists.txt b/fts/CMakeLists.txt index cd3fd431..cc19bf02 100644 --- a/fts/CMakeLists.txt +++ b/fts/CMakeLists.txt @@ -4,7 +4,21 @@ include_directories( src/include third_party/snowball/libstemmer ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include) + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include + ${PROJECT_SOURCE_DIR}/third_party/mecab/src) + +# cppjieba (jieba tokenizer dependency) is a header-only interface library; +# upstream references it but never vendored it, so define the target here. +if(NOT TARGET cppjieba) + add_library(cppjieba INTERFACE) + target_include_directories(cppjieba INTERFACE + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include) +endif() + +# MeCab (Japanese tokenizer): static lib + mecab-dict-index tool + build-time +# ipadic dictionary compilation (downloads the dictionary CSV at configure). +add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/mecab) add_subdirectory(src/catalog) add_subdirectory(src/function) @@ -15,16 +29,29 @@ add_subdirectory(src/utils) add_subdirectory(third_party/snowball) +# tokenizer.cpp includes mecab.h, which requires DLL_EXPORT to declare the +# MeCab API without __declspec(dllimport) on Windows. +target_compile_definitions(lbug_fts_utils PRIVATE DLL_EXPORT) + build_extension_lib(${BUILD_STATIC_EXTENSION} "fts") target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension PRIVATE snowball re2 - cppjieba) + cppjieba + mecab) # Copy Jieba dictionaries next to the built extension so runtime can find a default path add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/third_party/cppjieba/dict ${PROJECT_SOURCE_DIR}/extension/fts/build/dict) + +# Copy the compiled ipadic dictionary (UTF-8) next to the built extension so +# the default mecab_dict_dir resolves at runtime. +add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_BINARY_DIR}/mecab-dict/ipadic + ${PROJECT_SOURCE_DIR}/extension/fts/build/dict-ipadic) +add_dependencies(lbug_${EXTENSION_LIB_NAME}_extension mecab-ipadic-dict) From af3f95aff172687160f12940c9075d2c338130aa Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:57 +0800 Subject: [PATCH 099/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/include/function/fts_config.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fts/src/include/function/fts_config.h b/fts/src/include/function/fts_config.h index a2326d05..9b7b0b33 100644 --- a/fts/src/include/function/fts_config.h +++ b/fts/src/include/function/fts_config.h @@ -4,6 +4,7 @@ #include "common/types/types.h" #include "function/table/bind_input.h" +#include "utils/tokenizer.h" #include namespace lbug { @@ -65,9 +66,14 @@ struct Tokenizer { struct TokenizerInfo { std::string tokenizer = Tokenizer::DEFAULT_VALUE; - std::string jiebaDictDir = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY); - - TokenizerInfo() = default; + // Tokenizer-specific parameters (e.g. "jieba_dict_dir" for the jieba tokenizer). + TokenizerParams params; + + TokenizerInfo() { + params["jieba_dict_dir"] = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY); + params["mecab_dict_dir"] = + std::format("{}/extension/fts/build/dict-ipadic", LBUG_ROOT_DIRECTORY); + } }; struct FTSConfig; @@ -95,16 +101,19 @@ struct FTSConfig { std::string ignorePattern = ""; std::string ignorePatternQuery = ""; std::string tokenizer = ""; - std::string jiebaDictDir = ""; + // Serialized as an unordered map after a magic marker; entries of legacy + // catalogs (which stored a single "jiebaDictDir" string here) are folded + // into tokenizerParams["jieba_dict_dir"] at deserialization time. + TokenizerParams tokenizerParams = {}; FTSConfig() = default; FTSConfig(std::string stemmer, std::string stopWordsTableName, std::string stopWordsSource, std::string ignorePattern, std::string ignorePatternQuery, std::string tokenizer, - std::string jiebaDictDir) + TokenizerParams tokenizerParams) : stemmer{std::move(stemmer)}, stopWordsTableName{std::move(stopWordsTableName)}, stopWordsSource{std::move(stopWordsSource)}, ignorePattern{std::move(ignorePattern)}, ignorePatternQuery{std::move(ignorePatternQuery)}, tokenizer{std::move(tokenizer)}, - jiebaDictDir{std::move(jiebaDictDir)} {} + tokenizerParams{std::move(tokenizerParams)} {} void serialize(common::Serializer& serializer) const; From 02a5c26a82beacec1bc97be005c4744bdefdcb5b Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:41:59 +0800 Subject: [PATCH 100/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/function/fts_config.cpp | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/fts/src/function/fts_config.cpp b/fts/src/function/fts_config.cpp index d3b7c7cf..33eae4d3 100644 --- a/fts/src/function/fts_config.cpp +++ b/fts/src/function/fts_config.cpp @@ -156,8 +156,11 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_ Tokenizer::validate(tokenizerInfo.tokenizer); } else if (lowerCaseName == "jieba_dict_dir") { value.validateType(common::LogicalTypeID::STRING); - tokenizerInfo.jiebaDictDir = - common::StringUtils::getLower(value.getValue()); + // Note: the dict dir is a file path and must not be lower-cased. + tokenizerInfo.params["jieba_dict_dir"] = value.getValue(); + } else if (lowerCaseName == "mecab_dict_dir") { + value.validateType(common::LogicalTypeID::STRING); + tokenizerInfo.params["mecab_dict_dir"] = value.getValue(); } else { throw common::BinderException{"Unrecognized optional parameter: " + name}; } @@ -166,9 +169,16 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_ FTSConfig CreateFTSConfig::getFTSConfig() const { return FTSConfig{stemmer, stopWordsTableInfo.tableName, stopWordsTableInfo.stopWords, - ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.jiebaDictDir}; + ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.params}; } +// Magic marker written before the tokenizer params so that deserialization can +// tell apart new catalogs (marker + unordered map) from legacy ones (a single +// "jiebaDictDir" string field). +namespace { +constexpr const char* TOKENIZER_PARAMS_MAGIC = "lbug_tokenizer_params_v1"; +} // namespace + void FTSConfig::serialize(common::Serializer& serializer) const { serializer.serializeValue(stemmer); serializer.serializeValue(stopWordsTableName); @@ -176,7 +186,8 @@ void FTSConfig::serialize(common::Serializer& serializer) const { serializer.serializeValue(ignorePattern); serializer.serializeValue(ignorePatternQuery); serializer.serializeValue(tokenizer); - serializer.serializeValue(jiebaDictDir); + serializer.serializeValue(std::string{TOKENIZER_PARAMS_MAGIC}); + serializer.serializeUnorderedMap(tokenizerParams); } FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) { @@ -187,7 +198,14 @@ FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) { deserializer.deserializeValue(config.ignorePattern); deserializer.deserializeValue(config.ignorePatternQuery); deserializer.deserializeValue(config.tokenizer); - deserializer.deserializeValue(config.jiebaDictDir); + std::string tokenizerParamsField; + deserializer.deserializeValue(tokenizerParamsField); + if (tokenizerParamsField == TOKENIZER_PARAMS_MAGIC) { + deserializer.deserializeUnorderedMap(config.tokenizerParams); + } else { + // Legacy catalog: the field after tokenizer was the jieba dict dir. + config.tokenizerParams["jieba_dict_dir"] = tokenizerParamsField; + } return config; } @@ -214,7 +232,7 @@ void TopK::validate(uint64_t value) { } void Tokenizer::validate(const std::string& tokenizer) { - if (tokenizer == "simple" || tokenizer == "jieba") { + if (TokenizerRegistry::isSupported(tokenizer)) { return; } throw common::BinderException{ From 00c2b31692d90411d03d8da7ef7224a12b51e514 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:00 +0800 Subject: [PATCH 101/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/function/tokenize.cpp | 77 ++++++++++++++--------------------- 1 file changed, 31 insertions(+), 46 deletions(-) diff --git a/fts/src/function/tokenize.cpp b/fts/src/function/tokenize.cpp index 4f3e2af3..c094460f 100644 --- a/fts/src/function/tokenize.cpp +++ b/fts/src/function/tokenize.cpp @@ -6,10 +6,10 @@ #include "common/types/string_t.h" #include "common/types/types.h" #include "common/vector/value_vector.h" -#include "cppjieba/Jieba.hpp" #include "expression_evaluator/expression_evaluator_utils.h" #include "function/scalar_function.h" #include "re2.h" +#include "utils/tokenizer.h" namespace lbug { namespace fts_extension { @@ -17,16 +17,17 @@ namespace fts_extension { using namespace function; using namespace common; -struct JiebaBindData final : public FunctionBindData { - std::shared_ptr jieba; +struct TokenizerBindData final : public FunctionBindData { + std::shared_ptr tokenizer; - JiebaBindData(common::logical_type_vec_t paramTypes, std::shared_ptr jieba) + TokenizerBindData(common::logical_type_vec_t paramTypes, + std::shared_ptr tokenizer) : FunctionBindData{std::move(paramTypes), common::LogicalType::LIST(common::LogicalType::STRING())}, - jieba{std::move(jieba)} {} + tokenizer{std::move(tokenizer)} {} std::unique_ptr copy() const override { - return std::make_unique(copyVector(paramTypes), jieba); + return std::make_unique(copyVector(paramTypes), tokenizer); } }; @@ -38,21 +39,11 @@ static void addTokensToVector(const std::vector& tokens, list_entry } } -struct JiebaTokenizer { +struct TokenizeOp { static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/, list_entry_t& result, common::ValueVector& resultVector, void* dataPtr) { - std::vector tokens; - auto bindData = reinterpret_cast(dataPtr); - bindData->jieba->CutForSearch(text.getAsString(), tokens); - addTokensToVector(tokens, result, resultVector); - } -}; - -struct SimpleTokenizer { - static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/, - list_entry_t& result, common::ValueVector& resultVector, void* /*dataPtr*/) { - auto tokens = - StringUtils::split(text.getAsString(), " ", true /* ignoreEmptyStringParts */); + auto bindData = reinterpret_cast(dataPtr); + auto tokens = bindData->tokenizer->tokenize(text.getAsString()); addTokensToVector(tokens, result, resultVector); } }; @@ -62,37 +53,31 @@ static std::unique_ptr bindFunc(const ScalarBindFuncInput& inp throw BinderException{"The tokenizer parameter must be a literal expression."}; } if (input.arguments[2]->expressionType != ExpressionType::LITERAL) { - throw BinderException{"The path to the jieba dict directory must be a literal expression."}; + throw BinderException{"The tokenizer parameter must be a literal expression."}; } auto value = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression(input.arguments[1], input.context); - auto tokenizer = common::StringUtils::getLower(value.getValue()); - if (tokenizer == "jieba") { - std::string dictDir = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression( - input.arguments[2], input.context) - .getValue(); - std::string dict = dictDir + "/jieba.dict.utf8"; - std::string hmm = dictDir + "/hmm_model.utf8"; - std::string user = dictDir + "/user.dict.utf8"; // Contains custom AI/ML terms - std::string idf = dictDir + "/idf.utf8"; - std::string stop = dictDir + "/stop_words.utf8"; - auto jieba = std::make_unique(dict, hmm, user, idf, stop); - input.definition->ptrCast()->execFunc = - ScalarFunction::TernaryRegexExecFunction; - return std::make_unique( - binder::ExpressionUtil::getDataTypes(input.arguments), std::move(jieba)); - } else if (tokenizer == "simple" || tokenizer == "") { - input.definition->ptrCast()->execFunc = - ScalarFunction::TernaryRegexExecFunction; - return FunctionBindData::getSimpleBindData(input.arguments, - LogicalType::LIST(LogicalType::STRING())); - } else { - throw common::BinderException{ - "Unsupported tokenizer: " + tokenizer + - ".\nSupported tokenizers: 'simple' (default), 'jieba' (advanced Chinese)"}; + auto tokenizerName = common::StringUtils::getLower(value.getValue()); + if (tokenizerName.empty()) { + tokenizerName = "simple"; + } + // The third argument is the tokenizer's extra parameter; for 'jieba' it is + // the dictionary directory. + auto extraParam = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression( + input.arguments[2], input.context) + .getValue(); + TokenizerParams params; + if (tokenizerName == "jieba") { + params["jieba_dict_dir"] = extraParam; + } else if (tokenizerName == "mecab") { + params["mecab_dict_dir"] = extraParam; } + auto tokenizer = TokenizerPool::getOrCreate(tokenizerName, params); + input.definition->ptrCast()->execFunc = + ScalarFunction::TernaryRegexExecFunction; + return std::make_unique( + binder::ExpressionUtil::getDataTypes(input.arguments), std::move(tokenizer)); } function::function_set TokenizeFunction::getFunctionSet() { From 38255bb5dfd5654346c6ad2eb72432bd74202d2f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:02 +0800 Subject: [PATCH 102/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/function/create_fts_index.cpp | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index 2cc78ebc..8986b462 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -157,13 +157,22 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string query = ""; if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context), FTSUtils::getTokenizeMacroName(tableID, indexName))) { - // TOKENIZE(text, tokenizer, extra_param) + // TOKENIZE(text, tokenizer, extra_param); the extra param is the + // dictionary directory for dictionary-based tokenizers. Pick it by + // tokenizer name (the params map holds defaults for all tokenizers). + auto& tokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params; + auto& tokenizerName = ftsBindData->createFTSConfig.tokenizerInfo.tokenizer; + std::string extraParam = ""; + if (tokenizerName == "jieba") { + extraParam = tokenizerParams.at("jieba_dict_dir"); + } else if (tokenizerName == "mecab") { + extraParam = tokenizerParams.at("mecab_dict_dir"); + } query += std::format(R"(CREATE MACRO `{}`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)", FTSUtils::getTokenizeMacroName(tableID, indexName), formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), - ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, - ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); + ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, extraParam); } // Create the stop words table if not exists, or the user is not using the default english @@ -236,7 +245,17 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string params; params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer); params += - std::format("stopWords := '{}'", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + std::format("stopWords := '{}', ", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + // The internal index uses this config for incremental inserts/updates and + // queries, so the tokenizer (and its parameters) must be forwarded or it + // silently falls back to the default 'simple' tokenizer. + params += std::format("tokenizer := '{}'", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer); + auto& ftsTokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params; + if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "jieba") { + params += std::format(", jieba_dict_dir := '{}'", ftsTokenizerParams.at("jieba_dict_dir")); + } else if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "mecab") { + params += std::format(", mecab_dict_dir := '{}'", ftsTokenizerParams.at("mecab_dict_dir")); + } query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName, properties, params); query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName); From 7a77160e1611eeab8145ca2b4cf4f9d0717ef77a Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:03 +0800 Subject: [PATCH 103/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/include/utils/fts_utils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index 96ae0298..f43e5ab3 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -58,7 +58,8 @@ struct FTSUtils { return std::format("{}_tokenize", getInternalTablePrefix(tableID, indexName)); } - static std::vector tokenizeString(std::string& str, const FTSConfig& tokenizer); + static std::vector tokenizeString(const std::string& str, + const FTSConfig& tokenizer); }; } // namespace fts_extension From d912727010af9e0aa787874cfed0eccd0cc0f6ca Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:05 +0800 Subject: [PATCH 104/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/utils/fts_utils.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index fb6d68bc..ed4c02ea 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -1,12 +1,12 @@ #include "utils/fts_utils.h" #include "common/string_utils.h" -#include "cppjieba/Jieba.hpp" #include "function/stem.h" #include "libstemmer.h" #include "re2.h" #include "storage/storage_manager.h" #include "storage/table/node_table.h" +#include "utils/tokenizer.h" namespace lbug { namespace fts_extension { @@ -94,17 +94,11 @@ std::vector FTSUtils::stemTerms(std::vector terms, return result; } -std::vector FTSUtils::tokenizeString(std::string& str, const FTSConfig& config) { - std::vector terms; - if (config.tokenizer == "jieba") { - cppjieba::Jieba jieba(config.jiebaDictDir + "/jieba.dict.utf8", - config.jiebaDictDir + "/hmm_model.utf8", config.jiebaDictDir + "/user.dict.utf8", - config.jiebaDictDir + "/idf.utf8", config.jiebaDictDir + "/stop_words.utf8"); - jieba.CutForSearch(str, terms); - } else { - terms = StringUtils::split(str, " ", true /* ignoreEmptyStringParts */); - } - return terms; +std::vector FTSUtils::tokenizeString(const std::string& str, + const FTSConfig& config) { + // Tokenizer instances are cached in the pool, so this is cheap even though + // constructing a jieba tokenizer loads and indexes a dictionary. + return TokenizerPool::getOrCreate(config.tokenizer, config.tokenizerParams)->tokenize(str); } } // namespace fts_extension From ee066e82320367aae089667d03438919dfecc84a Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:06 +0800 Subject: [PATCH 105/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/utils/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fts/src/utils/CMakeLists.txt b/fts/src/utils/CMakeLists.txt index 1e5294f3..350a2560 100644 --- a/fts/src/utils/CMakeLists.txt +++ b/fts/src/utils/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(lbug_fts_utils OBJECT - fts_utils.cpp) + fts_utils.cpp + tokenizer.cpp) set(FTS_EXTENSION_OBJECT_FILES ${FTS_EXTENSION_OBJECT_FILES} $ From da235b9641b685841af11c2fd774b24d04ebe347 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:08 +0800 Subject: [PATCH 106/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/utils/tokenizer.cpp | 178 ++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 fts/src/utils/tokenizer.cpp diff --git a/fts/src/utils/tokenizer.cpp b/fts/src/utils/tokenizer.cpp new file mode 100644 index 00000000..2b93d098 --- /dev/null +++ b/fts/src/utils/tokenizer.cpp @@ -0,0 +1,178 @@ +#include "utils/tokenizer.h" + +#include "common/exception/binder.h" +#include "common/string_utils.h" +#include "cppjieba/Jieba.hpp" +#include "mecab.h" +#include +#include +#include + +namespace lbug { +namespace fts_extension { + +using namespace common; + +class SimpleTokenizer final : public ITokenizer { +public: + std::vector tokenize(const std::string& text) const override { + return StringUtils::split(text, " ", true /* ignoreEmptyStringParts */); + } + + std::string getName() const override { + return "simple"; + } +}; + +class JiebaTokenizer final : public ITokenizer { +public: + explicit JiebaTokenizer(const TokenizerParams& params) { + auto dictDir = + params.contains("jieba_dict_dir") ? params.at("jieba_dict_dir") : std::string{}; + jieba = std::make_shared(dictDir + "/jieba.dict.utf8", + dictDir + "/hmm_model.utf8", dictDir + "/user.dict.utf8", dictDir + "/idf.utf8", + dictDir + "/stop_words.utf8"); + } + + std::vector tokenize(const std::string& text) const override { + std::vector tokens; + jieba->CutForSearch(text, tokens); + return tokens; + } + + std::string getName() const override { + return "jieba"; + } + +private: + std::shared_ptr jieba; +}; + +std::unordered_map& TokenizerRegistry::getRegistry() { + static std::unordered_map registry; + return registry; +} + +// Japanese morphological analyzer. The ipadic dictionary is compiled to a +// UTF-8 binary dictionary at build time (see third_party/mecab). +class MeCabTokenizer final : public ITokenizer { +public: + explicit MeCabTokenizer(const TokenizerParams& params) { + auto dictDir = + params.contains("mecab_dict_dir") ? params.at("mecab_dict_dir") : std::string{}; + tagger.reset(MeCab::createTagger((std::string("-d ") + dictDir).c_str())); + if (!tagger) { + throw BinderException{ + std::format("Failed to create mecab tagger with dict dir: '{}'.", dictDir)}; + } + } + + std::vector tokenize(const std::string& text) const override { + std::vector tokens; + // parseToNode() returns nodes whose surface points into `text`, so the + // surface must be copied out while `text` is still alive. + const MeCab::Node* node = tagger->parseToNode(text.c_str()); + for (; node; node = node->next) { + if (node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE) { + continue; + } + if (node->surface == nullptr || node->length == 0) { + continue; + } + tokens.emplace_back(node->surface, node->length); + } + return tokens; + } + + std::string getName() const override { + return "mecab"; + } + +private: + std::unique_ptr tagger{nullptr, + MeCab::deleteTagger}; +}; + +void TokenizerRegistry::registerTokenizer(const std::string& name, TokenizerFactory factory) { + getRegistry()[name] = std::move(factory); +} + +bool TokenizerRegistry::isSupported(const std::string& name) { + return getRegistry().contains(name); +} + +std::vector TokenizerRegistry::getSupportedNames() { + std::vector names; + for (auto& [name, factory] : getRegistry()) { + names.push_back(name); + } + return names; +} + +std::unique_ptr TokenizerRegistry::create(const std::string& name, + const TokenizerParams& params) { + auto& registry = getRegistry(); + auto it = registry.find(name); + if (it == registry.end()) { + // Keep this message in sync with the registered tokenizers; the fts + // error.test asserts on it verbatim. + throw BinderException{"Unsupported tokenizer: " + name + + ".\nSupported tokenizers: 'simple' (default), 'jieba' (Chinese), 'mecab' " + "(Japanese)"}; + } + return it->second(params); +} + +namespace { + +struct BuiltinTokenizers { + BuiltinTokenizers() { + TokenizerRegistry::registerTokenizer("simple", + [](const TokenizerParams&) { return std::make_unique(); }); + TokenizerRegistry::registerTokenizer("jieba", + [](const TokenizerParams& params) { + return std::make_unique(params); + }); + TokenizerRegistry::registerTokenizer("mecab", + [](const TokenizerParams& params) { + return std::make_unique(params); + }); + } +}; + +} // namespace + +static BuiltinTokenizers builtinTokenizers; + +std::shared_ptr TokenizerPool::getOrCreate(const std::string& name, + const TokenizerParams& params) { + static std::mutex mutex; + static std::unordered_map> pool; + auto key = makeKey(name, params); + std::lock_guard guard{mutex}; + if (auto it = pool.find(key); it != pool.end()) { + if (auto instance = it->second.lock(); instance) { + return instance; + } + } + auto instance = std::shared_ptr(TokenizerRegistry::create(name, params)); + pool[key] = instance; + return instance; +} + +std::string TokenizerPool::makeKey(const std::string& name, const TokenizerParams& params) { + std::vector keys; + keys.reserve(params.size()); + for (auto& [key, value] : params) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end()); + std::string key = name; + for (auto& paramKey : keys) { + key += std::format("|{}={}", paramKey, params.at(paramKey)); + } + return key; +} + +} // namespace fts_extension +} // namespace lbug From ef8c2470915e2ba726a5f1fe932804b861aa3a50 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:10 +0800 Subject: [PATCH 107/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/src/include/utils/tokenizer.h | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 fts/src/include/utils/tokenizer.h diff --git a/fts/src/include/utils/tokenizer.h b/fts/src/include/utils/tokenizer.h new file mode 100644 index 00000000..ea697a48 --- /dev/null +++ b/fts/src/include/utils/tokenizer.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace lbug { +namespace fts_extension { + +// A tokenizer converts a piece of text (a document or a query) into a list of +// searchable terms. Each tokenizer is identified by a name and created from a +// set of string parameters (e.g. a dictionary directory). +class ITokenizer { +public: + virtual ~ITokenizer() = default; + + virtual std::vector tokenize(const std::string& text) const = 0; + + virtual std::string getName() const = 0; +}; + +using TokenizerParams = std::unordered_map; +using TokenizerFactory = + std::function(const TokenizerParams& params)>; + +// Tokenizers are registered by name. Adding a new tokenizer (e.g. 'mecab' for +// Japanese) only requires registering a factory here; TOKENIZE and +// CREATE_FTS_INDEX pick it up automatically. +class TokenizerRegistry { +public: + static void registerTokenizer(const std::string& name, TokenizerFactory factory); + + static bool isSupported(const std::string& name); + + static std::unique_ptr create(const std::string& name, + const TokenizerParams& params); + + static std::vector getSupportedNames(); + +private: + static std::unordered_map& getRegistry(); +}; + +// Constructing a tokenizer can be expensive (the jieba tokenizer loads and +// indexes a dictionary on construction), so instances are cached per +// (name, params) and shared across index inserts and queries. Tokenizer +// instances must be safe for concurrent use (jieba queries are read-only +// after construction). +class TokenizerPool { +public: + static std::shared_ptr getOrCreate(const std::string& name, + const TokenizerParams& params); + +private: + static std::string makeKey(const std::string& name, const TokenizerParams& params); +}; + +} // namespace fts_extension +} // namespace lbug From 159a5ec3e91fbe36edab33ec3ce4c74ebfe6a952 Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:11 +0800 Subject: [PATCH 108/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/test/test_files/error.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fts/test/test_files/error.test b/fts/test/test_files/error.test index f6852bad..8d4bb8de 100644 --- a/fts/test/test_files/error.test +++ b/fts/test/test_files/error.test @@ -206,4 +206,4 @@ Error: "missing ]: [a-z". -STATEMENT CALL CREATE_FTS_INDEX('person', 'personIdx', ['fName'], tokenizer := 'french') ---- error Binder exception: Unsupported tokenizer: french. -Supported tokenizers: 'simple' (default), 'jieba' (advanced Chinese) +Supported tokenizers: 'simple' (default), 'jieba' (Chinese), 'mecab' (Japanese) From 52d0e683b3b1107e5220b6e17fd62aaf23bb1b4d Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:13 +0800 Subject: [PATCH 109/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/test/test_files/fts_chinese.test | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/fts/test/test_files/fts_chinese.test b/fts/test/test_files/fts_chinese.test index 8905b03f..b6b4fb88 100644 --- a/fts/test/test_files/fts_chinese.test +++ b/fts/test/test_files/fts_chinese.test @@ -90,3 +90,22 @@ Python和机器学习结合开发AI应用 ---- 2 AI研究 NLP应用 + +# === Incremental Insert After Index Creation === +# Regression: the rewritten _CREATE_FTS_INDEX used to drop the tokenizer +# option, so incremental inserts fell back to 'simple' tokenization and the +# new rows' Chinese terms were never indexed. +-CASE ChineseIncrementalInsert +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE ZhDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:ZhDoc {text: '我爱北京天安门'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('ZhDoc', 'zh_idx', ['text'], tokenizer:='jieba', stemmer:='none'); +---- ok +-STATEMENT CREATE (d:ZhDoc {text: '北京欢迎你'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('ZhDoc', 'zh_idx', '北京', conjunctive:=false, top:=5) RETURN node.text ORDER BY node.text; +---- 2 +北京欢迎你 +我爱北京天安门 From 587cab11a5d06229ab4dd9f23e853ef8ea33d5bd Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 03:42:15 +0800 Subject: [PATCH 110/114] feat(fts): add MeCab Japanese tokenizer + tokenizer registry --- fts/test/test_files/fts_japanese.test | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 fts/test/test_files/fts_japanese.test diff --git a/fts/test/test_files/fts_japanese.test b/fts/test/test_files/fts_japanese.test new file mode 100644 index 00000000..b60725cf --- /dev/null +++ b/fts/test/test_files/fts_japanese.test @@ -0,0 +1,52 @@ +-DATASET CSV empty-db + +-- + +# Japanese FTS via the MeCab tokenizer with the compiled ipadic dictionary. +# The dictionary is generated at build time (third_party/mecab) and copied +# next to the extension; mecab_dict_dir defaults to that location. +-CASE JapaneseFTSBasic +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京タワーは赤い塔です'}); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京大学で勉強する'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '東京タワー', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京タワーは赤い塔です +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '勉強', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京大学で勉強する +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '東京', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京タワーは赤い塔です + +-CASE JapaneseIncrementalInsert +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京タワーは赤い塔です'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none'); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京大学で勉強する'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '勉強', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京大学で勉強する + +-CASE JapaneseCustomDictDir +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '日本語の全文検索テスト'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none', mecab_dict_dir := '${LBUG_ROOT_DIRECTORY}/extension/fts/build/dict-ipadic'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '全文検索', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +日本語の全文検索テスト From 42af2b09397b1559d07df82b11b765cdece6f7ee Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 04:23:16 +0800 Subject: [PATCH 111/114] fix(fts): resolve mecabrc path at runtime (-r) and fix out-of-tree add_subdirectory for CI --- fts/CMakeLists.txt | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/fts/CMakeLists.txt b/fts/CMakeLists.txt index cc19bf02..9dd136d4 100644 --- a/fts/CMakeLists.txt +++ b/fts/CMakeLists.txt @@ -1,24 +1,33 @@ +# Vendored dependencies live in the extensions repo root's third_party/. +# Use CMAKE_CURRENT_LIST_DIR (not PROJECT_SOURCE_DIR) so this also resolves +# when built under the engine's build system, where PROJECT_SOURCE_DIR points +# at the engine checkout. +set(FTS_EXTENSIONS_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..) + include_directories( ${PROJECT_SOURCE_DIR}/src/include ${CMAKE_BINARY_DIR}/src/include src/include third_party/snowball/libstemmer - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include - ${PROJECT_SOURCE_DIR}/third_party/mecab/src) + ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include + ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include + ${FTS_EXTENSIONS_ROOT}/third_party/mecab/src) # cppjieba (jieba tokenizer dependency) is a header-only interface library; -# upstream references it but never vendored it, so define the target here. +# upstream referenced it but never vendored it, so define the target here. if(NOT TARGET cppjieba) add_library(cppjieba INTERFACE) target_include_directories(cppjieba INTERFACE - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include) + ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include + ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include) endif() # MeCab (Japanese tokenizer): static lib + mecab-dict-index tool + build-time # ipadic dictionary compilation (downloads the dictionary CSV at configure). -add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/mecab) +# The source dir is outside this extension's tree, so an explicit binary dir +# is required (this also applies when built under the engine's build system). +add_subdirectory(${FTS_EXTENSIONS_ROOT}/third_party/mecab + ${CMAKE_CURRENT_BINARY_DIR}/mecab) add_subdirectory(src/catalog) add_subdirectory(src/function) From 25a358e5f00e445d017f70599ff88792e3784b8f Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Fri, 14 Aug 2026 04:23:19 +0800 Subject: [PATCH 112/114] fix(fts): resolve mecabrc path at runtime (-r) and fix out-of-tree add_subdirectory for CI --- fts/src/utils/tokenizer.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fts/src/utils/tokenizer.cpp b/fts/src/utils/tokenizer.cpp index 2b93d098..1b8c232c 100644 --- a/fts/src/utils/tokenizer.cpp +++ b/fts/src/utils/tokenizer.cpp @@ -60,10 +60,18 @@ class MeCabTokenizer final : public ITokenizer { explicit MeCabTokenizer(const TokenizerParams& params) { auto dictDir = params.contains("mecab_dict_dir") ? params.at("mecab_dict_dir") : std::string{}; - tagger.reset(MeCab::createTagger((std::string("-d ") + dictDir).c_str())); + // Pass the rc file explicitly. Without -r, MeCab falls back to + // MECAB_DEFAULT_RC, which is a build-time absolute path that does not + // exist in deployed environments (zip extracted elsewhere, build dir + // cleaned up) — the tagger then fails to create. The rc file ships + // alongside the dictionary (generated at build time). + tagger.reset(MeCab::createTagger( + (std::string("-d ") + dictDir + " -r " + dictDir + "/mecabrc").c_str())); if (!tagger) { - throw BinderException{ - std::format("Failed to create mecab tagger with dict dir: '{}'.", dictDir)}; + auto lastError = MeCab::getLastError(); + throw BinderException{std::format( + "Failed to create mecab tagger with dict dir: '{}'. (mecab error: {})", dictDir, + lastError ? lastError : "unknown")}; } } From a33519dc0825f970e517593386af5efcf97767ca Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Thu, 27 Aug 2026 16:50:29 +0800 Subject: [PATCH 113/114] fix(fts): vendor mecab in-tree like snowball; drop duplicate cppjieba; fix Linux build Restructure the vendored third_party layout to fix CI and follow the conventions already used in this repo: - Move third_party/mecab -> fts/third_party/mecab (same pattern as the vendored snowball stemmer). The previous repo-root location broke the CI build: under the engine build system the extensions repo is checked out at /extension, so the fts/CMakeLists.txt path resolved outside the extensions tree and add_subdirectory() failed with 'not an existing directory'. - Drop the vendored third_party/cppjieba copy: the engine repo already vendors cppjieba (with dictionaries) in its third_party, which the original fts/CMakeLists.txt referenced via PROJECT_SOURCE_DIR. The duplicate added here only carried headers, not the dict files, and was ~110 files of extra diff. - fts/CMakeLists.txt now only adds the mecab bits on top of the original upstream file (include dir, add_subdirectory, link, DLL_EXPORT for tokenizer.cpp, ipadic dict copy step). - mecab/CMakeLists.txt Linux fixes: * HAVE_WINDOWS_H only on Windows (utils.cpp #includes under it; defining it on Linux breaks the build) * HAVE_GCC_ATOMIC_OPS on non-Windows (tagger.cpp instantiates read_write_mutex, which only compiles under HAVE_ATOMIC_OPS) * HAVE_ICONV on non-Windows (mecab-dict-index needs glibc iconv to compile the EUC-JP ipadic CSV into UTF-8; without it the conversion is a silent no-op and the dictionary is corrupt) * MECAB_USE_UTF8_ONLY: skips the 4MB ucstable.h legacy-charset table (upstream --enable-utf8-only configuration). The extension only loads UTF-8 dictionaries. - Add files that were missing from the vendored tree and referenced by MECAB_SRCS / tokenizer.cpp: mecab.h, tagger.cpp, feature_index.cpp. --- fts/CMakeLists.txt | 34 +- fts/third_party/mecab/CMakeLists.txt | 160 ++ .../third_party}/mecab/LICENSE | 0 .../third_party}/mecab/src/Makefile.am | 0 .../third_party}/mecab/src/Makefile.msvc.in | 0 .../third_party}/mecab/src/char_property.cpp | 0 .../third_party}/mecab/src/char_property.h | 0 .../third_party}/mecab/src/common.h | 0 .../third_party}/mecab/src/connector.cpp | 0 .../third_party}/mecab/src/connector.h | 0 .../third_party}/mecab/src/context_id.cpp | 0 .../third_party}/mecab/src/context_id.h | 0 .../third_party}/mecab/src/darts.h | 0 .../third_party}/mecab/src/dictionary.cpp | 0 .../third_party}/mecab/src/dictionary.h | 0 .../mecab/src/dictionary_compiler.cpp | 0 .../mecab/src/dictionary_generator.cpp | 0 .../mecab/src/dictionary_rewriter.cpp | 0 .../mecab/src/dictionary_rewriter.h | 0 .../third_party}/mecab/src/eval.cpp | 0 fts/third_party/mecab/src/feature_index.cpp | 690 ++++++++ .../third_party}/mecab/src/feature_index.h | 0 .../third_party}/mecab/src/freelist.h | 0 .../third_party}/mecab/src/iconv_utils.cpp | 0 .../third_party}/mecab/src/iconv_utils.h | 0 .../third_party}/mecab/src/lbfgs.cpp | 0 .../third_party}/mecab/src/lbfgs.h | 0 .../third_party}/mecab/src/learner.cpp | 0 .../third_party}/mecab/src/learner_node.h | 0 .../third_party}/mecab/src/learner_tagger.cpp | 0 .../third_party}/mecab/src/learner_tagger.h | 0 .../third_party}/mecab/src/libmecab.cpp | 0 .../third_party}/mecab/src/make.bat | 0 .../mecab/src/mecab-cost-train.cpp | 0 .../third_party}/mecab/src/mecab-dict-gen.cpp | 0 .../mecab/src/mecab-dict-index.cpp | 0 .../mecab/src/mecab-system-eval.cpp | 0 .../third_party}/mecab/src/mecab-test-gen.cpp | 0 .../third_party}/mecab/src/mecab.cpp | 0 fts/third_party/mecab/src/mecab.h | 1509 ++++++++++++++++ .../third_party}/mecab/src/mmap.h | 0 .../mecab/src/nbest_generator.cpp | 0 .../third_party}/mecab/src/nbest_generator.h | 0 .../third_party}/mecab/src/param.cpp | 0 .../third_party}/mecab/src/param.h | 0 .../third_party}/mecab/src/scoped_ptr.h | 0 .../third_party}/mecab/src/stream_wrapper.h | 0 .../third_party}/mecab/src/string_buffer.cpp | 0 .../third_party}/mecab/src/string_buffer.h | 0 fts/third_party/mecab/src/tagger.cpp | 1277 ++++++++++++++ .../third_party}/mecab/src/thread.h | 0 .../third_party}/mecab/src/tokenizer.cpp | 0 .../third_party}/mecab/src/tokenizer.h | 0 .../third_party}/mecab/src/ucs.h | 0 .../third_party}/mecab/src/utils.cpp | 0 .../third_party}/mecab/src/utils.h | 0 .../third_party}/mecab/src/viterbi.cpp | 0 .../third_party}/mecab/src/viterbi.h | 0 .../third_party}/mecab/src/winmain.h | 0 .../third_party}/mecab/src/writer.cpp | 0 .../third_party}/mecab/src/writer.h | 0 third_party/cppjieba/.gitignore | 19 - third_party/cppjieba/CMakeLists.txt | 47 - third_party/cppjieba/LICENSE | 20 - third_party/cppjieba/VERSION | 2 - third_party/cppjieba/deps/limonp/.gitignore | 9 - third_party/cppjieba/deps/limonp/.gitmodules | 0 third_party/cppjieba/deps/limonp/CHANGELOG.md | 169 -- .../cppjieba/deps/limonp/CMakeLists.txt | 61 - third_party/cppjieba/deps/limonp/LICENSE | 20 - .../limonp/include/limonp/ArgvContext.hpp | 70 - .../deps/limonp/include/limonp/Closure.hpp | 206 --- .../deps/limonp/include/limonp/Colors.hpp | 31 - .../deps/limonp/include/limonp/Condition.hpp | 38 - .../deps/limonp/include/limonp/Config.hpp | 103 -- .../limonp/include/limonp/ForcePublic.hpp | 7 - .../limonp/include/limonp/LocalVector.hpp | 139 -- .../deps/limonp/include/limonp/Logging.hpp | 91 - .../limonp/include/limonp/NonCopyable.hpp | 21 - .../limonp/include/limonp/StdExtension.hpp | 157 -- .../deps/limonp/include/limonp/StringUtil.hpp | 367 ---- third_party/cppjieba/dict/README.md | 31 - .../cppjieba/dict/pos_dict/prob_start.utf8 | 259 --- third_party/cppjieba/dict/stop_words.utf8 | 1534 ----------------- third_party/cppjieba/dict/user.dict.utf8 | 10 - .../cppjieba/include/cppjieba/DictTrie.hpp | 280 --- .../cppjieba/include/cppjieba/FullSegment.hpp | 93 - .../cppjieba/include/cppjieba/HMMModel.hpp | 129 -- .../cppjieba/include/cppjieba/HMMSegment.hpp | 190 -- .../cppjieba/include/cppjieba/Jieba.hpp | 169 -- .../include/cppjieba/KeywordExtractor.hpp | 149 -- .../cppjieba/include/cppjieba/MPSegment.hpp | 137 -- .../cppjieba/include/cppjieba/MixSegment.hpp | 109 -- .../cppjieba/include/cppjieba/PosTagger.hpp | 77 - .../cppjieba/include/cppjieba/PreFilter.hpp | 54 - .../include/cppjieba/QuerySegment.hpp | 89 - .../cppjieba/include/cppjieba/SegmentBase.hpp | 46 - .../include/cppjieba/SegmentTagged.hpp | 23 - .../include/cppjieba/TextRankExtractor.hpp | 190 -- .../cppjieba/include/cppjieba/Trie.hpp | 193 --- .../cppjieba/include/cppjieba/Unicode.hpp | 227 --- third_party/mecab/CMakeLists.txt | 123 -- 102 files changed, 3645 insertions(+), 5714 deletions(-) create mode 100644 fts/third_party/mecab/CMakeLists.txt rename {third_party => fts/third_party}/mecab/LICENSE (100%) rename {third_party => fts/third_party}/mecab/src/Makefile.am (100%) rename {third_party => fts/third_party}/mecab/src/Makefile.msvc.in (100%) rename {third_party => fts/third_party}/mecab/src/char_property.cpp (100%) rename {third_party => fts/third_party}/mecab/src/char_property.h (100%) rename {third_party => fts/third_party}/mecab/src/common.h (100%) rename {third_party => fts/third_party}/mecab/src/connector.cpp (100%) rename {third_party => fts/third_party}/mecab/src/connector.h (100%) rename {third_party => fts/third_party}/mecab/src/context_id.cpp (100%) rename {third_party => fts/third_party}/mecab/src/context_id.h (100%) rename {third_party => fts/third_party}/mecab/src/darts.h (100%) rename {third_party => fts/third_party}/mecab/src/dictionary.cpp (100%) rename {third_party => fts/third_party}/mecab/src/dictionary.h (100%) rename {third_party => fts/third_party}/mecab/src/dictionary_compiler.cpp (100%) rename {third_party => fts/third_party}/mecab/src/dictionary_generator.cpp (100%) rename {third_party => fts/third_party}/mecab/src/dictionary_rewriter.cpp (100%) rename {third_party => fts/third_party}/mecab/src/dictionary_rewriter.h (100%) rename {third_party => fts/third_party}/mecab/src/eval.cpp (100%) create mode 100644 fts/third_party/mecab/src/feature_index.cpp rename {third_party => fts/third_party}/mecab/src/feature_index.h (100%) rename {third_party => fts/third_party}/mecab/src/freelist.h (100%) rename {third_party => fts/third_party}/mecab/src/iconv_utils.cpp (100%) rename {third_party => fts/third_party}/mecab/src/iconv_utils.h (100%) rename {third_party => fts/third_party}/mecab/src/lbfgs.cpp (100%) rename {third_party => fts/third_party}/mecab/src/lbfgs.h (100%) rename {third_party => fts/third_party}/mecab/src/learner.cpp (100%) rename {third_party => fts/third_party}/mecab/src/learner_node.h (100%) rename {third_party => fts/third_party}/mecab/src/learner_tagger.cpp (100%) rename {third_party => fts/third_party}/mecab/src/learner_tagger.h (100%) rename {third_party => fts/third_party}/mecab/src/libmecab.cpp (100%) rename {third_party => fts/third_party}/mecab/src/make.bat (100%) rename {third_party => fts/third_party}/mecab/src/mecab-cost-train.cpp (100%) rename {third_party => fts/third_party}/mecab/src/mecab-dict-gen.cpp (100%) rename {third_party => fts/third_party}/mecab/src/mecab-dict-index.cpp (100%) rename {third_party => fts/third_party}/mecab/src/mecab-system-eval.cpp (100%) rename {third_party => fts/third_party}/mecab/src/mecab-test-gen.cpp (100%) rename {third_party => fts/third_party}/mecab/src/mecab.cpp (100%) create mode 100644 fts/third_party/mecab/src/mecab.h rename {third_party => fts/third_party}/mecab/src/mmap.h (100%) rename {third_party => fts/third_party}/mecab/src/nbest_generator.cpp (100%) rename {third_party => fts/third_party}/mecab/src/nbest_generator.h (100%) rename {third_party => fts/third_party}/mecab/src/param.cpp (100%) rename {third_party => fts/third_party}/mecab/src/param.h (100%) rename {third_party => fts/third_party}/mecab/src/scoped_ptr.h (100%) rename {third_party => fts/third_party}/mecab/src/stream_wrapper.h (100%) rename {third_party => fts/third_party}/mecab/src/string_buffer.cpp (100%) rename {third_party => fts/third_party}/mecab/src/string_buffer.h (100%) create mode 100644 fts/third_party/mecab/src/tagger.cpp rename {third_party => fts/third_party}/mecab/src/thread.h (100%) rename {third_party => fts/third_party}/mecab/src/tokenizer.cpp (100%) rename {third_party => fts/third_party}/mecab/src/tokenizer.h (100%) rename {third_party => fts/third_party}/mecab/src/ucs.h (100%) rename {third_party => fts/third_party}/mecab/src/utils.cpp (100%) rename {third_party => fts/third_party}/mecab/src/utils.h (100%) rename {third_party => fts/third_party}/mecab/src/viterbi.cpp (100%) rename {third_party => fts/third_party}/mecab/src/viterbi.h (100%) rename {third_party => fts/third_party}/mecab/src/winmain.h (100%) rename {third_party => fts/third_party}/mecab/src/writer.cpp (100%) rename {third_party => fts/third_party}/mecab/src/writer.h (100%) delete mode 100644 third_party/cppjieba/.gitignore delete mode 100644 third_party/cppjieba/CMakeLists.txt delete mode 100644 third_party/cppjieba/LICENSE delete mode 100644 third_party/cppjieba/VERSION delete mode 100644 third_party/cppjieba/deps/limonp/.gitignore delete mode 100644 third_party/cppjieba/deps/limonp/.gitmodules delete mode 100644 third_party/cppjieba/deps/limonp/CHANGELOG.md delete mode 100644 third_party/cppjieba/deps/limonp/CMakeLists.txt delete mode 100644 third_party/cppjieba/deps/limonp/LICENSE delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Closure.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Config.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/Logging.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp delete mode 100644 third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp delete mode 100644 third_party/cppjieba/dict/README.md delete mode 100644 third_party/cppjieba/dict/pos_dict/prob_start.utf8 delete mode 100644 third_party/cppjieba/dict/stop_words.utf8 delete mode 100644 third_party/cppjieba/dict/user.dict.utf8 delete mode 100644 third_party/cppjieba/include/cppjieba/DictTrie.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/FullSegment.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/HMMModel.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/HMMSegment.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/Jieba.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/MPSegment.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/MixSegment.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/PosTagger.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/PreFilter.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/QuerySegment.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/SegmentBase.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/SegmentTagged.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/Trie.hpp delete mode 100644 third_party/cppjieba/include/cppjieba/Unicode.hpp delete mode 100644 third_party/mecab/CMakeLists.txt diff --git a/fts/CMakeLists.txt b/fts/CMakeLists.txt index 9dd136d4..21f94cf0 100644 --- a/fts/CMakeLists.txt +++ b/fts/CMakeLists.txt @@ -1,33 +1,11 @@ -# Vendored dependencies live in the extensions repo root's third_party/. -# Use CMAKE_CURRENT_LIST_DIR (not PROJECT_SOURCE_DIR) so this also resolves -# when built under the engine's build system, where PROJECT_SOURCE_DIR points -# at the engine checkout. -set(FTS_EXTENSIONS_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..) - include_directories( ${PROJECT_SOURCE_DIR}/src/include ${CMAKE_BINARY_DIR}/src/include src/include third_party/snowball/libstemmer - ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include - ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include - ${FTS_EXTENSIONS_ROOT}/third_party/mecab/src) - -# cppjieba (jieba tokenizer dependency) is a header-only interface library; -# upstream referenced it but never vendored it, so define the target here. -if(NOT TARGET cppjieba) - add_library(cppjieba INTERFACE) - target_include_directories(cppjieba INTERFACE - ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include - ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include) -endif() - -# MeCab (Japanese tokenizer): static lib + mecab-dict-index tool + build-time -# ipadic dictionary compilation (downloads the dictionary CSV at configure). -# The source dir is outside this extension's tree, so an explicit binary dir -# is required (this also applies when built under the engine's build system). -add_subdirectory(${FTS_EXTENSIONS_ROOT}/third_party/mecab - ${CMAKE_CURRENT_BINARY_DIR}/mecab) + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include + third_party/mecab/src) add_subdirectory(src/catalog) add_subdirectory(src/function) @@ -38,6 +16,12 @@ add_subdirectory(src/utils) add_subdirectory(third_party/snowball) +# MeCab (Japanese tokenizer), vendored in-tree like snowball: static lib + +# mecab-dict-index tool + ipadic dictionary compilation at build time +# (downloads the dictionary CSV at configure time when no prebuilt dictionary +# is bundled). +add_subdirectory(third_party/mecab) + # tokenizer.cpp includes mecab.h, which requires DLL_EXPORT to declare the # MeCab API without __declspec(dllimport) on Windows. target_compile_definitions(lbug_fts_utils PRIVATE DLL_EXPORT) diff --git a/fts/third_party/mecab/CMakeLists.txt b/fts/third_party/mecab/CMakeLists.txt new file mode 100644 index 00000000..079d8978 --- /dev/null +++ b/fts/third_party/mecab/CMakeLists.txt @@ -0,0 +1,160 @@ +# MeCab morphological analyzer - vendored from https://github.com/taku910/mecab +# (BSD-3-Clause, see LICENSE). Used by the FTS extension for Japanese +# tokenization. The ipadic dictionary (EUC-JP CSV) is compiled to the native +# binary dictionary format (UTF-8) at build time via mecab-dict-index. + +set(MECAB_SRCS + src/char_property.cpp + src/connector.cpp + src/context_id.cpp + src/dictionary.cpp + src/dictionary_compiler.cpp + src/dictionary_generator.cpp + src/dictionary_rewriter.cpp + src/eval.cpp + src/feature_index.cpp + src/iconv_utils.cpp + src/lbfgs.cpp + src/learner.cpp + src/learner_tagger.cpp + src/libmecab.cpp + src/nbest_generator.cpp + src/param.cpp + src/string_buffer.cpp + src/tagger.cpp + src/tokenizer.cpp + src/utils.cpp + src/viterbi.cpp + src/writer.cpp) + +# Same defines the upstream Makefile.msvc uses; VERSION/DIC_VERSION come from +# configure.in (0.996, 102). DLL_EXPORT is omitted: we link statically. +set(MECAB_DEFS + _CRT_SECURE_NO_DEPRECATE + MECAB_USE_THREAD + HAVE_GETENV + # UTF-8-only build (upstream's --enable-utf8-only): drops the 4MB + # ucstable.h legacy-charset table. The FTS extension only ever loads + # UTF-8 dictionaries (ipadic is compiled to UTF-8 at build time); + # legacy-charset input CSVs are converted with iconv before use. + MECAB_USE_UTF8_ONLY + DIC_VERSION=102 + VERSION=\"0.996\" + PACKAGE=\"mecab\" + # Default rc file: an empty file generated alongside the dictionary. + # (An empty string would make Param::load() try to open "" and fail.) + MECAB_DEFAULT_RC=\"${CMAKE_BINARY_DIR}/mecab-dict/ipadic/mecabrc\" + # Gate for StringBuffer::operator<<(unsigned long long) in string_buffer.h. + HAVE_UNSIGNED_LONG_LONG_INT) + +# utils.cpp includes under HAVE_WINDOWS_H without a _WIN32 guard, +# so only define it on Windows (defining it on Linux breaks the build). +if(WIN32) + list(APPEND MECAB_DEFS HAVE_WINDOWS_H) +endif() + +if(NOT WIN32) + # tagger.cpp instantiates read_write_mutex, which only compiles when + # HAVE_ATOMIC_OPS is defined - use the GCC/Clang __sync builtins (what + # upstream's autoconf configure detects on Linux). + # HAVE_ICONV: glibc provides iconv(3) in libc; mecab-dict-index needs it + # to compile the EUC-JP ipadic CSV into UTF-8. + list(APPEND MECAB_DEFS HAVE_GCC_ATOMIC_OPS HAVE_ICONV) +endif() + +add_library(mecab STATIC ${MECAB_SRCS}) +target_include_directories(mecab PUBLIC src) +target_compile_definitions(mecab PRIVATE ${MECAB_DEFS}) +# mecab.h declares MECAB_DLL_EXTERN as __declspec(dllimport) on _WIN32 unless +# DLL_EXPORT is defined. We link statically, so export (not import) must be +# declared consistently in both the library and its consumers. +target_compile_definitions(mecab PUBLIC DLL_EXPORT) +if(MSVC) + # Upstream suppresses these in Makefile.msvc; the code is C++98-era. + target_compile_options(mecab PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Build-time tool that compiles dictionary CSV files into the binary dictionary +# format (sys.dic/sys.dat/sys.cha/unk.*/char.bin/matrix.bin). +add_executable(mecab-dict-index src/mecab-dict-index.cpp ${MECAB_SRCS}) +target_include_directories(mecab-dict-index PRIVATE src) +target_compile_definitions(mecab-dict-index PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-dict-index PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Compile the ipadic dictionary (EUC-JP CSV) into a UTF-8 binary dictionary. +# Primary source: a prebuilt dictionary bundled at prebuilt-ipadic/ (generated +# by mecab-dict-index on a little-endian host; x86_64/aarch64 are +# little-endian, so it works offline on all mainstream platforms). +# Fallback: download the 54MB CSV from GitHub and compile it at build time +# (needed on big-endian hosts, or when the prebuilt dir is absent). +# ipadic has no standalone repo; it lives in taku910/mecab's mecab-ipadic/ +# subdirectory, so we download the mecab archive and keep that subtree. +set(MECAB_IPADIC_DIR ${CMAKE_BINARY_DIR}/mecab-ipadic-src) +set(MECAB_PREBUILT_IPADIC_DIR ${CMAKE_CURRENT_LIST_DIR}/prebuilt-ipadic) +if(EXISTS ${MECAB_PREBUILT_IPADIC_DIR}/dicrc) + set(MECAB_USE_PREBUILT_IPADIC TRUE) + message(STATUS "Using bundled prebuilt mecab-ipadic dictionary (offline mode)") +else() + if(NOT EXISTS ${MECAB_IPADIC_DIR}/dicrc) + set(MECAB_IPADIC_ZIP ${CMAKE_BINARY_DIR}/mecab-ipadic.zip) + set(MECAB_IPADIC_EXTRACT ${CMAKE_BINARY_DIR}/mecab-ipadic-extract) + message(STATUS "Downloading mecab-ipadic dictionary sources (54MB CSV)...") + file(DOWNLOAD https://github.com/taku910/mecab/archive/refs/heads/master.zip + ${MECAB_IPADIC_ZIP} STATUS MECAB_DL_STATUS) + list(GET MECAB_DL_STATUS 0 MECAB_DL_CODE) + if(NOT MECAB_DL_CODE EQUAL 0) + list(GET MECAB_DL_STATUS 1 MECAB_DL_ERROR) + message(FATAL_ERROR "Failed to download mecab-ipadic: ${MECAB_DL_ERROR} " + "(no bundled prebuilt-ipadic/ found either)") + endif() + file(MAKE_DIRECTORY ${MECAB_IPADIC_EXTRACT}) + file(ARCHIVE_EXTRACT INPUT ${MECAB_IPADIC_ZIP} DESTINATION ${MECAB_IPADIC_EXTRACT}) + set(MECAB_IPADIC_SUBDIR ${MECAB_IPADIC_EXTRACT}/mecab-master/mecab-ipadic) + if(NOT IS_DIRECTORY "${MECAB_IPADIC_SUBDIR}") + message(FATAL_ERROR "Unexpected mecab archive layout (mecab-ipadic dir missing)") + endif() + file(RENAME ${MECAB_IPADIC_SUBDIR} ${MECAB_IPADIC_DIR}) + file(REMOVE_RECURSE ${MECAB_IPADIC_EXTRACT}) + file(REMOVE ${MECAB_IPADIC_ZIP}) + endif() +endif() +set(MECAB_DICT_OUT_DIR ${CMAKE_BINARY_DIR}/mecab-dict/ipadic) +set(MECAB_DICT_OUTPUTS + ${MECAB_DICT_OUT_DIR}/sys.dic + ${MECAB_DICT_OUT_DIR}/unk.dic + ${MECAB_DICT_OUT_DIR}/char.bin + ${MECAB_DICT_OUT_DIR}/matrix.bin + ${MECAB_DICT_OUT_DIR}/dicrc + ${MECAB_DICT_OUT_DIR}/mecabrc) + +if(MECAB_USE_PREBUILT_IPADIC) + add_custom_command( + OUTPUT ${MECAB_DICT_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${MECAB_PREBUILT_IPADIC_DIR} ${MECAB_DICT_OUT_DIR} + COMMENT "Copying bundled prebuilt ipadic dictionary (little-endian)") +else() + add_custom_command( + OUTPUT ${MECAB_DICT_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} + COMMAND $ -f euc-jp -t utf-8 + -d ${MECAB_IPADIC_DIR} -o ${MECAB_DICT_OUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${MECAB_IPADIC_DIR}/dicrc ${MECAB_DICT_OUT_DIR}/dicrc + COMMAND ${CMAKE_COMMAND} -E touch ${MECAB_DICT_OUT_DIR}/mecabrc + DEPENDS mecab-dict-index + COMMENT "Compiling ipadic dictionary (EUC-JP -> UTF-8)") +endif() + +add_custom_target(mecab-ipadic-dict + DEPENDS ${MECAB_DICT_OUT_DIR}/sys.dic) + +# CLI for ad-hoc tokenization checks during development. +add_executable(mecab-cli src/mecab.cpp ${MECAB_SRCS}) +target_include_directories(mecab-cli PRIVATE src) +target_compile_definitions(mecab-cli PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-cli PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() diff --git a/third_party/mecab/LICENSE b/fts/third_party/mecab/LICENSE similarity index 100% rename from third_party/mecab/LICENSE rename to fts/third_party/mecab/LICENSE diff --git a/third_party/mecab/src/Makefile.am b/fts/third_party/mecab/src/Makefile.am similarity index 100% rename from third_party/mecab/src/Makefile.am rename to fts/third_party/mecab/src/Makefile.am diff --git a/third_party/mecab/src/Makefile.msvc.in b/fts/third_party/mecab/src/Makefile.msvc.in similarity index 100% rename from third_party/mecab/src/Makefile.msvc.in rename to fts/third_party/mecab/src/Makefile.msvc.in diff --git a/third_party/mecab/src/char_property.cpp b/fts/third_party/mecab/src/char_property.cpp similarity index 100% rename from third_party/mecab/src/char_property.cpp rename to fts/third_party/mecab/src/char_property.cpp diff --git a/third_party/mecab/src/char_property.h b/fts/third_party/mecab/src/char_property.h similarity index 100% rename from third_party/mecab/src/char_property.h rename to fts/third_party/mecab/src/char_property.h diff --git a/third_party/mecab/src/common.h b/fts/third_party/mecab/src/common.h similarity index 100% rename from third_party/mecab/src/common.h rename to fts/third_party/mecab/src/common.h diff --git a/third_party/mecab/src/connector.cpp b/fts/third_party/mecab/src/connector.cpp similarity index 100% rename from third_party/mecab/src/connector.cpp rename to fts/third_party/mecab/src/connector.cpp diff --git a/third_party/mecab/src/connector.h b/fts/third_party/mecab/src/connector.h similarity index 100% rename from third_party/mecab/src/connector.h rename to fts/third_party/mecab/src/connector.h diff --git a/third_party/mecab/src/context_id.cpp b/fts/third_party/mecab/src/context_id.cpp similarity index 100% rename from third_party/mecab/src/context_id.cpp rename to fts/third_party/mecab/src/context_id.cpp diff --git a/third_party/mecab/src/context_id.h b/fts/third_party/mecab/src/context_id.h similarity index 100% rename from third_party/mecab/src/context_id.h rename to fts/third_party/mecab/src/context_id.h diff --git a/third_party/mecab/src/darts.h b/fts/third_party/mecab/src/darts.h similarity index 100% rename from third_party/mecab/src/darts.h rename to fts/third_party/mecab/src/darts.h diff --git a/third_party/mecab/src/dictionary.cpp b/fts/third_party/mecab/src/dictionary.cpp similarity index 100% rename from third_party/mecab/src/dictionary.cpp rename to fts/third_party/mecab/src/dictionary.cpp diff --git a/third_party/mecab/src/dictionary.h b/fts/third_party/mecab/src/dictionary.h similarity index 100% rename from third_party/mecab/src/dictionary.h rename to fts/third_party/mecab/src/dictionary.h diff --git a/third_party/mecab/src/dictionary_compiler.cpp b/fts/third_party/mecab/src/dictionary_compiler.cpp similarity index 100% rename from third_party/mecab/src/dictionary_compiler.cpp rename to fts/third_party/mecab/src/dictionary_compiler.cpp diff --git a/third_party/mecab/src/dictionary_generator.cpp b/fts/third_party/mecab/src/dictionary_generator.cpp similarity index 100% rename from third_party/mecab/src/dictionary_generator.cpp rename to fts/third_party/mecab/src/dictionary_generator.cpp diff --git a/third_party/mecab/src/dictionary_rewriter.cpp b/fts/third_party/mecab/src/dictionary_rewriter.cpp similarity index 100% rename from third_party/mecab/src/dictionary_rewriter.cpp rename to fts/third_party/mecab/src/dictionary_rewriter.cpp diff --git a/third_party/mecab/src/dictionary_rewriter.h b/fts/third_party/mecab/src/dictionary_rewriter.h similarity index 100% rename from third_party/mecab/src/dictionary_rewriter.h rename to fts/third_party/mecab/src/dictionary_rewriter.h diff --git a/third_party/mecab/src/eval.cpp b/fts/third_party/mecab/src/eval.cpp similarity index 100% rename from third_party/mecab/src/eval.cpp rename to fts/third_party/mecab/src/eval.cpp diff --git a/fts/third_party/mecab/src/feature_index.cpp b/fts/third_party/mecab/src/feature_index.cpp new file mode 100644 index 00000000..051bdf8b --- /dev/null +++ b/fts/third_party/mecab/src/feature_index.cpp @@ -0,0 +1,690 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "feature_index.h" +#include "param.h" +#include "iconv_utils.h" +#include "learner_node.h" +#include "scoped_ptr.h" +#include "string_buffer.h" +#include "utils.h" + +#define BUFSIZE (2048) +#define POSSIZE (64) + +#define ADDB(b) do { const int id = this->id((b)); \ + if (id != -1) feature_.push_back(id); } while (0) + +#define COPY_FEATURE(ptr) do { \ + feature_.push_back(-1); \ + (ptr) = feature_freelist_.alloc(feature_.size()); \ + std::copy(feature_.begin(), feature_.end(), const_cast(ptr)); \ + feature_.clear(); } while (0) + +namespace MeCab { + +const char* FeatureIndex::getIndex(char **p, char **column, size_t max) { + ++(*p); + + bool flg = false; + + if (**p == '?') { + flg = true; + ++(*p); + } // undef flg + + CHECK_DIE(**p =='[') << "getIndex(): unmatched '['"; + + size_t n = 0; + ++(*p); + + for (;; ++(*p)) { + switch (**p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n + (**p - '0'); + break; + case ']': + if (n >= max) { + return 0; + } + + if (flg == true && ((std::strcmp("*", column[n]) == 0) + || column[n][0] == '\0')) { + return 0; + } + return column[n]; // return; + break; + default: + CHECK_DIE(false) << "unmatched '['"; + } + } + + return 0; +} + +void FeatureIndex::set_alpha(const double *alpha) { + alpha_ = alpha; +} + +bool FeatureIndex::openTemplate(const Param ¶m) { + std::string filename = create_filename(param.get("dicdir"), + FEATURE_FILE); + std::ifstream ifs(WPATH(filename.c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + + scoped_fixed_array buf; + char *column[4]; + + unigram_templs_.clear(); + bigram_templs_.clear(); + + while (ifs.getline(buf.get(), buf.size())) { + if (buf[0] == '\0' || buf[0] == '#' || buf[0] == ' ') { + continue; + } + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " <strdup(column[1])); + } else if (std::strcmp(column[0], "BIGRAM") == 0) { + bigram_templs_.push_back(this->strdup(column[1])); + } else { + CHECK_DIE(false) << "format error: " << filename; + } + } + + // second, open rewrite rules + filename = create_filename(param.get("dicdir"), + REWRITE_FILE); + rewrite_.open(filename.c_str()); + + return true; +} + +bool EncoderFeatureIndex::open(const Param ¶m) { + return openTemplate(param); +} + +bool DecoderFeatureIndex::open(const Param ¶m) { + const std::string modelfile = param.get("model"); + // open the file as binary mode again and fallback to text file + if (!openBinaryModel(param)) { + std::cout << modelfile + << " is not a binary model. reopen it as text mode..." + << std::endl; + CHECK_DIE(openTextModel(param)) << + "no such file or directory: " << modelfile; + } + + if (!openTemplate(param)) { + close(); + return false; + } + + return true; +} + + +bool DecoderFeatureIndex::openFromArray(const char *begin, const char *end) { + const char *ptr = begin; + unsigned int maxid = 0; + read_static(&ptr, maxid); + maxid_ = static_cast(maxid); + const size_t file_size = static_cast(end - begin); + const size_t expected_file_size = + (sizeof(double) + sizeof(uint64_t)) * maxid_ + sizeof(maxid) + 32; + if (expected_file_size != file_size) { + return false; + } + charset_ = ptr; + ptr += 32; + alpha_ = reinterpret_cast(ptr); + ptr += (sizeof(alpha_[0]) * maxid_); + key_ = reinterpret_cast(ptr); + return true; +} + +bool DecoderFeatureIndex::openBinaryModel(const Param ¶m) { + const std::string modelfile = param.get("model"); + CHECK_DIE(mmap_.open(modelfile.c_str())) << mmap_.what(); + if (!openFromArray(mmap_.begin(), mmap_.end())) { + mmap_.close(); + return false; + } + const std::string to = param.get("charset"); + CHECK_DIE(decode_charset(charset_) == + decode_charset(to.c_str())) + << "model charset and dictionary charset are different. " + << "model_charset=" << charset_ + << " dictionary_charset=" << to; + + return true; +} + +bool DecoderFeatureIndex::openTextModel(const Param ¶m) { + const std::string modelfile = param.get("model"); + CHECK_DIE(FeatureIndex::convert(param, modelfile.c_str(), &model_buffer_)); + return openFromArray(model_buffer_.data(), + model_buffer_.data() + model_buffer_.size()); +} + +void DecoderFeatureIndex::clear() { + feature_freelist_.free(); +} + +void EncoderFeatureIndex::clear() {} + +void EncoderFeatureIndex::clearcache() { + feature_cache_.clear(); + rewrite_.clear(); +} + +void EncoderFeatureIndex::close() { + dic_.clear(); + feature_cache_.clear(); + maxid_ = 0; +} + +void DecoderFeatureIndex::close() { + mmap_.close(); + model_buffer_.clear(); + maxid_ = 0; +} + +void FeatureIndex::calcCost(LearnerNode *node) { + node->wcost = 0.0; + if (node->stat == MECAB_EOS_NODE) return; + for (const int *f = node->fvector; *f != -1; ++f) { + node->wcost += alpha_[*f]; + } +} + +void FeatureIndex::calcCost(LearnerPath *path) { + if (is_empty(path)) return; + path->cost = path->rnode->wcost; + for (const int *f = path->fvector; *f != -1; ++f) { + path->cost += alpha_[*f]; + } +} + +const char *FeatureIndex::strdup(const char *p) { + size_t len = std::strlen(p); + char *q = char_freelist_.alloc(len + 1); + std::strncpy(q, p, len + 1); + return q; +} + +bool DecoderFeatureIndex::buildFeature(LearnerPath *path) { + path->rnode->wcost = path->cost = 0.0; + + std::string ufeature1; + std::string lfeature1; + std::string rfeature1; + std::string ufeature2; + std::string lfeature2; + std::string rfeature2; + + CHECK_DIE(rewrite_.rewrite2(path->lnode->feature, + &ufeature1, + &lfeature1, + &rfeature1)) + << " cannot rewrite pattern: " + << path->lnode->feature; + + CHECK_DIE(rewrite_.rewrite2(path->rnode->feature, + &ufeature2, + &lfeature2, + &rfeature2)) + << " cannot rewrite pattern: " + << path->rnode->feature; + + if (!buildUnigramFeature(path, ufeature2.c_str())) { + return false; + } + + if (!buildBigramFeature(path, rfeature1.c_str(), lfeature2.c_str())) { + return false; + } + + return true; +} + +bool EncoderFeatureIndex::buildFeature(LearnerPath *path) { + path->rnode->wcost = path->cost = 0.0; + + std::string ufeature1; + std::string lfeature1; + std::string rfeature1; + std::string ufeature2; + std::string lfeature2; + std::string rfeature2; + + CHECK_DIE(rewrite_.rewrite2(path->lnode->feature, + &ufeature1, + &lfeature1, + &rfeature1)) + << " cannot rewrite pattern: " + << path->lnode->feature; + + CHECK_DIE(rewrite_.rewrite2(path->rnode->feature, + &ufeature2, + &lfeature2, + &rfeature2)) + << " cannot rewrite pattern: " + << path->rnode->feature; + + { + os_.clear(); + os_ << ufeature2 << ' ' << path->rnode->char_type << '\0'; + const std::string key(os_.str()); + std::map >::iterator + it = feature_cache_.find(key); + if (it != feature_cache_.end()) { + path->rnode->fvector = it->second.first; + it->second.second++; + } else { + if (!buildUnigramFeature(path, ufeature2.c_str())) { + return false; + } + feature_cache_.insert(std::pair + > + (key, + std::pair + (path->rnode->fvector, 1))); + } + } + + { + os_.clear(); + os_ << rfeature1 << ' ' << lfeature2 << '\0'; + std::string key(os_.str()); + std::map >::iterator + it = feature_cache_.find(key); + if (it != feature_cache_.end()) { + path->fvector = it->second.first; + it->second.second++; + } else { + if (!buildBigramFeature(path, rfeature1.c_str(), lfeature2.c_str())) + return false; + feature_cache_.insert(std::pair + > + (key, std::pair + (path->fvector, 1))); + } + } + + CHECK_DIE(path->fvector) << " fvector is NULL"; + CHECK_DIE(path->rnode->fvector) << "fevector is NULL"; + + return true; +} + +bool FeatureIndex::buildUnigramFeature(LearnerPath *path, + const char *ufeature) { + scoped_fixed_array ubuf; + scoped_fixed_array F; + + feature_.clear(); + std::strncpy(ubuf.get(), ufeature, ubuf.size()); + const size_t usize = tokenizeCSV(ubuf.get(), F.get(), F.size()); + + for (std::vector::const_iterator it = unigram_templs_.begin(); + it != unigram_templs_.end(); ++it) { + const char *p = *it; + os_.clear(); + + for (; *p; p++) { + switch (*p) { + default: os_ << *p; break; + case '\\': os_ << getEscapedChar(*++p); break; + case '%': { + switch (*++p) { + case 'F': { + const char *r = getIndex(const_cast(&p), F.get(), usize); + if (!r) goto NEXT; + os_ << r; + } break; + case 't': os_ << (size_t)path->rnode->char_type; break; + case 'u': os_ << ufeature; break; + case 'w': + if (path->rnode->stat == MECAB_NOR_NODE) { + os_.write(path->rnode->surface, path->rnode->length); + } + default: + CHECK_DIE(false) << "unknown meta char: " << *p; + } + } + } + } + + os_ << '\0'; + ADDB(os_.str()); + + NEXT: continue; + } + + COPY_FEATURE(path->rnode->fvector); + + return true; +} + +bool FeatureIndex::buildBigramFeature(LearnerPath *path, + const char *rfeature, + const char *lfeature) { + scoped_fixed_array rbuf; + scoped_fixed_array lbuf; + scoped_fixed_array R; + scoped_fixed_array L; + + feature_.clear(); + std::strncpy(lbuf.get(), rfeature, lbuf.size()); + std::strncpy(rbuf.get(), lfeature, rbuf.size()); + + const size_t lsize = tokenizeCSV(lbuf.get(), L.get(), L.size()); + const size_t rsize = tokenizeCSV(rbuf.get(), R.get(), R.size()); + + for (std::vector::const_iterator it = bigram_templs_.begin(); + it != bigram_templs_.end(); ++it) { + const char *p = *it; + os_.clear(); + + for (; *p; p++) { + switch (*p) { + default: os_ << *p; break; + case '\\': os_ << getEscapedChar(*++p); break; + case '%': { + switch (*++p) { + case 'L': { + const char *r = getIndex(const_cast(&p), L.get(), lsize); + if (!r) goto NEXT; + os_ << r; + } break; + case 'R': { + const char *r = getIndex(const_cast(&p), R.get(), rsize); + if (!r) goto NEXT; + os_ << r; + } break; + case 'l': os_ << lfeature; break; // use lfeature as it is + case 'r': os_ << rfeature; break; + default: + CHECK_DIE(false) << "unknown meta char: " << *p; + } + } + } + } + + os_ << '\0'; + + ADDB(os_.str()); + + NEXT: continue; + } + + COPY_FEATURE(path->fvector); + + return true; +} + +int DecoderFeatureIndex::id(const char *key) { + const uint64_t fp = fingerprint(key, std::strlen(key)); + const uint64_t *result = std::lower_bound(key_, + key_ + maxid_, + fp); + if (result == key_ + maxid_ || *result != fp) { + return -1; + } + const int n = static_cast(result - key_); + CHECK_DIE(key_[n] == fp); + return n; +} + +int EncoderFeatureIndex::id(const char *key) { + std::map::const_iterator it = dic_.find(key); + if (it == dic_.end()) { + dic_.insert(std::pair(std::string(key), maxid_)); + return maxid_++; + } else { + return it->second; + } + return -1; +} + +void EncoderFeatureIndex::shrink(size_t freq, + std::vector *observed) { + std::vector freqv; + // count fvector + freqv.resize(maxid_); + for (std::map >::const_iterator + it = feature_cache_.begin(); + it != feature_cache_.end(); ++it) { + for (const int *f = it->second.first; *f != -1; ++f) { + freqv[*f] += it->second.second; // freq + } + } + + if (freq <= 1) { + return; + } + + // make old2new map + maxid_ = 0; + std::map old2new; + for (size_t i = 0; i < freqv.size(); ++i) { + if (freqv[i] >= freq) { + old2new.insert(std::pair(i, maxid_++)); + } + } + + // update dic_ + for (std::map::iterator + it = dic_.begin(); it != dic_.end();) { + std::map::const_iterator it2 = old2new.find(it->second); + if (it2 != old2new.end()) { + it->second = it2->second; + ++it; + } else { + dic_.erase(it++); + } + } + + // update all fvector + for (std::map >::const_iterator + it = feature_cache_.begin(); it != feature_cache_.end(); ++it) { + int *to = const_cast(it->second.first); + for (const int *f = it->second.first; *f != -1; ++f) { + std::map::const_iterator it2 = old2new.find(*f); + if (it2 != old2new.end()) { + *to = it2->second; + ++to; + } + } + *to = -1; + } + + // update observed vector + std::vector observed_new(maxid_); + for (size_t i = 0; i < observed->size(); ++i) { + std::map::const_iterator it = old2new.find(static_cast(i)); + if (it != old2new.end()) { + observed_new[it->second] = (*observed)[i]; + } + } + + // copy + *observed = observed_new; + + return; +} + +bool FeatureIndex::compile(const Param ¶m, + const char* txtfile, const char *binfile) { + std::string buf; + FeatureIndex::convert(param, txtfile, &buf); + std::ofstream ofs(WPATH(binfile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << binfile; + ofs.write(buf.data(), buf.size()); + return true; +} + +bool FeatureIndex::convert(const Param ¶m, + const char* txtfile, std::string *output) { + std::ifstream ifs(WPATH(txtfile)); + CHECK_DIE(ifs) << "no such file or directory: " << txtfile; + scoped_fixed_array buf; + char *column[4]; + std::vector > dic; + std::string model_charset; + + while (ifs.getline(buf.get(), buf.size())) { + if (std::strlen(buf.get()) == 0) { + break; + } + CHECK_DIE(tokenize2(buf.get(), ":", column, 2) == 2) + << "format error: " << buf.get(); + if (std::string(column[0]) == "charset") { + model_charset = column[1] + 1; + } + } + + std::string from = param.get("dictionary-charset"); + std::string to = param.get("charset"); + + if (!from.empty()) { + CHECK_DIE(decode_charset(model_charset.c_str()) + == decode_charset(from.c_str())) + << "dictionary charset and model charset are different. " + << "dictionary_charset=" << from + << " model_charset=" << model_charset; + } else { + from = model_charset; + } + + if (to.empty()) { + to = from; + } + + Iconv iconv; + CHECK_DIE(iconv.open(from.c_str(), to.c_str())) + << "cannot create model from=" << from + << " to=" << to; + + while (ifs.getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t", column, 2) == 2) + << "format error: " << buf.get(); + std::string feature = column[1]; + CHECK_DIE(iconv.convert(&feature)); + const uint64_t fp = fingerprint(feature); + const double alpha = atof(column[0]); + dic.push_back(std::pair(fp, alpha)); + } + + output->clear(); + unsigned int size = static_cast(dic.size()); + output->append(reinterpret_cast(&size), sizeof(size)); + + char charset_buf[32]; + std::fill(charset_buf, charset_buf + sizeof(charset_buf), '\0'); + std::strncpy(charset_buf, to.c_str(), 31); + output->append(reinterpret_cast(charset_buf), + sizeof(charset_buf)); + + std::sort(dic.begin(), dic.end()); + + for (size_t i = 0; i < dic.size(); ++i) { + const double alpha = dic[i].second; + output->append(reinterpret_cast(&alpha), sizeof(alpha)); + } + + for (size_t i = 0; i < dic.size(); ++i) { + const uint64_t fp = dic[i].first; + output->append(reinterpret_cast(&fp), sizeof(fp)); + } + + return true; +} + +// TODO(taku): consider charset +bool EncoderFeatureIndex::reopen(const char *filename, + const char *dic_charset, + std::vector *alpha, + Param *param) { + close(); + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + return false; + } + + scoped_fixed_array buf; + char *column[8]; + + std::string model_charset; + + while (ifs.getline(buf.get(), buf.size())) { + if (std::strlen(buf.get()) == 0) { + break; + } + CHECK_DIE(tokenize2(buf.get(), ":", column, 2) == 2) + << "format error: " << buf.get(); + if (std::string(column[0]) == "charset") { + model_charset = column[1] + 1; + } else { + param->set(column[0], column[1] + 1, true); + } + } + + CHECK_DIE(dic_charset); + CHECK_DIE(!model_charset.empty()) << "charset is empty"; + + Iconv iconv; + CHECK_DIE(iconv.open(model_charset.c_str(), dic_charset)) + << "cannot create model from=" << model_charset + << " to=" << dic_charset; + + alpha->clear(); + CHECK_DIE(maxid_ == 0); + CHECK_DIE(dic_.empty()); + + while (ifs.getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t", column, 2) == 2) + << "format error: " << buf.get(); + std::string feature = column[1]; + CHECK_DIE(iconv.convert(&feature)); + dic_.insert(std::make_pair(feature, maxid_++)); + alpha->push_back(atof(column[0])); + } + + return true; +} + +bool EncoderFeatureIndex::save(const char *filename, const char *header) const { + CHECK_DIE(header); + CHECK_DIE(alpha_); + + std::ofstream ofs(WPATH(filename)); + if (!ofs) { + return false; + } + + ofs.setf(std::ios::fixed, std::ios::floatfield); + ofs.precision(16); + + ofs << header; + ofs << std::endl; + + for (std::map::const_iterator it = dic_.begin(); + it != dic_.end(); ++it) { + ofs << alpha_[it->second] << '\t' << it->first << '\n'; + } + + return true; +} +} diff --git a/third_party/mecab/src/feature_index.h b/fts/third_party/mecab/src/feature_index.h similarity index 100% rename from third_party/mecab/src/feature_index.h rename to fts/third_party/mecab/src/feature_index.h diff --git a/third_party/mecab/src/freelist.h b/fts/third_party/mecab/src/freelist.h similarity index 100% rename from third_party/mecab/src/freelist.h rename to fts/third_party/mecab/src/freelist.h diff --git a/third_party/mecab/src/iconv_utils.cpp b/fts/third_party/mecab/src/iconv_utils.cpp similarity index 100% rename from third_party/mecab/src/iconv_utils.cpp rename to fts/third_party/mecab/src/iconv_utils.cpp diff --git a/third_party/mecab/src/iconv_utils.h b/fts/third_party/mecab/src/iconv_utils.h similarity index 100% rename from third_party/mecab/src/iconv_utils.h rename to fts/third_party/mecab/src/iconv_utils.h diff --git a/third_party/mecab/src/lbfgs.cpp b/fts/third_party/mecab/src/lbfgs.cpp similarity index 100% rename from third_party/mecab/src/lbfgs.cpp rename to fts/third_party/mecab/src/lbfgs.cpp diff --git a/third_party/mecab/src/lbfgs.h b/fts/third_party/mecab/src/lbfgs.h similarity index 100% rename from third_party/mecab/src/lbfgs.h rename to fts/third_party/mecab/src/lbfgs.h diff --git a/third_party/mecab/src/learner.cpp b/fts/third_party/mecab/src/learner.cpp similarity index 100% rename from third_party/mecab/src/learner.cpp rename to fts/third_party/mecab/src/learner.cpp diff --git a/third_party/mecab/src/learner_node.h b/fts/third_party/mecab/src/learner_node.h similarity index 100% rename from third_party/mecab/src/learner_node.h rename to fts/third_party/mecab/src/learner_node.h diff --git a/third_party/mecab/src/learner_tagger.cpp b/fts/third_party/mecab/src/learner_tagger.cpp similarity index 100% rename from third_party/mecab/src/learner_tagger.cpp rename to fts/third_party/mecab/src/learner_tagger.cpp diff --git a/third_party/mecab/src/learner_tagger.h b/fts/third_party/mecab/src/learner_tagger.h similarity index 100% rename from third_party/mecab/src/learner_tagger.h rename to fts/third_party/mecab/src/learner_tagger.h diff --git a/third_party/mecab/src/libmecab.cpp b/fts/third_party/mecab/src/libmecab.cpp similarity index 100% rename from third_party/mecab/src/libmecab.cpp rename to fts/third_party/mecab/src/libmecab.cpp diff --git a/third_party/mecab/src/make.bat b/fts/third_party/mecab/src/make.bat similarity index 100% rename from third_party/mecab/src/make.bat rename to fts/third_party/mecab/src/make.bat diff --git a/third_party/mecab/src/mecab-cost-train.cpp b/fts/third_party/mecab/src/mecab-cost-train.cpp similarity index 100% rename from third_party/mecab/src/mecab-cost-train.cpp rename to fts/third_party/mecab/src/mecab-cost-train.cpp diff --git a/third_party/mecab/src/mecab-dict-gen.cpp b/fts/third_party/mecab/src/mecab-dict-gen.cpp similarity index 100% rename from third_party/mecab/src/mecab-dict-gen.cpp rename to fts/third_party/mecab/src/mecab-dict-gen.cpp diff --git a/third_party/mecab/src/mecab-dict-index.cpp b/fts/third_party/mecab/src/mecab-dict-index.cpp similarity index 100% rename from third_party/mecab/src/mecab-dict-index.cpp rename to fts/third_party/mecab/src/mecab-dict-index.cpp diff --git a/third_party/mecab/src/mecab-system-eval.cpp b/fts/third_party/mecab/src/mecab-system-eval.cpp similarity index 100% rename from third_party/mecab/src/mecab-system-eval.cpp rename to fts/third_party/mecab/src/mecab-system-eval.cpp diff --git a/third_party/mecab/src/mecab-test-gen.cpp b/fts/third_party/mecab/src/mecab-test-gen.cpp similarity index 100% rename from third_party/mecab/src/mecab-test-gen.cpp rename to fts/third_party/mecab/src/mecab-test-gen.cpp diff --git a/third_party/mecab/src/mecab.cpp b/fts/third_party/mecab/src/mecab.cpp similarity index 100% rename from third_party/mecab/src/mecab.cpp rename to fts/third_party/mecab/src/mecab.cpp diff --git a/fts/third_party/mecab/src/mecab.h b/fts/third_party/mecab/src/mecab.h new file mode 100644 index 00000000..ce7f265c --- /dev/null +++ b/fts/third_party/mecab/src/mecab.h @@ -0,0 +1,1509 @@ +/* + MeCab -- Yet Another Part-of-Speech and Morphological Analyzer + + Copyright(C) 2001-2011 Taku Kudo + Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +*/ +#ifndef MECAB_MECAB_H_ +#define MECAB_MECAB_H_ + +/* C/C++ common data structures */ + +/** + * DictionaryInfo structure + */ +struct mecab_dictionary_info_t { + /** + * filename of dictionary + * On Windows, filename is stored in UTF-8 encoding + */ + const char *filename; + + /** + * character set of the dictionary. e.g., "SHIFT-JIS", "UTF-8" + */ + const char *charset; + + /** + * How many words are registered in this dictionary. + */ + unsigned int size; + + /** + * dictionary type + * this value should be MECAB_USR_DIC, MECAB_SYS_DIC, or MECAB_UNK_DIC. + */ + int type; + + /** + * left attributes size + */ + unsigned int lsize; + + /** + * right attributes size + */ + unsigned int rsize; + + /** + * version of this dictionary + */ + unsigned short version; + + /** + * pointer to the next dictionary info. + */ + struct mecab_dictionary_info_t *next; +}; + +/** + * Path structure + */ +struct mecab_path_t { + /** + * pointer to the right node + */ + struct mecab_node_t* rnode; + + /** + * pointer to the next right path + */ + struct mecab_path_t* rnext; + + /** + * pointer to the left node + */ + struct mecab_node_t* lnode; + + /** + * pointer to the next left path + */ + + struct mecab_path_t* lnext; + + /** + * local cost + */ + int cost; + + /** + * marginal probability + */ + float prob; +}; + +/** + * Node structure + */ +struct mecab_node_t { + /** + * pointer to the previous node. + */ + struct mecab_node_t *prev; + + /** + * pointer to the next node. + */ + struct mecab_node_t *next; + + /** + * pointer to the node which ends at the same position. + */ + struct mecab_node_t *enext; + + /** + * pointer to the node which starts at the same position. + */ + struct mecab_node_t *bnext; + + /** + * pointer to the right path. + * this value is NULL if MECAB_ONE_BEST mode. + */ + struct mecab_path_t *rpath; + + /** + * pointer to the right path. + * this value is NULL if MECAB_ONE_BEST mode. + */ + struct mecab_path_t *lpath; + + /** + * surface string. + * this value is not 0 terminated. + * You can get the length with length/rlength members. + */ + const char *surface; + + /** + * feature string + */ + const char *feature; + + /** + * unique node id + */ + unsigned int id; + + /** + * length of the surface form. + */ + unsigned short length; + + /** + * length of the surface form including white space before the morph. + */ + unsigned short rlength; + + /** + * right attribute id + */ + unsigned short rcAttr; + + /** + * left attribute id + */ + unsigned short lcAttr; + + /** + * unique part of speech id. This value is defined in "pos.def" file. + */ + unsigned short posid; + + /** + * character type + */ + unsigned char char_type; + + /** + * status of this model. + * This value is MECAB_NOR_NODE, MECAB_UNK_NODE, MECAB_BOS_NODE, MECAB_EOS_NODE, or MECAB_EON_NODE. + */ + unsigned char stat; + + /** + * set 1 if this node is best node. + */ + unsigned char isbest; + + /** + * forward accumulative log summation. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float alpha; + + /** + * backward accumulative log summation. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float beta; + + /** + * marginal probability. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float prob; + + /** + * word cost. + */ + short wcost; + + /** + * best accumulative cost from bos node to this node. + */ + long cost; +}; + +/** + * Parameters for MeCab::Node::stat + */ +enum { + /** + * Normal node defined in the dictionary. + */ + MECAB_NOR_NODE = 0, + /** + * Unknown node not defined in the dictionary. + */ + MECAB_UNK_NODE = 1, + /** + * Virtual node representing a beginning of the sentence. + */ + MECAB_BOS_NODE = 2, + /** + * Virtual node representing a end of the sentence. + */ + MECAB_EOS_NODE = 3, + + /** + * Virtual node representing a end of the N-best enumeration. + */ + MECAB_EON_NODE = 4 +}; + +/** + * Parameters for MeCab::DictionaryInfo::type + */ +enum { + /** + * This is a system dictionary. + */ + MECAB_SYS_DIC = 0, + + /** + * This is a user dictionary. + */ + MECAB_USR_DIC = 1, + + /** + * This is a unknown word dictionary. + */ + MECAB_UNK_DIC = 2 +}; + +/** + * Parameters for MeCab::Lattice::request_type + */ +enum { + /** + * One best result is obtained (default mode) + */ + MECAB_ONE_BEST = 1, + /** + * Set this flag if you want to obtain N best results. + */ + MECAB_NBEST = 2, + /** + * Set this flag if you want to enable a partial parsing mode. + * When this flag is set, the input |sentence| needs to be written + * in partial parsing format. + */ + MECAB_PARTIAL = 4, + /** + * Set this flag if you want to obtain marginal probabilities. + * Marginal probability is set in MeCab::Node::prob. + * The parsing speed will get 3-5 times slower than the default mode. + */ + MECAB_MARGINAL_PROB = 8, + /** + * Set this flag if you want to obtain alternative results. + * Not implemented. + */ + MECAB_ALTERNATIVE = 16, + /** + * When this flag is set, the result linked-list (Node::next/prev) + * traverses all nodes in the lattice. + */ + MECAB_ALL_MORPHS = 32, + + /** + * When this flag is set, tagger internally copies the body of passed + * sentence into internal buffer. + */ + MECAB_ALLOCATE_SENTENCE = 64 +}; + +/** + * Parameters for MeCab::Lattice::boundary_constraint_type + */ +enum { + /** + * The token boundary is not specified. + */ + MECAB_ANY_BOUNDARY = 0, + + /** + * The position is a strong token boundary. + */ + MECAB_TOKEN_BOUNDARY = 1, + + /** + * The position is not a token boundary. + */ + MECAB_INSIDE_TOKEN = 2 +}; + +/* C interface */ +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _WIN32 +#include +# ifdef DLL_EXPORT +# define MECAB_DLL_EXTERN __declspec(dllexport) +# define MECAB_DLL_CLASS_EXTERN __declspec(dllexport) +# else +# define MECAB_DLL_EXTERN __declspec(dllimport) +# endif +#endif + +#ifndef MECAB_DLL_EXTERN +# define MECAB_DLL_EXTERN extern +#endif + +#ifndef MECAB_DLL_CLASS_EXTERN +# define MECAB_DLL_CLASS_EXTERN +#endif + + typedef struct mecab_t mecab_t; + typedef struct mecab_model_t mecab_model_t; + typedef struct mecab_lattice_t mecab_lattice_t; + typedef struct mecab_dictionary_info_t mecab_dictionary_info_t; + typedef struct mecab_node_t mecab_node_t; + typedef struct mecab_path_t mecab_path_t; + +#ifndef SWIG + /* C interface */ + + /* old mecab interface */ + /** + * C wrapper of MeCab::Tagger::create(argc, argv) + */ + MECAB_DLL_EXTERN mecab_t* mecab_new(int argc, char **argv); + + /** + * C wrapper of MeCab::Tagger::create(arg) + */ + MECAB_DLL_EXTERN mecab_t* mecab_new2(const char *arg); + + /** + * C wrapper of MeCab::Tagger::version() + */ + MECAB_DLL_EXTERN const char* mecab_version(); + + /** + * C wrapper of MeCab::getLastError() + */ + MECAB_DLL_EXTERN const char* mecab_strerror(mecab_t *mecab); + + /** + * C wrapper of MeCab::deleteTagger(tagger) + */ + MECAB_DLL_EXTERN void mecab_destroy(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger:set_partial() + */ + MECAB_DLL_EXTERN int mecab_get_partial(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::partial() + */ + MECAB_DLL_EXTERN void mecab_set_partial(mecab_t *mecab, int partial); + + /** + * C wrapper of MeCab::Tagger::theta() + */ + MECAB_DLL_EXTERN float mecab_get_theta(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_theta() + */ + MECAB_DLL_EXTERN void mecab_set_theta(mecab_t *mecab, float theta); + + /** + * C wrapper of MeCab::Tagger::lattice_level() + */ + MECAB_DLL_EXTERN int mecab_get_lattice_level(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_lattice_level() + */ + MECAB_DLL_EXTERN void mecab_set_lattice_level(mecab_t *mecab, int level); + + /** + * C wrapper of MeCab::Tagger::all_morphs() + */ + MECAB_DLL_EXTERN int mecab_get_all_morphs(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_all_moprhs() + */ + MECAB_DLL_EXTERN void mecab_set_all_morphs(mecab_t *mecab, int all_morphs); + + /** + * C wrapper of MeCab::Tagger::parse(MeCab::Lattice *lattice) + */ + MECAB_DLL_EXTERN int mecab_parse_lattice(mecab_t *mecab, mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str) + */ + MECAB_DLL_EXTERN const char* mecab_sparse_tostr(mecab_t *mecab, const char *str); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str, size_t len) + */ + MECAB_DLL_EXTERN const char* mecab_sparse_tostr2(mecab_t *mecab, const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str, char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_sparse_tostr3(mecab_t *mecab, const char *str, size_t len, + char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::parseToNode(const char *str) + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_sparse_tonode(mecab_t *mecab, const char*); + + /** + * C wrapper of MeCab::Tagger::parseToNode(const char *str, size_t len) + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_sparse_tonode2(mecab_t *mecab, const char*, size_t); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str) + */ + MECAB_DLL_EXTERN const char* mecab_nbest_sparse_tostr(mecab_t *mecab, size_t N, const char *str); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str, size_t len) + */ + MECAB_DLL_EXTERN const char* mecab_nbest_sparse_tostr2(mecab_t *mecab, size_t N, + const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str, char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_nbest_sparse_tostr3(mecab_t *mecab, size_t N, + const char *str, size_t len, + char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::parseNBestInit(const char *str) + */ + MECAB_DLL_EXTERN int mecab_nbest_init(mecab_t *mecab, const char *str); + + /** + * C wrapper of MeCab::Tagger::parseNBestInit(const char *str, size_t len) + */ + MECAB_DLL_EXTERN int mecab_nbest_init2(mecab_t *mecab, const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::next() + */ + MECAB_DLL_EXTERN const char* mecab_nbest_next_tostr(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::next(char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_nbest_next_tostr2(mecab_t *mecab, char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::nextNode() + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_nbest_next_tonode(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::formatNode(const Node *node) + */ + MECAB_DLL_EXTERN const char* mecab_format_node(mecab_t *mecab, const mecab_node_t *node); + + /** + * C wrapper of MeCab::Tagger::dictionary_info() + */ + MECAB_DLL_EXTERN const mecab_dictionary_info_t* mecab_dictionary_info(mecab_t *mecab); + + /* lattice interface */ + /** + * C wrapper of MeCab::createLattice() + */ + MECAB_DLL_EXTERN mecab_lattice_t *mecab_lattice_new(); + + /** + * C wrapper of MeCab::deleteLattice(lattice) + */ + MECAB_DLL_EXTERN void mecab_lattice_destroy(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::clear() + */ + MECAB_DLL_EXTERN void mecab_lattice_clear(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::is_available() + */ + + MECAB_DLL_EXTERN int mecab_lattice_is_available(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::bos_node() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_bos_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::eos_node() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_eos_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::begin_nodes() + */ + + MECAB_DLL_EXTERN mecab_node_t **mecab_lattice_get_all_begin_nodes(mecab_lattice_t *lattice); + /** + * C wrapper of MeCab::Lattice::end_nodes() + */ + MECAB_DLL_EXTERN mecab_node_t **mecab_lattice_get_all_end_nodes(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::begin_nodes(pos) + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_begin_nodes(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::end_nodes(pos) + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_end_nodes(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::sentence() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_get_sentence(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_sentence(sentence) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_sentence(mecab_lattice_t *lattice, const char *sentence); + + /** + * C wrapper of MeCab::Lattice::set_sentence(sentence, len) + */ + + MECAB_DLL_EXTERN void mecab_lattice_set_sentence2(mecab_lattice_t *lattice, const char *sentence, size_t len); + + /** + * C wrapper of MeCab::Lattice::size() + */ + MECAB_DLL_EXTERN size_t mecab_lattice_get_size(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::Z() + */ + MECAB_DLL_EXTERN double mecab_lattice_get_z(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_Z() + */ + MECAB_DLL_EXTERN void mecab_lattice_set_z(mecab_lattice_t *lattice, double Z); + + /** + * C wrapper of MeCab::Lattice::theta() + */ + MECAB_DLL_EXTERN double mecab_lattice_get_theta(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_theta() + */ + + MECAB_DLL_EXTERN void mecab_lattice_set_theta(mecab_lattice_t *lattice, double theta); + + /** + * C wrapper of MeCab::Lattice::next() + */ + MECAB_DLL_EXTERN int mecab_lattice_next(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::request_type() + */ + MECAB_DLL_EXTERN int mecab_lattice_get_request_type(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::has_request_type() + */ + MECAB_DLL_EXTERN int mecab_lattice_has_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::set_request_type() + */ + MECAB_DLL_EXTERN void mecab_lattice_set_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::add_request_type() + */ + + MECAB_DLL_EXTERN void mecab_lattice_add_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::remove_request_type() + */ + MECAB_DLL_EXTERN void mecab_lattice_remove_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::newNode(); + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_new_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::toString() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_tostr(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::toString(buf, size) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_tostr2(mecab_lattice_t *lattice, char *buf, size_t size); + + /** + * C wrapper of MeCab::Lattice::enumNBestAsString(N) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_nbest_tostr(mecab_lattice_t *lattice, size_t N); + + /** + * C wrapper of MeCab::Lattice::enumNBestAsString(N, buf, size) + */ + + MECAB_DLL_EXTERN const char *mecab_lattice_nbest_tostr2(mecab_lattice_t *lattice, size_t N, char *buf, size_t size); + + /** + * C wrapper of MeCab::Lattice::has_constraint() + */ + MECAB_DLL_EXTERN int mecab_lattice_has_constraint(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::boundary_constraint(pos) + */ + MECAB_DLL_EXTERN int mecab_lattice_get_boundary_constraint(mecab_lattice_t *lattice, size_t pos); + + + /** + * C wrapper of MeCab::Lattice::feature_constraint(pos) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_get_feature_constraint(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::boundary_constraint(pos, type) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_boundary_constraint(mecab_lattice_t *lattice, size_t pos, int boundary_type); + + /** + * C wrapper of MeCab::Lattice::set_feature_constraint(begin_pos, end_pos, feature) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_feature_constraint(mecab_lattice_t *lattice, size_t begin_pos, size_t end_pos, const char *feature); + + /** + * C wrapper of MeCab::Lattice::set_result(result); + */ + MECAB_DLL_EXTERN void mecab_lattice_set_result(mecab_lattice_t *lattice, const char *result); + + /** + * C wrapper of MeCab::Lattice::what() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_strerror(mecab_lattice_t *lattice); + + + /* model interface */ + /** + * C wapper of MeCab::Model::create(argc, argv) + */ + MECAB_DLL_EXTERN mecab_model_t *mecab_model_new(int argc, char **argv); + + /** + * C wapper of MeCab::Model::create(arg) + */ + MECAB_DLL_EXTERN mecab_model_t *mecab_model_new2(const char *arg); + + /** + * C wapper of MeCab::deleteModel(model) + */ + + MECAB_DLL_EXTERN void mecab_model_destroy(mecab_model_t *model); + + /** + * C wapper of MeCab::Model::createTagger() + */ + MECAB_DLL_EXTERN mecab_t *mecab_model_new_tagger(mecab_model_t *model); + + /** + * C wapper of MeCab::Model::createLattice() + */ + MECAB_DLL_EXTERN mecab_lattice_t *mecab_model_new_lattice(mecab_model_t *model); + + /** + * C wrapper of MeCab::Model::swap() + */ + MECAB_DLL_EXTERN int mecab_model_swap(mecab_model_t *model, mecab_model_t *new_model); + + /** + * C wapper of MeCab::Model::dictionary_info() + */ + MECAB_DLL_EXTERN const mecab_dictionary_info_t* mecab_model_dictionary_info(mecab_model_t *model); + + /** + * C wrapper of MeCab::Model::transition_cost() + */ + MECAB_DLL_EXTERN int mecab_model_transition_cost(mecab_model_t *model, + unsigned short rcAttr, + unsigned short lcAttr); + + /** + * C wrapper of MeCab::Model::lookup() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_model_lookup(mecab_model_t *model, + const char *begin, + const char *end, + mecab_lattice_t *lattice); + + /* static functions */ + MECAB_DLL_EXTERN int mecab_do(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_dict_index(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_dict_gen(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_cost_train(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_system_eval(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_test_gen(int argc, char **argv); +#endif + +#ifdef __cplusplus +} +#endif + +/* C++ interface */ +#ifdef __cplusplus + +namespace MeCab { +typedef struct mecab_dictionary_info_t DictionaryInfo; +typedef struct mecab_path_t Path; +typedef struct mecab_node_t Node; + +template class Allocator; +class Tagger; + +/** + * Lattice class + */ +class MECAB_DLL_CLASS_EXTERN Lattice { +public: + /** + * Clear all internal lattice data. + */ + virtual void clear() = 0; + + /** + * Return true if result object is available. + * @return boolean + */ + virtual bool is_available() const = 0; + + /** + * Return bos (begin of sentence) node. + * You can obtain all nodes via "for (const Node *node = lattice->bos_node(); node; node = node->next) {}" + * @return bos node object + */ + virtual Node *bos_node() const = 0; + + /** + * Return eos (end of sentence) node. + * @return eos node object + */ + virtual Node *eos_node() const = 0; + +#ifndef SWIG + /** + * This method is used internally. + */ + virtual Node **begin_nodes() const = 0; + + /** + * This method is used internally. + */ + virtual Node **end_nodes() const = 0; +#endif + + /** + * Return node linked list ending at |pos|. + * You can obtain all nodes via "for (const Node *node = lattice->end_nodes(pos); node; node = node->enext) {}" + * @param pos position of nodes. 0 <= pos < size() + * @return node linked list + */ + virtual Node *end_nodes(size_t pos) const = 0; + + /** + * Return node linked list starting at |pos|. + * You can obtain all nodes via "for (const Node *node = lattice->begin_nodes(pos); node; node = node->bnext) {}" + * @param pos position of nodes. 0 <= pos < size() + * @return node linked list + */ + virtual Node *begin_nodes(size_t pos) const = 0; + + /** + * Return sentence. + * If MECAB_NBEST or MECAB_PARTIAL mode is off, the returned poiner is the same as the one set by set_sentence(). + * @return sentence + */ + virtual const char *sentence() const = 0; + + /** + * Set sentence. This method does not take the ownership of the object. + * @param sentence sentence + */ + virtual void set_sentence(const char *sentence) = 0; + +#ifndef SWIG + /** + * Set sentence. This method does not take the ownership of the object. + * @param sentence sentence + * @param len length of the sentence + */ + virtual void set_sentence(const char *sentence, size_t len) = 0; +#endif + + /** + * Return sentence size. + * @return sentence size + */ + virtual size_t size() const = 0; + + /** + * Set normalization factor of CRF. + * @param Z new normalization factor. + */ + virtual void set_Z(double Z) = 0; + + /** + * return normalization factor of CRF. + * @return normalization factor. + */ + virtual double Z() const = 0; + + /** + * Set temparature parameter theta. + * @param theta temparature parameter. + */ + virtual void set_theta(float theta) = 0; + + /** + * Return temparature parameter theta. + * @return temparature parameter. + */ + virtual float theta() const = 0; + + /** + * Obtain next-best result. The internal linked list structure is updated. + * You should set MECAB_NBEST reques_type in advance. + * Return false if no more results are available or request_type is invalid. + * @return boolean + */ + virtual bool next() = 0; + + /** + * Return the current request type. + * @return request type + */ + virtual int request_type() const = 0; + + /** + * Return true if the object has a specified request type. + * @return boolean + */ + virtual bool has_request_type(int request_type) const = 0; + + /** + * Set request type. + * @param request_type new request type assigned + */ + virtual void set_request_type(int request_type) = 0; + + /** + * Add request type. + * @param request_type new request type added + */ + virtual void add_request_type(int request_type) = 0; + + /** + * Remove request type. + * @param request_type new request type removed + */ + virtual void remove_request_type(int request_type) = 0; + +#ifndef SWIG + /** + * This method is used internally. + */ + virtual Allocator *allocator() const = 0; +#endif + + /** + * Return new node. Lattice objects has the ownership of the node. + * @return new node object + */ + virtual Node *newNode() = 0; + + /** + * Return string representation of the lattice. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the lattice + */ + virtual const char *toString() = 0; + + /** + * Return string representation of the node. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the node + * @param node node object + */ + virtual const char *toString(const Node *node) = 0; + + /** + * Return string representation of the N-best results. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the node + * @param N how many results you want to obtain + */ + virtual const char *enumNBestAsString(size_t N) = 0; + +#ifndef SWIG + /** + * Return string representation of the lattice. + * Result is saved in the specified buffer. + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *toString(char *buf, size_t size) = 0; + + /** + * Return string representation of the node. + * Result is saved in the specified buffer. + * @param node node object + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *toString(const Node *node, + char *buf, size_t size) = 0; + + /** + * Return string representation of the N-best result. + * Result is saved in the specified. + * @param N how many results you want to obtain + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *enumNBestAsString(size_t N, char *buf, size_t size) = 0; +#endif + + /** + * Returns true if any parsing constraint is set + */ + virtual bool has_constraint() const = 0; + + /** + * Returns the boundary constraint at the position. + * @param pos the position of constraint + * @return boundary constraint type + */ + virtual int boundary_constraint(size_t pos) const = 0; + + /** + * Returns the token constraint at the position. + * @param pos the beginning position of constraint. + * @return constrained node starting at the position. + */ + virtual const char *feature_constraint(size_t pos) const = 0; + + /** + * Set parsing constraint for partial parsing mode. + * @param pos the position of the boundary + * @param boundary_constraint_type the type of boundary + */ + virtual void set_boundary_constraint(size_t pos, + int boundary_constraint_type) = 0; + + /** + * Set parsing constraint for partial parsing mode. + * @param begin_pos the starting position of the constrained token. + * @param end_pos the the ending position of the constrained token. + * @param feature the feature of the constrained token. + */ + virtual void set_feature_constraint( + size_t begin_pos, size_t end_pos, + const char *feature) = 0; + + /** + * Set golden parsing results for unittesting. + * @param result the parsing result written in the standard mecab output. + */ + virtual void set_result(const char *result) = 0; + + /** + * Return error string. + * @return error string + */ + virtual const char *what() const = 0; + + /** + * Set error string. given string is copied to the internal buffer. + * @param str new error string + */ + virtual void set_what(const char *str) = 0; + +#ifndef SWIG + /** + * Create new Lattice object + * @return new Lattice object + */ + static Lattice *create(); +#endif + + virtual ~Lattice() {} +}; + +/** + * Model class + */ +class MECAB_DLL_CLASS_EXTERN Model { +public: + /** + * Return DictionaryInfo linked list. + * @return DictionaryInfo linked list + */ + virtual const DictionaryInfo *dictionary_info() const = 0; + + /** + * Return transtion cost from rcAttr to lcAttr. + * @return transtion cost + */ + virtual int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const = 0; + + /** + * perform common prefix search from the range [begin, end). + * |lattice| takes the ownership of return value. + * @return node linked list. + */ + virtual Node *lookup(const char *begin, const char *end, + Lattice *lattice) const = 0; + + /** + * Create a new Tagger object. + * All returned tagger object shares this model object as a parsing model. + * Never delete this model object before deleting tagger object. + * @return new Tagger object + */ + virtual Tagger *createTagger() const = 0; + + /** + * Create a new Lattice object. + * @return new Lattice object + */ + virtual Lattice *createLattice() const = 0; + + /** + * Swap the instance with |model|. + * The ownership of |model| always moves to this instance, + * meaning that passed |model| will no longer be accessible after calling this method. + * return true if new model is swapped successfully. + * This method is thread safe. All taggers created by + * Model::createTagger() method will also be updated asynchronously. + * No need to stop the parsing thread excplicitly before swapping model object. + * @return boolean + * @param model new model which is going to be swapped with the current model. + */ + virtual bool swap(Model *model) = 0; + + /** + * Return a version string + * @return version string + */ + static const char *version(); + + virtual ~Model() {} + +#ifndef SWIG + /** + * Factory method to create a new Model with a specified main's argc/argv-style parameters. + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param argc number of parameters + * @param argv parameter list + */ + static Model* create(int argc, char **argv); + + /** + * Factory method to create a new Model with a string parameter representation, i.e., + * "-d /user/local/mecab/dic/ipadic -Ochasen". + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param arg single string representation of the argment. + */ + static Model* create(const char *arg); +#endif +}; + +/** + * Tagger class + */ +class MECAB_DLL_CLASS_EXTERN Tagger { +public: + /** + * Handy static method. + * Return true if lattice is parsed successfully. + * This function is equivalent to + * { + * Tagger *tagger = model.createModel(); + * cosnt bool result = tagger->parse(lattice); + * delete tagger; + * return result; + * } + * @return boolean + */ + static bool parse(const Model &model, Lattice *lattice); + + /** + * Parse lattice object. + * Return true if lattice is parsed successfully. + * A sentence must be set to the lattice with Lattice:set_sentence object before calling this method. + * Parsed node object can be obtained with Lattice:bos_node. + * This method is thread safe. + * @return lattice lattice object + * @return boolean + */ + virtual bool parse(Lattice *lattice) const = 0; + + /** + * Parse given sentence and return parsed result as string. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is NOT thread safe. + * @param str sentence + * @return parsed result + */ + virtual const char* parse(const char *str) = 0; + + /** + * Parse given sentence and return Node object. + * You should not delete the returned node object. The returned buffer + * is overwritten when parse method is called again. + * You can traverse all nodes via Node::next member. + * This method is NOT thread safe. + * @param str sentence + * @return bos node object + */ + virtual const Node* parseToNode(const char *str) = 0; + + /** + * Parse given sentence and obtain N-best results as a string format. + * Currently, N must be 1 <= N <= 512 due to the limitation of the buffer size. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is DEPRECATED. Use Lattice class. + * @param N how many results you want to obtain + * @param str sentence + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str) = 0; + + /** + * Initialize N-best enumeration with a sentence. + * Return true if initialization finishes successfully. + * N-best result is obtained by calling next() or nextNode() in sequence. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @param str sentence + * @return boolean + */ + virtual bool parseNBestInit(const char *str) = 0; + + /** + * Return next-best parsed result. You must call parseNBestInit() in advance. + * Return NULL if no more reuslt is available. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @return node object + */ + virtual const Node* nextNode() = 0; + + /** + * Return next-best parsed result. You must call parseNBestInit() in advance. + * Return NULL if no more reuslt is available. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @return parsed result + */ + virtual const char* next() = 0; + + /** + * Return formatted node object. The format is specified with + * --unk-format, --bos-format, --eos-format, and --eon-format respectively. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @param node node object. + * @return parsed result + */ + virtual const char* formatNode(const Node *node) = 0; + +#ifndef SWIG + /** + * The same as parse() method, but input length and output buffer are passed. + * Return parsed result as string. The result pointer is the same as |ostr|. + * Return NULL, if parsed result string cannot be stored within |olen| bytes. + * @param str sentence + * @param len sentence length + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* parse(const char *str, size_t len, char *ostr, size_t olen) = 0; + + /** + * The same as parse() method, but input length can be passed. + * @param str sentence + * @param len sentence length + * @return parsed result + */ + virtual const char* parse(const char *str, size_t len) = 0; + + /** + * The same as parseToNode(), but input lenth can be passed. + * @param str sentence + * @param len sentence length + * @return node object + */ + virtual const Node* parseToNode(const char *str, size_t len) = 0; + + /** + * The same as parseNBest(), but input length can be passed. + * @param N how many results you want to obtain + * @param str sentence + * @param len sentence length + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str, size_t len) = 0; + + /** + * The same as parseNBestInit(), but input length can be passed. + * @param str sentence + * @param len sentence length + * @return boolean + * @return parsed result + */ + virtual bool parseNBestInit(const char *str, size_t len) = 0; + + /** + * The same as next(), but output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* next(char *ostr , size_t olen) = 0; + + /** + * The same as parseNBest(), but input length and output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param N how many results you want to obtain + * @param str input sentence + * @param len input sentence length + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str, + size_t len, char *ostr, size_t olen) = 0; + + /** + * The same as formatNode(), but output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param node node object + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* formatNode(const Node *node, char *ostr, size_t olen) = 0; +#endif + + /** + * Set request type. + * This method is DEPRECATED. Use Lattice::set_request_type(MECAB_PARTIAL). + * @param request_type new request type assigned + */ + virtual void set_request_type(int request_type) = 0; + + /** + * Return the current request type. + * This method is DEPRECATED. Use Lattice class. + * @return request type + */ + virtual int request_type() const = 0; + + /** + * Return true if partial parsing mode is on. + * This method is DEPRECATED. Use Lattice::has_request_type(MECAB_PARTIAL). + * @return boolean + */ + virtual bool partial() const = 0; + + /** + * set partial parsing mode. + * This method is DEPRECATED. Use Lattice::add_request_type(MECAB_PARTIAL) or Lattice::remove_request_type(MECAB_PARTIAL) + * @param partial partial mode + */ + virtual void set_partial(bool partial) = 0; + + /** + * Return lattice level. + * This method is DEPRECATED. Use Lattice::*_request_type() + * @return int lattice level + */ + virtual int lattice_level() const = 0; + + /** + * Set lattice level. + * This method is DEPRECATED. Use Lattice::*_request_type() + * @param level lattice level + */ + virtual void set_lattice_level(int level) = 0; + + /** + * Return true if all morphs output mode is on. + * This method is DEPRECATED. Use Lattice::has_request_type(MECAB_ALL_MORPHS). + * @return boolean + */ + virtual bool all_morphs() const = 0; + + /** + * set all-morphs output mode. + * This method is DEPRECATED. Use Lattice::add_request_type(MECAB_ALL_MORPHS) or Lattice::remove_request_type(MECAB_ALL_MORPHS) + * @param all_morphs + */ + virtual void set_all_morphs(bool all_morphs) = 0; + + /** + * Set temparature parameter theta. + * @param theta temparature parameter. + */ + virtual void set_theta(float theta) = 0; + + /** + * Return temparature parameter theta. + * @return temparature parameter. + */ + virtual float theta() const = 0; + + /** + * Return DictionaryInfo linked list. + * @return DictionaryInfo linked list + */ + virtual const DictionaryInfo* dictionary_info() const = 0; + + /** + * Return error string. + * @return error string + */ + virtual const char* what() const = 0; + + virtual ~Tagger() {} + +#ifndef SWIG + /** + * Factory method to create a new Tagger with a specified main's argc/argv-style parameters. + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Tagger object + * @param argc number of parameters + * @param argv parameter list + */ + static Tagger *create(int argc, char **argv); + + /** + * Factory method to create a new Tagger with a string parameter representation, i.e., + * "-d /user/local/mecab/dic/ipadic -Ochasen". + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param arg single string representation of the argment. + */ + static Tagger *create(const char *arg); +#endif + + /** + * Return a version string + * @return version string + */ + static const char *version(); +}; + +#ifndef SWIG +/** + * Alias of Lattice::create() + */ +MECAB_DLL_EXTERN Lattice *createLattice(); + +/** + * Alias of Mode::create(argc, argv) + */ +MECAB_DLL_EXTERN Model *createModel(int argc, char **argv); + +/** + * Alias of Mode::create(arg) + */ +MECAB_DLL_EXTERN Model *createModel(const char *arg); + +/** + * Alias of Tagger::create(argc, argv) + */ +MECAB_DLL_EXTERN Tagger *createTagger(int argc, char **argv); + +/** + * Alias of Tagger::create(arg) + */ +MECAB_DLL_EXTERN Tagger *createTagger(const char *arg); + +/** + * delete Lattice object. + * This method calles "delete lattice". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param lattice lattice object + */ +MECAB_DLL_EXTERN void deleteLattice(Lattice *lattice); + + +/** + * delete Model object. + * This method calles "delete model". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param model model object + */ +MECAB_DLL_EXTERN void deleteModel(Model *model); + +/** + * delete Tagger object. + * This method calles "delete tagger". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param tagger tagger object + */ +MECAB_DLL_EXTERN void deleteTagger(Tagger *tagger); + +/** + * Return last error string. + * @return error string + */ +MECAB_DLL_EXTERN const char* getLastError(); + +/** + * An alias of getLastError. + * It is kept for backward compatibility. + * @return error string + */ +MECAB_DLL_EXTERN const char* getTaggerError(); +#endif +} +#endif +#endif /* MECAB_MECAB_H_ */ diff --git a/third_party/mecab/src/mmap.h b/fts/third_party/mecab/src/mmap.h similarity index 100% rename from third_party/mecab/src/mmap.h rename to fts/third_party/mecab/src/mmap.h diff --git a/third_party/mecab/src/nbest_generator.cpp b/fts/third_party/mecab/src/nbest_generator.cpp similarity index 100% rename from third_party/mecab/src/nbest_generator.cpp rename to fts/third_party/mecab/src/nbest_generator.cpp diff --git a/third_party/mecab/src/nbest_generator.h b/fts/third_party/mecab/src/nbest_generator.h similarity index 100% rename from third_party/mecab/src/nbest_generator.h rename to fts/third_party/mecab/src/nbest_generator.h diff --git a/third_party/mecab/src/param.cpp b/fts/third_party/mecab/src/param.cpp similarity index 100% rename from third_party/mecab/src/param.cpp rename to fts/third_party/mecab/src/param.cpp diff --git a/third_party/mecab/src/param.h b/fts/third_party/mecab/src/param.h similarity index 100% rename from third_party/mecab/src/param.h rename to fts/third_party/mecab/src/param.h diff --git a/third_party/mecab/src/scoped_ptr.h b/fts/third_party/mecab/src/scoped_ptr.h similarity index 100% rename from third_party/mecab/src/scoped_ptr.h rename to fts/third_party/mecab/src/scoped_ptr.h diff --git a/third_party/mecab/src/stream_wrapper.h b/fts/third_party/mecab/src/stream_wrapper.h similarity index 100% rename from third_party/mecab/src/stream_wrapper.h rename to fts/third_party/mecab/src/stream_wrapper.h diff --git a/third_party/mecab/src/string_buffer.cpp b/fts/third_party/mecab/src/string_buffer.cpp similarity index 100% rename from third_party/mecab/src/string_buffer.cpp rename to fts/third_party/mecab/src/string_buffer.cpp diff --git a/third_party/mecab/src/string_buffer.h b/fts/third_party/mecab/src/string_buffer.h similarity index 100% rename from third_party/mecab/src/string_buffer.h rename to fts/third_party/mecab/src/string_buffer.h diff --git a/fts/third_party/mecab/src/tagger.cpp b/fts/third_party/mecab/src/tagger.cpp new file mode 100644 index 00000000..2780dbb2 --- /dev/null +++ b/fts/third_party/mecab/src/tagger.cpp @@ -0,0 +1,1277 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include "common.h" +#include "connector.h" +#include "mecab.h" +#include "nbest_generator.h" +#include "param.h" +#include "scoped_ptr.h" +#include "stream_wrapper.h" +#include "string_buffer.h" +#include "thread.h" +#include "tokenizer.h" +#include "viterbi.h" +#include "writer.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +const char *getGlobalError(); +void setGlobalError(const char *str); + +namespace MeCab { +namespace { + +const float kDefaultTheta = 0.75; + +const MeCab::Option long_options[] = { + { "rcfile", 'r', 0, "FILE", "use FILE as resource file" }, + { "dicdir", 'd', 0, "DIR", "set DIR as a system dicdir" }, + { "userdic", 'u', 0, "FILE", "use FILE as a user dictionary" }, + { "lattice-level", 'l', "0", "INT", + "lattice information level (DEPRECATED)" }, + { "dictionary-info", 'D', 0, 0, "show dictionary information and exit" }, + { "output-format-type", 'O', 0, "TYPE", + "set output format type (wakati,none,...)" }, + { "all-morphs", 'a', 0, 0, "output all morphs(default false)" }, + { "nbest", 'N', "1", + "INT", "output N best results (default 1)" }, + { "partial", 'p', 0, 0, + "partial parsing mode (default false)" }, + { "marginal", 'm', 0, 0, + "output marginal probability (default false)" }, + { "max-grouping-size", 'M', "24", + "INT", "maximum grouping size for unknown words (default 24)" }, + { "node-format", 'F', "%m\\t%H\\n", "STR", + "use STR as the user-defined node format" }, + { "unk-format", 'U', "%m\\t%H\\n", "STR", + "use STR as the user-defined unknown node format" }, + { "bos-format", 'B', "", "STR", + "use STR as the user-defined beginning-of-sentence format" }, + { "eos-format", 'E', "EOS\\n", "STR", + "use STR as the user-defined end-of-sentence format" }, + { "eon-format", 'S', "", "STR", + "use STR as the user-defined end-of-NBest format" }, + { "unk-feature", 'x', 0, "STR", + "use STR as the feature for unknown word" }, + { "input-buffer-size", 'b', 0, "INT", + "set input buffer size (default 8192)" }, + { "dump-config", 'P', 0, 0, "dump MeCab parameters" }, + { "allocate-sentence", 'C', 0, 0, + "allocate new memory for input sentence" }, + { "theta", 't', "0.75", "FLOAT", + "set temparature parameter theta (default 0.75)" }, + { "cost-factor", 'c', "700", "INT", + "set cost factor (default 700)" }, + { "output", 'o', 0, "FILE", "set the output file name" }, + { "version", 'v', 0, 0, "show the version and exit." }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } +}; + +class ModelImpl: public Model { + public: + ModelImpl(); + virtual ~ModelImpl(); + + bool open(int argc, char **argv); + bool open(const char *arg); + bool open(const Param ¶m); + + bool swap(Model *model); + + bool is_available() const { + return (viterbi_ && writer_.get()); + } + + int request_type() const { + return request_type_; + } + + double theta() const { + return theta_; + } + + const DictionaryInfo *dictionary_info() const { + return viterbi_->tokenizer() ? + viterbi_->tokenizer()->dictionary_info() : 0; + } + + int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const { + return viterbi_->connector()->transition_cost(rcAttr, lcAttr); + } + + Node *lookup(const char *begin, const char *end, + Lattice *lattice) const { + return viterbi_->tokenizer()->lookup( + begin, end, + lattice->allocator(), lattice); + } + + Tagger *createTagger() const; + + Lattice *createLattice() const; + + const Viterbi *viterbi() const { + return viterbi_; + } + + // moves the owership. + Viterbi *take_viterbi() { + Viterbi *result = viterbi_; + viterbi_ = 0; + return result; + } + + const Writer *writer() const { + return writer_.get(); + } + +#ifdef HAVE_ATOMIC_OPS + read_write_mutex *mutex() const { + return &mutex_; + } +#endif + + private: + Viterbi *viterbi_; + scoped_ptr writer_; + int request_type_; + double theta_; + +#ifdef HAVE_ATOMIC_OPS + mutable read_write_mutex mutex_; +#endif +}; + +class TaggerImpl: public Tagger { + public: + bool open(int argc, char **argv); + bool open(const char *arg); + bool open(const ModelImpl &model); + + bool parse(Lattice *lattice) const; + + void set_request_type(int request_type); + int request_type() const; + + const char* parse(const char*); + const char* parse(const char*, size_t); + const char* parse(const char*, size_t, char*, size_t); + const Node* parseToNode(const char*); + const Node* parseToNode(const char*, size_t = 0); + const char* parseNBest(size_t, const char*); + const char* parseNBest(size_t, const char*, size_t); + const char* parseNBest(size_t, const char*, + size_t, char *, size_t); + bool parseNBestInit(const char*); + bool parseNBestInit(const char*, size_t); + const Node* nextNode(); + const char* next(); + const char* next(char*, size_t); + + const char *formatNode(const Node *); + const char *formatNode(const Node *, char *, size_t); + + const DictionaryInfo *dictionary_info() const; + + void set_partial(bool partial); + bool partial() const; + void set_theta(float theta); + float theta() const; + void set_lattice_level(int level); + int lattice_level() const; + void set_all_morphs(bool all_morphs); + bool all_morphs() const; + + const char* what() const; + + TaggerImpl(); + virtual ~TaggerImpl(); + + private: + const ModelImpl *model() const { return current_model_; } + + void set_what(const char *str) { + what_.assign(str); + } + + void initRequestType() { + mutable_lattice()->set_request_type(request_type_); + mutable_lattice()->set_theta(theta_); + } + + Lattice *mutable_lattice() { + if (!lattice_.get()) { + lattice_.reset(model()->createLattice()); + } + return lattice_.get(); + } + + const ModelImpl *current_model_; + scoped_ptr model_; + scoped_ptr lattice_; + int request_type_; + double theta_; + std::string what_; +}; + +class LatticeImpl : public Lattice { + public: + explicit LatticeImpl(const Writer *writer = 0); + ~LatticeImpl(); + + // clear internal lattice + void clear(); + + bool is_available() const { + return (sentence_ && + !begin_nodes_.empty() && + !end_nodes_.empty()); + } + + // nbest; + bool next(); + + // return bos/eos node + Node *bos_node() const { return end_nodes_[0]; } + Node *eos_node() const { return begin_nodes_[size()]; } + Node **begin_nodes() const { return const_cast(&begin_nodes_[0]); } + Node **end_nodes() const { return const_cast(&end_nodes_[0]); } + Node *begin_nodes(size_t pos) const { return begin_nodes_[pos]; } + Node *end_nodes(size_t pos) const { return end_nodes_[pos]; } + + const char *sentence() const { return sentence_; } + void set_sentence(const char *sentence); + void set_sentence(const char *sentence, size_t len); + size_t size() const { return size_; } + + void set_Z(double Z) { Z_ = Z; } + double Z() const { return Z_; } + + float theta() const { return theta_; } + void set_theta(float theta) { theta_ = theta; } + + int request_type() const { return request_type_; } + + void set_request_type(int request_type) { + request_type_ = request_type; + } + bool has_request_type(int request_type) const { + return request_type & request_type_; + } + void add_request_type(int request_type) { + request_type_ |= request_type; + } + void remove_request_type(int request_type) { + request_type_ &= ~request_type; + } + + Allocator *allocator() const { + return allocator_.get(); + } + + Node *newNode() { + return allocator_->newNode(); + } + + bool has_constraint() const; + int boundary_constraint(size_t pos) const; + const char *feature_constraint(size_t begin_pos) const; + + void set_boundary_constraint(size_t pos, + int boundary_constraint_type); + + void set_feature_constraint(size_t begin_pos, size_t end_pos, + const char *feature); + + void set_result(const char *result); + + const char *what() const { return what_.c_str(); } + + void set_what(const char *str) { + what_.assign(str); + } + + const char *toString(); + const char *toString(char *buf, size_t size); + const char *toString(const Node *node); + const char *toString(const Node *node, + char *buf, size_t size); + const char *enumNBestAsString(size_t N); + const char *enumNBestAsString(size_t N, char *buf, size_t size); + + private: + const char *sentence_; + size_t size_; + double theta_; + double Z_; + int request_type_; + std::string what_; + std::vector end_nodes_; + std::vector begin_nodes_; + std::vector feature_constraint_; + std::vector boundary_constraint_; + const Writer *writer_; + scoped_ptr ostrs_; + scoped_ptr > allocator_; + + StringBuffer *stream() { + if (!ostrs_.get()) { + ostrs_.reset(new StringBuffer); + } + return ostrs_.get(); + } + + const char *toStringInternal(StringBuffer *os); + const char *toStringInternal(const Node *node, StringBuffer *os); + const char *enumNBestAsStringInternal(size_t N, StringBuffer *os); +}; + +ModelImpl::ModelImpl() + : viterbi_(new Viterbi), writer_(new Writer), + request_type_(MECAB_ONE_BEST), theta_(0.0) {} + +ModelImpl::~ModelImpl() { + delete viterbi_; + viterbi_ = 0; +} + +bool ModelImpl::open(int argc, char **argv) { + Param param; + if (!param.open(argc, argv, long_options) || + !load_dictionary_resource(¶m)) { + setGlobalError(param.what()); + return false; + } + return open(param); +} + +bool ModelImpl::open(const char *arg) { + Param param; + if (!param.open(arg, long_options) || + !load_dictionary_resource(¶m)) { + setGlobalError(param.what()); + return false; + } + return open(param); +} + +bool ModelImpl::open(const Param ¶m) { + if (!writer_->open(param) || !viterbi_->open(param)) { + std::string error = viterbi_->what(); + if (!error.empty()) { + error.append(" "); + } + error.append(writer_->what()); + setGlobalError(error.c_str()); + return false; + } + + request_type_ = load_request_type(param); + theta_ = param.get("theta"); + + return is_available(); +} + +bool ModelImpl::swap(Model *model) { + scoped_ptr model_data(model); + + if (!is_available()) { + setGlobalError("current model is not available"); + return false; + } +#ifndef HAVE_ATOMIC_OPS + setGlobalError("atomic model replacement is not supported"); + return false; +#else + ModelImpl *m = static_cast(model_data.get()); + if (!m) { + setGlobalError("Invalid model is passed"); + return false; + } + + if (!m->is_available()) { + setGlobalError("Passed model is not available"); + return false; + } + + Viterbi *current_viterbi = viterbi_; + { + scoped_writer_lock l(mutex()); + viterbi_ = m->take_viterbi(); + request_type_ = m->request_type(); + theta_ = m->theta(); + } + + delete current_viterbi; + + return true; +#endif +} + +Tagger *ModelImpl::createTagger() const { + if (!is_available()) { + setGlobalError("Model is not available"); + return 0; + } + TaggerImpl *tagger = new TaggerImpl; + if (!tagger->open(*this)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + tagger->set_theta(theta_); + tagger->set_request_type(request_type_); + return tagger; +} + +Lattice *ModelImpl::createLattice() const { + if (!is_available()) { + setGlobalError("Model is not available"); + return 0; + } + return new LatticeImpl(writer_.get()); +} + +TaggerImpl::TaggerImpl() + : current_model_(0), + request_type_(MECAB_ONE_BEST), theta_(kDefaultTheta) {} + +TaggerImpl::~TaggerImpl() {} + +const char *TaggerImpl::what() const { + return what_.c_str(); +} + +bool TaggerImpl::open(int argc, char **argv) { + model_.reset(new ModelImpl); + if (!model_->open(argc, argv)) { + model_.reset(0); + return false; + } + current_model_ = model_.get(); + request_type_ = model()->request_type(); + theta_ = model()->theta(); + return true; +} + +bool TaggerImpl::open(const char *arg) { + model_.reset(new ModelImpl); + if (!model_->open(arg)) { + model_.reset(0); + return false; + } + current_model_ = model_.get(); + request_type_ = model()->request_type(); + theta_ = model()->theta(); + return true; +} + +bool TaggerImpl::open(const ModelImpl &model) { + if (!model.is_available()) { + return false; + } + model_.reset(0); + current_model_ = &model; + request_type_ = current_model_->request_type(); + theta_ = current_model_->theta(); + return true; +} + +void TaggerImpl::set_request_type(int request_type) { + request_type_ = request_type; +} + +int TaggerImpl::request_type() const { + return request_type_; +} + +void TaggerImpl::set_partial(bool partial) { + if (partial) { + request_type_ |= MECAB_PARTIAL; + } else { + request_type_ &= ~MECAB_PARTIAL; + } +} + +bool TaggerImpl::partial() const { + return request_type_ & MECAB_PARTIAL; +} + +void TaggerImpl::set_theta(float theta) { + theta_ = theta; +} + +float TaggerImpl::theta() const { + return theta_; +} + +void TaggerImpl::set_lattice_level(int level) { + switch (level) { + case 0: request_type_ |= MECAB_ONE_BEST; + break; + case 1: request_type_ |= MECAB_NBEST; + break; + case 2: request_type_ |= MECAB_MARGINAL_PROB; + break; + default: + break; + } +} + +int TaggerImpl::lattice_level() const { + if (request_type_ & MECAB_MARGINAL_PROB) { + return 2; + } else if (request_type_ & MECAB_NBEST) { + return 1; + } else { + return 0; + } +} + +void TaggerImpl::set_all_morphs(bool all_morphs) { + if (all_morphs) { + request_type_ |= MECAB_ALL_MORPHS; + } else { + request_type_ &= ~MECAB_ALL_MORPHS; + } +} + +bool TaggerImpl::all_morphs() const { + return request_type_ & MECAB_ALL_MORPHS; +} + +bool TaggerImpl::parse(Lattice *lattice) const { +#ifdef HAVE_ATOMIC_OPS + scoped_reader_lock l(model()->mutex()); +#endif + + return model()->viterbi()->analyze(lattice); +} + +const char *TaggerImpl::parse(const char *str) { + return parse(str, std::strlen(str)); +} + +const char *TaggerImpl::parse(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + const char *result = lattice->toString(); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char *TaggerImpl::parse(const char *str, size_t len, + char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + const char *result = lattice->toString(out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const Node *TaggerImpl::parseToNode(const char *str) { + return parseToNode(str, std::strlen(str)); +} + +const Node *TaggerImpl::parseToNode(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + return lattice->bos_node(); +} + +bool TaggerImpl::parseNBestInit(const char *str) { + return parseNBestInit(str, std::strlen(str)); +} + +bool TaggerImpl::parseNBestInit(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return false; + } + return true; +} + +const Node* TaggerImpl::nextNode() { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + return lattice->bos_node(); +} + +const char* TaggerImpl::next() { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + const char *result = lattice->toString(); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::next(char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + const char *result = lattice->toString(out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::parseNBest(size_t N, const char* str) { + return parseNBest(N, str, std::strlen(str)); +} + +const char* TaggerImpl::parseNBest(size_t N, + const char* str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + + const char *result = lattice->enumNBestAsString(N); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::parseNBest(size_t N, const char* str, size_t len, + char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + + const char *result = lattice->enumNBestAsString(N, out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::formatNode(const Node* node) { + const char *result = mutable_lattice()->toString(node); + if (!result) { + set_what(mutable_lattice()->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::formatNode(const Node* node, + char *out, size_t len) { + const char *result = mutable_lattice()->toString(node, out, len); + if (!result) { + set_what(mutable_lattice()->what()); + return 0; + } + return result; +} + +const DictionaryInfo *TaggerImpl::dictionary_info() const { + return model()->dictionary_info(); +} + +LatticeImpl::LatticeImpl(const Writer *writer) + : sentence_(0), size_(0), theta_(kDefaultTheta), Z_(0.0), + request_type_(MECAB_ONE_BEST), + writer_(writer), + ostrs_(0), + allocator_(new Allocator) { + begin_nodes_.reserve(MIN_INPUT_BUFFER_SIZE); + end_nodes_.reserve(MIN_INPUT_BUFFER_SIZE); +} + +LatticeImpl::~LatticeImpl() {} + +void LatticeImpl::clear() { + allocator_->free(); + if (ostrs_.get()) { + ostrs_->clear(); + } + begin_nodes_.clear(); + end_nodes_.clear(); + feature_constraint_.clear(); + boundary_constraint_.clear(); + size_ = 0; + theta_ = kDefaultTheta; + Z_ = 0.0; + sentence_ = 0; +} + +void LatticeImpl::set_sentence(const char *sentence) { + return set_sentence(sentence, strlen(sentence)); +} + +void LatticeImpl::set_sentence(const char *sentence, size_t len) { + clear(); + end_nodes_.resize(len + 4); + begin_nodes_.resize(len + 4); + + if (has_request_type(MECAB_ALLOCATE_SENTENCE) || + has_request_type(MECAB_PARTIAL)) { + char *new_sentence = allocator()->strdup(sentence, len); + sentence_ = new_sentence; + } else { + sentence_ = sentence; + } + + size_ = len; + std::memset(&end_nodes_[0], 0, + sizeof(end_nodes_[0]) * (len + 4)); + std::memset(&begin_nodes_[0], 0, + sizeof(begin_nodes_[0]) * (len + 4)); +} + +bool LatticeImpl::next() { + if (!has_request_type(MECAB_NBEST)) { + set_what("MECAB_NBEST request type is not set"); + return false; + } + + if (!allocator()->nbest_generator()->next()) { + return false; + } + + Viterbi::buildResultForNBest(this); + return true; +} + +void LatticeImpl::set_result(const char *result) { + char *str = allocator()->strdup(result, std::strlen(result)); + std::vector lines; + const size_t lsize = tokenize(str, "\n", + std::back_inserter(lines), + std::strlen(result)); + CHECK_DIE(lsize == lines.size()); + + std::string sentence; + std::vector surfaces, features; + for (size_t i = 0; i < lines.size(); ++i) { + if (::strcmp("EOS", lines[i]) == 0) { + break; + } + char *cols[2]; + if (tokenize(lines[i], "\t", cols, 2) != 2) { + break; + } + sentence += cols[0]; + surfaces.push_back(cols[0]); + features.push_back(cols[1]); + } + + CHECK_DIE(features.size() == surfaces.size()); + + set_sentence(allocator()->strdup(sentence.c_str(), sentence.size())); + + Node *bos_node = allocator()->newNode(); + bos_node->surface = const_cast(BOS_KEY); // dummy + bos_node->feature = "BOS/EOS"; + bos_node->isbest = 1; + bos_node->stat = MECAB_BOS_NODE; + + Node *eos_node = allocator()->newNode(); + eos_node->surface = const_cast(BOS_KEY); // dummy + eos_node->feature = "BOS/EOS"; + eos_node->isbest = 1; + eos_node->stat = MECAB_EOS_NODE; + + bos_node->surface = sentence_; + end_nodes_[0] = bos_node; + + size_t offset = 0; + Node *prev = bos_node; + for (size_t i = 0; i < surfaces.size(); ++i) { + Node *node = allocator()->newNode(); + node->prev = prev; + prev->next = node; + node->surface = sentence_ + offset; + node->length = surfaces[i].size(); + node->rlength = surfaces[i].size(); + node->isbest = 1; + node->stat = MECAB_NOR_NODE; + node->wcost = 0; + node->cost = 0; + node->feature = allocator()->strdup(features[i].c_str(), + features[i].size()); + begin_nodes_[offset] = node; + end_nodes_[offset + node->length] = node; + offset += node->length; + prev = node; + } + + prev->next = eos_node; + eos_node->prev = prev; +} + +// default implementation of Lattice formatter. +namespace { +void writeLattice(Lattice *lattice, StringBuffer *os) { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature; + *os << '\n'; + } + *os << "EOS\n"; +} +} // namespace + +const char *LatticeImpl::toString() { + return toStringInternal(stream()); +} + +const char *LatticeImpl::toString(char *buf, size_t size) { + StringBuffer os(buf, size); + return toStringInternal(&os); +} + +const char *LatticeImpl::toStringInternal(StringBuffer *os) { + os->clear(); + if (writer_) { + if (!writer_->write(this, os)) { + return 0; + } + } else { + writeLattice(this, os); + } + *os << '\0'; + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + return os->str(); +} + +const char *LatticeImpl::toString(const Node *node) { + return toStringInternal(node, stream()); +} + +const char *LatticeImpl::toString(const Node *node, + char *buf, size_t size) { + StringBuffer os(buf, size); + return toStringInternal(node, &os); +} + +const char *LatticeImpl::toStringInternal(const Node *node, + StringBuffer *os) { + os->clear(); + if (!node) { + set_what("node is NULL"); + return 0; + } + if (writer_) { + if (!writer_->writeNode(this, node, os)) { + return 0; + } + } else { + os->write(node->surface, node->length); + *os << '\t' << node->feature; + } + *os << '\0'; + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + return os->str(); +} + +const char *LatticeImpl::enumNBestAsString(size_t N) { + return enumNBestAsStringInternal(N, stream()); +} + +const char *LatticeImpl::enumNBestAsString(size_t N, char *buf, size_t size) { + StringBuffer os(buf, size); + return enumNBestAsStringInternal(N, &os); +} + +const char *LatticeImpl::enumNBestAsStringInternal(size_t N, + StringBuffer *os) { + os->clear(); + + if (N == 0 || N > NBEST_MAX) { + set_what("nbest size must be 1 <= nbest <= 512"); + return 0; + } + + for (size_t i = 0; i < N; ++i) { + if (!next()) { + break; + } + if (writer_) { + if (!writer_->write(this, os)) { + return 0; + } + } else { + writeLattice(this, os); + } + } + + // make a dummy node for EON + if (writer_) { + Node eon_node; + memset(&eon_node, 0, sizeof(eon_node)); + eon_node.stat = MECAB_EON_NODE; + eon_node.next = 0; + eon_node.surface = this->sentence() + this->size(); + if (!writer_->writeNode(this, &eon_node, os)) { + return 0; + } + } + *os << '\0'; + + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + + return os->str(); +} + +bool LatticeImpl::has_constraint() const { + return !boundary_constraint_.empty(); +} + +int LatticeImpl::boundary_constraint(size_t pos) const { + if (!boundary_constraint_.empty()) { + return boundary_constraint_[pos]; + } + return MECAB_ANY_BOUNDARY; +} + +const char *LatticeImpl::feature_constraint(size_t begin_pos) const { + if (!feature_constraint_.empty()) { + return feature_constraint_[begin_pos]; + } + return 0; +} + +void LatticeImpl::set_boundary_constraint(size_t pos, + int boundary_constraint_type) { + if (boundary_constraint_.empty()) { + boundary_constraint_.resize(size() + 4, MECAB_ANY_BOUNDARY); + } + boundary_constraint_[pos] = boundary_constraint_type; +} + +void LatticeImpl::set_feature_constraint(size_t begin_pos, size_t end_pos, + const char *feature) { + if (begin_pos >= end_pos || !feature) { + return; + } + + if (feature_constraint_.empty()) { + feature_constraint_.resize(size() + 4, 0); + } + + end_pos = std::min(end_pos, size()); + + set_boundary_constraint(begin_pos, MECAB_TOKEN_BOUNDARY); + set_boundary_constraint(end_pos, MECAB_TOKEN_BOUNDARY); + for (size_t i = begin_pos + 1; i < end_pos; ++i) { + set_boundary_constraint(i, MECAB_INSIDE_TOKEN); + } + + feature_constraint_[begin_pos] = feature; +} +} // namespace + +Tagger *Tagger::create(int argc, char **argv) { + return createTagger(argc, argv); +} + +Tagger *Tagger::create(const char *arg) { + return createTagger(arg); +} + +const char *Tagger::version() { + return VERSION; +} + +Tagger *createTagger(int argc, char **argv) { + TaggerImpl *tagger = new TaggerImpl(); + if (!tagger->open(argc, argv)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + return tagger; +} + +Tagger *createTagger(const char *argv) { + TaggerImpl *tagger = new TaggerImpl(); + if (!tagger->open(argv)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + return tagger; +} + +void deleteTagger(Tagger *tagger) { + delete tagger; +} + +const char *getTaggerError() { + return getLastError(); +} + +const char *getLastError() { + return getGlobalError(); +} + +Model *createModel(int argc, char **argv) { + ModelImpl *model = new ModelImpl; + if (!model->open(argc, argv)) { + delete model; + return 0; + } + return model; +} + +Model *createModel(const char *arg) { + ModelImpl *model = new ModelImpl; + if (!model->open(arg)) { + delete model; + return 0; + } + return model; +} + +void deleteModel(Model *model) { + delete model; +} + +Model *Model::create(int argc, char **argv) { + return createModel(argc, argv); +} + +Model *Model::create(const char *arg) { + return createModel(arg); +} + +const char *Model::version() { + return VERSION; +} + +bool Tagger::parse(const Model &model, Lattice *lattice) { + scoped_ptr tagger(model.createTagger()); + return tagger->parse(lattice); +} + +Lattice *Lattice::create() { + return createLattice(); +} + +Lattice *createLattice() { + return new LatticeImpl; +} + +void deleteLattice(Lattice *lattice) { + delete lattice; +} +} // MeCab + +int mecab_do(int argc, char **argv) { +#define WHAT_ERROR(msg) do { \ + std::cout << msg << std::endl; \ + return EXIT_FAILURE; } \ + while (0); + + MeCab::Param param; + if (!param.open(argc, argv, MeCab::long_options)) { + std::cout << param.what() << std::endl; + return EXIT_FAILURE; + } + + if (param.get("help")) { + std::cout << param.help() << std::endl; + return EXIT_SUCCESS; + } + + if (param.get("version")) { + std::cout << param.version() << std::endl; + return EXIT_SUCCESS; + } + + if (!load_dictionary_resource(¶m)) { + std::cout << param.what() << std::endl; + return EXIT_SUCCESS; + } + + if (param.get("lattice-level") >= 1) { + std::cerr << "lattice-level is DEPERCATED. " + << "use --marginal or --nbest." << std::endl; + } + + MeCab::scoped_ptr model(new MeCab::ModelImpl); + if (!model->open(param)) { + std::cout << MeCab::getLastError() << std::endl; + return EXIT_FAILURE; + } + + std::string ofilename = param.get("output"); + if (ofilename.empty()) { + ofilename = "-"; + } + + const int nbest = param.get("nbest"); + if (nbest <= 0 || nbest > NBEST_MAX) { + WHAT_ERROR("invalid N value"); + } + + MeCab::ostream_wrapper ofs(ofilename.c_str()); + if (!*ofs) { + WHAT_ERROR("no such file or directory: " << ofilename); + } + + if (param.get("dump-config")) { + param.dump_config(&*ofs); + return EXIT_FAILURE; + } + + if (param.get("dictionary-info")) { + for (const MeCab::DictionaryInfo *d = model->dictionary_info(); + d; d = d->next) { + *ofs << "filename:\t" << d->filename << std::endl; + *ofs << "version:\t" << d->version << std::endl; + *ofs << "charset:\t" << d->charset << std::endl; + *ofs << "type:\t" << d->type << std::endl; + *ofs << "size:\t" << d->size << std::endl; + *ofs << "left size:\t" << d->lsize << std::endl; + *ofs << "right size:\t" << d->rsize << std::endl; + *ofs << std::endl; + } + return EXIT_FAILURE; + } + + const std::vector& rest_ = param.rest_args(); + std::vector rest = rest_; + + if (rest.empty()) { + rest.push_back("-"); + } + + size_t ibufsize = std::min(MAX_INPUT_BUFFER_SIZE, + std::max(param.get + ("input-buffer-size"), + MIN_INPUT_BUFFER_SIZE)); + + const bool partial = param.get("partial"); + if (partial) { + ibufsize *= 8; + } + + MeCab::scoped_array ibuf_data(new char[ibufsize]); + char *ibuf = ibuf_data.get(); + + MeCab::scoped_ptr tagger(model->createTagger()); + + if (!tagger.get()) { + WHAT_ERROR("cannot create tagger"); + } + + for (size_t i = 0; i < rest.size(); ++i) { + MeCab::istream_wrapper ifs(rest[i].c_str()); + if (!*ifs) { + WHAT_ERROR("no such file or directory: " << rest[i]); + } + + while (true) { + if (!partial) { + ifs->getline(ibuf, ibufsize); + } else { + std::string sentence; + MeCab::scoped_fixed_array line; + for (;;) { + if (!ifs->getline(line.get(), line.size())) { + ifs->clear(std::ios::eofbit|std::ios::badbit); + break; + } + sentence += line.get(); + sentence += '\n'; + if (std::strcmp(line.get(), "EOS") == 0 || line[0] == '\0') { + break; + } + } + std::strncpy(ibuf, sentence.c_str(), ibufsize); + } + if (ifs->eof() && !ibuf[0]) { + return false; + } + if (ifs->fail()) { + std::cerr << "input-buffer overflow. " + << "The line is split. use -b #SIZE option." << std::endl; + ifs->clear(); + } + const char *r = (nbest >= 2) ? tagger->parseNBest(nbest, ibuf) : + tagger->parse(ibuf); + if (!r) { + WHAT_ERROR(tagger->what()); + } + *ofs << r << std::flush; + } + } + + return EXIT_SUCCESS; + +#undef WHAT_ERROR +} diff --git a/third_party/mecab/src/thread.h b/fts/third_party/mecab/src/thread.h similarity index 100% rename from third_party/mecab/src/thread.h rename to fts/third_party/mecab/src/thread.h diff --git a/third_party/mecab/src/tokenizer.cpp b/fts/third_party/mecab/src/tokenizer.cpp similarity index 100% rename from third_party/mecab/src/tokenizer.cpp rename to fts/third_party/mecab/src/tokenizer.cpp diff --git a/third_party/mecab/src/tokenizer.h b/fts/third_party/mecab/src/tokenizer.h similarity index 100% rename from third_party/mecab/src/tokenizer.h rename to fts/third_party/mecab/src/tokenizer.h diff --git a/third_party/mecab/src/ucs.h b/fts/third_party/mecab/src/ucs.h similarity index 100% rename from third_party/mecab/src/ucs.h rename to fts/third_party/mecab/src/ucs.h diff --git a/third_party/mecab/src/utils.cpp b/fts/third_party/mecab/src/utils.cpp similarity index 100% rename from third_party/mecab/src/utils.cpp rename to fts/third_party/mecab/src/utils.cpp diff --git a/third_party/mecab/src/utils.h b/fts/third_party/mecab/src/utils.h similarity index 100% rename from third_party/mecab/src/utils.h rename to fts/third_party/mecab/src/utils.h diff --git a/third_party/mecab/src/viterbi.cpp b/fts/third_party/mecab/src/viterbi.cpp similarity index 100% rename from third_party/mecab/src/viterbi.cpp rename to fts/third_party/mecab/src/viterbi.cpp diff --git a/third_party/mecab/src/viterbi.h b/fts/third_party/mecab/src/viterbi.h similarity index 100% rename from third_party/mecab/src/viterbi.h rename to fts/third_party/mecab/src/viterbi.h diff --git a/third_party/mecab/src/winmain.h b/fts/third_party/mecab/src/winmain.h similarity index 100% rename from third_party/mecab/src/winmain.h rename to fts/third_party/mecab/src/winmain.h diff --git a/third_party/mecab/src/writer.cpp b/fts/third_party/mecab/src/writer.cpp similarity index 100% rename from third_party/mecab/src/writer.cpp rename to fts/third_party/mecab/src/writer.cpp diff --git a/third_party/mecab/src/writer.h b/fts/third_party/mecab/src/writer.h similarity index 100% rename from third_party/mecab/src/writer.h rename to fts/third_party/mecab/src/writer.h diff --git a/third_party/cppjieba/.gitignore b/third_party/cppjieba/.gitignore deleted file mode 100644 index 4a1921d3..00000000 --- a/third_party/cppjieba/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -tags -*.demo -*swp -*.out -*.o -*.d -*.ut -log -main -lib*.a -*_demo -segdict* -prior.gbk -tmp -t.* -*.pid -build -Testing/Temporary/CTestCostData.txt -Testing/Temporary/LastTest.log diff --git a/third_party/cppjieba/CMakeLists.txt b/third_party/cppjieba/CMakeLists.txt deleted file mode 100644 index f7fbd231..00000000 --- a/third_party/cppjieba/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -CMAKE_MINIMUM_REQUIRED (VERSION 3.10) -PROJECT(CPPJIEBA) - -# Use vendored limonp -set(LIMONP_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/deps/limonp/include") -INCLUDE_DIRECTORIES("${LIMONP_INCLUDE_DIR}" - ${PROJECT_SOURCE_DIR}/include) - -if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) -endif() -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -ADD_DEFINITIONS(-O3 -g) - -# Define a variable to check if this is the top-level project -if(NOT DEFINED CPPJIEBA_TOP_LEVEL_PROJECT) - if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - set(CPPJIEBA_TOP_LEVEL_PROJECT ON) - else() - set(CPPJIEBA_TOP_LEVEL_PROJECT OFF) - endif() -endif() - -if(NOT TARGET cppjieba) - add_library(cppjieba INTERFACE) - target_include_directories(cppjieba INTERFACE - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_SOURCE_DIR}/deps/limonp/include - ) -endif() - -include(GNUInstallDirs) -install(DIRECTORY include/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -install(DIRECTORY dict/ - DESTINATION ${CMAKE_INSTALL_DATADIR}/cppjieba/dict) - -if(CPPJIEBA_TOP_LEVEL_PROJECT) - ENABLE_TESTING() - - message(STATUS "MSVC value: ${MSVC}") - ADD_SUBDIRECTORY(test) - ADD_TEST(NAME ./test/test.run COMMAND ./test/test.run) - ADD_TEST(NAME ./load_test COMMAND ./load_test) -endif() diff --git a/third_party/cppjieba/LICENSE b/third_party/cppjieba/LICENSE deleted file mode 100644 index 6308e939..00000000 --- a/third_party/cppjieba/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/cppjieba/VERSION b/third_party/cppjieba/VERSION deleted file mode 100644 index 55a34b06..00000000 --- a/third_party/cppjieba/VERSION +++ /dev/null @@ -1,2 +0,0 @@ -v5.6.0 -git@github.com:yanyiwu/cppjieba.git diff --git a/third_party/cppjieba/deps/limonp/.gitignore b/third_party/cppjieba/deps/limonp/.gitignore deleted file mode 100644 index ad7223aa..00000000 --- a/third_party/cppjieba/deps/limonp/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.o -*.ut -libcm.a -tags -*.d -build -t.cpp -a.out -*.swp diff --git a/third_party/cppjieba/deps/limonp/.gitmodules b/third_party/cppjieba/deps/limonp/.gitmodules deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/cppjieba/deps/limonp/CHANGELOG.md b/third_party/cppjieba/deps/limonp/CHANGELOG.md deleted file mode 100644 index 17427cef..00000000 --- a/third_party/cppjieba/deps/limonp/CHANGELOG.md +++ /dev/null @@ -1,169 +0,0 @@ -# CHANGELOG - -## v1.0.1 - -+ [CI] Update GitHub Actions configurations - - Add stale issues workflow - - Update checkout action from v2 to v4 - - Update macOS test environments (remove macOS-12, add macOS-15) -+ [dep] Update googletest to release-1.12.1 -+ [doc] Add build instructions to README.md - -## v1.0.0 - -+ rm thread pool demo -+ deleted: include/limonp/BlockingQueue.hpp -+ deleted: include/limonp/BoundedBlockingQueue.hpp -+ deleted: include/limonp/BoundedQueue.hpp -+ deleted: include/limonp/MutexLock.hpp -+ deleted: include/limonp/Thread.hpp -+ deleted: include/limonp/ThreadPool.hpp -+ deleted: test/unittest/TBlockingQueue.cpp -+ deleted: test/unittest/TBoundedQueue.cpp -+ deleted: test/unittest/TMutexLock.cpp -+ deleted: test/unittest/TThread.cpp -+ deleted: test/unittest/TThreadPool.cpp -+ rm FileLock -+ rm Md5.hpp - -## v0.9.0 - -+ [c++20] compatibility -+ [c++17] compatibility - -## v0.8.1 - -+ [CI] fix windows gtest thread link error -+ [submodule] rm test/googletest -+ [CMake] FetchContent googletest -+ [CMake] required 3.5 -> 3.14 - -## v0.8.0 - -+ [StringUtil] Fix windows assert typo -+ [CMake] find_package(Threads REQUIRED); target_link_libraries(... Threads::Threads) -+ [CMAKE][CI] windows: 2019,2022 -+ [CMAKE][CI] matrix.build_type[Release, Debug] -+ [unittest] disable #TMd5.cpp -+ [unittest] disable #TFileLock.cpp -+ [unittest] disable #TBoundedQueue.cpp -+ [unittest] disable #TMutexLock.cpp -+ [unittest] disable #TBlockingQueue.cpp -+ [unittest] disable #TThread.cpp -+ [unittest] disable #TThreadPool.cpp - -## v0.7.2 - -+ [CI] ubuntu version from 20 to 22, macos version from 12 to 14 -+ [test/unittes] uint->size_t -+ [googletest] v1.6.0->v1.10.0 -+ [CMake] version required 3.0 -> 3.5 - -## v0.7.1 - -+ [CMake] fix CMAKE_CXX_STANDARD passed from github/actions and [c++11, c++14] only - -## v0.7.0 - -+ [CI] Added os.macos and cpp_version=[c++98, c++03, c++11, c++14, c++17, c++20] -+ [git submodule] Added googletest-release-v1.6.0 - -## v0.6.7 - -+ Merged [pr35](https://github.com/yanyiwu/limonp/pull/35) -+ Merged [pr33](https://github.com/yanyiwu/limonp/pull/33) -+ Merged [pr32](https://github.com/yanyiwu/limonp/pull/32) - -## v0.6.6 - -+ Merged [pr-31 To be compatible with cpp17 and later, use lambda instead of std::not1 & std::bind2nd #31](https://github.com/yanyiwu/limonp/pull/31) - -## v0.6.5 - -+ Merged [pr-25 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/25) -+ Merged [pr-26 fix license for end of line sequence and remove useless signature](https://github.com/yanyiwu/limonp/pull/26) -+ Merged [pr-27 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/27) -+ Merged [pr-28 add a target to be ready to support installation](https://github.com/yanyiwu/limonp/pull/28) -+ Merged [pr-29 Installable by cmake](https://github.com/yanyiwu/limonp/pull/29) -+ Merged [pr-30 Replace localtime with localtime_s on Windows and localtime_r on Linux](https://github.com/yanyiwu/limonp/pull/30) - -## v0.6.4 - -+ merge [fixup gcc8 warnings](https://github.com/yanyiwu/gojieba/pull/70) - -## v0.6.3 - -+ remove compiler conplained macro - -## v0.6.2 - -+ merge [pr-18](https://github.com/yanyiwu/limonp/pull/18/files) - -## v0.6.1 - -add Specialized template for vector - -when it is `vector`, print like this: ["hello", "world"]; (special case) -when it is `vector`, print like this: [1, 10, 1000]; (common cases) - -## v0.6.0 - -+ remove Trim out of Split. - -## v0.5.6 - -+ fix hidden trouble. - -## v0.5.5 - -+ macro name LOG and CHECK in Logging.hpp is so easy to confict with other lib, so I have to rename them to XLOG and XCHECK for avoiding those macro name conflicts. - -## v0.5.4 - -+ add ForcePublic.hpp -+ Add Utf8ToUnicode32 and Unicode32ToUtf8 in StringUtil.hpp - -## v0.5.3 - -+ Fix incompatibility problem about 'time.h' in Windows. - -## v0.5.2 - -+ Fix incompatibility problem about `enum {INFO ...}` name conflicts in Windows . -+ So from this version begin: the compile flags: `-DLOGGING_LEVEL=WARNING` must be changed to `-DLOGGING_LEVEL=LL_WARNING` - -## v0.5.1 - -+ add `ThreadPool::Stop()` to wait util all the threads finished. -If Stop() has not been called, it will be called when the ThreadPool destructing. - -## v0.5.0 - -+ Reorganized directories: include/ -> include/limonp/ ... and so on. -+ Add `NewClosure` in Closure.hpp, 0~3 arguments have been supported. -+ Update ThreadPool, use `NewClosure` instead of `CreateTask` - -## v0.4.1 - -+ `CHECK(exp) << "log message"` supported; - -## v0.4.0 - -+ add test/demo.cc as example. -+ move `print` macro to StdExtension.hpp -+ BigChange: rewrite `log` module, use `LOG(INFO) << "xxx" ` instead `LogInfo` . -+ remove HandyMacro.hpp, add CHECK in Logging.hpp instead. - -## v0.3.0 - -+ remove 'MysqlClient.hpp', 'InitOnOff.hpp', 'CastFloat.hpp' -+ add 'Closure.hpp' -+ uniform code style - -## v0.2.0 - -+ `namespace limonp`, not `Limonp` . - -## v0.1.0 - -+ Basic functions diff --git a/third_party/cppjieba/deps/limonp/CMakeLists.txt b/third_party/cppjieba/deps/limonp/CMakeLists.txt deleted file mode 100644 index 77c4210d..00000000 --- a/third_party/cppjieba/deps/limonp/CMakeLists.txt +++ /dev/null @@ -1,61 +0,0 @@ -cmake_minimum_required(VERSION 3.14) - -PROJECT(limonp - LANGUAGES CXX) - -################ -# cmake config # -################ - -if(NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) -endif() -message(STATUS "CMAKE_CXX_STANDARD is ${CMAKE_CXX_STANDARD}") - -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - - -############## -# dependency # -############## -include(GNUInstallDirs) - -########## -# target # -########## - -add_library(${PROJECT_NAME} INTERFACE) -add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) - -target_include_directories(${PROJECT_NAME} - INTERFACE - $ - $) - -######## -# test # -######## - -ENABLE_TESTING() - -ADD_SUBDIRECTORY(test) -ADD_TEST(NAME ./test/demo COMMAND ./test/demo) -ADD_TEST(NAME ./test/test.run COMMAND ./test/test.run) - -########### -# install # -########### - -include(GNUInstallDirs) - -install(TARGETS ${PROJECT_NAME} - EXPORT ${PROJECT_NAME}) - -install(EXPORT ${PROJECT_NAME} - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}/ - NAMESPACE ${PROJECT_NAME}:: - FILE ${PROJECT_NAME}-config.cmake) - -install(DIRECTORY include/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/third_party/cppjieba/deps/limonp/LICENSE b/third_party/cppjieba/deps/limonp/LICENSE deleted file mode 100644 index 6308e939..00000000 --- a/third_party/cppjieba/deps/limonp/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp b/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp deleted file mode 100644 index ba3abe06..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/************************************ - * file enc : ascii - * author : wuyanyi09@gmail.com - ************************************/ - -#ifndef LIMONP_ARGV_FUNCTS_H -#define LIMONP_ARGV_FUNCTS_H - -#include -#include -#include "StringUtil.hpp" - -namespace limonp { - -using namespace std; - -class ArgvContext { - public : - ArgvContext(int argc, const char* const * argv) { - for(int i = 0; i < argc; i++) { - if(StartsWith(argv[i], "-")) { - if(i + 1 < argc && !StartsWith(argv[i + 1], "-")) { - mpss_[argv[i]] = argv[i+1]; - i++; - } else { - sset_.insert(argv[i]); - } - } else { - args_.push_back(argv[i]); - } - } - } - ~ArgvContext() { - } - - friend ostream& operator << (ostream& os, const ArgvContext& args); - string operator [](size_t i) const { - if(i < args_.size()) { - return args_[i]; - } - return ""; - } - string operator [](const string& key) const { - map::const_iterator it = mpss_.find(key); - if(it != mpss_.end()) { - return it->second; - } - return ""; - } - - bool HasKey(const string& key) const { - if(mpss_.find(key) != mpss_.end() || sset_.find(key) != sset_.end()) { - return true; - } - return false; - } - - private: - vector args_; - map mpss_; - set sset_; -}; // class ArgvContext - -inline ostream& operator << (ostream& os, const ArgvContext& args) { - return os< -class Closure0: public ClosureInterface { - public: - Closure0(Funct fun) { - fun_ = fun; - } - virtual ~Closure0() { - } - virtual void Run() { - (*fun_)(); - } - private: - Funct fun_; -}; - -template -class Closure1: public ClosureInterface { - public: - Closure1(Funct fun, Arg1 arg1) { - fun_ = fun; - arg1_ = arg1; - } - virtual ~Closure1() { - } - virtual void Run() { - (*fun_)(arg1_); - } - private: - Funct fun_; - Arg1 arg1_; -}; - -template -class Closure2: public ClosureInterface { - public: - Closure2(Funct fun, Arg1 arg1, Arg2 arg2) { - fun_ = fun; - arg1_ = arg1; - arg2_ = arg2; - } - virtual ~Closure2() { - } - virtual void Run() { - (*fun_)(arg1_, arg2_); - } - private: - Funct fun_; - Arg1 arg1_; - Arg2 arg2_; -}; - -template -class Closure3: public ClosureInterface { - public: - Closure3(Funct fun, Arg1 arg1, Arg2 arg2, Arg3 arg3) { - fun_ = fun; - arg1_ = arg1; - arg2_ = arg2; - arg3_ = arg3; - } - virtual ~Closure3() { - } - virtual void Run() { - (*fun_)(arg1_, arg2_, arg3_); - } - private: - Funct fun_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; -}; - -template -class ObjClosure0: public ClosureInterface { - public: - ObjClosure0(Obj* p, Funct fun) { - p_ = p; - fun_ = fun; - } - virtual ~ObjClosure0() { - } - virtual void Run() { - (p_->*fun_)(); - } - private: - Obj* p_; - Funct fun_; -}; - -template -class ObjClosure1: public ClosureInterface { - public: - ObjClosure1(Obj* p, Funct fun, Arg1 arg1) { - p_ = p; - fun_ = fun; - arg1_ = arg1; - } - virtual ~ObjClosure1() { - } - virtual void Run() { - (p_->*fun_)(arg1_); - } - private: - Obj* p_; - Funct fun_; - Arg1 arg1_; -}; - -template -class ObjClosure2: public ClosureInterface { - public: - ObjClosure2(Obj* p, Funct fun, Arg1 arg1, Arg2 arg2) { - p_ = p; - fun_ = fun; - arg1_ = arg1; - arg2_ = arg2; - } - virtual ~ObjClosure2() { - } - virtual void Run() { - (p_->*fun_)(arg1_, arg2_); - } - private: - Obj* p_; - Funct fun_; - Arg1 arg1_; - Arg2 arg2_; -}; -template -class ObjClosure3: public ClosureInterface { - public: - ObjClosure3(Obj* p, Funct fun, Arg1 arg1, Arg2 arg2, Arg3 arg3) { - p_ = p; - fun_ = fun; - arg1_ = arg1; - arg2_ = arg2; - arg3_ = arg3; - } - virtual ~ObjClosure3() { - } - virtual void Run() { - (p_->*fun_)(arg1_, arg2_, arg3_); - } - private: - Obj* p_; - Funct fun_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; -}; - -template -ClosureInterface* NewClosure(R (*fun)()) { - return new Closure0(fun); -} - -template -ClosureInterface* NewClosure(R (*fun)(Arg1), Arg1 arg1) { - return new Closure1(fun, arg1); -} - -template -ClosureInterface* NewClosure(R (*fun)(Arg1, Arg2), Arg1 arg1, Arg2 arg2) { - return new Closure2(fun, arg1, arg2); -} - -template -ClosureInterface* NewClosure(R (*fun)(Arg1, Arg2, Arg3), Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new Closure3(fun, arg1, arg2, arg3); -} - -template -ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)()) { - return new ObjClosure0(obj, fun); -} - -template -ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1), Arg1 arg1) { - return new ObjClosure1(obj, fun, arg1); -} - -template -ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1, Arg2), Arg1 arg1, Arg2 arg2) { - return new ObjClosure2(obj, fun, arg1, arg2); -} - -template -ClosureInterface* NewClosure(Obj* obj, R (Obj::* fun)(Arg1, Arg2, Arg3), Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new ObjClosure3(obj, fun, arg1, arg2, arg3); -} - -} // namespace limonp - -#endif // LIMONP_CLOSURE_HPP diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp deleted file mode 100644 index 04edd7eb..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/Colors.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef LIMONP_COLOR_PRINT_HPP -#define LIMONP_COLOR_PRINT_HPP - -#include -#include - -namespace limonp { - -using std::string; - -enum Color { - BLACK = 30, - RED, - GREEN, - YELLOW, - BLUE, - PURPLE -}; // enum Color - -static void ColorPrintln(enum Color color, const char * fmt, ...) { - va_list ap; - printf("\033[0;%dm", color); - va_start(ap, fmt); - vprintf(fmt, ap); - va_end(ap); - printf("\033[0m\n"); // if not \n , in some situation , the next lines will be set the same color unexpectedly -} - -} // namespace limonp - -#endif // LIMONP_COLOR_PRINT_HPP diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp deleted file mode 100644 index 656a61d7..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/Condition.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef LIMONP_CONDITION_HPP -#define LIMONP_CONDITION_HPP - -#include "MutexLock.hpp" - -namespace limonp { - -class Condition : NonCopyable { - public: - explicit Condition(MutexLock& mutex) - : mutex_(mutex) { - XCHECK(!pthread_cond_init(&pcond_, NULL)); - } - - ~Condition() { - XCHECK(!pthread_cond_destroy(&pcond_)); - } - - void Wait() { - XCHECK(!pthread_cond_wait(&pcond_, mutex_.GetPthreadMutex())); - } - - void Notify() { - XCHECK(!pthread_cond_signal(&pcond_)); - } - - void NotifyAll() { - XCHECK(!pthread_cond_broadcast(&pcond_)); - } - - private: - MutexLock& mutex_; - pthread_cond_t pcond_; -}; // class Condition - -} // namespace limonp - -#endif // LIMONP_CONDITION_HPP diff --git a/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp b/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp deleted file mode 100644 index c98f2227..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/Config.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/************************************ - * file enc : utf8 - * author : wuyanyi09@gmail.com - ************************************/ -#ifndef LIMONP_CONFIG_H -#define LIMONP_CONFIG_H - -#include -#include -#include -#include -#include "StringUtil.hpp" - -namespace limonp { - -using namespace std; - -class Config { - public: - explicit Config(const string& filePath) { - LoadFile(filePath); - } - - operator bool () { - return !map_.empty(); - } - - string Get(const string& key, const string& defaultvalue) const { - map::const_iterator it = map_.find(key); - if(map_.end() != it) { - return it->second; - } - return defaultvalue; - } - int Get(const string& key, int defaultvalue) const { - string str = Get(key, ""); - if("" == str) { - return defaultvalue; - } - return atoi(str.c_str()); - } - const char* operator [] (const char* key) const { - if(NULL == key) { - return NULL; - } - map::const_iterator it = map_.find(key); - if(map_.end() != it) { - return it->second.c_str(); - } - return NULL; - } - - string GetConfigInfo() const { - string res; - res << *this; - return res; - } - - private: - void LoadFile(const string& filePath) { - ifstream ifs(filePath.c_str()); - assert(ifs); - string line; - vector vecBuf; - size_t lineno = 0; - while(getline(ifs, line)) { - lineno ++; - Trim(line); - if(line.empty() || StartsWith(line, "#")) { - continue; - } - vecBuf.clear(); - Split(line, vecBuf, "="); - if(2 != vecBuf.size()) { - fprintf(stderr, "line[%s] illegal.\n", line.c_str()); - assert(false); - continue; - } - string& key = vecBuf[0]; - string& value = vecBuf[1]; - Trim(key); - Trim(value); - if(!map_.insert(make_pair(key, value)).second) { - fprintf(stderr, "key[%s] already exits.\n", key.c_str()); - assert(false); - continue; - } - } - ifs.close(); - } - - friend ostream& operator << (ostream& os, const Config& config); - - map map_; -}; // class Config - -inline ostream& operator << (ostream& os, const Config& config) { - return os << config.map_; -} - -} // namespace limonp - -#endif // LIMONP_CONFIG_H diff --git a/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp b/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp deleted file mode 100644 index 20766820..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef LIMONP_FORCE_PUBLIC_H -#define LIMONP_FORCE_PUBLIC_H - -#define private public -#define protected public - -#endif // LIMONP_FORCE_PUBLIC_H diff --git a/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp b/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp deleted file mode 100644 index 11339cc8..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/LocalVector.hpp +++ /dev/null @@ -1,139 +0,0 @@ -#ifndef LIMONP_LOCAL_VECTOR_HPP -#define LIMONP_LOCAL_VECTOR_HPP - -#include -#include -#include -#include - -namespace limonp { -using namespace std; -/* - * LocalVector : T must be primitive type (char , int, size_t), if T is struct or class, LocalVector may be dangerous.. - * LocalVector is simple and not well-tested. - */ -const size_t LOCAL_VECTOR_BUFFER_SIZE = 16; -template -class LocalVector { - public: - typedef const T* const_iterator ; - typedef T value_type; - typedef size_t size_type; - private: - T buffer_[LOCAL_VECTOR_BUFFER_SIZE]; - T * ptr_; - size_t size_; - size_t capacity_; - public: - LocalVector() { - init_(); - }; - LocalVector(const LocalVector& vec) { - init_(); - *this = vec; - } - LocalVector(const_iterator begin, const_iterator end) { // TODO: make it faster - init_(); - while(begin != end) { - push_back(*begin++); - } - } - LocalVector(size_t size, const T& t) { // TODO: make it faster - init_(); - while(size--) { - push_back(t); - } - } - ~LocalVector() { - if(ptr_ != buffer_) { - free(ptr_); - } - }; - public: - LocalVector& operator = (const LocalVector& vec) { - clear(); - size_ = vec.size(); - capacity_ = vec.capacity(); - if(vec.buffer_ == vec.ptr_) { - memcpy(static_cast(buffer_), vec.buffer_, sizeof(T) * size_); - ptr_ = buffer_; - } else { - ptr_ = (T*) malloc(vec.capacity() * sizeof(T)); - assert(ptr_); - memcpy(static_cast(ptr_), vec.ptr_, vec.size() * sizeof(T)); - } - return *this; - } - private: - void init_() { - ptr_ = buffer_; - size_ = 0; - capacity_ = LOCAL_VECTOR_BUFFER_SIZE; - } - public: - T& operator [] (size_t i) { - return ptr_[i]; - } - const T& operator [] (size_t i) const { - return ptr_[i]; - } - void push_back(const T& t) { - if(size_ == capacity_) { - assert(capacity_); - reserve(capacity_ * 2); - } - ptr_[size_ ++ ] = t; - } - void reserve(size_t size) { - if(size <= capacity_) { - return; - } - T * next = (T*)malloc(sizeof(T) * size); - assert(next); - T * old = ptr_; - ptr_ = next; - memcpy(static_cast(ptr_), old, sizeof(T) * capacity_); - capacity_ = size; - if(old != buffer_) { - free(old); - } - } - bool empty() const { - return 0 == size(); - } - size_t size() const { - return size_; - } - size_t capacity() const { - return capacity_; - } - const_iterator begin() const { - return ptr_; - } - const_iterator end() const { - return ptr_ + size_; - } - void clear() { - if(ptr_ != buffer_) { - free(ptr_); - } - init_(); - } -}; - -template -ostream & operator << (ostream& os, const LocalVector& vec) { - if(vec.empty()) { - return os << "[]"; - } - os<<"[\""< -#include -#include -#include -#include - -#ifdef XLOG -#error "XLOG has been defined already" -#endif // XLOG -#ifdef XCHECK -#error "XCHECK has been defined already" -#endif // XCHECK - -#define XLOG(level) limonp::Logger(limonp::LL_##level, __FILE__, __LINE__).Stream() -#define XCHECK(exp) if(!(exp)) XLOG(FATAL) << "exp: ["#exp << "] false. " - -namespace limonp { - -enum { - LL_DEBUG = 0, - LL_INFO = 1, - LL_WARNING = 2, - LL_ERROR = 3, - LL_FATAL = 4, -}; // enum - -static const char * LOG_LEVEL_ARRAY[] = {"DEBUG","INFO","WARN","ERROR","FATAL"}; -static const char * LOG_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"; - -class Logger { - public: - Logger(size_t level, const char* filename, int lineno) - : level_(level) { -#ifdef LOGGING_LEVEL - if (level_ < LOGGING_LEVEL) { - return; - } -#endif - assert(level_ <= sizeof(LOG_LEVEL_ARRAY)/sizeof(*LOG_LEVEL_ARRAY)); - - char buf[32]; - - time_t timeNow; - time(&timeNow); - - struct tm tmNow; - - #if defined(_WIN32) || defined(_WIN64) - errno_t e = localtime_s(&tmNow, &timeNow); - assert(e == 0); - #else - struct tm * tm_tmp = localtime_r(&timeNow, &tmNow); - (void)tm_tmp; - assert(tm_tmp != nullptr); - #endif - - strftime(buf, sizeof(buf), LOG_TIME_FORMAT, &tmNow); - - stream_ << buf - << " " << filename - << ":" << lineno - << " " << LOG_LEVEL_ARRAY[level_] - << " "; - } - ~Logger() { -#ifdef LOGGING_LEVEL - if (level_ < LOGGING_LEVEL) { - return; - } -#endif - std::cerr << stream_.str() << std::endl; - if (level_ == LL_FATAL) { - abort(); - } - } - - std::ostream& Stream() { - return stream_; - } - - private: - std::ostringstream stream_; - size_t level_; -}; // class Logger - -} // namespace limonp - -#endif // LIMONP_LOGGING_HPP diff --git a/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp b/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp deleted file mode 100644 index 145400f4..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp +++ /dev/null @@ -1,21 +0,0 @@ -/************************************ - ************************************/ -#ifndef LIMONP_NONCOPYABLE_H -#define LIMONP_NONCOPYABLE_H - -namespace limonp { - -class NonCopyable { - protected: - NonCopyable() { - } - ~NonCopyable() { - } - private: - NonCopyable(const NonCopyable& ); - const NonCopyable& operator=(const NonCopyable& ); -}; // class NonCopyable - -} // namespace limonp - -#endif // LIMONP_NONCOPYABLE_H diff --git a/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp b/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp deleted file mode 100644 index cf00e941..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/StdExtension.hpp +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef LIMONP_STD_EXTEMSION_HPP -#define LIMONP_STD_EXTEMSION_HPP - -#include - -#ifdef __APPLE__ -#include -#include -#elif(__cplusplus >= 201103L) -#include -#include -#elif defined _MSC_VER -#include -#include -#else -#include -#include -namespace std { -using std::tr1::unordered_map; -using std::tr1::unordered_set; -} - -#endif - -#include -#include -#include -#include -#include -#include - -namespace std { - -template -ostream& operator << (ostream& os, const vector& v) { - if(v.empty()) { - return os << "[]"; - } - os<<"["< -inline ostream& operator << (ostream& os, const vector& v) { - if(v.empty()) { - return os << "[]"; - } - os<<"[\""< -ostream& operator << (ostream& os, const deque& dq) { - if(dq.empty()) { - return os << "[]"; - } - os<<"[\""< -ostream& operator << (ostream& os, const pair& pr) { - os << pr.first << ":" << pr.second ; - return os; -} - - -template -string& operator << (string& str, const T& obj) { - stringstream ss; - ss << obj; // call ostream& operator << (ostream& os, - return str = ss.str(); -} - -template -ostream& operator << (ostream& os, const map& mp) { - if(mp.empty()) { - os<<"{}"; - return os; - } - os<<'{'; - typename map::const_iterator it = mp.begin(); - os<<*it; - it++; - while(it != mp.end()) { - os<<", "<<*it; - it++; - } - os<<'}'; - return os; -} -template -ostream& operator << (ostream& os, const std::unordered_map& mp) { - if(mp.empty()) { - return os << "{}"; - } - os<<'{'; - typename std::unordered_map::const_iterator it = mp.begin(); - os<<*it; - it++; - while(it != mp.end()) { - os<<", "<<*it++; - } - return os<<'}'; -} - -template -ostream& operator << (ostream& os, const set& st) { - if(st.empty()) { - os << "{}"; - return os; - } - os<<'{'; - typename set::const_iterator it = st.begin(); - os<<*it; - it++; - while(it != st.end()) { - os<<", "<<*it; - it++; - } - os<<'}'; - return os; -} - -template -bool IsIn(const ContainType& contain, const KeyType& key) { - return contain.end() != contain.find(key); -} - -template -basic_string & operator << (basic_string & s, ifstream & ifs) { - return s.assign((istreambuf_iterator(ifs)), istreambuf_iterator()); -} - -template -ofstream & operator << (ofstream & ofs, const basic_string& s) { - ostreambuf_iterator itr (ofs); - copy(s.begin(), s.end(), itr); - return ofs; -} - -} // namespace std - -#endif diff --git a/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp b/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp deleted file mode 100644 index 11869b42..00000000 --- a/third_party/cppjieba/deps/limonp/include/limonp/StringUtil.hpp +++ /dev/null @@ -1,367 +0,0 @@ -/************************************ - * file enc : ascii - * author : wuyanyi09@gmail.com - ************************************/ -#ifndef LIMONP_STR_FUNCTS_H -#define LIMONP_STR_FUNCTS_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "StdExtension.hpp" - -namespace limonp { -using namespace std; - -template -void Join(T begin, T end, string& res, const string& connector) { - if(begin == end) { - return; - } - stringstream ss; - ss<<*begin; - begin++; - while(begin != end) { - ss << connector << *begin; - begin ++; - } - res = ss.str(); -} - -template -string Join(T begin, T end, const string& connector) { - string res; - Join(begin ,end, res, connector); - return res; -} - -inline string& Upper(string& str) { - transform(str.begin(), str.end(), str.begin(), (int (*)(int))toupper); - return str; -} - -inline string& Lower(string& str) { - transform(str.begin(), str.end(), str.begin(), (int (*)(int))tolower); - return str; -} - -inline bool IsSpace(unsigned c) { - // when passing large int as the argument of isspace, it core dump, so here need a type cast. - return c > 0xff ? false : std::isspace(c & 0xff) != 0; -} - -inline std::string& LTrim(std::string &s) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { - return !std::isspace(ch); - })); - return s; -} - -inline std::string& RTrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { - return !std::isspace(ch); - }).base(), s.end()); - return s; -} - -inline std::string& Trim(std::string &s) { - return LTrim(RTrim(s)); -} - -inline std::string& LTrim(std::string& s, char x) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), - [x](unsigned char c) { return !std::isspace(c) && c != x; })); - return s; -} - -inline std::string& RTrim(std::string& s, char x) { - s.erase(std::find_if(s.rbegin(), s.rend(), - [x](unsigned char c) { return !std::isspace(c) && c != x; }).base(), s.end()); - return s; -} - -inline std::string& Trim(std::string &s, char x) { - return LTrim(RTrim(s, x), x); -} - -inline void Split(const string& src, vector& res, const string& pattern, size_t maxsplit = string::npos) { - res.clear(); - size_t Start = 0; - size_t end = 0; - string sub; - while(Start < src.size()) { - end = src.find_first_of(pattern, Start); - if(string::npos == end || res.size() >= maxsplit) { - sub = src.substr(Start); - res.push_back(sub); - return; - } - sub = src.substr(Start, end - Start); - res.push_back(sub); - Start = end + 1; - } - return; -} - -inline vector Split(const string& src, const string& pattern, size_t maxsplit = string::npos) { - vector res; - Split(src, res, pattern, maxsplit); - return res; -} - -inline bool StartsWith(const string& str, const string& prefix) { - if(prefix.length() > str.length()) { - return false; - } - return 0 == str.compare(0, prefix.length(), prefix); -} - -inline bool EndsWith(const string& str, const string& suffix) { - if(suffix.length() > str.length()) { - return false; - } - return 0 == str.compare(str.length() - suffix.length(), suffix.length(), suffix); -} - -inline bool IsInStr(const string& str, char ch) { - return str.find(ch) != string::npos; -} - -inline uint16_t TwocharToUint16(char high, char low) { - return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff)); -} - -template -bool Utf8ToUnicode(const char * const str, size_t len, Uint16Container& vec) { - if(!str) { - return false; - } - char ch1, ch2; - uint16_t tmp; - vec.clear(); - for(size_t i = 0; i < len;) { - if(!(str[i] & 0x80)) { // 0xxxxxxx - vec.push_back(str[i]); - i++; - } else if ((uint8_t)str[i] <= 0xdf && i + 1 < len) { // 110xxxxxx - ch1 = (str[i] >> 2) & 0x07; - ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 ); - tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff)); - vec.push_back(tmp); - i += 2; - } else if((uint8_t)str[i] <= 0xef && i + 2 < len) { - ch1 = ((uint8_t)str[i] << 4) | ((str[i+1] >> 2) & 0x0f ); - ch2 = (((uint8_t)str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f); - tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff)); - vec.push_back(tmp); - i += 3; - } else { - return false; - } - } - return true; -} - -template -bool Utf8ToUnicode(const string& str, Uint16Container& vec) { - return Utf8ToUnicode(str.c_str(), str.size(), vec); -} - -template -bool Utf8ToUnicode32(const string& str, Uint32Container& vec) { - uint32_t tmp; - vec.clear(); - for(size_t i = 0; i < str.size();) { - if(!(str[i] & 0x80)) { // 0xxxxxxx - // 7bit, total 7bit - tmp = (uint8_t)(str[i]) & 0x7f; - i++; - } else if ((uint8_t)str[i] <= 0xdf && i + 1 < str.size()) { // 110xxxxxx - // 5bit, total 5bit - tmp = (uint8_t)(str[i]) & 0x1f; - - // 6bit, total 11bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+1]) & 0x3f; - i += 2; - } else if((uint8_t)str[i] <= 0xef && i + 2 < str.size()) { // 1110xxxxxx - // 4bit, total 4bit - tmp = (uint8_t)(str[i]) & 0x0f; - - // 6bit, total 10bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+1]) & 0x3f; - - // 6bit, total 16bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+2]) & 0x3f; - - i += 3; - } else if((uint8_t)str[i] <= 0xf7 && i + 3 < str.size()) { // 11110xxxx - // 3bit, total 3bit - tmp = (uint8_t)(str[i]) & 0x07; - - // 6bit, total 9bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+1]) & 0x3f; - - // 6bit, total 15bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+2]) & 0x3f; - - // 6bit, total 21bit - tmp <<= 6; - tmp |= (uint8_t)(str[i+3]) & 0x3f; - - i += 4; - } else { - return false; - } - vec.push_back(tmp); - } - return true; -} - -template -void Unicode32ToUtf8(Uint32ContainerConIter begin, Uint32ContainerConIter end, string& res) { - res.clear(); - uint32_t ui; - while(begin != end) { - ui = *begin; - if(ui <= 0x7f) { - res += char(ui); - } else if(ui <= 0x7ff) { - res += char(((ui >> 6) & 0x1f) | 0xc0); - res += char((ui & 0x3f) | 0x80); - } else if(ui <= 0xffff) { - res += char(((ui >> 12) & 0x0f) | 0xe0); - res += char(((ui >> 6) & 0x3f) | 0x80); - res += char((ui & 0x3f) | 0x80); - } else { - res += char(((ui >> 18) & 0x03) | 0xf0); - res += char(((ui >> 12) & 0x3f) | 0x80); - res += char(((ui >> 6) & 0x3f) | 0x80); - res += char((ui & 0x3f) | 0x80); - } - begin ++; - } -} - -template -void UnicodeToUtf8(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) { - res.clear(); - uint16_t ui; - while(begin != end) { - ui = *begin; - if(ui <= 0x7f) { - res += char(ui); - } else if(ui <= 0x7ff) { - res += char(((ui>>6) & 0x1f) | 0xc0); - res += char((ui & 0x3f) | 0x80); - } else { - res += char(((ui >> 12) & 0x0f )| 0xe0); - res += char(((ui>>6) & 0x3f )| 0x80 ); - res += char((ui & 0x3f) | 0x80); - } - begin ++; - } -} - - -template -bool GBKTrans(const char* const str, size_t len, Uint16Container& vec) { - vec.clear(); - if(!str) { - return true; - } - size_t i = 0; - while(i < len) { - if(0 == (str[i] & 0x80)) { - vec.push_back(uint16_t(str[i])); - i++; - } else { - if(i + 1 < len) { //&& (str[i+1] & 0x80)) - uint16_t tmp = (((uint16_t(str[i]) & 0x00ff ) << 8) | (uint16_t(str[i+1]) & 0x00ff)); - vec.push_back(tmp); - i += 2; - } else { - return false; - } - } - } - return true; -} - -template -bool GBKTrans(const string& str, Uint16Container& vec) { - return GBKTrans(str.c_str(), str.size(), vec); -} - -template -void GBKTrans(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) { - res.clear(); - //pair pa; - char first, second; - while(begin != end) { - //pa = uint16ToChar2(*begin); - first = ((*begin)>>8) & 0x00ff; - second = (*begin) & 0x00ff; - if(first & 0x80) { - res += first; - res += second; - } else { - res += second; - } - begin++; - } -} - -/* - * format example: "%Y-%m-%d %H:%M:%S" - */ -inline void GetTime(const string& format, string& timeStr) { - time_t timeNow; - time(&timeNow); - - struct tm tmNow; - - #if defined(_WIN32) || defined(_WIN64) - errno_t e = localtime_s(&tmNow, &timeNow); - assert(e == 0); - #else - struct tm * tm_tmp = localtime_r(&timeNow, &tmNow); - (void)tm_tmp; - assert(tm_tmp != nullptr); - #endif - - timeStr.resize(64); - - size_t len = strftime((char*)timeStr.c_str(), timeStr.size(), format.c_str(), &tmNow); - - timeStr.resize(len); -} - -inline string PathJoin(const string& path1, const string& path2) { - if(EndsWith(path1, "/")) { - return path1 + path2; - } - return path1 + "/" + path2; -} - -} -#endif diff --git a/third_party/cppjieba/dict/README.md b/third_party/cppjieba/dict/README.md deleted file mode 100644 index 88791189..00000000 --- a/third_party/cppjieba/dict/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# CppJieba字典 - -文件后缀名代表的是词典的编码方式。 -比如filename.utf8 是 utf8编码，filename.gbk 是 gbk编码方式。 - - -## 分词 - -### jieba.dict.utf8/gbk - -作为最大概率法(MPSegment: Max Probability)分词所使用的词典。 - -### hmm_model.utf8/gbk - -作为隐式马尔科夫模型(HMMSegment: Hidden Markov Model)分词所使用的词典。 - -__对于MixSegment(混合MPSegment和HMMSegment两者)则同时使用以上两个词典__ - - -## 关键词抽取 - -### idf.utf8 - -IDF(Inverse Document Frequency) -在KeywordExtractor中，使用的是经典的TF-IDF算法，所以需要这么一个词典提供IDF信息。 - -### stop_words.utf8 - -停用词词典 - - diff --git a/third_party/cppjieba/dict/pos_dict/prob_start.utf8 b/third_party/cppjieba/dict/pos_dict/prob_start.utf8 deleted file mode 100644 index 433750d5..00000000 --- a/third_party/cppjieba/dict/pos_dict/prob_start.utf8 +++ /dev/null @@ -1,259 +0,0 @@ -#初始状态的概率 -#格式 -#状态:概率 -B,a:-4.7623052146 -B,ad:-6.68006603678 -B,ag:-3.14e+100 -B,an:-8.69708322302 -B,b:-5.01837436211 -B,bg:-3.14e+100 -B,c:-3.42388018495 -B,d:-3.97504752976 -B,df:-8.88897423083 -B,dg:-3.14e+100 -B,e:-8.56355183039 -B,en:-3.14e+100 -B,f:-5.49163041848 -B,g:-3.14e+100 -B,h:-13.53336513 -B,i:-6.11578472756 -B,in:-3.14e+100 -B,j:-5.05761912847 -B,jn:-3.14e+100 -B,k:-3.14e+100 -B,l:-4.90588358466 -B,ln:-3.14e+100 -B,m:-3.6524299819 -B,mg:-3.14e+100 -B,mq:-6.7869530014 -B,n:-1.69662577975 -B,ng:-3.14e+100 -B,nr:-2.23104959138 -B,nrfg:-5.87372217541 -B,nrt:-4.98564273352 -B,ns:-2.8228438315 -B,nt:-4.84609166818 -B,nz:-3.94698846058 -B,o:-8.43349870215 -B,p:-4.20098413209 -B,q:-6.99812385896 -B,qe:-3.14e+100 -B,qg:-3.14e+100 -B,r:-3.40981877908 -B,rg:-3.14e+100 -B,rr:-12.4347528413 -B,rz:-7.94611647157 -B,s:-5.52267359084 -B,t:-3.36474790945 -B,tg:-3.14e+100 -B,u:-9.1639172775 -B,ud:-3.14e+100 -B,ug:-3.14e+100 -B,uj:-3.14e+100 -B,ul:-3.14e+100 -B,uv:-3.14e+100 -B,uz:-3.14e+100 -B,v:-2.67405848743 -B,vd:-9.04472876024 -B,vg:-3.14e+100 -B,vi:-12.4347528413 -B,vn:-4.33156108902 -B,vq:-12.1470707689 -B,w:-3.14e+100 -B,x:-3.14e+100 -B,y:-9.84448567586 -B,yg:-3.14e+100 -B,z:-7.04568111149 -B,zg:-3.14e+100 -E,a:-3.14e+100 -E,ad:-3.14e+100 -E,ag:-3.14e+100 -E,an:-3.14e+100 -E,b:-3.14e+100 -E,bg:-3.14e+100 -E,c:-3.14e+100 -E,d:-3.14e+100 -E,df:-3.14e+100 -E,dg:-3.14e+100 -E,e:-3.14e+100 -E,en:-3.14e+100 -E,f:-3.14e+100 -E,g:-3.14e+100 -E,h:-3.14e+100 -E,i:-3.14e+100 -E,in:-3.14e+100 -E,j:-3.14e+100 -E,jn:-3.14e+100 -E,k:-3.14e+100 -E,l:-3.14e+100 -E,ln:-3.14e+100 -E,m:-3.14e+100 -E,mg:-3.14e+100 -E,mq:-3.14e+100 -E,n:-3.14e+100 -E,ng:-3.14e+100 -E,nr:-3.14e+100 -E,nrfg:-3.14e+100 -E,nrt:-3.14e+100 -E,ns:-3.14e+100 -E,nt:-3.14e+100 -E,nz:-3.14e+100 -E,o:-3.14e+100 -E,p:-3.14e+100 -E,q:-3.14e+100 -E,qe:-3.14e+100 -E,qg:-3.14e+100 -E,r:-3.14e+100 -E,rg:-3.14e+100 -E,rr:-3.14e+100 -E,rz:-3.14e+100 -E,s:-3.14e+100 -E,t:-3.14e+100 -E,tg:-3.14e+100 -E,u:-3.14e+100 -E,ud:-3.14e+100 -E,ug:-3.14e+100 -E,uj:-3.14e+100 -E,ul:-3.14e+100 -E,uv:-3.14e+100 -E,uz:-3.14e+100 -E,v:-3.14e+100 -E,vd:-3.14e+100 -E,vg:-3.14e+100 -E,vi:-3.14e+100 -E,vn:-3.14e+100 -E,vq:-3.14e+100 -E,w:-3.14e+100 -E,x:-3.14e+100 -E,y:-3.14e+100 -E,yg:-3.14e+100 -E,z:-3.14e+100 -E,zg:-3.14e+100 -M,a:-3.14e+100 -M,ad:-3.14e+100 -M,ag:-3.14e+100 -M,an:-3.14e+100 -M,b:-3.14e+100 -M,bg:-3.14e+100 -M,c:-3.14e+100 -M,d:-3.14e+100 -M,df:-3.14e+100 -M,dg:-3.14e+100 -M,e:-3.14e+100 -M,en:-3.14e+100 -M,f:-3.14e+100 -M,g:-3.14e+100 -M,h:-3.14e+100 -M,i:-3.14e+100 -M,in:-3.14e+100 -M,j:-3.14e+100 -M,jn:-3.14e+100 -M,k:-3.14e+100 -M,l:-3.14e+100 -M,ln:-3.14e+100 -M,m:-3.14e+100 -M,mg:-3.14e+100 -M,mq:-3.14e+100 -M,n:-3.14e+100 -M,ng:-3.14e+100 -M,nr:-3.14e+100 -M,nrfg:-3.14e+100 -M,nrt:-3.14e+100 -M,ns:-3.14e+100 -M,nt:-3.14e+100 -M,nz:-3.14e+100 -M,o:-3.14e+100 -M,p:-3.14e+100 -M,q:-3.14e+100 -M,qe:-3.14e+100 -M,qg:-3.14e+100 -M,r:-3.14e+100 -M,rg:-3.14e+100 -M,rr:-3.14e+100 -M,rz:-3.14e+100 -M,s:-3.14e+100 -M,t:-3.14e+100 -M,tg:-3.14e+100 -M,u:-3.14e+100 -M,ud:-3.14e+100 -M,ug:-3.14e+100 -M,uj:-3.14e+100 -M,ul:-3.14e+100 -M,uv:-3.14e+100 -M,uz:-3.14e+100 -M,v:-3.14e+100 -M,vd:-3.14e+100 -M,vg:-3.14e+100 -M,vi:-3.14e+100 -M,vn:-3.14e+100 -M,vq:-3.14e+100 -M,w:-3.14e+100 -M,x:-3.14e+100 -M,y:-3.14e+100 -M,yg:-3.14e+100 -M,z:-3.14e+100 -M,zg:-3.14e+100 -S,a:-3.90253968313 -S,ad:-11.0484584802 -S,ag:-6.95411391796 -S,an:-12.8402179494 -S,b:-6.47288876397 -S,bg:-3.14e+100 -S,c:-4.78696679586 -S,d:-3.90391976418 -S,df:-3.14e+100 -S,dg:-8.9483976513 -S,e:-5.94251300628 -S,en:-3.14e+100 -S,f:-5.19482024998 -S,g:-6.50782681533 -S,h:-8.65056320738 -S,i:-3.14e+100 -S,in:-3.14e+100 -S,j:-4.91199211964 -S,jn:-3.14e+100 -S,k:-6.94032059583 -S,l:-3.14e+100 -S,ln:-3.14e+100 -S,m:-3.26920065212 -S,mg:-10.8253149289 -S,mq:-3.14e+100 -S,n:-3.85514838976 -S,ng:-4.9134348611 -S,nr:-4.48366310396 -S,nrfg:-3.14e+100 -S,nrt:-3.14e+100 -S,ns:-3.14e+100 -S,nt:-12.1470707689 -S,nz:-3.14e+100 -S,o:-8.46446092775 -S,p:-2.98684018136 -S,q:-4.88865861826 -S,qe:-3.14e+100 -S,qg:-3.14e+100 -S,r:-2.76353367841 -S,rg:-10.2752685919 -S,rr:-3.14e+100 -S,rz:-3.14e+100 -S,s:-3.14e+100 -S,t:-3.14e+100 -S,tg:-6.27284253188 -S,u:-6.94032059583 -S,ud:-7.72823016105 -S,ug:-7.53940370266 -S,uj:-6.85251045118 -S,ul:-8.41537131755 -S,uv:-8.15808672229 -S,uz:-9.29925862537 -S,v:-3.05329230341 -S,vd:-3.14e+100 -S,vg:-5.94301818437 -S,vi:-3.14e+100 -S,vn:-11.4539235883 -S,vq:-3.14e+100 -S,w:-3.14e+100 -S,x:-8.42741965607 -S,y:-6.19707946995 -S,yg:-13.53336513 -S,z:-3.14e+100 -S,zg:-3.14e+100 diff --git a/third_party/cppjieba/dict/stop_words.utf8 b/third_party/cppjieba/dict/stop_words.utf8 deleted file mode 100644 index 32ac9e67..00000000 --- a/third_party/cppjieba/dict/stop_words.utf8 +++ /dev/null @@ -1,1534 +0,0 @@ -" -. -。 -, -、 -！ -？ -： -； -` -﹑ -• -＂ -^ -… -‘ -’ -“ -” -〝 -〞 -~ -\ -∕ -| -¦ -‖ -—　 -( -) -〈 -〉 -﹞ -﹝ -「 -」 -‹ -› -〖 -〗 -】 -【 -» -« -』 -『 -〕 -〔 -》 -《 -} -{ -] -[ -﹐ -¸ -﹕ -︰ -﹔ -; -！ -¡ -？ -¿ -﹖ -﹌ -﹏ -﹋ -＇ -´ -ˊ -ˋ -- -― -﹫ -@ -︳ -︴ -_ -¯ -＿ -￣ -﹢ -+ -﹦ -= -﹤ -‐ -< -­ -˜ -~ -﹟ -# -﹩ -$ -﹠ -& -﹪ -% -﹡ -* -﹨ -\ -﹍ -﹉ -﹎ -﹊ -ˇ -︵ -︶ -︷ -︸ -︹ -︿ -﹀ -︺ -︽ -︾ -_ -ˉ -﹁ -﹂ -﹃ -﹄ -︻ -︼ -的 -了 -the -a -an -that -those -this -that -$ -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -? -_ -“ -” -、 -。 -《 -》 -一 -一些 -一何 -一切 -一则 -一方面 -一旦 -一来 -一样 -一般 -一转眼 -万一 -上 -上下 -下 -不 -不仅 -不但 -不光 -不单 -不只 -不外乎 -不如 -不妨 -不尽 -不尽然 -不得 -不怕 -不惟 -不成 -不拘 -不料 -不是 -不比 -不然 -不特 -不独 -不管 -不至于 -不若 -不论 -不过 -不问 -与 -与其 -与其说 -与否 -与此同时 -且 -且不说 -且说 -两者 -个 -个别 -临 -为 -为了 -为什么 -为何 -为止 -为此 -为着 -乃 -乃至 -乃至于 -么 -之 -之一 -之所以 -之类 -乌乎 -乎 -乘 -也 -也好 -也罢 -了 -二来 -于 -于是 -于是乎 -云云 -云尔 -些 -亦 -人 -人们 -人家 -什么 -什么样 -今 -介于 -仍 -仍旧 -从 -从此 -从而 -他 -他人 -他们 -以 -以上 -以为 -以便 -以免 -以及 -以故 -以期 -以来 -以至 -以至于 -以致 -们 -任 -任何 -任凭 -似的 -但 -但凡 -但是 -何 -何以 -何况 -何处 -何时 -余外 -作为 -你 -你们 -使 -使得 -例如 -依 -依据 -依照 -便于 -俺 -俺们 -倘 -倘使 -倘或 -倘然 -倘若 -借 -假使 -假如 -假若 -傥然 -像 -儿 -先不先 -光是 -全体 -全部 -兮 -关于 -其 -其一 -其中 -其二 -其他 -其余 -其它 -其次 -具体地说 -具体说来 -兼之 -内 -再 -再其次 -再则 -再有 -再者 -再者说 -再说 -冒 -冲 -况且 -几 -几时 -凡 -凡是 -凭 -凭借 -出于 -出来 -分别 -则 -则甚 -别 -别人 -别处 -别是 -别的 -别管 -别说 -到 -前后 -前此 -前者 -加之 -加以 -即 -即令 -即使 -即便 -即如 -即或 -即若 -却 -去 -又 -又及 -及 -及其 -及至 -反之 -反而 -反过来 -反过来说 -受到 -另 -另一方面 -另外 -另悉 -只 -只当 -只怕 -只是 -只有 -只消 -只要 -只限 -叫 -叮咚 -可 -可以 -可是 -可见 -各 -各个 -各位 -各种 -各自 -同 -同时 -后 -后者 -向 -向使 -向着 -吓 -吗 -否则 -吧 -吧哒 -吱 -呀 -呃 -呕 -呗 -呜 -呜呼 -呢 -呵 -呵呵 -呸 -呼哧 -咋 -和 -咚 -咦 -咧 -咱 -咱们 -咳 -哇 -哈 -哈哈 -哉 -哎 -哎呀 -哎哟 -哗 -哟 -哦 -哩 -哪 -哪个 -哪些 -哪儿 -哪天 -哪年 -哪怕 -哪样 -哪边 -哪里 -哼 -哼唷 -唉 -唯有 -啊 -啐 -啥 -啦 -啪达 -啷当 -喂 -喏 -喔唷 -喽 -嗡 -嗡嗡 -嗬 -嗯 -嗳 -嘎 -嘎登 -嘘 -嘛 -嘻 -嘿 -嘿嘿 -因 -因为 -因了 -因此 -因着 -因而 -固然 -在 -在下 -在于 -地 -基于 -处在 -多 -多么 -多少 -大 -大家 -她 -她们 -好 -如 -如上 -如上所述 -如下 -如何 -如其 -如同 -如是 -如果 -如此 -如若 -始而 -孰料 -孰知 -宁 -宁可 -宁愿 -宁肯 -它 -它们 -对 -对于 -对待 -对方 -对比 -将 -小 -尔 -尔后 -尔尔 -尚且 -就 -就是 -就是了 -就是说 -就算 -就要 -尽 -尽管 -尽管如此 -岂但 -己 -已 -已矣 -巴 -巴巴 -并 -并且 -并非 -庶乎 -庶几 -开外 -开始 -归 -归齐 -当 -当地 -当然 -当着 -彼 -彼时 -彼此 -往 -待 -很 -得 -得了 -怎 -怎么 -怎么办 -怎么样 -怎奈 -怎样 -总之 -总的来看 -总的来说 -总的说来 -总而言之 -恰恰相反 -您 -惟其 -慢说 -我 -我们 -或 -或则 -或是 -或曰 -或者 -截至 -所 -所以 -所在 -所幸 -所有 -才 -才能 -打 -打从 -把 -抑或 -拿 -按 -按照 -换句话说 -换言之 -据 -据此 -接着 -故 -故此 -故而 -旁人 -无 -无宁 -无论 -既 -既往 -既是 -既然 -时候 -是 -是以 -是的 -曾 -替 -替代 -最 -有 -有些 -有关 -有及 -有时 -有的 -望 -朝 -朝着 -本 -本人 -本地 -本着 -本身 -来 -来着 -来自 -来说 -极了 -果然 -果真 -某 -某个 -某些 -某某 -根据 -欤 -正值 -正如 -正巧 -正是 -此 -此地 -此处 -此外 -此时 -此次 -此间 -毋宁 -每 -每当 -比 -比及 -比如 -比方 -没奈何 -沿 -沿着 -漫说 -焉 -然则 -然后 -然而 -照 -照着 -犹且 -犹自 -甚且 -甚么 -甚或 -甚而 -甚至 -甚至于 -用 -用来 -由 -由于 -由是 -由此 -由此可见 -的 -的确 -的话 -直到 -相对而言 -省得 -看 -眨眼 -着 -着呢 -矣 -矣乎 -矣哉 -离 -竟而 -第 -等 -等到 -等等 -简言之 -管 -类如 -紧接着 -纵 -纵令 -纵使 -纵然 -经 -经过 -结果 -给 -继之 -继后 -继而 -综上所述 -罢了 -者 -而 -而且 -而况 -而后 -而外 -而已 -而是 -而言 -能 -能否 -腾 -自 -自个儿 -自从 -自各儿 -自后 -自家 -自己 -自打 -自身 -至 -至于 -至今 -至若 -致 -般的 -若 -若夫 -若是 -若果 -若非 -莫不然 -莫如 -莫若 -虽 -虽则 -虽然 -虽说 -被 -要 -要不 -要不是 -要不然 -要么 -要是 -譬喻 -譬如 -让 -许多 -论 -设使 -设或 -设若 -诚如 -诚然 -该 -说来 -诸 -诸位 -诸如 -谁 -谁人 -谁料 -谁知 -贼死 -赖以 -赶 -起 -起见 -趁 -趁着 -越是 -距 -跟 -较 -较之 -边 -过 -还 -还是 -还有 -还要 -这 -这一来 -这个 -这么 -这么些 -这么样 -这么点儿 -这些 -这会儿 -这儿 -这就是说 -这时 -这样 -这次 -这般 -这边 -这里 -进而 -连 -连同 -逐步 -通过 -遵循 -遵照 -那 -那个 -那么 -那么些 -那么样 -那些 -那会儿 -那儿 -那时 -那样 -那般 -那边 -那里 -都 -鄙人 -鉴于 -针对 -阿 -除 -除了 -除外 -除开 -除此之外 -除非 -随 -随后 -随时 -随着 -难道说 -非但 -非徒 -非特 -非独 -靠 -顺 -顺着 -首先 -！ -， -： -； -？ -to -can -could -dare -do -did -does -may -might -would -should -must -will -ought -shall -need -is -a -am -are -about -according -after -against -all -almost -also -although -among -an -and -another -any -anything -approximately -as -asked -at -back -because -before -besides -between -both -but -by -call -called -currently -despite -did -do -dr -during -each -earlier -eight -even -eventually -every -everything -five -for -four -from -he -her -here -his -how -however -i -if -in -indeed -instead -it -its -just -last -like -major -many -may -maybe -meanwhile -more -moreover -most -mr -mrs -ms -much -my -neither -net -never -nevertheless -nine -no -none -not -nothing -now -of -on -once -one -only -or -other -our -over -partly -perhaps -prior -regarding -separately -seven -several -she -should -similarly -since -six -so -some -somehow -still -such -ten -that -the -their -then -there -therefore -these -they -this -those -though -three -to -two -under -unless -unlike -until -volume -we -what -whatever -whats -when -where -which -while -why -with -without -yesterday -yet -you -your -aboard -about -above -according to -across -afore -after -against -agin -along -alongside -amid -amidst -among -amongst -anent -around -as -aslant -astride -at -athwart -bar -because of -before -behind -below -beneath -beside -besides -between -betwixt -beyond -but -by -circa -despite -down -during -due to -ere -except -for -from -in -inside -into -less -like -mid -midst -minus -near -next -nigh -nigher -nighest -notwithstanding -of -off -on -on to -onto -out -out of -outside -over -past -pending -per -plus -qua -re -round -sans -save -since -through -throughout -thru -till -to -toward -towards -under -underneath -unlike -until -unto -up -upon -versus -via -vice -with -within -without -he -her -herself -hers -him -himself -his -I -it -its -itself -me -mine -my -myself -ours -she -their -theirs -them -themselves -they -us -we -our -ourselves -you -your -yours -yourselves -yourself -this -that -these -those -" -' -'' -( -) -*LRB* -*RRB* - - - - - -@ -& -[ -] -` -`` -e.g., -{ -} -" -“ -” --RRB- --LRB- --- -a -about -above -across -after -afterwards -again -against -all -almost -alone -along -already -also -although -always -am -among -amongst -amoungst -amount -an -and -another -any -anyhow -anyone -anything -anyway -anywhere -are -around -as -at -back -be -became -because -become -becomes -becoming -been -before -beforehand -behind -being -below -beside -besides -between -beyond -bill -both -bottom -but -by -call -can -cannot -cant -co -computer -con -could -couldnt -cry -de -describe -detail -do -done -down -due -during -each -eg -eight -either -eleven -else -elsewhere -empty -enough -etc -even -ever -every -everyone -everything -everywhere -except -few -fifteen -fify -fill -find -fire -first -five -for -former -formerly -forty -found -four -from -front -full -further -get -give -go -had -has -hasnt -have -he -hence -her -here -hereafter -hereby -herein -hereupon -hers -herself -him -himself -his -how -however -hundred -i -ie -if -in -inc -indeed -interest -into -is -it -its -itself -keep -last -latter -latterly -least -less -ltd -made -many -may -me -meanwhile -might -mill -mine -more -moreover -most -mostly -move -much -must -my -myself -name -namely -neither -never -nevertheless -next -nine -no -nobody -none -noone -nor -not -nothing -now -nowhere -of -off -often -on -once -one -only -onto -or -other -others -otherwise -our -ours -ourselves -out -over -own -part -per -perhaps -please -put -rather -re -same -see -seem -seemed -seeming -seems -serious -several -she -should -show -side -since -sincere -six -sixty -so -some -somehow -someone -something -sometime -sometimes -somewhere -still -such -system -take -ten -than -that -the -their -them -themselves -then -thence -there -thereafter -thereby -therefore -therein -thereupon -these -they -thick -thin -third -this -those -though -three -through -throughout -thru -thus -to -together -too -top -toward -towards -twelve -twenty -two -un -under -until -up -upon -us -very -via -was -we -well -were -what -whatever -when -whence -whenever -where -whereafter -whereas -whereby -wherein -whereupon -wherever -whether -which -while -whither -who -whoever -whole -whom -whose -why -will -with -within -without -would -yet -you -your -yours -yourself -yourselves - - -: -/ -（ -> -） -< -! diff --git a/third_party/cppjieba/dict/user.dict.utf8 b/third_party/cppjieba/dict/user.dict.utf8 deleted file mode 100644 index f4a1a4b8..00000000 --- a/third_party/cppjieba/dict/user.dict.utf8 +++ /dev/null @@ -1,10 +0,0 @@ -云计算 -区块链 10 nz -机器学习 10 n -深度学习 10 n -神经网络 10 n -人工智能 10 n -数据挖掘 10 n -自然语言处理 10 n -大模型 10 n -大语言模型 10 n \ No newline at end of file diff --git a/third_party/cppjieba/include/cppjieba/DictTrie.hpp b/third_party/cppjieba/include/cppjieba/DictTrie.hpp deleted file mode 100644 index ea8916c1..00000000 --- a/third_party/cppjieba/include/cppjieba/DictTrie.hpp +++ /dev/null @@ -1,280 +0,0 @@ -#ifndef CPPJIEBA_DICT_TRIE_HPP -#define CPPJIEBA_DICT_TRIE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "limonp/StringUtil.hpp" -#include "limonp/Logging.hpp" -#include "Unicode.hpp" -#include "Trie.hpp" - -namespace cppjieba { - -const double MIN_DOUBLE = -3.14e+100; -const double MAX_DOUBLE = 3.14e+100; -const size_t DICT_COLUMN_NUM = 3; -const char* const UNKNOWN_TAG = ""; - -class DictTrie { - public: - enum UserWordWeightOption { - WordWeightMin, - WordWeightMedian, - WordWeightMax, - }; // enum UserWordWeightOption - - DictTrie(const std::string& dict_path, const std::string& user_dict_paths = "", UserWordWeightOption user_word_weight_opt = WordWeightMedian) { - Init(dict_path, user_dict_paths, user_word_weight_opt); - } - - ~DictTrie() { - delete trie_; - } - - bool InsertUserWord(const std::string& word, const std::string& tag = UNKNOWN_TAG) { - DictUnit node_info; - if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) { - return false; - } - active_node_infos_.push_back(node_info); - trie_->InsertNode(node_info.word, &active_node_infos_.back()); - return true; - } - - bool InsertUserWord(const std::string& word,int freq, const std::string& tag = UNKNOWN_TAG) { - DictUnit node_info; - double weight = freq ? log(1.0 * freq / freq_sum_) : user_word_default_weight_ ; - if (!MakeNodeInfo(node_info, word, weight , tag)) { - return false; - } - active_node_infos_.push_back(node_info); - trie_->InsertNode(node_info.word, &active_node_infos_.back()); - return true; - } - - bool DeleteUserWord(const std::string& word, const std::string& tag = UNKNOWN_TAG) { - DictUnit node_info; - if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) { - return false; - } - trie_->DeleteNode(node_info.word, &node_info); - return true; - } - - const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { - return trie_->Find(begin, end); - } - - void Find(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator end, - std::vector&res, - size_t max_word_len = MAX_WORD_LENGTH) const { - trie_->Find(begin, end, res, max_word_len); - } - - bool Find(const std::string& word) - { - const DictUnit *tmp = NULL; - RuneStrArray runes; - if (!DecodeUTF8RunesInString(word, runes)) - { - XLOG(ERROR) << "Decode failed."; - } - tmp = Find(runes.begin(), runes.end()); - if (tmp == NULL) - { - return false; - } - else - { - return true; - } - } - - bool IsUserDictSingleChineseWord(const Rune& word) const { - return IsIn(user_dict_single_chinese_word_, word); - } - - double GetMinWeight() const { - return min_weight_; - } - - void InserUserDictNode(const std::string& line) { - std::vector buf; - DictUnit node_info; - limonp::Split(line, buf, " "); - if(buf.size() == 1){ - MakeNodeInfo(node_info, - buf[0], - user_word_default_weight_, - UNKNOWN_TAG); - } else if (buf.size() == 2) { - MakeNodeInfo(node_info, - buf[0], - user_word_default_weight_, - buf[1]); - } else if (buf.size() == 3) { - int freq = atoi(buf[1].c_str()); - assert(freq_sum_ > 0.0); - double weight = log(1.0 * freq / freq_sum_); - MakeNodeInfo(node_info, buf[0], weight, buf[2]); - } - static_node_infos_.push_back(node_info); - if (node_info.word.size() == 1) { - user_dict_single_chinese_word_.insert(node_info.word[0]); - } - } - - void LoadUserDict(const std::vector& buf) { - for (size_t i = 0; i < buf.size(); i++) { - InserUserDictNode(buf[i]); - } - } - - void LoadUserDict(const std::set& buf) { - std::set::const_iterator iter; - for (iter = buf.begin(); iter != buf.end(); iter++){ - InserUserDictNode(*iter); - } - } - - void LoadUserDict(const std::string& filePaths) { - std::vector files = limonp::Split(filePaths, "|;"); - for (size_t i = 0; i < files.size(); i++) { - std::ifstream ifs(files[i].c_str()); - XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; - std::string line; - - while(getline(ifs, line)) { - if (line.size() == 0) { - continue; - } - InserUserDictNode(line); - } - } - } - - - private: - void Init(const std::string& dict_path, const std::string& user_dict_paths, UserWordWeightOption user_word_weight_opt) { - LoadDict(dict_path); - freq_sum_ = CalcFreqSum(static_node_infos_); - CalculateWeight(static_node_infos_, freq_sum_); - SetStaticWordWeights(user_word_weight_opt); - - if (user_dict_paths.size()) { - LoadUserDict(user_dict_paths); - } - Shrink(static_node_infos_); - CreateTrie(static_node_infos_); - } - - void CreateTrie(const std::vector& dictUnits) { - assert(dictUnits.size()); - std::vector words; - std::vector valuePointers; - for (size_t i = 0 ; i < dictUnits.size(); i ++) { - words.push_back(dictUnits[i].word); - valuePointers.push_back(&dictUnits[i]); - } - - trie_ = new Trie(words, valuePointers); - } - - bool MakeNodeInfo(DictUnit& node_info, - const std::string& word, - double weight, - const std::string& tag) { - if (!DecodeUTF8RunesInString(word, node_info.word)) { - XLOG(ERROR) << "UTF-8 decode failed for dict word: " << word; - return false; - } - node_info.weight = weight; - node_info.tag = tag; - return true; - } - - void LoadDict(const std::string& filePath) { - std::ifstream ifs(filePath.c_str()); - XCHECK(ifs.is_open()) << "open " << filePath << " failed."; - std::string line; - std::vector buf; - - DictUnit node_info; - while (getline(ifs, line)) { - limonp::Split(line, buf, " "); - XCHECK(buf.size() == DICT_COLUMN_NUM) << "split result illegal, line:" << line; - MakeNodeInfo(node_info, - buf[0], - atof(buf[1].c_str()), - buf[2]); - static_node_infos_.push_back(node_info); - } - } - - static bool WeightCompare(const DictUnit& lhs, const DictUnit& rhs) { - return lhs.weight < rhs.weight; - } - - void SetStaticWordWeights(UserWordWeightOption option) { - XCHECK(!static_node_infos_.empty()); - std::vector x = static_node_infos_; - std::sort(x.begin(), x.end(), WeightCompare); - min_weight_ = x[0].weight; - max_weight_ = x[x.size() - 1].weight; - median_weight_ = x[x.size() / 2].weight; - switch (option) { - case WordWeightMin: - user_word_default_weight_ = min_weight_; - break; - case WordWeightMedian: - user_word_default_weight_ = median_weight_; - break; - default: - user_word_default_weight_ = max_weight_; - break; - } - } - - double CalcFreqSum(const std::vector& node_infos) const { - double sum = 0.0; - for (size_t i = 0; i < node_infos.size(); i++) { - sum += node_infos[i].weight; - } - return sum; - } - - void CalculateWeight(std::vector& node_infos, double sum) const { - assert(sum > 0.0); - for (size_t i = 0; i < node_infos.size(); i++) { - DictUnit& node_info = node_infos[i]; - assert(node_info.weight > 0.0); - node_info.weight = log(double(node_info.weight)/sum); - } - } - - void Shrink(std::vector& units) const { - std::vector(units.begin(), units.end()).swap(units); - } - - std::vector static_node_infos_; - std::deque active_node_infos_; // must not be std::vector - Trie * trie_; - - double freq_sum_; - double min_weight_; - double max_weight_; - double median_weight_; - double user_word_default_weight_; - std::unordered_set user_dict_single_chinese_word_; -}; -} - -#endif diff --git a/third_party/cppjieba/include/cppjieba/FullSegment.hpp b/third_party/cppjieba/include/cppjieba/FullSegment.hpp deleted file mode 100644 index 79d5211e..00000000 --- a/third_party/cppjieba/include/cppjieba/FullSegment.hpp +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef CPPJIEBA_FULLSEGMENT_H -#define CPPJIEBA_FULLSEGMENT_H - -#include -#include -#include -#include "limonp/Logging.hpp" -#include "DictTrie.hpp" -#include "SegmentBase.hpp" -#include "Unicode.hpp" - -namespace cppjieba { -class FullSegment: public SegmentBase { - public: - FullSegment(const string& dictPath) { - dictTrie_ = new DictTrie(dictPath); - isNeedDestroy_ = true; - } - FullSegment(const DictTrie* dictTrie) - : dictTrie_(dictTrie), isNeedDestroy_(false) { - assert(dictTrie_); - } - ~FullSegment() { - if (isNeedDestroy_) { - delete dictTrie_; - } - } - void Cut(const string& sentence, - vector& words) const { - vector tmp; - Cut(sentence, tmp); - GetStringsFromWords(tmp, words); - } - void Cut(const string& sentence, - vector& words) const { - PreFilter pre_filter(symbols_, sentence); - PreFilter::Range range; - vector wrs; - wrs.reserve(sentence.size()/2); - while (pre_filter.HasNext()) { - range = pre_filter.Next(); - Cut(range.begin, range.end, wrs); - } - words.clear(); - words.reserve(wrs.size()); - GetWordsFromWordRanges(sentence, wrs, words); - } - void Cut(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator end, - vector& res) const { - // result of searching in trie tree - LocalVector > tRes; - - // max index of res's words - size_t maxIdx = 0; - - // always equals to (uItr - begin) - size_t uIdx = 0; - - // tmp variables - size_t wordLen = 0; - assert(dictTrie_); - vector dags; - dictTrie_->Find(begin, end, dags); - for (size_t i = 0; i < dags.size(); i++) { - for (size_t j = 0; j < dags[i].nexts.size(); j++) { - size_t nextoffset = dags[i].nexts[j].first; - assert(nextoffset < dags.size()); - const DictUnit* du = dags[i].nexts[j].second; - if (du == NULL) { - if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) { - WordRange wr(begin + i, begin + nextoffset); - res.push_back(wr); - } - } else { - wordLen = du->word.size(); - if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) { - WordRange wr(begin + i, begin + nextoffset); - res.push_back(wr); - } - } - maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx; - } - uIdx++; - } - } - private: - const DictTrie* dictTrie_; - bool isNeedDestroy_; -}; -} - -#endif diff --git a/third_party/cppjieba/include/cppjieba/HMMModel.hpp b/third_party/cppjieba/include/cppjieba/HMMModel.hpp deleted file mode 100644 index 3921faaf..00000000 --- a/third_party/cppjieba/include/cppjieba/HMMModel.hpp +++ /dev/null @@ -1,129 +0,0 @@ -#ifndef CPPJIEBA_HMMMODEL_H -#define CPPJIEBA_HMMMODEL_H - -#include "limonp/StringUtil.hpp" -#include "Trie.hpp" - -namespace cppjieba { - -using namespace limonp; -typedef unordered_map EmitProbMap; - -struct HMMModel { - /* - * STATUS: - * 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S - * */ - enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4}; - - HMMModel(const string& modelPath) { - memset(startProb, 0, sizeof(startProb)); - memset(transProb, 0, sizeof(transProb)); - statMap[0] = 'B'; - statMap[1] = 'E'; - statMap[2] = 'M'; - statMap[3] = 'S'; - emitProbVec.push_back(&emitProbB); - emitProbVec.push_back(&emitProbE); - emitProbVec.push_back(&emitProbM); - emitProbVec.push_back(&emitProbS); - LoadModel(modelPath); - } - ~HMMModel() { - } - void LoadModel(const string& filePath) { - ifstream ifile(filePath.c_str()); - XCHECK(ifile.is_open()) << "open " << filePath << " failed"; - string line; - vector tmp; - vector tmp2; - //Load startProb - XCHECK(GetLine(ifile, line)); - Split(line, tmp, " "); - XCHECK(tmp.size() == STATUS_SUM); - for (size_t j = 0; j< tmp.size(); j++) { - startProb[j] = atof(tmp[j].c_str()); - } - - //Load transProb - for (size_t i = 0; i < STATUS_SUM; i++) { - XCHECK(GetLine(ifile, line)); - Split(line, tmp, " "); - XCHECK(tmp.size() == STATUS_SUM); - for (size_t j =0; j < STATUS_SUM; j++) { - transProb[i][j] = atof(tmp[j].c_str()); - } - } - - //Load emitProbB - XCHECK(GetLine(ifile, line)); - XCHECK(LoadEmitProb(line, emitProbB)); - - //Load emitProbE - XCHECK(GetLine(ifile, line)); - XCHECK(LoadEmitProb(line, emitProbE)); - - //Load emitProbM - XCHECK(GetLine(ifile, line)); - XCHECK(LoadEmitProb(line, emitProbM)); - - //Load emitProbS - XCHECK(GetLine(ifile, line)); - XCHECK(LoadEmitProb(line, emitProbS)); - } - double GetEmitProb(const EmitProbMap* ptMp, Rune key, - double defVal)const { - EmitProbMap::const_iterator cit = ptMp->find(key); - if (cit == ptMp->end()) { - return defVal; - } - return cit->second; - } - bool GetLine(ifstream& ifile, string& line) { - while (getline(ifile, line)) { - Trim(line); - if (line.empty()) { - continue; - } - if (StartsWith(line, "#")) { - continue; - } - return true; - } - return false; - } - bool LoadEmitProb(const string& line, EmitProbMap& mp) { - if (line.empty()) { - return false; - } - vector tmp, tmp2; - Unicode unicode; - Split(line, tmp, ","); - for (size_t i = 0; i < tmp.size(); i++) { - Split(tmp[i], tmp2, ":"); - if (2 != tmp2.size()) { - XLOG(ERROR) << "emitProb illegal."; - return false; - } - if (!DecodeUTF8RunesInString(tmp2[0], unicode) || unicode.size() != 1) { - XLOG(ERROR) << "TransCode failed."; - return false; - } - mp[unicode[0]] = atof(tmp2[1].c_str()); - } - return true; - } - - char statMap[STATUS_SUM]; - double startProb[STATUS_SUM]; - double transProb[STATUS_SUM][STATUS_SUM]; - EmitProbMap emitProbB; - EmitProbMap emitProbE; - EmitProbMap emitProbM; - EmitProbMap emitProbS; - vector emitProbVec; -}; // struct HMMModel - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/HMMSegment.hpp b/third_party/cppjieba/include/cppjieba/HMMSegment.hpp deleted file mode 100644 index d515c049..00000000 --- a/third_party/cppjieba/include/cppjieba/HMMSegment.hpp +++ /dev/null @@ -1,190 +0,0 @@ -#ifndef CPPJIBEA_HMMSEGMENT_H -#define CPPJIBEA_HMMSEGMENT_H - -#include -#include -#include -#include -#include "HMMModel.hpp" -#include "SegmentBase.hpp" - -namespace cppjieba { -class HMMSegment: public SegmentBase { - public: - HMMSegment(const string& filePath) - : model_(new HMMModel(filePath)), isNeedDestroy_(true) { - } - HMMSegment(const HMMModel* model) - : model_(model), isNeedDestroy_(false) { - } - ~HMMSegment() { - if (isNeedDestroy_) { - delete model_; - } - } - - void Cut(const string& sentence, - vector& words) const { - vector tmp; - Cut(sentence, tmp); - GetStringsFromWords(tmp, words); - } - void Cut(const string& sentence, - vector& words) const { - PreFilter pre_filter(symbols_, sentence); - PreFilter::Range range; - vector wrs; - wrs.reserve(sentence.size()/2); - while (pre_filter.HasNext()) { - range = pre_filter.Next(); - Cut(range.begin, range.end, wrs); - } - words.clear(); - words.reserve(wrs.size()); - GetWordsFromWordRanges(sentence, wrs, words); - } - void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res) const { - RuneStrArray::const_iterator left = begin; - RuneStrArray::const_iterator right = begin; - while (right != end) { - if (right->rune < 0x80) { - if (left != right) { - InternalCut(left, right, res); - } - left = right; - do { - right = SequentialLetterRule(left, end); - if (right != left) { - break; - } - right = NumbersRule(left, end); - if (right != left) { - break; - } - right ++; - } while (false); - WordRange wr(left, right - 1); - res.push_back(wr); - left = right; - } else { - right++; - } - } - if (left != right) { - InternalCut(left, right, res); - } - } - private: - // sequential letters rule - RuneStrArray::const_iterator SequentialLetterRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { - Rune x = begin->rune; - if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) { - begin ++; - } else { - return begin; - } - while (begin != end) { - x = begin->rune; - if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) { - begin ++; - } else { - break; - } - } - return begin; - } - // - RuneStrArray::const_iterator NumbersRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { - Rune x = begin->rune; - if ('0' <= x && x <= '9') { - begin ++; - } else { - return begin; - } - while (begin != end) { - x = begin->rune; - if ( ('0' <= x && x <= '9') || x == '.') { - begin++; - } else { - break; - } - } - return begin; - } - void InternalCut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res) const { - vector status; - Viterbi(begin, end, status); - - RuneStrArray::const_iterator left = begin; - RuneStrArray::const_iterator right; - for (size_t i = 0; i < status.size(); i++) { - if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i]) - right = begin + i + 1; - WordRange wr(left, right - 1); - res.push_back(wr); - left = right; - } - } - } - - void Viterbi(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator end, - vector& status) const { - size_t Y = HMMModel::STATUS_SUM; - size_t X = end - begin; - - size_t XYSize = X * Y; - size_t now, old, stat; - double tmp, endE, endS; - - vector path(XYSize); - vector weight(XYSize); - - //start - for (size_t y = 0; y < Y; y++) { - weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE); - path[0 + y * X] = -1; - } - - double emitProb; - - for (size_t x = 1; x < X; x++) { - for (size_t y = 0; y < Y; y++) { - now = x + y*X; - weight[now] = MIN_DOUBLE; - path[now] = HMMModel::E; // warning - emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE); - for (size_t preY = 0; preY < Y; preY++) { - old = x - 1 + preY * X; - tmp = weight[old] + model_->transProb[preY][y] + emitProb; - if (tmp > weight[now]) { - weight[now] = tmp; - path[now] = preY; - } - } - } - } - - endE = weight[X-1+HMMModel::E*X]; - endS = weight[X-1+HMMModel::S*X]; - stat = 0; - if (endE >= endS) { - stat = HMMModel::E; - } else { - stat = HMMModel::S; - } - - status.resize(X); - for (int x = X -1 ; x >= 0; x--) { - status[x] = stat; - stat = path[x + stat*X]; - } - } - - const HMMModel* model_; - bool isNeedDestroy_; -}; // class HMMSegment - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/Jieba.hpp b/third_party/cppjieba/include/cppjieba/Jieba.hpp deleted file mode 100644 index 01fea361..00000000 --- a/third_party/cppjieba/include/cppjieba/Jieba.hpp +++ /dev/null @@ -1,169 +0,0 @@ -#ifndef CPPJIEAB_JIEBA_H -#define CPPJIEAB_JIEBA_H - -#include "QuerySegment.hpp" -#include "KeywordExtractor.hpp" - -namespace cppjieba { - -class Jieba { - public: - Jieba(const string& dict_path = "", - const string& model_path = "", - const string& user_dict_path = "", - const string& idf_path = "", - const string& stop_word_path = "") - : dict_trie_(getPath(dict_path, "jieba.dict.utf8"), getPath(user_dict_path, "user.dict.utf8")), - model_(getPath(model_path, "hmm_model.utf8")), - mp_seg_(&dict_trie_), - hmm_seg_(&model_), - mix_seg_(&dict_trie_, &model_), - full_seg_(&dict_trie_), - query_seg_(&dict_trie_, &model_), - extractor(&dict_trie_, &model_, - getPath(idf_path, "idf.utf8"), - getPath(stop_word_path, "stop_words.utf8")) { - } - ~Jieba() { - } - - struct LocWord { - string word; - size_t begin; - size_t end; - }; // struct LocWord - - void Cut(const string& sentence, vector& words, bool hmm = true) const { - mix_seg_.Cut(sentence, words, hmm); - } - void Cut(const string& sentence, vector& words, bool hmm = true) const { - mix_seg_.Cut(sentence, words, hmm); - } - void CutAll(const string& sentence, vector& words) const { - full_seg_.Cut(sentence, words); - } - void CutAll(const string& sentence, vector& words) const { - full_seg_.Cut(sentence, words); - } - void CutForSearch(const string& sentence, vector& words, bool hmm = true) const { - query_seg_.Cut(sentence, words, hmm); - } - void CutForSearch(const string& sentence, vector& words, bool hmm = true) const { - query_seg_.Cut(sentence, words, hmm); - } - void CutHMM(const string& sentence, vector& words) const { - hmm_seg_.Cut(sentence, words); - } - void CutHMM(const string& sentence, vector& words) const { - hmm_seg_.Cut(sentence, words); - } - void CutSmall(const string& sentence, vector& words, size_t max_word_len) const { - mp_seg_.Cut(sentence, words, max_word_len); - } - void CutSmall(const string& sentence, vector& words, size_t max_word_len) const { - mp_seg_.Cut(sentence, words, max_word_len); - } - - void Tag(const string& sentence, vector >& words) const { - mix_seg_.Tag(sentence, words); - } - string LookupTag(const string &str) const { - return mix_seg_.LookupTag(str); - } - bool InsertUserWord(const string& word, const string& tag = UNKNOWN_TAG) { - return dict_trie_.InsertUserWord(word, tag); - } - - bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) { - return dict_trie_.InsertUserWord(word,freq, tag); - } - - bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) { - return dict_trie_.DeleteUserWord(word, tag); - } - - bool Find(const string& word) - { - return dict_trie_.Find(word); - } - - void ResetSeparators(const string& s) { - //TODO - mp_seg_.ResetSeparators(s); - hmm_seg_.ResetSeparators(s); - mix_seg_.ResetSeparators(s); - full_seg_.ResetSeparators(s); - query_seg_.ResetSeparators(s); - } - - const DictTrie* GetDictTrie() const { - return &dict_trie_; - } - - const HMMModel* GetHMMModel() const { - return &model_; - } - - void LoadUserDict(const vector& buf) { - dict_trie_.LoadUserDict(buf); - } - - void LoadUserDict(const set& buf) { - dict_trie_.LoadUserDict(buf); - } - - void LoadUserDict(const string& path) { - dict_trie_.LoadUserDict(path); - } - - private: - static string pathJoin(const string& dir, const string& filename) { - if (dir.empty()) { - return filename; - } - - char last_char = dir[dir.length() - 1]; - if (last_char == '/' || last_char == '\\') { - return dir + filename; - } else { - #ifdef _WIN32 - return dir + '\\' + filename; - #else - return dir + '/' + filename; - #endif - } - } - - static string getCurrentDirectory() { - string path(__FILE__); - size_t pos = path.find_last_of("/\\"); - return (pos == string::npos) ? "" : path.substr(0, pos); - } - - static string getPath(const string& path, const string& default_file) { - if (path.empty()) { - string current_dir = getCurrentDirectory(); - string parent_dir = current_dir.substr(0, current_dir.find_last_of("/\\")); - string grandparent_dir = parent_dir.substr(0, parent_dir.find_last_of("/\\")); - return pathJoin(pathJoin(grandparent_dir, "dict"), default_file); - } - return path; - } - - DictTrie dict_trie_; - HMMModel model_; - - // They share the same dict trie and model - MPSegment mp_seg_; - HMMSegment hmm_seg_; - MixSegment mix_seg_; - FullSegment full_seg_; - QuerySegment query_seg_; - - public: - KeywordExtractor extractor; -}; // class Jieba - -} // namespace cppjieba - -#endif // CPPJIEAB_JIEBA_H diff --git a/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp b/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp deleted file mode 100644 index 24b2c409..00000000 --- a/third_party/cppjieba/include/cppjieba/KeywordExtractor.hpp +++ /dev/null @@ -1,149 +0,0 @@ -#ifndef CPPJIEBA_KEYWORD_EXTRACTOR_H -#define CPPJIEBA_KEYWORD_EXTRACTOR_H - -#include -#include -#include -#include "MixSegment.hpp" - -namespace cppjieba { - -/*utf8*/ -class KeywordExtractor { - public: - struct Word { - std::string word; - std::vector offsets; - double weight; - }; // struct Word - - KeywordExtractor(const std::string& dictPath, - const std::string& hmmFilePath, - const std::string& idfPath, - const std::string& stopWordPath, - const std::string& userDict = "") - : segment_(dictPath, hmmFilePath, userDict) { - LoadIdfDict(idfPath); - LoadStopWordDict(stopWordPath); - } - KeywordExtractor(const DictTrie* dictTrie, - const HMMModel* model, - const std::string& idfPath, - const std::string& stopWordPath) - : segment_(dictTrie, model) { - LoadIdfDict(idfPath); - LoadStopWordDict(stopWordPath); - } - ~KeywordExtractor() { - } - - void Extract(const std::string& sentence, std::vector& keywords, size_t topN) const { - std::vector topWords; - Extract(sentence, topWords, topN); - for (size_t i = 0; i < topWords.size(); i++) { - keywords.push_back(topWords[i].word); - } - } - - void Extract(const std::string& sentence, std::vector >& keywords, size_t topN) const { - std::vector topWords; - Extract(sentence, topWords, topN); - for (size_t i = 0; i < topWords.size(); i++) { - keywords.push_back(pair(topWords[i].word, topWords[i].weight)); - } - } - - void Extract(const std::string& sentence, std::vector& keywords, size_t topN) const { - std::vector words; - segment_.Cut(sentence, words); - - std::map wordmap; - size_t offset = 0; - for (size_t i = 0; i < words.size(); ++i) { - size_t t = offset; - offset += words[i].size(); - if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) { - continue; - } - wordmap[words[i]].offsets.push_back(t); - wordmap[words[i]].weight += 1.0; - } - if (offset != sentence.size()) { - XLOG(ERROR) << "words illegal"; - return; - } - - keywords.clear(); - keywords.reserve(wordmap.size()); - for (std::map::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) { - std::unordered_map::const_iterator cit = idfMap_.find(itr->first); - if (cit != idfMap_.end()) { - itr->second.weight *= cit->second; - } else { - itr->second.weight *= idfAverage_; - } - itr->second.word = itr->first; - keywords.push_back(itr->second); - } - topN = min(topN, keywords.size()); - std::partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare); - keywords.resize(topN); - } - private: - void LoadIdfDict(const std::string& idfPath) { - std::ifstream ifs(idfPath.c_str()); - XCHECK(ifs.is_open()) << "open " << idfPath << " failed"; - std::string line ; - std::vector buf; - double idf = 0.0; - double idfSum = 0.0; - size_t lineno = 0; - for (; getline(ifs, line); lineno++) { - buf.clear(); - if (line.empty()) { - XLOG(ERROR) << "lineno: " << lineno << " empty. skipped."; - continue; - } - limonp::Split(line, buf, " "); - if (buf.size() != 2) { - XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " empty. skipped."; - continue; - } - idf = atof(buf[1].c_str()); - idfMap_[buf[0]] = idf; - idfSum += idf; - - } - - assert(lineno); - idfAverage_ = idfSum / lineno; - assert(idfAverage_ > 0.0); - } - void LoadStopWordDict(const std::string& filePath) { - std::ifstream ifs(filePath.c_str()); - XCHECK(ifs.is_open()) << "open " << filePath << " failed"; - std::string line ; - while (getline(ifs, line)) { - stopWords_.insert(line); - } - assert(stopWords_.size()); - } - - static bool Compare(const Word& lhs, const Word& rhs) { - return lhs.weight > rhs.weight; - } - - MixSegment segment_; - std::unordered_map idfMap_; - double idfAverage_; - - std::unordered_set stopWords_; -}; // class KeywordExtractor - -inline std::ostream& operator << (std::ostream& os, const KeywordExtractor::Word& word) { - return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; -} - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/MPSegment.hpp b/third_party/cppjieba/include/cppjieba/MPSegment.hpp deleted file mode 100644 index bcbfaba6..00000000 --- a/third_party/cppjieba/include/cppjieba/MPSegment.hpp +++ /dev/null @@ -1,137 +0,0 @@ -#ifndef CPPJIEBA_MPSEGMENT_H -#define CPPJIEBA_MPSEGMENT_H - -#include -#include -#include -#include "limonp/Logging.hpp" -#include "DictTrie.hpp" -#include "SegmentTagged.hpp" -#include "PosTagger.hpp" - -namespace cppjieba { - -class MPSegment: public SegmentTagged { - public: - MPSegment(const string& dictPath, const string& userDictPath = "") - : dictTrie_(new DictTrie(dictPath, userDictPath)), isNeedDestroy_(true) { - } - MPSegment(const DictTrie* dictTrie) - : dictTrie_(dictTrie), isNeedDestroy_(false) { - assert(dictTrie_); - } - ~MPSegment() { - if (isNeedDestroy_) { - delete dictTrie_; - } - } - - void Cut(const string& sentence, vector& words) const { - Cut(sentence, words, MAX_WORD_LENGTH); - } - - void Cut(const string& sentence, - vector& words, - size_t max_word_len) const { - vector tmp; - Cut(sentence, tmp, max_word_len); - GetStringsFromWords(tmp, words); - } - void Cut(const string& sentence, - vector& words, - size_t max_word_len = MAX_WORD_LENGTH) const { - PreFilter pre_filter(symbols_, sentence); - PreFilter::Range range; - vector wrs; - wrs.reserve(sentence.size()/2); - while (pre_filter.HasNext()) { - range = pre_filter.Next(); - Cut(range.begin, range.end, wrs, max_word_len); - } - words.clear(); - words.reserve(wrs.size()); - GetWordsFromWordRanges(sentence, wrs, words); - } - void Cut(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator end, - vector& words, - size_t max_word_len = MAX_WORD_LENGTH) const { - vector dags; - dictTrie_->Find(begin, - end, - dags, - max_word_len); - CalcDP(dags); - CutByDag(begin, end, dags, words); - } - - const DictTrie* GetDictTrie() const { - return dictTrie_; - } - - bool Tag(const string& src, vector >& res) const { - return tagger_.Tag(src, res, *this); - } - - bool IsUserDictSingleChineseWord(const Rune& value) const { - return dictTrie_->IsUserDictSingleChineseWord(value); - } - private: - void CalcDP(vector& dags) const { - size_t nextPos; - const DictUnit* p; - double val; - - for (vector::reverse_iterator rit = dags.rbegin(); rit != dags.rend(); rit++) { - rit->pInfo = NULL; - rit->weight = MIN_DOUBLE; - assert(!rit->nexts.empty()); - for (LocalVector >::const_iterator it = rit->nexts.begin(); it != rit->nexts.end(); it++) { - nextPos = it->first; - p = it->second; - val = 0.0; - if (nextPos + 1 < dags.size()) { - val += dags[nextPos + 1].weight; - } - - if (p) { - val += p->weight; - } else { - val += dictTrie_->GetMinWeight(); - } - if (val > rit->weight) { - rit->pInfo = p; - rit->weight = val; - } - } - } - } - void CutByDag(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator /*end*/, - const vector& dags, - vector& words) const { - size_t i = 0; - while (i < dags.size()) { - const DictUnit* p = dags[i].pInfo; - if (p) { - assert(p->word.size() >= 1); - WordRange wr(begin + i, begin + i + p->word.size() - 1); - words.push_back(wr); - i += p->word.size(); - } else { //single chinese word - WordRange wr(begin + i, begin + i); - words.push_back(wr); - i++; - } - } - } - - const DictTrie* dictTrie_; - bool isNeedDestroy_; - PosTagger tagger_; - -}; // class MPSegment - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/MixSegment.hpp b/third_party/cppjieba/include/cppjieba/MixSegment.hpp deleted file mode 100644 index 8fd24e90..00000000 --- a/third_party/cppjieba/include/cppjieba/MixSegment.hpp +++ /dev/null @@ -1,109 +0,0 @@ -#ifndef CPPJIEBA_MIXSEGMENT_H -#define CPPJIEBA_MIXSEGMENT_H - -#include -#include "MPSegment.hpp" -#include "HMMSegment.hpp" -#include "limonp/StringUtil.hpp" -#include "PosTagger.hpp" - -namespace cppjieba { -class MixSegment: public SegmentTagged { - public: - MixSegment(const string& mpSegDict, const string& hmmSegDict, - const string& userDict = "") - : mpSeg_(mpSegDict, userDict), - hmmSeg_(hmmSegDict) { - } - MixSegment(const DictTrie* dictTrie, const HMMModel* model) - : mpSeg_(dictTrie), hmmSeg_(model) { - } - ~MixSegment() { - } - - void Cut(const string& sentence, vector& words) const { - Cut(sentence, words, true); - } - void Cut(const string& sentence, vector& words, bool hmm) const { - vector tmp; - Cut(sentence, tmp, hmm); - GetStringsFromWords(tmp, words); - } - void Cut(const string& sentence, vector& words, bool hmm = true) const { - PreFilter pre_filter(symbols_, sentence); - PreFilter::Range range; - vector wrs; - wrs.reserve(sentence.size() / 2); - while (pre_filter.HasNext()) { - range = pre_filter.Next(); - Cut(range.begin, range.end, wrs, hmm); - } - words.clear(); - words.reserve(wrs.size()); - GetWordsFromWordRanges(sentence, wrs, words); - } - - void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res, bool hmm) const { - if (!hmm) { - mpSeg_.Cut(begin, end, res); - return; - } - vector words; - assert(end >= begin); - words.reserve(end - begin); - mpSeg_.Cut(begin, end, words); - - vector hmmRes; - hmmRes.reserve(end - begin); - for (size_t i = 0; i < words.size(); i++) { - //if mp Get a word, it's ok, put it into result - if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) { - res.push_back(words[i]); - continue; - } - - // if mp Get a single one and it is not in userdict, collect it in sequence - size_t j = i; - while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) { - j++; - } - - // Cut the sequence with hmm - assert(j - 1 >= i); - // TODO - hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes); - //put hmm result to result - for (size_t k = 0; k < hmmRes.size(); k++) { - res.push_back(hmmRes[k]); - } - - //clear tmp vars - hmmRes.clear(); - - //let i jump over this piece - i = j - 1; - } - } - - const DictTrie* GetDictTrie() const { - return mpSeg_.GetDictTrie(); - } - - bool Tag(const string& src, vector >& res) const { - return tagger_.Tag(src, res, *this); - } - - string LookupTag(const string &str) const { - return tagger_.LookupTag(str, *this); - } - - private: - MPSegment mpSeg_; - HMMSegment hmmSeg_; - PosTagger tagger_; - -}; // class MixSegment - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/PosTagger.hpp b/third_party/cppjieba/include/cppjieba/PosTagger.hpp deleted file mode 100644 index 15863306..00000000 --- a/third_party/cppjieba/include/cppjieba/PosTagger.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef CPPJIEBA_POS_TAGGING_H -#define CPPJIEBA_POS_TAGGING_H - -#include "limonp/StringUtil.hpp" -#include "SegmentTagged.hpp" -#include "DictTrie.hpp" - -namespace cppjieba { -using namespace limonp; - -static const char* const POS_M = "m"; -static const char* const POS_ENG = "eng"; -static const char* const POS_X = "x"; - -class PosTagger { - public: - PosTagger() { - } - ~PosTagger() { - } - - bool Tag(const string& src, vector >& res, const SegmentTagged& segment) const { - vector CutRes; - segment.Cut(src, CutRes); - - for (vector::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) { - res.push_back(make_pair(*itr, LookupTag(*itr, segment))); - } - return !res.empty(); - } - - string LookupTag(const string &str, const SegmentTagged& segment) const { - const DictUnit *tmp = NULL; - RuneStrArray runes; - const DictTrie * dict = segment.GetDictTrie(); - assert(dict != NULL); - if (!DecodeUTF8RunesInString(str, runes)) { - XLOG(ERROR) << "UTF-8 decode failed for word: " << str; - return POS_X; - } - tmp = dict->Find(runes.begin(), runes.end()); - if (tmp == NULL || tmp->tag.empty()) { - return SpecialRule(runes); - } else { - return tmp->tag; - } - } - - private: - const char* SpecialRule(const RuneStrArray& unicode) const { - size_t m = 0; - size_t eng = 0; - for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) { - if (unicode[i].rune < 0x80) { - eng ++; - if ('0' <= unicode[i].rune && unicode[i].rune <= '9') { - m++; - } - } - } - // ascii char is not found - if (eng == 0) { - return POS_X; - } - // all the ascii is number char - if (m == eng) { - return POS_M; - } - // the ascii chars contain english letter - return POS_ENG; - } - -}; // class PosTagger - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/PreFilter.hpp b/third_party/cppjieba/include/cppjieba/PreFilter.hpp deleted file mode 100644 index deb750b5..00000000 --- a/third_party/cppjieba/include/cppjieba/PreFilter.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef CPPJIEBA_PRE_FILTER_H -#define CPPJIEBA_PRE_FILTER_H - -#include "Trie.hpp" -#include "limonp/Logging.hpp" - -namespace cppjieba { - -class PreFilter { - public: - //TODO use WordRange instead of Range - struct Range { - RuneStrArray::const_iterator begin; - RuneStrArray::const_iterator end; - }; // struct Range - - PreFilter(const unordered_set& symbols, - const string& sentence) - : symbols_(symbols) { - if (!DecodeUTF8RunesInString(sentence, sentence_)) { - XLOG(ERROR) << "UTF-8 decode failed for input sentence"; - } - cursor_ = sentence_.begin(); - } - ~PreFilter() { - } - bool HasNext() const { - return cursor_ != sentence_.end(); - } - Range Next() { - Range range; - range.begin = cursor_; - while (cursor_ != sentence_.end()) { - if (IsIn(symbols_, cursor_->rune)) { - if (range.begin == cursor_) { - cursor_ ++; - } - range.end = cursor_; - return range; - } - cursor_ ++; - } - range.end = sentence_.end(); - return range; - } - private: - RuneStrArray::const_iterator cursor_; - RuneStrArray sentence_; - const unordered_set& symbols_; -}; // class PreFilter - -} // namespace cppjieba - -#endif // CPPJIEBA_PRE_FILTER_H diff --git a/third_party/cppjieba/include/cppjieba/QuerySegment.hpp b/third_party/cppjieba/include/cppjieba/QuerySegment.hpp deleted file mode 100644 index 6be886ab..00000000 --- a/third_party/cppjieba/include/cppjieba/QuerySegment.hpp +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef CPPJIEBA_QUERYSEGMENT_H -#define CPPJIEBA_QUERYSEGMENT_H - -#include -#include -#include -#include "limonp/Logging.hpp" -#include "DictTrie.hpp" -#include "SegmentBase.hpp" -#include "FullSegment.hpp" -#include "MixSegment.hpp" -#include "Unicode.hpp" - -namespace cppjieba { -class QuerySegment: public SegmentBase { - public: - QuerySegment(const string& dict, const string& model, const string& userDict = "") - : mixSeg_(dict, model, userDict), - trie_(mixSeg_.GetDictTrie()) { - } - QuerySegment(const DictTrie* dictTrie, const HMMModel* model) - : mixSeg_(dictTrie, model), trie_(dictTrie) { - } - ~QuerySegment() { - } - - void Cut(const string& sentence, vector& words) const { - Cut(sentence, words, true); - } - void Cut(const string& sentence, vector& words, bool hmm) const { - vector tmp; - Cut(sentence, tmp, hmm); - GetStringsFromWords(tmp, words); - } - void Cut(const string& sentence, vector& words, bool hmm = true) const { - PreFilter pre_filter(symbols_, sentence); - PreFilter::Range range; - vector wrs; - wrs.reserve(sentence.size()/2); - while (pre_filter.HasNext()) { - range = pre_filter.Next(); - Cut(range.begin, range.end, wrs, hmm); - } - words.clear(); - words.reserve(wrs.size()); - GetWordsFromWordRanges(sentence, wrs, words); - } - void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector& res, bool hmm) const { - //use mix Cut first - vector mixRes; - mixSeg_.Cut(begin, end, mixRes, hmm); - - vector fullRes; - for (vector::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) { - if (mixResItr->Length() > 2) { - for (size_t i = 0; i + 1 < mixResItr->Length(); i++) { - WordRange wr(mixResItr->left + i, mixResItr->left + i + 1); - if (trie_->Find(wr.left, wr.right + 1) != NULL) { - res.push_back(wr); - } - } - } - if (mixResItr->Length() > 3) { - for (size_t i = 0; i + 2 < mixResItr->Length(); i++) { - WordRange wr(mixResItr->left + i, mixResItr->left + i + 2); - if (trie_->Find(wr.left, wr.right + 1) != NULL) { - res.push_back(wr); - } - } - } - res.push_back(*mixResItr); - } - } - private: - bool IsAllAscii(const Unicode& s) const { - for(size_t i = 0; i < s.size(); i++) { - if (s[i] >= 0x80) { - return false; - } - } - return true; - } - MixSegment mixSeg_; - const DictTrie* trie_; -}; // QuerySegment - -} // namespace cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/SegmentBase.hpp b/third_party/cppjieba/include/cppjieba/SegmentBase.hpp deleted file mode 100644 index 130b2128..00000000 --- a/third_party/cppjieba/include/cppjieba/SegmentBase.hpp +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef CPPJIEBA_SEGMENTBASE_H -#define CPPJIEBA_SEGMENTBASE_H - -#include "limonp/Logging.hpp" -#include "PreFilter.hpp" -#include - - -namespace cppjieba { - -const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82"; - -using namespace limonp; - -class SegmentBase { - public: - SegmentBase() { - XCHECK(ResetSeparators(SPECIAL_SEPARATORS)); - } - virtual ~SegmentBase() { - } - - virtual void Cut(const string& sentence, vector& words) const = 0; - - bool ResetSeparators(const string& s) { - symbols_.clear(); - RuneStrArray runes; - if (!DecodeUTF8RunesInString(s, runes)) { - XLOG(ERROR) << "UTF-8 decode failed for separators: " << s; - return false; - } - for (size_t i = 0; i < runes.size(); i++) { - if (!symbols_.insert(runes[i].rune).second) { - XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists"; - return false; - } - } - return true; - } - protected: - unordered_set symbols_; -}; // class SegmentBase - -} // cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp b/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp deleted file mode 100644 index 4d99a31a..00000000 --- a/third_party/cppjieba/include/cppjieba/SegmentTagged.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef CPPJIEBA_SEGMENTTAGGED_H -#define CPPJIEBA_SEGMENTTAGGED_H - -#include "SegmentBase.hpp" - -namespace cppjieba { - -class SegmentTagged : public SegmentBase{ - public: - SegmentTagged() { - } - virtual ~SegmentTagged() { - } - - virtual bool Tag(const string& src, vector >& res) const = 0; - - virtual const DictTrie* GetDictTrie() const = 0; - -}; // class SegmentTagged - -} // cppjieba - -#endif diff --git a/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp b/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp deleted file mode 100644 index 292d0a8f..00000000 --- a/third_party/cppjieba/include/cppjieba/TextRankExtractor.hpp +++ /dev/null @@ -1,190 +0,0 @@ -#ifndef CPPJIEBA_TEXTRANK_EXTRACTOR_H -#define CPPJIEBA_TEXTRANK_EXTRACTOR_H - -#include -#include "Jieba.hpp" - -namespace cppjieba { - using namespace limonp; - using namespace std; - - class TextRankExtractor { - public: - typedef struct _Word {string word;vector offsets;double weight;} Word; // struct Word - private: - typedef std::map WordMap; - - class WordGraph{ - private: - typedef double Score; - typedef string Node; - typedef std::set NodeSet; - - typedef std::map Edges; - typedef std::map Graph; - //typedef std::unordered_map Edges; - //typedef std::unordered_map Graph; - - double d; - Graph graph; - NodeSet nodeSet; - public: - WordGraph(): d(0.85) {}; - WordGraph(double in_d): d(in_d) {}; - - void addEdge(Node start,Node end,double weight){ - Edges temp; - Edges::iterator gotEdges; - nodeSet.insert(start); - nodeSet.insert(end); - graph[start][end]+=weight; - graph[end][start]+=weight; - } - - void rank(WordMap &ws,size_t rankTime=10){ - WordMap outSum; - Score wsdef, min_rank, max_rank; - - if( graph.size() == 0) - return; - - wsdef = 1.0 / graph.size(); - - for(Graph::iterator edges=graph.begin();edges!=graph.end();++edges){ - // edges->first start节点；edge->first end节点；edge->second 权重 - ws[edges->first].word=edges->first; - ws[edges->first].weight=wsdef; - outSum[edges->first].weight=0; - for(Edges::iterator edge=edges->second.begin();edge!=edges->second.end();++edge){ - outSum[edges->first].weight+=edge->second; - } - } - //sort(nodeSet.begin(),nodeSet.end()); 是否需要排序? - for( size_t i=0; ifirst end节点；edge->second 权重 - s += edge->second / outSum[edge->first].weight * ws[edge->first].weight; - ws[*node].weight = (1 - d) + d * s; - } - } - - min_rank=max_rank=ws.begin()->second.weight; - for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){ - if( i->second.weight < min_rank ){ - min_rank = i->second.weight; - } - if( i->second.weight > max_rank ){ - max_rank = i->second.weight; - } - } - for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){ - ws[i->first].weight = (i->second.weight - min_rank / 10.0) / (max_rank - min_rank / 10.0); - } - } - }; - - public: - TextRankExtractor(const string& dictPath, - const string& hmmFilePath, - const string& stopWordPath, - const string& userDict = "") - : segment_(dictPath, hmmFilePath, userDict) { - LoadStopWordDict(stopWordPath); - } - TextRankExtractor(const DictTrie* dictTrie, - const HMMModel* model, - const string& stopWordPath) - : segment_(dictTrie, model) { - LoadStopWordDict(stopWordPath); - } - TextRankExtractor(const Jieba& jieba, const string& stopWordPath) : segment_(jieba.GetDictTrie(), jieba.GetHMMModel()) { - LoadStopWordDict(stopWordPath); - } - ~TextRankExtractor() { - } - - void Extract(const string& sentence, vector& keywords, size_t topN) const { - vector topWords; - Extract(sentence, topWords, topN); - for (size_t i = 0; i < topWords.size(); i++) { - keywords.push_back(topWords[i].word); - } - } - - void Extract(const string& sentence, vector >& keywords, size_t topN) const { - vector topWords; - Extract(sentence, topWords, topN); - for (size_t i = 0; i < topWords.size(); i++) { - keywords.push_back(pair(topWords[i].word, topWords[i].weight)); - } - } - - void Extract(const string& sentence, vector& keywords, size_t topN, size_t span=5,size_t rankTime=10) const { - vector words; - segment_.Cut(sentence, words); - - TextRankExtractor::WordGraph graph; - WordMap wordmap; - size_t offset = 0; - - for(size_t i=0; i < words.size(); i++){ - size_t t = offset; - offset += words[i].size(); - if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) { - continue; - } - for(size_t j=i+1,skip=0;jsecond); - } - - topN = min(topN, keywords.size()); - partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare); - keywords.resize(topN); - } - private: - void LoadStopWordDict(const string& filePath) { - ifstream ifs(filePath.c_str()); - XCHECK(ifs.is_open()) << "open " << filePath << " failed"; - string line ; - while (getline(ifs, line)) { - stopWords_.insert(line); - } - assert(stopWords_.size()); - } - - static bool Compare(const Word &x,const Word &y){ - return x.weight > y.weight; - } - - MixSegment segment_; - unordered_set stopWords_; - }; // class TextRankExtractor - - inline ostream& operator << (ostream& os, const TextRankExtractor::Word& word) { - return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; - } -} // namespace cppjieba - -#endif - - diff --git a/third_party/cppjieba/include/cppjieba/Trie.hpp b/third_party/cppjieba/include/cppjieba/Trie.hpp deleted file mode 100644 index dc3c78a2..00000000 --- a/third_party/cppjieba/include/cppjieba/Trie.hpp +++ /dev/null @@ -1,193 +0,0 @@ -#ifndef CPPJIEBA_TRIE_HPP -#define CPPJIEBA_TRIE_HPP - -#include -#include -#include "limonp/StdExtension.hpp" -#include "Unicode.hpp" - -namespace cppjieba { - -using namespace std; - -const size_t MAX_WORD_LENGTH = 512; - -struct DictUnit { - Unicode word; - double weight; - string tag; -}; // struct DictUnit - -struct Dag { - RuneStr runestr; - // [offset, nexts.first] - limonp::LocalVector > nexts; - const DictUnit * pInfo; - double weight; - size_t nextPos; // TODO - Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) { - } -}; // struct Dag - -typedef Rune TrieKey; - -class TrieNode { - public : - TrieNode(): next(NULL), ptValue(NULL) { - } - public: - typedef unordered_map NextMap; - NextMap *next; - const DictUnit *ptValue; -}; - -class Trie { - public: - Trie(const vector& keys, const vector& valuePointers) - : root_(new TrieNode) { - CreateTrie(keys, valuePointers); - } - ~Trie() { - DeleteNode(root_); - } - - const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const { - if (begin == end) { - return NULL; - } - - const TrieNode* ptNode = root_; - TrieNode::NextMap::const_iterator citer; - for (RuneStrArray::const_iterator it = begin; it != end; it++) { - if (NULL == ptNode->next) { - return NULL; - } - citer = ptNode->next->find(it->rune); - if (ptNode->next->end() == citer) { - return NULL; - } - ptNode = citer->second; - } - return ptNode->ptValue; - } - - void Find(RuneStrArray::const_iterator begin, - RuneStrArray::const_iterator end, - vector&res, - size_t max_word_len = MAX_WORD_LENGTH) const { - assert(root_ != NULL); - res.resize(end - begin); - - const TrieNode *ptNode = NULL; - TrieNode::NextMap::const_iterator citer; - for (size_t i = 0; i < size_t(end - begin); i++) { - res[i].runestr = *(begin + i); - - if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) { - ptNode = citer->second; - } else { - ptNode = NULL; - } - if (ptNode != NULL) { - res[i].nexts.push_back(pair(i, ptNode->ptValue)); - } else { - res[i].nexts.push_back(pair(i, static_cast(NULL))); - } - - for (size_t j = i + 1; j < size_t(end - begin) && (j - i + 1) <= max_word_len; j++) { - if (ptNode == NULL || ptNode->next == NULL) { - break; - } - citer = ptNode->next->find((begin + j)->rune); - if (ptNode->next->end() == citer) { - break; - } - ptNode = citer->second; - if (NULL != ptNode->ptValue) { - res[i].nexts.push_back(pair(j, ptNode->ptValue)); - } - } - } - } - - void InsertNode(const Unicode& key, const DictUnit* ptValue) { - if (key.begin() == key.end()) { - return; - } - - TrieNode::NextMap::const_iterator kmIter; - TrieNode *ptNode = root_; - for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) { - if (NULL == ptNode->next) { - ptNode->next = new TrieNode::NextMap; - } - kmIter = ptNode->next->find(*citer); - if (ptNode->next->end() == kmIter) { - TrieNode *nextNode = new TrieNode; - - ptNode->next->insert(make_pair(*citer, nextNode)); - ptNode = nextNode; - } else { - ptNode = kmIter->second; - } - } - assert(ptNode != NULL); - ptNode->ptValue = ptValue; - } - void DeleteNode(const Unicode& key, const DictUnit* /*ptValue*/) { - if (key.begin() == key.end()) { - return; - } - //定义一个NextMap迭代器 - TrieNode::NextMap::const_iterator kmIter; - //定义一个指向root的TrieNode指针 - TrieNode *ptNode = root_; - for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) { - //链表不存在元素 - if (NULL == ptNode->next) { - return; - } - kmIter = ptNode->next->find(*citer); - //如果map中不存在,跳出循环 - if (ptNode->next->end() == kmIter) { - break; - } - //从unordered_map中擦除该项 - ptNode->next->erase(*citer); - //删除该node - ptNode = kmIter->second; - delete ptNode; - break; - } - return; - } - private: - void CreateTrie(const vector& keys, const vector& valuePointers) { - if (valuePointers.empty() || keys.empty()) { - return; - } - assert(keys.size() == valuePointers.size()); - - for (size_t i = 0; i < keys.size(); i++) { - InsertNode(keys[i], valuePointers[i]); - } - } - - void DeleteNode(TrieNode* node) { - if (NULL == node) { - return; - } - if (NULL != node->next) { - for (TrieNode::NextMap::iterator it = node->next->begin(); it != node->next->end(); ++it) { - DeleteNode(it->second); - } - delete node->next; - } - delete node; - } - - TrieNode* root_; -}; // class Trie -} // namespace cppjieba - -#endif // CPPJIEBA_TRIE_HPP diff --git a/third_party/cppjieba/include/cppjieba/Unicode.hpp b/third_party/cppjieba/include/cppjieba/Unicode.hpp deleted file mode 100644 index 9adec2ca..00000000 --- a/third_party/cppjieba/include/cppjieba/Unicode.hpp +++ /dev/null @@ -1,227 +0,0 @@ -#ifndef CPPJIEBA_UNICODE_H -#define CPPJIEBA_UNICODE_H - -#include -#include -#include -#include -#include -#include "limonp/LocalVector.hpp" - -namespace cppjieba { - -using std::string; -using std::vector; - -typedef uint32_t Rune; - -struct Word { - string word; - uint32_t offset; - uint32_t unicode_offset; - uint32_t unicode_length; - Word(const string& w, uint32_t o) - : word(w), offset(o) { - } - Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length) - : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) { - } -}; // struct Word - -inline std::ostream& operator << (std::ostream& os, const Word& w) { - return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}"; -} - -struct RuneStr { - Rune rune; - uint32_t offset; - uint32_t len; - uint32_t unicode_offset; - uint32_t unicode_length; - RuneStr(): rune(0), offset(0), len(0), unicode_offset(0), unicode_length(0) { - } - RuneStr(Rune r, uint32_t o, uint32_t l) - : rune(r), offset(o), len(l), unicode_offset(0), unicode_length(0) { - } - RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length) - : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) { - } -}; // struct RuneStr - -inline std::ostream& operator << (std::ostream& os, const RuneStr& r) { - return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}"; -} - -typedef limonp::LocalVector Unicode; -typedef limonp::LocalVector RuneStrArray; - -// [left, right] -struct WordRange { - RuneStrArray::const_iterator left; - RuneStrArray::const_iterator right; - WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r) - : left(l), right(r) { - } - size_t Length() const { - return right - left + 1; - } - bool IsAllAscii() const { - for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) { - if (iter->rune >= 0x80) { - return false; - } - } - return true; - } -}; // struct WordRange - -struct RuneStrLite { - uint32_t rune; - uint32_t len; - RuneStrLite(): rune(0), len(0) { - } - RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) { - } -}; // struct RuneStrLite - -inline RuneStrLite DecodeUTF8ToRune(const char* str, size_t len) { - RuneStrLite rp(0, 0); - if (str == NULL || len == 0) { - return rp; - } - if (!(str[0] & 0x80)) { // 0xxxxxxx - // 7bit, total 7bit - rp.rune = (uint8_t)(str[0]) & 0x7f; - rp.len = 1; - } else if ((uint8_t)str[0] <= 0xdf && 1 < len) { - // 110xxxxxx - // 5bit, total 5bit - rp.rune = (uint8_t)(str[0]) & 0x1f; - - // 6bit, total 11bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[1]) & 0x3f; - rp.len = 2; - } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx - // 4bit, total 4bit - rp.rune = (uint8_t)(str[0]) & 0x0f; - - // 6bit, total 10bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[1]) & 0x3f; - - // 6bit, total 16bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[2]) & 0x3f; - - rp.len = 3; - } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx - // 3bit, total 3bit - rp.rune = (uint8_t)(str[0]) & 0x07; - - // 6bit, total 9bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[1]) & 0x3f; - - // 6bit, total 15bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[2]) & 0x3f; - - // 6bit, total 21bit - rp.rune <<= 6; - rp.rune |= (uint8_t)(str[3]) & 0x3f; - - rp.len = 4; - } else { - rp.rune = 0; - rp.len = 0; - } - return rp; -} - -inline bool DecodeUTF8RunesInString(const char* s, size_t len, RuneStrArray& runes) { - runes.clear(); - runes.reserve(len / 2); - for (uint32_t i = 0, j = 0; i < len;) { - RuneStrLite rp = DecodeUTF8ToRune(s + i, len - i); - if (rp.len == 0) { - runes.clear(); - return false; - } - RuneStr x(rp.rune, i, rp.len, j, 1); - runes.push_back(x); - i += rp.len; - ++j; - } - return true; -} - -inline bool DecodeUTF8RunesInString(const string& s, RuneStrArray& runes) { - return DecodeUTF8RunesInString(s.c_str(), s.size(), runes); -} - -inline bool DecodeUTF8RunesInString(const char* s, size_t len, Unicode& unicode) { - unicode.clear(); - RuneStrArray runes; - if (!DecodeUTF8RunesInString(s, len, runes)) { - return false; - } - unicode.reserve(runes.size()); - for (size_t i = 0; i < runes.size(); i++) { - unicode.push_back(runes[i].rune); - } - return true; -} - -inline bool IsSingleWord(const string& str) { - RuneStrLite rp = DecodeUTF8ToRune(str.c_str(), str.size()); - return rp.len == str.size(); -} - -inline bool DecodeUTF8RunesInString(const string& s, Unicode& unicode) { - return DecodeUTF8RunesInString(s.c_str(), s.size(), unicode); -} - -inline Unicode DecodeUTF8RunesInString(const string& s) { - Unicode result; - DecodeUTF8RunesInString(s, result); - return result; -} - - -// [left, right] -inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) { - assert(right->offset >= left->offset); - uint32_t len = right->offset - left->offset + right->len; - uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length; - return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length); -} - -inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) { - assert(right->offset >= left->offset); - uint32_t len = right->offset - left->offset + right->len; - return s.substr(left->offset, len); -} - -inline void GetWordsFromWordRanges(const string& s, const vector& wrs, vector& words) { - for (size_t i = 0; i < wrs.size(); i++) { - words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right)); - } -} - -inline vector GetWordsFromWordRanges(const string& s, const vector& wrs) { - vector result; - GetWordsFromWordRanges(s, wrs, result); - return result; -} - -inline void GetStringsFromWords(const vector& words, vector& strs) { - strs.resize(words.size()); - for (size_t i = 0; i < words.size(); ++i) { - strs[i] = words[i].word; - } -} - -} // namespace cppjieba - -#endif // CPPJIEBA_UNICODE_H diff --git a/third_party/mecab/CMakeLists.txt b/third_party/mecab/CMakeLists.txt deleted file mode 100644 index 9a53833f..00000000 --- a/third_party/mecab/CMakeLists.txt +++ /dev/null @@ -1,123 +0,0 @@ -# MeCab morphological analyzer — vendored from https://github.com/taku910/mecab -# (BSD-3-Clause, see LICENSE). Used by the FTS extension for Japanese -# tokenization. The ipadic dictionary (EUC-JP CSV) is compiled to the native -# binary dictionary format (UTF-8) at build time via mecab-dict-index. - -set(MECAB_SRCS - src/char_property.cpp - src/connector.cpp - src/context_id.cpp - src/dictionary.cpp - src/dictionary_compiler.cpp - src/dictionary_generator.cpp - src/dictionary_rewriter.cpp - src/eval.cpp - src/feature_index.cpp - src/iconv_utils.cpp - src/lbfgs.cpp - src/learner.cpp - src/learner_tagger.cpp - src/libmecab.cpp - src/nbest_generator.cpp - src/param.cpp - src/string_buffer.cpp - src/tagger.cpp - src/tokenizer.cpp - src/utils.cpp - src/viterbi.cpp - src/writer.cpp) - -# Same defines the upstream Makefile.msvc uses; VERSION/DIC_VERSION come from -# configure.in (0.996, 102). DLL_EXPORT is omitted: we link statically. -set(MECAB_DEFS - _CRT_SECURE_NO_DEPRECATE - MECAB_USE_THREAD - HAVE_GETENV - HAVE_WINDOWS_H - DIC_VERSION=102 - VERSION=\"0.996\" - PACKAGE=\"mecab\" - # Default rc file: an empty file generated alongside the dictionary. - # (An empty string would make Param::load() try to open "" and fail.) - MECAB_DEFAULT_RC=\"${CMAKE_BINARY_DIR}/mecab-dict/ipadic/mecabrc\" - # Gate for StringBuffer::operator<<(unsigned long long) in string_buffer.h. - HAVE_UNSIGNED_LONG_LONG_INT) - -add_library(mecab STATIC ${MECAB_SRCS}) -target_include_directories(mecab PUBLIC src) -target_compile_definitions(mecab PRIVATE ${MECAB_DEFS}) -# mecab.h declares MECAB_DLL_EXTERN as __declspec(dllimport) on _WIN32 unless -# DLL_EXPORT is defined. We link statically, so export (not import) must be -# declared consistently in both the library and its consumers. -target_compile_definitions(mecab PUBLIC DLL_EXPORT) -if(MSVC) - # Upstream suppresses these in Makefile.msvc; the code is C++98-era. - target_compile_options(mecab PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) -endif() - -# Build-time tool that compiles dictionary CSV files into the binary dictionary -# format (sys.dic/sys.dat/sys.cha/unk.*/char.bin/matrix.bin). -add_executable(mecab-dict-index src/mecab-dict-index.cpp ${MECAB_SRCS}) -target_include_directories(mecab-dict-index PRIVATE src) -target_compile_definitions(mecab-dict-index PRIVATE ${MECAB_DEFS}) -if(MSVC) - target_compile_options(mecab-dict-index PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) -endif() - -# Compile the ipadic dictionary (EUC-JP CSV) into a UTF-8 binary dictionary. -# The dictionary is not vendored (54MB CSV): it is downloaded from -# https://github.com/taku910/mecab-ipadic at configure time. The compiled -# dictionary files are host-endian, so they are generated at build time. -# ipadic has no standalone repo; it lives in taku910/mecab's mecab-ipadic/ -# subdirectory, so we download the mecab archive and keep that subtree. -set(MECAB_IPADIC_DIR ${CMAKE_BINARY_DIR}/mecab-ipadic-src) -if(NOT EXISTS ${MECAB_IPADIC_DIR}/dicrc) - set(MECAB_IPADIC_ZIP ${CMAKE_BINARY_DIR}/mecab-ipadic.zip) - set(MECAB_IPADIC_EXTRACT ${CMAKE_BINARY_DIR}/mecab-ipadic-extract) - message(STATUS "Downloading mecab-ipadic dictionary sources (54MB CSV)...") - file(DOWNLOAD https://github.com/taku910/mecab/archive/refs/heads/master.zip - ${MECAB_IPADIC_ZIP} STATUS MECAB_DL_STATUS) - list(GET MECAB_DL_STATUS 0 MECAB_DL_CODE) - if(NOT MECAB_DL_CODE EQUAL 0) - list(GET MECAB_DL_STATUS 1 MECAB_DL_ERROR) - message(FATAL_ERROR "Failed to download mecab-ipadic: ${MECAB_DL_ERROR}") - endif() - file(MAKE_DIRECTORY ${MECAB_IPADIC_EXTRACT}) - file(ARCHIVE_EXTRACT INPUT ${MECAB_IPADIC_ZIP} DESTINATION ${MECAB_IPADIC_EXTRACT}) - set(MECAB_IPADIC_SUBDIR ${MECAB_IPADIC_EXTRACT}/mecab-master/mecab-ipadic) - if(NOT IS_DIRECTORY "${MECAB_IPADIC_SUBDIR}") - message(FATAL_ERROR "Unexpected mecab archive layout (mecab-ipadic dir missing)") - endif() - file(RENAME ${MECAB_IPADIC_SUBDIR} ${MECAB_IPADIC_DIR}) - file(REMOVE_RECURSE ${MECAB_IPADIC_EXTRACT}) - file(REMOVE ${MECAB_IPADIC_ZIP}) -endif() -set(MECAB_DICT_OUT_DIR ${CMAKE_BINARY_DIR}/mecab-dict/ipadic) -set(MECAB_DICT_OUTPUTS - ${MECAB_DICT_OUT_DIR}/sys.dic - ${MECAB_DICT_OUT_DIR}/unk.dic - ${MECAB_DICT_OUT_DIR}/char.bin - ${MECAB_DICT_OUT_DIR}/matrix.bin - ${MECAB_DICT_OUT_DIR}/dicrc - ${MECAB_DICT_OUT_DIR}/mecabrc) - -add_custom_command( - OUTPUT ${MECAB_DICT_OUTPUTS} - COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} - COMMAND $ -f euc-jp -t utf-8 - -d ${MECAB_IPADIC_DIR} -o ${MECAB_DICT_OUT_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${MECAB_IPADIC_DIR}/dicrc ${MECAB_DICT_OUT_DIR}/dicrc - COMMAND ${CMAKE_COMMAND} -E touch ${MECAB_DICT_OUT_DIR}/mecabrc - DEPENDS mecab-dict-index - COMMENT "Compiling ipadic dictionary (EUC-JP -> UTF-8)") - -add_custom_target(mecab-ipadic-dict - DEPENDS ${MECAB_DICT_OUT_DIR}/sys.dic) - -# CLI for ad-hoc tokenization checks during development. -add_executable(mecab-cli src/mecab.cpp ${MECAB_SRCS}) -target_include_directories(mecab-cli PRIVATE src) -target_compile_definitions(mecab-cli PRIVATE ${MECAB_DEFS}) -if(MSVC) - target_compile_options(mecab-cli PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) -endif() From 0b549041f1f871ae7609555ead218a29fd41e86c Mon Sep 17 00:00:00 2001 From: chiangchenghsin-hash Date: Thu, 27 Aug 2026 17:26:44 +0800 Subject: [PATCH 114/114] fix(mecab): define the POSIX HAVE_* macros on non-Windows The vendored sources follow upstream's autoconf convention and guard every system header/feature behind HAVE_* checks (config.h normally supplies them). Without them GCC fell back to MSVC-only paths and the build failed with: utils.h:35: error: expected initializer before 'uint64_t' (falls into 'typedef unsigned __int64 uint64_t' - MSVC-only) mmap.h:146: error: 'O_RDONLY' was not declared in this scope mmap.h:152: error: '::open' has not been declared (fcntl.h / unistd.h / sys/stat.h never included) Added the defines upstream's configure detects on Linux: HAVE_STDINT_H, HAVE_SYS_TYPES_H, HAVE_SYS_STAT_H, HAVE_FCNTL_H, HAVE_STRING_H, HAVE_SYS_MMAN_H, HAVE_UNISTD_H, HAVE_DIRENT_H, HAVE_MMAP, HAVE_TLS_KEYWORD. --- fts/third_party/mecab/CMakeLists.txt | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/fts/third_party/mecab/CMakeLists.txt b/fts/third_party/mecab/CMakeLists.txt index 079d8978..c1df7be0 100644 --- a/fts/third_party/mecab/CMakeLists.txt +++ b/fts/third_party/mecab/CMakeLists.txt @@ -54,9 +54,23 @@ if(WIN32) endif() if(NOT WIN32) + # The defines upstream's autoconf configure would detect on Linux/macOS. + # The sources guard every system header / feature behind HAVE_* checks: + # without them GCC falls back to MSVC-only paths (e.g. 'unsigned __int64') + # or misses the POSIX includes (, , ...) entirely. + list(APPEND MECAB_DEFS + HAVE_STDINT_H + HAVE_SYS_TYPES_H + HAVE_SYS_STAT_H + HAVE_FCNTL_H + HAVE_STRING_H + HAVE_SYS_MMAN_H + HAVE_UNISTD_H + HAVE_DIRENT_H + HAVE_MMAP + HAVE_TLS_KEYWORD) # tagger.cpp instantiates read_write_mutex, which only compiles when - # HAVE_ATOMIC_OPS is defined - use the GCC/Clang __sync builtins (what - # upstream's autoconf configure detects on Linux). + # HAVE_ATOMIC_OPS is defined - use the GCC/Clang __sync builtins. # HAVE_ICONV: glibc provides iconv(3) in libc; mecab-dict-index needs it # to compile the EUC-JP ipadic CSV into UTF-8. list(APPEND MECAB_DEFS HAVE_GCC_ATOMIC_OPS HAVE_ICONV)