diff --git a/fts/CMakeLists.txt b/fts/CMakeLists.txt index cd3fd431..21f94cf0 100644 --- a/fts/CMakeLists.txt +++ b/fts/CMakeLists.txt @@ -4,7 +4,8 @@ include_directories( src/include third_party/snowball/libstemmer ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include - ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include) + ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include + third_party/mecab/src) add_subdirectory(src/catalog) add_subdirectory(src/function) @@ -15,16 +16,35 @@ add_subdirectory(src/utils) add_subdirectory(third_party/snowball) +# MeCab (Japanese tokenizer), vendored in-tree like snowball: static lib + +# mecab-dict-index tool + ipadic dictionary compilation at build time +# (downloads the dictionary CSV at configure time when no prebuilt dictionary +# is bundled). +add_subdirectory(third_party/mecab) + +# tokenizer.cpp includes mecab.h, which requires DLL_EXPORT to declare the +# MeCab API without __declspec(dllimport) on Windows. +target_compile_definitions(lbug_fts_utils PRIVATE DLL_EXPORT) + build_extension_lib(${BUILD_STATIC_EXTENSION} "fts") target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension PRIVATE snowball re2 - cppjieba) + cppjieba + mecab) # Copy Jieba dictionaries next to the built extension so runtime can find a default path add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/third_party/cppjieba/dict ${PROJECT_SOURCE_DIR}/extension/fts/build/dict) + +# Copy the compiled ipadic dictionary (UTF-8) next to the built extension so +# the default mecab_dict_dir resolves at runtime. +add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_BINARY_DIR}/mecab-dict/ipadic + ${PROJECT_SOURCE_DIR}/extension/fts/build/dict-ipadic) +add_dependencies(lbug_${EXTENSION_LIB_NAME}_extension mecab-ipadic-dict) diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index 2cc78ebc..8986b462 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -157,13 +157,22 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string query = ""; if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context), FTSUtils::getTokenizeMacroName(tableID, indexName))) { - // TOKENIZE(text, tokenizer, extra_param) + // TOKENIZE(text, tokenizer, extra_param); the extra param is the + // dictionary directory for dictionary-based tokenizers. Pick it by + // tokenizer name (the params map holds defaults for all tokenizers). + auto& tokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params; + auto& tokenizerName = ftsBindData->createFTSConfig.tokenizerInfo.tokenizer; + std::string extraParam = ""; + if (tokenizerName == "jieba") { + extraParam = tokenizerParams.at("jieba_dict_dir"); + } else if (tokenizerName == "mecab") { + extraParam = tokenizerParams.at("mecab_dict_dir"); + } query += std::format(R"(CREATE MACRO `{}`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)", FTSUtils::getTokenizeMacroName(tableID, indexName), formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), - ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, - ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); + ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, extraParam); } // Create the stop words table if not exists, or the user is not using the default english @@ -236,7 +245,17 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string params; params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer); params += - std::format("stopWords := '{}'", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + std::format("stopWords := '{}', ", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + // The internal index uses this config for incremental inserts/updates and + // queries, so the tokenizer (and its parameters) must be forwarded or it + // silently falls back to the default 'simple' tokenizer. + params += std::format("tokenizer := '{}'", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer); + auto& ftsTokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params; + if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "jieba") { + params += std::format(", jieba_dict_dir := '{}'", ftsTokenizerParams.at("jieba_dict_dir")); + } else if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "mecab") { + params += std::format(", mecab_dict_dir := '{}'", ftsTokenizerParams.at("mecab_dict_dir")); + } query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName, properties, params); query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName); diff --git a/fts/src/function/fts_config.cpp b/fts/src/function/fts_config.cpp index d3b7c7cf..33eae4d3 100644 --- a/fts/src/function/fts_config.cpp +++ b/fts/src/function/fts_config.cpp @@ -156,8 +156,11 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_ Tokenizer::validate(tokenizerInfo.tokenizer); } else if (lowerCaseName == "jieba_dict_dir") { value.validateType(common::LogicalTypeID::STRING); - tokenizerInfo.jiebaDictDir = - common::StringUtils::getLower(value.getValue()); + // Note: the dict dir is a file path and must not be lower-cased. + tokenizerInfo.params["jieba_dict_dir"] = value.getValue(); + } else if (lowerCaseName == "mecab_dict_dir") { + value.validateType(common::LogicalTypeID::STRING); + tokenizerInfo.params["mecab_dict_dir"] = value.getValue(); } else { throw common::BinderException{"Unrecognized optional parameter: " + name}; } @@ -166,9 +169,16 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_ FTSConfig CreateFTSConfig::getFTSConfig() const { return FTSConfig{stemmer, stopWordsTableInfo.tableName, stopWordsTableInfo.stopWords, - ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.jiebaDictDir}; + ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.params}; } +// Magic marker written before the tokenizer params so that deserialization can +// tell apart new catalogs (marker + unordered map) from legacy ones (a single +// "jiebaDictDir" string field). +namespace { +constexpr const char* TOKENIZER_PARAMS_MAGIC = "lbug_tokenizer_params_v1"; +} // namespace + void FTSConfig::serialize(common::Serializer& serializer) const { serializer.serializeValue(stemmer); serializer.serializeValue(stopWordsTableName); @@ -176,7 +186,8 @@ void FTSConfig::serialize(common::Serializer& serializer) const { serializer.serializeValue(ignorePattern); serializer.serializeValue(ignorePatternQuery); serializer.serializeValue(tokenizer); - serializer.serializeValue(jiebaDictDir); + serializer.serializeValue(std::string{TOKENIZER_PARAMS_MAGIC}); + serializer.serializeUnorderedMap(tokenizerParams); } FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) { @@ -187,7 +198,14 @@ FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) { deserializer.deserializeValue(config.ignorePattern); deserializer.deserializeValue(config.ignorePatternQuery); deserializer.deserializeValue(config.tokenizer); - deserializer.deserializeValue(config.jiebaDictDir); + std::string tokenizerParamsField; + deserializer.deserializeValue(tokenizerParamsField); + if (tokenizerParamsField == TOKENIZER_PARAMS_MAGIC) { + deserializer.deserializeUnorderedMap(config.tokenizerParams); + } else { + // Legacy catalog: the field after tokenizer was the jieba dict dir. + config.tokenizerParams["jieba_dict_dir"] = tokenizerParamsField; + } return config; } @@ -214,7 +232,7 @@ void TopK::validate(uint64_t value) { } void Tokenizer::validate(const std::string& tokenizer) { - if (tokenizer == "simple" || tokenizer == "jieba") { + if (TokenizerRegistry::isSupported(tokenizer)) { return; } throw common::BinderException{ diff --git a/fts/src/function/tokenize.cpp b/fts/src/function/tokenize.cpp index 4f3e2af3..c094460f 100644 --- a/fts/src/function/tokenize.cpp +++ b/fts/src/function/tokenize.cpp @@ -6,10 +6,10 @@ #include "common/types/string_t.h" #include "common/types/types.h" #include "common/vector/value_vector.h" -#include "cppjieba/Jieba.hpp" #include "expression_evaluator/expression_evaluator_utils.h" #include "function/scalar_function.h" #include "re2.h" +#include "utils/tokenizer.h" namespace lbug { namespace fts_extension { @@ -17,16 +17,17 @@ namespace fts_extension { using namespace function; using namespace common; -struct JiebaBindData final : public FunctionBindData { - std::shared_ptr jieba; +struct TokenizerBindData final : public FunctionBindData { + std::shared_ptr tokenizer; - JiebaBindData(common::logical_type_vec_t paramTypes, std::shared_ptr jieba) + TokenizerBindData(common::logical_type_vec_t paramTypes, + std::shared_ptr tokenizer) : FunctionBindData{std::move(paramTypes), common::LogicalType::LIST(common::LogicalType::STRING())}, - jieba{std::move(jieba)} {} + tokenizer{std::move(tokenizer)} {} std::unique_ptr copy() const override { - return std::make_unique(copyVector(paramTypes), jieba); + return std::make_unique(copyVector(paramTypes), tokenizer); } }; @@ -38,21 +39,11 @@ static void addTokensToVector(const std::vector& tokens, list_entry } } -struct JiebaTokenizer { +struct TokenizeOp { static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/, list_entry_t& result, common::ValueVector& resultVector, void* dataPtr) { - std::vector tokens; - auto bindData = reinterpret_cast(dataPtr); - bindData->jieba->CutForSearch(text.getAsString(), tokens); - addTokensToVector(tokens, result, resultVector); - } -}; - -struct SimpleTokenizer { - static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/, - list_entry_t& result, common::ValueVector& resultVector, void* /*dataPtr*/) { - auto tokens = - StringUtils::split(text.getAsString(), " ", true /* ignoreEmptyStringParts */); + auto bindData = reinterpret_cast(dataPtr); + auto tokens = bindData->tokenizer->tokenize(text.getAsString()); addTokensToVector(tokens, result, resultVector); } }; @@ -62,37 +53,31 @@ static std::unique_ptr bindFunc(const ScalarBindFuncInput& inp throw BinderException{"The tokenizer parameter must be a literal expression."}; } if (input.arguments[2]->expressionType != ExpressionType::LITERAL) { - throw BinderException{"The path to the jieba dict directory must be a literal expression."}; + throw BinderException{"The tokenizer parameter must be a literal expression."}; } auto value = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression(input.arguments[1], input.context); - auto tokenizer = common::StringUtils::getLower(value.getValue()); - if (tokenizer == "jieba") { - std::string dictDir = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression( - input.arguments[2], input.context) - .getValue(); - std::string dict = dictDir + "/jieba.dict.utf8"; - std::string hmm = dictDir + "/hmm_model.utf8"; - std::string user = dictDir + "/user.dict.utf8"; // Contains custom AI/ML terms - std::string idf = dictDir + "/idf.utf8"; - std::string stop = dictDir + "/stop_words.utf8"; - auto jieba = std::make_unique(dict, hmm, user, idf, stop); - input.definition->ptrCast()->execFunc = - ScalarFunction::TernaryRegexExecFunction; - return std::make_unique( - binder::ExpressionUtil::getDataTypes(input.arguments), std::move(jieba)); - } else if (tokenizer == "simple" || tokenizer == "") { - input.definition->ptrCast()->execFunc = - ScalarFunction::TernaryRegexExecFunction; - return FunctionBindData::getSimpleBindData(input.arguments, - LogicalType::LIST(LogicalType::STRING())); - } else { - throw common::BinderException{ - "Unsupported tokenizer: " + tokenizer + - ".\nSupported tokenizers: 'simple' (default), 'jieba' (advanced Chinese)"}; + auto tokenizerName = common::StringUtils::getLower(value.getValue()); + if (tokenizerName.empty()) { + tokenizerName = "simple"; + } + // The third argument is the tokenizer's extra parameter; for 'jieba' it is + // the dictionary directory. + auto extraParam = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression( + input.arguments[2], input.context) + .getValue(); + TokenizerParams params; + if (tokenizerName == "jieba") { + params["jieba_dict_dir"] = extraParam; + } else if (tokenizerName == "mecab") { + params["mecab_dict_dir"] = extraParam; } + auto tokenizer = TokenizerPool::getOrCreate(tokenizerName, params); + input.definition->ptrCast()->execFunc = + ScalarFunction::TernaryRegexExecFunction; + return std::make_unique( + binder::ExpressionUtil::getDataTypes(input.arguments), std::move(tokenizer)); } function::function_set TokenizeFunction::getFunctionSet() { diff --git a/fts/src/include/function/fts_config.h b/fts/src/include/function/fts_config.h index a2326d05..9b7b0b33 100644 --- a/fts/src/include/function/fts_config.h +++ b/fts/src/include/function/fts_config.h @@ -4,6 +4,7 @@ #include "common/types/types.h" #include "function/table/bind_input.h" +#include "utils/tokenizer.h" #include namespace lbug { @@ -65,9 +66,14 @@ struct Tokenizer { struct TokenizerInfo { std::string tokenizer = Tokenizer::DEFAULT_VALUE; - std::string jiebaDictDir = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY); - - TokenizerInfo() = default; + // Tokenizer-specific parameters (e.g. "jieba_dict_dir" for the jieba tokenizer). + TokenizerParams params; + + TokenizerInfo() { + params["jieba_dict_dir"] = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY); + params["mecab_dict_dir"] = + std::format("{}/extension/fts/build/dict-ipadic", LBUG_ROOT_DIRECTORY); + } }; struct FTSConfig; @@ -95,16 +101,19 @@ struct FTSConfig { std::string ignorePattern = ""; std::string ignorePatternQuery = ""; std::string tokenizer = ""; - std::string jiebaDictDir = ""; + // Serialized as an unordered map after a magic marker; entries of legacy + // catalogs (which stored a single "jiebaDictDir" string here) are folded + // into tokenizerParams["jieba_dict_dir"] at deserialization time. + TokenizerParams tokenizerParams = {}; FTSConfig() = default; FTSConfig(std::string stemmer, std::string stopWordsTableName, std::string stopWordsSource, std::string ignorePattern, std::string ignorePatternQuery, std::string tokenizer, - std::string jiebaDictDir) + TokenizerParams tokenizerParams) : stemmer{std::move(stemmer)}, stopWordsTableName{std::move(stopWordsTableName)}, stopWordsSource{std::move(stopWordsSource)}, ignorePattern{std::move(ignorePattern)}, ignorePatternQuery{std::move(ignorePatternQuery)}, tokenizer{std::move(tokenizer)}, - jiebaDictDir{std::move(jiebaDictDir)} {} + tokenizerParams{std::move(tokenizerParams)} {} void serialize(common::Serializer& serializer) const; diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index 96ae0298..f43e5ab3 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -58,7 +58,8 @@ struct FTSUtils { return std::format("{}_tokenize", getInternalTablePrefix(tableID, indexName)); } - static std::vector tokenizeString(std::string& str, const FTSConfig& tokenizer); + static std::vector tokenizeString(const std::string& str, + const FTSConfig& tokenizer); }; } // namespace fts_extension diff --git a/fts/src/include/utils/tokenizer.h b/fts/src/include/utils/tokenizer.h new file mode 100644 index 00000000..ea697a48 --- /dev/null +++ b/fts/src/include/utils/tokenizer.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace lbug { +namespace fts_extension { + +// A tokenizer converts a piece of text (a document or a query) into a list of +// searchable terms. Each tokenizer is identified by a name and created from a +// set of string parameters (e.g. a dictionary directory). +class ITokenizer { +public: + virtual ~ITokenizer() = default; + + virtual std::vector tokenize(const std::string& text) const = 0; + + virtual std::string getName() const = 0; +}; + +using TokenizerParams = std::unordered_map; +using TokenizerFactory = + std::function(const TokenizerParams& params)>; + +// Tokenizers are registered by name. Adding a new tokenizer (e.g. 'mecab' for +// Japanese) only requires registering a factory here; TOKENIZE and +// CREATE_FTS_INDEX pick it up automatically. +class TokenizerRegistry { +public: + static void registerTokenizer(const std::string& name, TokenizerFactory factory); + + static bool isSupported(const std::string& name); + + static std::unique_ptr create(const std::string& name, + const TokenizerParams& params); + + static std::vector getSupportedNames(); + +private: + static std::unordered_map& getRegistry(); +}; + +// Constructing a tokenizer can be expensive (the jieba tokenizer loads and +// indexes a dictionary on construction), so instances are cached per +// (name, params) and shared across index inserts and queries. Tokenizer +// instances must be safe for concurrent use (jieba queries are read-only +// after construction). +class TokenizerPool { +public: + static std::shared_ptr getOrCreate(const std::string& name, + const TokenizerParams& params); + +private: + static std::string makeKey(const std::string& name, const TokenizerParams& params); +}; + +} // namespace fts_extension +} // namespace lbug diff --git a/fts/src/utils/CMakeLists.txt b/fts/src/utils/CMakeLists.txt index 1e5294f3..350a2560 100644 --- a/fts/src/utils/CMakeLists.txt +++ b/fts/src/utils/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(lbug_fts_utils OBJECT - fts_utils.cpp) + fts_utils.cpp + tokenizer.cpp) set(FTS_EXTENSION_OBJECT_FILES ${FTS_EXTENSION_OBJECT_FILES} $ diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index fb6d68bc..ed4c02ea 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -1,12 +1,12 @@ #include "utils/fts_utils.h" #include "common/string_utils.h" -#include "cppjieba/Jieba.hpp" #include "function/stem.h" #include "libstemmer.h" #include "re2.h" #include "storage/storage_manager.h" #include "storage/table/node_table.h" +#include "utils/tokenizer.h" namespace lbug { namespace fts_extension { @@ -94,17 +94,11 @@ std::vector FTSUtils::stemTerms(std::vector terms, return result; } -std::vector FTSUtils::tokenizeString(std::string& str, const FTSConfig& config) { - std::vector terms; - if (config.tokenizer == "jieba") { - cppjieba::Jieba jieba(config.jiebaDictDir + "/jieba.dict.utf8", - config.jiebaDictDir + "/hmm_model.utf8", config.jiebaDictDir + "/user.dict.utf8", - config.jiebaDictDir + "/idf.utf8", config.jiebaDictDir + "/stop_words.utf8"); - jieba.CutForSearch(str, terms); - } else { - terms = StringUtils::split(str, " ", true /* ignoreEmptyStringParts */); - } - return terms; +std::vector FTSUtils::tokenizeString(const std::string& str, + const FTSConfig& config) { + // Tokenizer instances are cached in the pool, so this is cheap even though + // constructing a jieba tokenizer loads and indexes a dictionary. + return TokenizerPool::getOrCreate(config.tokenizer, config.tokenizerParams)->tokenize(str); } } // namespace fts_extension diff --git a/fts/src/utils/tokenizer.cpp b/fts/src/utils/tokenizer.cpp new file mode 100644 index 00000000..1b8c232c --- /dev/null +++ b/fts/src/utils/tokenizer.cpp @@ -0,0 +1,186 @@ +#include "utils/tokenizer.h" + +#include "common/exception/binder.h" +#include "common/string_utils.h" +#include "cppjieba/Jieba.hpp" +#include "mecab.h" +#include +#include +#include + +namespace lbug { +namespace fts_extension { + +using namespace common; + +class SimpleTokenizer final : public ITokenizer { +public: + std::vector tokenize(const std::string& text) const override { + return StringUtils::split(text, " ", true /* ignoreEmptyStringParts */); + } + + std::string getName() const override { + return "simple"; + } +}; + +class JiebaTokenizer final : public ITokenizer { +public: + explicit JiebaTokenizer(const TokenizerParams& params) { + auto dictDir = + params.contains("jieba_dict_dir") ? params.at("jieba_dict_dir") : std::string{}; + jieba = std::make_shared(dictDir + "/jieba.dict.utf8", + dictDir + "/hmm_model.utf8", dictDir + "/user.dict.utf8", dictDir + "/idf.utf8", + dictDir + "/stop_words.utf8"); + } + + std::vector tokenize(const std::string& text) const override { + std::vector tokens; + jieba->CutForSearch(text, tokens); + return tokens; + } + + std::string getName() const override { + return "jieba"; + } + +private: + std::shared_ptr jieba; +}; + +std::unordered_map& TokenizerRegistry::getRegistry() { + static std::unordered_map registry; + return registry; +} + +// Japanese morphological analyzer. The ipadic dictionary is compiled to a +// UTF-8 binary dictionary at build time (see third_party/mecab). +class MeCabTokenizer final : public ITokenizer { +public: + explicit MeCabTokenizer(const TokenizerParams& params) { + auto dictDir = + params.contains("mecab_dict_dir") ? params.at("mecab_dict_dir") : std::string{}; + // Pass the rc file explicitly. Without -r, MeCab falls back to + // MECAB_DEFAULT_RC, which is a build-time absolute path that does not + // exist in deployed environments (zip extracted elsewhere, build dir + // cleaned up) — the tagger then fails to create. The rc file ships + // alongside the dictionary (generated at build time). + tagger.reset(MeCab::createTagger( + (std::string("-d ") + dictDir + " -r " + dictDir + "/mecabrc").c_str())); + if (!tagger) { + auto lastError = MeCab::getLastError(); + throw BinderException{std::format( + "Failed to create mecab tagger with dict dir: '{}'. (mecab error: {})", dictDir, + lastError ? lastError : "unknown")}; + } + } + + std::vector tokenize(const std::string& text) const override { + std::vector tokens; + // parseToNode() returns nodes whose surface points into `text`, so the + // surface must be copied out while `text` is still alive. + const MeCab::Node* node = tagger->parseToNode(text.c_str()); + for (; node; node = node->next) { + if (node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE) { + continue; + } + if (node->surface == nullptr || node->length == 0) { + continue; + } + tokens.emplace_back(node->surface, node->length); + } + return tokens; + } + + std::string getName() const override { + return "mecab"; + } + +private: + std::unique_ptr tagger{nullptr, + MeCab::deleteTagger}; +}; + +void TokenizerRegistry::registerTokenizer(const std::string& name, TokenizerFactory factory) { + getRegistry()[name] = std::move(factory); +} + +bool TokenizerRegistry::isSupported(const std::string& name) { + return getRegistry().contains(name); +} + +std::vector TokenizerRegistry::getSupportedNames() { + std::vector names; + for (auto& [name, factory] : getRegistry()) { + names.push_back(name); + } + return names; +} + +std::unique_ptr TokenizerRegistry::create(const std::string& name, + const TokenizerParams& params) { + auto& registry = getRegistry(); + auto it = registry.find(name); + if (it == registry.end()) { + // Keep this message in sync with the registered tokenizers; the fts + // error.test asserts on it verbatim. + throw BinderException{"Unsupported tokenizer: " + name + + ".\nSupported tokenizers: 'simple' (default), 'jieba' (Chinese), 'mecab' " + "(Japanese)"}; + } + return it->second(params); +} + +namespace { + +struct BuiltinTokenizers { + BuiltinTokenizers() { + TokenizerRegistry::registerTokenizer("simple", + [](const TokenizerParams&) { return std::make_unique(); }); + TokenizerRegistry::registerTokenizer("jieba", + [](const TokenizerParams& params) { + return std::make_unique(params); + }); + TokenizerRegistry::registerTokenizer("mecab", + [](const TokenizerParams& params) { + return std::make_unique(params); + }); + } +}; + +} // namespace + +static BuiltinTokenizers builtinTokenizers; + +std::shared_ptr TokenizerPool::getOrCreate(const std::string& name, + const TokenizerParams& params) { + static std::mutex mutex; + static std::unordered_map> pool; + auto key = makeKey(name, params); + std::lock_guard guard{mutex}; + if (auto it = pool.find(key); it != pool.end()) { + if (auto instance = it->second.lock(); instance) { + return instance; + } + } + auto instance = std::shared_ptr(TokenizerRegistry::create(name, params)); + pool[key] = instance; + return instance; +} + +std::string TokenizerPool::makeKey(const std::string& name, const TokenizerParams& params) { + std::vector keys; + keys.reserve(params.size()); + for (auto& [key, value] : params) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end()); + std::string key = name; + for (auto& paramKey : keys) { + key += std::format("|{}={}", paramKey, params.at(paramKey)); + } + return key; +} + +} // namespace fts_extension +} // namespace lbug diff --git a/fts/test/test_files/error.test b/fts/test/test_files/error.test index f6852bad..8d4bb8de 100644 --- a/fts/test/test_files/error.test +++ b/fts/test/test_files/error.test @@ -206,4 +206,4 @@ Error: "missing ]: [a-z". -STATEMENT CALL CREATE_FTS_INDEX('person', 'personIdx', ['fName'], tokenizer := 'french') ---- error Binder exception: Unsupported tokenizer: french. -Supported tokenizers: 'simple' (default), 'jieba' (advanced Chinese) +Supported tokenizers: 'simple' (default), 'jieba' (Chinese), 'mecab' (Japanese) diff --git a/fts/test/test_files/fts_chinese.test b/fts/test/test_files/fts_chinese.test index 8905b03f..b6b4fb88 100644 --- a/fts/test/test_files/fts_chinese.test +++ b/fts/test/test_files/fts_chinese.test @@ -90,3 +90,22 @@ Python和机器学习结合开发AI应用 ---- 2 AI研究 NLP应用 + +# === Incremental Insert After Index Creation === +# Regression: the rewritten _CREATE_FTS_INDEX used to drop the tokenizer +# option, so incremental inserts fell back to 'simple' tokenization and the +# new rows' Chinese terms were never indexed. +-CASE ChineseIncrementalInsert +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE ZhDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:ZhDoc {text: '我爱北京天安门'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('ZhDoc', 'zh_idx', ['text'], tokenizer:='jieba', stemmer:='none'); +---- ok +-STATEMENT CREATE (d:ZhDoc {text: '北京欢迎你'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('ZhDoc', 'zh_idx', '北京', conjunctive:=false, top:=5) RETURN node.text ORDER BY node.text; +---- 2 +北京欢迎你 +我爱北京天安门 diff --git a/fts/test/test_files/fts_japanese.test b/fts/test/test_files/fts_japanese.test new file mode 100644 index 00000000..b60725cf --- /dev/null +++ b/fts/test/test_files/fts_japanese.test @@ -0,0 +1,52 @@ +-DATASET CSV empty-db + +-- + +# Japanese FTS via the MeCab tokenizer with the compiled ipadic dictionary. +# The dictionary is generated at build time (third_party/mecab) and copied +# next to the extension; mecab_dict_dir defaults to that location. +-CASE JapaneseFTSBasic +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京タワーは赤い塔です'}); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京大学で勉強する'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '東京タワー', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京タワーは赤い塔です +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '勉強', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京大学で勉強する +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '東京', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京タワーは赤い塔です + +-CASE JapaneseIncrementalInsert +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京タワーは赤い塔です'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none'); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '東京大学で勉強する'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '勉強', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +東京大学で勉強する + +-CASE JapaneseCustomDictDir +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE JpDoc (ID SERIAL, text STRING, PRIMARY KEY (ID)); +---- ok +-STATEMENT CREATE (d:JpDoc {text: '日本語の全文検索テスト'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('JpDoc', 'jp_idx', ['text'], tokenizer:='mecab', stemmer:='none', mecab_dict_dir := '${LBUG_ROOT_DIRECTORY}/extension/fts/build/dict-ipadic'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('JpDoc', 'jp_idx', '全文検索', conjunctive:=false, top:=5) RETURN node.text; +---- 1 +日本語の全文検索テスト diff --git a/fts/third_party/mecab/CMakeLists.txt b/fts/third_party/mecab/CMakeLists.txt new file mode 100644 index 00000000..c1df7be0 --- /dev/null +++ b/fts/third_party/mecab/CMakeLists.txt @@ -0,0 +1,174 @@ +# MeCab morphological analyzer - vendored from https://github.com/taku910/mecab +# (BSD-3-Clause, see LICENSE). Used by the FTS extension for Japanese +# tokenization. The ipadic dictionary (EUC-JP CSV) is compiled to the native +# binary dictionary format (UTF-8) at build time via mecab-dict-index. + +set(MECAB_SRCS + src/char_property.cpp + src/connector.cpp + src/context_id.cpp + src/dictionary.cpp + src/dictionary_compiler.cpp + src/dictionary_generator.cpp + src/dictionary_rewriter.cpp + src/eval.cpp + src/feature_index.cpp + src/iconv_utils.cpp + src/lbfgs.cpp + src/learner.cpp + src/learner_tagger.cpp + src/libmecab.cpp + src/nbest_generator.cpp + src/param.cpp + src/string_buffer.cpp + src/tagger.cpp + src/tokenizer.cpp + src/utils.cpp + src/viterbi.cpp + src/writer.cpp) + +# Same defines the upstream Makefile.msvc uses; VERSION/DIC_VERSION come from +# configure.in (0.996, 102). DLL_EXPORT is omitted: we link statically. +set(MECAB_DEFS + _CRT_SECURE_NO_DEPRECATE + MECAB_USE_THREAD + HAVE_GETENV + # UTF-8-only build (upstream's --enable-utf8-only): drops the 4MB + # ucstable.h legacy-charset table. The FTS extension only ever loads + # UTF-8 dictionaries (ipadic is compiled to UTF-8 at build time); + # legacy-charset input CSVs are converted with iconv before use. + MECAB_USE_UTF8_ONLY + DIC_VERSION=102 + VERSION=\"0.996\" + PACKAGE=\"mecab\" + # Default rc file: an empty file generated alongside the dictionary. + # (An empty string would make Param::load() try to open "" and fail.) + MECAB_DEFAULT_RC=\"${CMAKE_BINARY_DIR}/mecab-dict/ipadic/mecabrc\" + # Gate for StringBuffer::operator<<(unsigned long long) in string_buffer.h. + HAVE_UNSIGNED_LONG_LONG_INT) + +# utils.cpp includes under HAVE_WINDOWS_H without a _WIN32 guard, +# so only define it on Windows (defining it on Linux breaks the build). +if(WIN32) + list(APPEND MECAB_DEFS HAVE_WINDOWS_H) +endif() + +if(NOT WIN32) + # The defines upstream's autoconf configure would detect on Linux/macOS. + # The sources guard every system header / feature behind HAVE_* checks: + # without them GCC falls back to MSVC-only paths (e.g. 'unsigned __int64') + # or misses the POSIX includes (, , ...) entirely. + list(APPEND MECAB_DEFS + HAVE_STDINT_H + HAVE_SYS_TYPES_H + HAVE_SYS_STAT_H + HAVE_FCNTL_H + HAVE_STRING_H + HAVE_SYS_MMAN_H + HAVE_UNISTD_H + HAVE_DIRENT_H + HAVE_MMAP + HAVE_TLS_KEYWORD) + # tagger.cpp instantiates read_write_mutex, which only compiles when + # HAVE_ATOMIC_OPS is defined - use the GCC/Clang __sync builtins. + # HAVE_ICONV: glibc provides iconv(3) in libc; mecab-dict-index needs it + # to compile the EUC-JP ipadic CSV into UTF-8. + list(APPEND MECAB_DEFS HAVE_GCC_ATOMIC_OPS HAVE_ICONV) +endif() + +add_library(mecab STATIC ${MECAB_SRCS}) +target_include_directories(mecab PUBLIC src) +target_compile_definitions(mecab PRIVATE ${MECAB_DEFS}) +# mecab.h declares MECAB_DLL_EXTERN as __declspec(dllimport) on _WIN32 unless +# DLL_EXPORT is defined. We link statically, so export (not import) must be +# declared consistently in both the library and its consumers. +target_compile_definitions(mecab PUBLIC DLL_EXPORT) +if(MSVC) + # Upstream suppresses these in Makefile.msvc; the code is C++98-era. + target_compile_options(mecab PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Build-time tool that compiles dictionary CSV files into the binary dictionary +# format (sys.dic/sys.dat/sys.cha/unk.*/char.bin/matrix.bin). +add_executable(mecab-dict-index src/mecab-dict-index.cpp ${MECAB_SRCS}) +target_include_directories(mecab-dict-index PRIVATE src) +target_compile_definitions(mecab-dict-index PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-dict-index PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() + +# Compile the ipadic dictionary (EUC-JP CSV) into a UTF-8 binary dictionary. +# Primary source: a prebuilt dictionary bundled at prebuilt-ipadic/ (generated +# by mecab-dict-index on a little-endian host; x86_64/aarch64 are +# little-endian, so it works offline on all mainstream platforms). +# Fallback: download the 54MB CSV from GitHub and compile it at build time +# (needed on big-endian hosts, or when the prebuilt dir is absent). +# ipadic has no standalone repo; it lives in taku910/mecab's mecab-ipadic/ +# subdirectory, so we download the mecab archive and keep that subtree. +set(MECAB_IPADIC_DIR ${CMAKE_BINARY_DIR}/mecab-ipadic-src) +set(MECAB_PREBUILT_IPADIC_DIR ${CMAKE_CURRENT_LIST_DIR}/prebuilt-ipadic) +if(EXISTS ${MECAB_PREBUILT_IPADIC_DIR}/dicrc) + set(MECAB_USE_PREBUILT_IPADIC TRUE) + message(STATUS "Using bundled prebuilt mecab-ipadic dictionary (offline mode)") +else() + if(NOT EXISTS ${MECAB_IPADIC_DIR}/dicrc) + set(MECAB_IPADIC_ZIP ${CMAKE_BINARY_DIR}/mecab-ipadic.zip) + set(MECAB_IPADIC_EXTRACT ${CMAKE_BINARY_DIR}/mecab-ipadic-extract) + message(STATUS "Downloading mecab-ipadic dictionary sources (54MB CSV)...") + file(DOWNLOAD https://github.com/taku910/mecab/archive/refs/heads/master.zip + ${MECAB_IPADIC_ZIP} STATUS MECAB_DL_STATUS) + list(GET MECAB_DL_STATUS 0 MECAB_DL_CODE) + if(NOT MECAB_DL_CODE EQUAL 0) + list(GET MECAB_DL_STATUS 1 MECAB_DL_ERROR) + message(FATAL_ERROR "Failed to download mecab-ipadic: ${MECAB_DL_ERROR} " + "(no bundled prebuilt-ipadic/ found either)") + endif() + file(MAKE_DIRECTORY ${MECAB_IPADIC_EXTRACT}) + file(ARCHIVE_EXTRACT INPUT ${MECAB_IPADIC_ZIP} DESTINATION ${MECAB_IPADIC_EXTRACT}) + set(MECAB_IPADIC_SUBDIR ${MECAB_IPADIC_EXTRACT}/mecab-master/mecab-ipadic) + if(NOT IS_DIRECTORY "${MECAB_IPADIC_SUBDIR}") + message(FATAL_ERROR "Unexpected mecab archive layout (mecab-ipadic dir missing)") + endif() + file(RENAME ${MECAB_IPADIC_SUBDIR} ${MECAB_IPADIC_DIR}) + file(REMOVE_RECURSE ${MECAB_IPADIC_EXTRACT}) + file(REMOVE ${MECAB_IPADIC_ZIP}) + endif() +endif() +set(MECAB_DICT_OUT_DIR ${CMAKE_BINARY_DIR}/mecab-dict/ipadic) +set(MECAB_DICT_OUTPUTS + ${MECAB_DICT_OUT_DIR}/sys.dic + ${MECAB_DICT_OUT_DIR}/unk.dic + ${MECAB_DICT_OUT_DIR}/char.bin + ${MECAB_DICT_OUT_DIR}/matrix.bin + ${MECAB_DICT_OUT_DIR}/dicrc + ${MECAB_DICT_OUT_DIR}/mecabrc) + +if(MECAB_USE_PREBUILT_IPADIC) + add_custom_command( + OUTPUT ${MECAB_DICT_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${MECAB_PREBUILT_IPADIC_DIR} ${MECAB_DICT_OUT_DIR} + COMMENT "Copying bundled prebuilt ipadic dictionary (little-endian)") +else() + add_custom_command( + OUTPUT ${MECAB_DICT_OUTPUTS} + COMMAND ${CMAKE_COMMAND} -E make_directory ${MECAB_DICT_OUT_DIR} + COMMAND $ -f euc-jp -t utf-8 + -d ${MECAB_IPADIC_DIR} -o ${MECAB_DICT_OUT_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${MECAB_IPADIC_DIR}/dicrc ${MECAB_DICT_OUT_DIR}/dicrc + COMMAND ${CMAKE_COMMAND} -E touch ${MECAB_DICT_OUT_DIR}/mecabrc + DEPENDS mecab-dict-index + COMMENT "Compiling ipadic dictionary (EUC-JP -> UTF-8)") +endif() + +add_custom_target(mecab-ipadic-dict + DEPENDS ${MECAB_DICT_OUT_DIR}/sys.dic) + +# CLI for ad-hoc tokenization checks during development. +add_executable(mecab-cli src/mecab.cpp ${MECAB_SRCS}) +target_include_directories(mecab-cli PRIVATE src) +target_compile_definitions(mecab-cli PRIVATE ${MECAB_DEFS}) +if(MSVC) + target_compile_options(mecab-cli PRIVATE /wd4800 /wd4305 /wd4244 /wd4996) +endif() diff --git a/fts/third_party/mecab/LICENSE b/fts/third_party/mecab/LICENSE new file mode 100644 index 00000000..71d7d805 --- /dev/null +++ b/fts/third_party/mecab/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2001-2008, Taku Kudo +Copyright (c) 2004-2008, Nippon Telegraph and Telephone Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are +permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + + * Neither the name of the Nippon Telegraph and Telegraph Corporation + nor the names of its contributors may be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/fts/third_party/mecab/src/Makefile.am b/fts/third_party/mecab/src/Makefile.am new file mode 100644 index 00000000..0b9342fb --- /dev/null +++ b/fts/third_party/mecab/src/Makefile.am @@ -0,0 +1,41 @@ +AUTOMAKE_OPTIONS = no-dependencies +lib_LTLIBRARIES = libmecab.la +EXTRA_DIST = Makefile.msvc.in make.bat +pkglibexecdir = ${libexecdir}/mecab +INCLUDES = -DDIC_VERSION=$(DIC_VERSION) $(MECAB_NO_TLS) $(MECAB_USE_UTF8_ONLY) -DMECAB_DEFAULT_RC="\"$(MECAB_DEFAULT_RC)\"" +libmecab_la_LDFLAGS = -no-undefined -version-info $(LTVERSION) +libmecab_la_SOURCES = viterbi.cpp tagger.cpp utils.cpp utils.h eval.cpp iconv_utils.cpp iconv_utils.h \ + dictionary_rewriter.h dictionary_rewriter.cpp dictionary_generator.cpp \ + dictionary_compiler.cpp context_id.h context_id.cpp \ + winmain.h thread.h connector.cpp nbest_generator.h nbest_generator.cpp connector.h \ + writer.h writer.cpp mmap.h ucs.h \ + string_buffer.h string_buffer.cpp \ + tokenizer.h stream_wrapper.h common.h darts.h char_property.h ucstable.h \ + freelist.h viterbi.h param.cpp tokenizer.cpp \ + ucstable.h char_property.cpp dictionary.h scoped_ptr.h \ + param.h mecab.h dictionary.cpp \ + feature_index.cpp feature_index.h lbfgs.cpp \ + lbfgs.h learner_tagger.cpp learner_tagger.h learner.cpp \ + learner_node.h libmecab.cpp + +include_HEADERS = mecab.h +bin_PROGRAMS = mecab +pkglibexec_PROGRAMS = mecab-dict-index mecab-dict-gen mecab-cost-train mecab-system-eval mecab-test-gen + +mecab_dict_index_SOURCES = mecab-dict-index.cpp +mecab_dict_index_LDADD = libmecab.la + +mecab_dict_gen_SOURCES = mecab-dict-gen.cpp +mecab_dict_gen_LDADD = libmecab.la + +mecab_system_eval_SOURCES = mecab-system-eval.cpp +mecab_system_eval_LDADD = libmecab.la + +mecab_cost_train_SOURCES = mecab-cost-train.cpp +mecab_cost_train_LDADD = libmecab.la + +mecab_test_gen_SOURCES = mecab-test-gen.cpp +mecab_test_gen_LDADD = libmecab.la + +mecab_SOURCES = mecab.cpp +mecab_LDADD = libmecab.la diff --git a/fts/third_party/mecab/src/Makefile.msvc.in b/fts/third_party/mecab/src/Makefile.msvc.in new file mode 100644 index 00000000..eec583df --- /dev/null +++ b/fts/third_party/mecab/src/Makefile.msvc.in @@ -0,0 +1,53 @@ +CC = cl.exe +CXXC = cl.exe +LINK=link.exe + +CFLAGS = /EHsc /O2 /GL /GA /Ob2 /nologo /W3 /MT /Zi /wd4800 /wd4305 /wd4244 +LDFLAGS = /nologo /OPT:REF /OPT:ICF /LTCG /NXCOMPAT /DYNAMICBASE /MACHINE:X86 ADVAPI32.LIB +DEFS = -D_CRT_SECURE_NO_DEPRECATE -DMECAB_USE_THREAD \ + -DDLL_EXPORT -DHAVE_GETENV -DHAVE_WINDOWS_H -DDIC_VERSION=@DIC_VERSION@ \ + -DVERSION="\"@VERSION@\"" -DPACKAGE="\"mecab\"" \ + -DUNICODE -D_UNICODE \ + -DMECAB_DEFAULT_RC="\"c:\\Program Files\\mecab\\etc\\mecabrc\"" +INC = -I. -I.. +DEL = del + +OBJ = feature_index.obj param.obj learner.obj string_buffer.obj \ + char_property.obj learner_tagger.obj tagger.obj \ + connector.obj tokenizer.obj \ + context_id.obj dictionary.obj utils.obj \ + dictionary_compiler.obj viterbi.obj \ + dictionary_generator.obj writer.obj iconv_utils.obj \ + dictionary_rewriter.obj lbfgs.obj eval.obj nbest_generator.obj + +.c.obj: + $(CC) $(CFLAGS) $(INC) $(DEFS) -c $< + +.cpp.obj: + $(CC) $(CFLAGS) $(INC) $(DEFS) -c $< + +all: libmecab mecab mecab-dict-index mecab-dict-gen mecab-cost-train mecab-system-eval mecab-test-gen + +mecab: $(OBJ) mecab.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab.obj libmecab.lib + +mecab-dict-index: $(OBJ) mecab-dict-index.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-dict-index.obj libmecab.lib + +mecab-dict-gen: $(OBJ) mecab-dict-gen.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-dict-gen.obj libmecab.lib + +mecab-cost-train: $(OBJ) mecab-cost-train.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-cost-train.obj libmecab.lib + +mecab-system-eval: $(OBJ) mecab-system-eval.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-system-eval.obj libmecab.lib + +mecab-test-gen: mecab-test-gen.obj + $(LINK) $(LDFLAGS) /out:$@.exe mecab-test-gen.obj libmecab.lib + +libmecab: $(OBJ) libmecab.obj + $(LINK) $(LDFLAGS) /out:$@.dll $(OBJ) libmecab.obj /dll + +clean: + $(DEL) *.exe *.obj *.dll *.a *.lib *.o *.exp *.def diff --git a/fts/third_party/mecab/src/char_property.cpp b/fts/third_party/mecab/src/char_property.cpp new file mode 100644 index 00000000..1029a11a --- /dev/null +++ b/fts/third_party/mecab/src/char_property.cpp @@ -0,0 +1,279 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include +#include "char_property.h" +#include "common.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +namespace { +struct Range { + int low; + int high; + std::vector c; +}; + +int atohex(const char *s) { + int n = 0; + + CHECK_DIE(std::strlen(s) >= 3 + && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) + << "no hex value: " << s; + + const char *p = s; + s += 2; + while (*s) { + int r = 0; + if (*s >= '0' && *s <= '9') + r = *s - '0'; + else if (*s >= 'A' && *s <= 'F') + r = *s - 'A' + 10; + else if (*s >= 'a' && *s <= 'f') + r = *s - 'a' + 10; + else + CHECK_DIE(false) << "no hex value: " << p; + + n = 16 * n + r; + s++; + } + + return n; +} + +CharInfo encode(const std::vector &c, + std::map *category) { + CHECK_DIE(c.size()) << "category size is empty"; + + std::map::const_iterator it = category->find(c[0]); + CHECK_DIE(it != category->end()) + << "category [" << c[0] << "] is undefined"; + + CharInfo base = it->second; + for (size_t i = 0; i < c.size(); ++i) { + std::map::const_iterator it = + category->find(c[i]); + CHECK_DIE(it != category->end()) + << "category [" << c[i] << "] is undefined"; + base.type += (1 << it->second.default_type); + } + + return base; +} +} + +bool CharProperty::open(const Param ¶m) { + const std::string prefix = param.get("dicdir"); + const std::string filename = create_filename(prefix, CHAR_PROPERTY_FILE); + return open(filename.c_str()); +} + +bool CharProperty::open(const char *filename) { + std::ostringstream error; + CHECK_FALSE(cmmap_->open(filename, "r")); + + const char *ptr = cmmap_->begin(); + unsigned int csize; + read_static(&ptr, csize); + + size_t fsize = sizeof(unsigned int) + + (32 * csize) + sizeof(unsigned int) * 0xffff; + + CHECK_FALSE(fsize == cmmap_->size()) + << "invalid file size: " << filename; + + clist_.clear(); + for (unsigned int i = 0; i < csize; ++i) { + const char *s = read_ptr(&ptr, 32); + clist_.push_back(s); + } + + map_ = reinterpret_cast(ptr); + + return true; +} + +void CharProperty::close() { + cmmap_->close(); +} + +size_t CharProperty::size() const { return clist_.size(); } + +const char *CharProperty::name(size_t i) const { + return const_cast(clist_[i]); +} + +// this function must be rewritten. +void CharProperty::set_charset(const char *ct) { + charset_ = decode_charset(ct); +} + +int CharProperty::id(const char *key) const { + for (int i = 0; i < static_cast(clist_.size()); ++i) { + if (std::strcmp(key, clist_[i]) == 0) { + return i; + } + } + return -1; +} + +bool CharProperty::compile(const char *cfile, + const char *ufile, + const char *ofile) { + scoped_fixed_array line; + scoped_fixed_array col; + size_t id = 0; + std::vector range; + std::map category; + std::vector category_ary; + std::ifstream ifs(WPATH(cfile)); + std::istringstream iss(CHAR_PROPERTY_DEF_DEFAULT); + std::istream *is = &ifs; + + if (!ifs) { + std::cerr << cfile + << " is not found. minimum setting is used" << std::endl; + is = &iss; + } + + while (is->getline(line.get(), line.size())) { + if (std::strlen(line.get()) == 0 || line[0] == '#') { + continue; + } + const size_t size = tokenize2(line.get(), "\t ", col.get(), col.size()); + CHECK_DIE(size >= 2) << "format error: " << line.get(); + + // 0xFFFF..0xFFFF hoge hoge hgoe # + if (std::strncmp(col[0], "0x", 2) == 0) { + std::string low = col[0]; + std::string high; + size_t pos = low.find(".."); + + if (pos != std::string::npos) { + high = low.substr(pos + 2, low.size() - pos - 2); + low = low.substr(0, pos); + } else { + high = low; + } + + Range r; + r.low = atohex(low.c_str()); + r.high = atohex(high.c_str()); + + CHECK_DIE(r.low >= 0 && r.low < 0xffff && + r.high >= 0 && r.high < 0xffff && + r.low <= r.high) + << "range error: low=" << r.low << " high=" << r.high; + + for (size_t i = 1; i < size; ++i) { + if (col[i][0] == '#') { + break; // skip comments + } + CHECK_DIE(category.find(std::string(col[i])) != category.end()) + << "category [" << col[i] << "] is undefined"; + r.c.push_back(col[i]); + } + range.push_back(r); + } else { + CHECK_DIE(size >= 4) << "format error: " << line.get(); + + std::string key = col[0]; + CHECK_DIE(category.find(key) == category.end()) + << "category " << key << " is already defined"; + + CharInfo c; + std::memset(&c, 0, sizeof(c)); + c.invoke = std::atoi(col[1]); + c.group = std::atoi(col[2]); + c.length = std::atoi(col[3]); + c.default_type = id++; + + category.insert(std::pair(key, c)); + category_ary.push_back(key); + } + } + + CHECK_DIE(category.size() < 18) << "too many categories(>= 18)"; + + CHECK_DIE(category.find("DEFAULT") != category.end()) + << "category [DEFAULT] is undefined"; + + CHECK_DIE(category.find("SPACE") != category.end()) + << "category [SPACE] is undefined"; + + std::istringstream iss2(UNK_DEF_DEFAULT); + std::ifstream ifs2(WPATH(ufile)); + std::istream *is2 = &ifs2; + + if (!ifs2) { + std::cerr << ufile + << " is not found. minimum setting is used." << std::endl; + is2 = &iss2; + } + + std::set unk; + while (is2->getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col.get(), 2); + CHECK_DIE(n >= 1) << "format error: " << line.get(); + const std::string key = col[0]; + CHECK_DIE(category.find(key) != category.end()) + << "category [" << key << "] is undefined in " << cfile; + unk.insert(key); + } + + for (std::map::const_iterator it = category.begin(); + it != category.end(); + ++it) { + CHECK_DIE(unk.find(it->first) != unk.end()) + << "category [" << it->first << "] is undefined in " << ufile; + } + + std::vector table(0xffff); + { + std::vector tmp; + tmp.push_back("DEFAULT"); + const CharInfo c = encode(tmp, &category); + std::fill(table.begin(), table.end(), c); + } + + for (std::vector::const_iterator it = range.begin(); + it != range.end(); + ++it) { + const CharInfo c = encode(it->c, &category); + for (int i = it->low; i <= it->high; ++i) { + table[i] = c; + } + } + + // output binary table + { + std::ofstream ofs(WPATH(ofile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << ofile; + + unsigned int size = static_cast(category.size()); + ofs.write(reinterpret_cast(&size), sizeof(size)); + for (std::vector::const_iterator it = category_ary.begin(); + it != category_ary.end(); + ++it) { + char buf[32]; + std::fill(buf, buf + sizeof(buf), '\0'); + std::strncpy(buf, it->c_str(), sizeof(buf) - 1); + ofs.write(reinterpret_cast(buf), sizeof(buf)); + } + ofs.write(reinterpret_cast(&table[0]), + sizeof(CharInfo) * table.size()); + ofs.close(); + } + + return true; +} +} diff --git a/fts/third_party/mecab/src/char_property.h b/fts/third_party/mecab/src/char_property.h new file mode 100644 index 00000000..9c904ba0 --- /dev/null +++ b/fts/third_party/mecab/src/char_property.h @@ -0,0 +1,92 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CHARACTER_CATEGORY_H_ +#define MECAB_CHARACTER_CATEGORY_H_ + +#include "mmap.h" +#include "scoped_ptr.h" +#include "ucs.h" +#include "utils.h" + +namespace MeCab { +class Param; + +struct CharInfo { + unsigned int type: 18; + unsigned int default_type: 8; + unsigned int length: 4; + unsigned int group: 1; + unsigned int invoke: 1; + CharInfo() : type(0), default_type(0), length(0), group(0), invoke(0) {} + bool isKindOf(CharInfo c) const { return type & c.type; } +}; + +class CharProperty { + public: + bool open(const Param &); + bool open(const char*); + void close(); + size_t size() const; + void set_charset(const char *charset); + int id(const char *) const; + const char *name(size_t i) const; + const char *what() { return what_.str(); } + + inline const char *seekToOtherType(const char *begin, const char *end, + CharInfo c, CharInfo *fail, + size_t *mblen, size_t *clen) const { + const char *p = begin; + *clen = 0; + while (p != end && c.isKindOf(*fail = getCharInfo(p, end, mblen))) { + p += *mblen; + ++(*clen); + c = *fail; + } + return p; + } + + inline CharInfo getCharInfo(const char *begin, + const char *end, + size_t *mblen) const { + unsigned short int t = 0; +#ifndef MECAB_USE_UTF8_ONLY + switch (charset_) { + case EUC_JP: t = euc_to_ucs2(begin, end, mblen); break; + case CP932: t = cp932_to_ucs2(begin, end, mblen); break; + case UTF8: t = utf8_to_ucs2(begin, end, mblen); break; + case UTF16: t = utf16_to_ucs2(begin, end, mblen); break; + case UTF16LE: t = utf16le_to_ucs2(begin, end, mblen); break; + case UTF16BE: t = utf16be_to_ucs2(begin, end, mblen); break; + case ASCII: t = ascii_to_ucs2(begin, end, mblen); break; + default: t = utf8_to_ucs2(begin, end, mblen); break; + } +#else + switch (charset_) { + case UTF8: t = utf8_to_ucs2(begin, end, mblen); break; + case UTF16: t = utf16_to_ucs2(begin, end, mblen); break; + case UTF16LE: t = utf16le_to_ucs2(begin, end, mblen); break; + case UTF16BE: t = utf16be_to_ucs2(begin, end, mblen); break; + default: t = utf8_to_ucs2(begin, end, mblen); break; + } +#endif + return map_[t]; + } + + inline CharInfo getCharInfo(size_t id) const { return map_[id]; } + + static bool compile(const char *, const char *, const char*); + + CharProperty(): cmmap_(new Mmap), map_(0), charset_(0) {} + virtual ~CharProperty() { this->close(); } + + private: + scoped_ptr > cmmap_; + std::vector clist_; + const CharInfo *map_; + int charset_; + whatlog what_; +}; +} +#endif // MECAB_CHARACTER_CATEGORY_H_ diff --git a/fts/third_party/mecab/src/common.h b/fts/third_party/mecab/src/common.h new file mode 100644 index 00000000..2e452a76 --- /dev/null +++ b/fts/third_party/mecab/src/common.h @@ -0,0 +1,143 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_COMMON_H_ +#define MECAB_COMMON_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __CYGWIN__ +#define _GLIBCXX_EXPORT_TEMPLATE +#endif + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#if defined(_MSC_VER) || defined(__CYGWIN__) +#define NOMINMAX +#define snprintf _snprintf +#endif + +#define COPYRIGHT "MeCab: Yet Another Part-of-Speech and Morphological Analyzer\n\ +\nCopyright(C) 2001-2012 Taku Kudo \nCopyright(C) 2004-2008 Nippon Telegraph and Telephone Corporation\n" + +#define SYS_DIC_FILE "sys.dic" +#define UNK_DEF_FILE "unk.def" +#define UNK_DIC_FILE "unk.dic" +#define MATRIX_DEF_FILE "matrix.def" +#define MATRIX_FILE "matrix.bin" +#define CHAR_PROPERTY_DEF_FILE "char.def" +#define CHAR_PROPERTY_FILE "char.bin" +#define FEATURE_FILE "feature.def" +#define REWRITE_FILE "rewrite.def" +#define LEFT_ID_FILE "left-id.def" +#define RIGHT_ID_FILE "right-id.def" +#define POS_ID_FILE "pos-id.def" +#define MODEL_DEF_FILE "model.def" +#define MODEL_FILE "model.bin" +#define DICRC "dicrc" +#define BOS_KEY "BOS/EOS" + +#define DEFAULT_MAX_GROUPING_SIZE 24 + +#define CHAR_PROPERTY_DEF_DEFAULT "DEFAULT 1 0 0\nSPACE 0 1 0\n0x0020 SPACE\n" +#define UNK_DEF_DEFAULT "DEFAULT,0,0,0,*\nSPACE,0,0,0,*\n" +#define MATRIX_DEF_DEFAULT "1 1\n0 0 0\n" + +#ifdef MECAB_USE_UTF8_ONLY +#define MECAB_DEFAULT_CHARSET "UTF-8" +#endif + +#ifndef MECAB_DEFAULT_CHARSET +#if defined(_WIN32) && !defined(__CYGWIN__) +#define MECAB_DEFAULT_CHARSET "SHIFT-JIS" +#else +#define MECAB_DEFAULT_CHARSET "EUC-JP" +#endif +#endif + +#define NBEST_MAX 512 +#define NODE_FREELIST_SIZE 512 +#define PATH_FREELIST_SIZE 2048 +#define MIN_INPUT_BUFFER_SIZE 8192 +#define MAX_INPUT_BUFFER_SIZE (8192*640) +#define BUF_SIZE 8192 + +#ifndef EXIT_FAILURE +#define EXIT_FAILURE 1 +#endif + +#ifndef EXIT_SUCCESS +#define EXIT_SUCCESS 0 +#endif + +#ifdef _WIN32 +#ifdef __GNUC__ +#define WPATH_FORCE(path) (MeCab::Utf8ToWide(path).c_str()) +#define WPATH(path) (path) +#else +// Upstream bug: the MSVC branch only defines WPATH, but call sites use +// WPATH_FORCE directly (e.g. mmap.h). Define it like the MinGW branch. +#define WPATH_FORCE(path) (MeCab::Utf8ToWide(path).c_str()) +#define WPATH(path) WPATH_FORCE(path) +#endif +#else +#define WPATH_FORCE(path) (path) +#define WPATH(path) (path) +#endif + +namespace MeCab { +class die { + public: + die() {} + ~die() { + std::cerr << std::endl; + exit(-1); + } + int operator&(std::ostream&) { return 0; } +}; + +struct whatlog { + std::ostringstream stream_; + std::string str_; + const char *str() { + str_ = stream_.str(); + return str_.c_str(); + } +}; + +class wlog { + public: + wlog(whatlog *what) : what_(what) { + what_->stream_.clear(); + } + bool operator&(std::ostream &) { + return false; + } + private: + whatlog *what_; +}; +} // MeCab + +#define WHAT what_.stream_ + +#define CHECK_FALSE(condition) \ + if (condition) {} else return \ + wlog(&what_) & what_.stream_ << \ + __FILE__ << "(" << __LINE__ << ") [" << #condition << "] " + +#define CHECK_DIE(condition) \ +(condition) ? 0 : die() & std::cerr << __FILE__ << \ +"(" << __LINE__ << ") [" << #condition << "] " + +#endif // MECAB_COMMON_H_ diff --git a/fts/third_party/mecab/src/connector.cpp b/fts/third_party/mecab/src/connector.cpp new file mode 100644 index 00000000..56900221 --- /dev/null +++ b/fts/third_party/mecab/src/connector.cpp @@ -0,0 +1,113 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "connector.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +bool Connector::open(const Param ¶m) { + const std::string filename = create_filename + (param.get("dicdir"), MATRIX_FILE); + return open(filename.c_str()); +} + +bool Connector::open(const char* filename, + const char *mode) { + CHECK_FALSE(cmmap_->open(filename, mode)) + << "cannot open: " << filename; + + matrix_ = cmmap_->begin(); + + CHECK_FALSE(matrix_) << "matrix is NULL" ; + CHECK_FALSE(cmmap_->size() >= 2) + << "file size is invalid: " << filename; + + lsize_ = static_cast((*cmmap_)[0]); + rsize_ = static_cast((*cmmap_)[1]); + + CHECK_FALSE(static_cast(lsize_ * rsize_ + 2) + == cmmap_->size()) + << "file size is invalid: " << filename; + + matrix_ = cmmap_->begin() + 2; + return true; +} + +void Connector::close() { + cmmap_->close(); +} + +bool Connector::openText(const char *filename) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + WHAT << "no such file or directory: " << filename; + return false; + } + char *column[2]; + scoped_fixed_array buf; + ifs.getline(buf.get(), buf.size()); + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " << buf.get(); + lsize_ = std::atoi(column[0]); + rsize_ = std::atoi(column[1]); + return true; +} + +bool Connector::compile(const char *ifile, const char *ofile) { + std::ifstream ifs(WPATH(ifile)); + std::istringstream iss(MATRIX_DEF_DEFAULT); + std::istream *is = &ifs; + + if (!ifs) { + std::cerr << ifile + << " is not found. minimum setting is used." << std::endl; + is = &iss; + } + + + char *column[4]; + scoped_fixed_array buf; + + is->getline(buf.get(), buf.size()); + + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " << buf.get(); + + const unsigned short lsize = std::atoi(column[0]); + const unsigned short rsize = std::atoi(column[1]); + std::vector matrix(lsize * rsize); + std::fill(matrix.begin(), matrix.end(), 0); + + std::cout << "reading " << ifile << " ... " + << lsize << "x" << rsize << std::endl; + + while (is->getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 3) == 3) + << "format error: " << buf.get(); + const size_t l = std::atoi(column[0]); + const size_t r = std::atoi(column[1]); + const int c = std::atoi(column[2]); + CHECK_DIE(l < lsize && r < rsize) << "index values are out of range"; + progress_bar("emitting matrix ", l + 1, lsize); + matrix[(l + lsize * r)] = static_cast(c); + } + + std::ofstream ofs(WPATH(ofile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << ofile; + ofs.write(reinterpret_cast(&lsize), sizeof(unsigned short)); + ofs.write(reinterpret_cast(&rsize), sizeof(unsigned short)); + ofs.write(reinterpret_cast(&matrix[0]), + lsize * rsize * sizeof(short)); + ofs.close(); + + return true; +} +} diff --git a/fts/third_party/mecab/src/connector.h b/fts/third_party/mecab/src/connector.h new file mode 100644 index 00000000..8a687170 --- /dev/null +++ b/fts/third_party/mecab/src/connector.h @@ -0,0 +1,67 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CONNECTOR_H_ +#define MECAB_CONNECTOR_H_ + +#include "mecab.h" +#include "mmap.h" +#include "common.h" +#include "scoped_ptr.h" + +namespace MeCab { +class Param; + +class Connector { + private: + scoped_ptr > cmmap_; + short *matrix_; + unsigned short lsize_; + unsigned short rsize_; + whatlog what_; + + public: + + bool open(const Param ¶m); + void close(); + void clear() {} + + const char *what() { return what_.str(); } + + size_t left_size() const { return static_cast(lsize_); } + size_t right_size() const { return static_cast(rsize_); } + + void set_left_size(size_t lsize) { lsize_ = lsize; } + void set_right_size(size_t rsize) { rsize_ = rsize; } + + inline int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const { + return matrix_[rcAttr + lsize_ * lcAttr]; + } + + inline int cost(const Node *lNode, const Node *rNode) const { + return matrix_[lNode->rcAttr + lsize_ * rNode->lcAttr] + rNode->wcost; + } + + // access to raw matrix + short *mutable_matrix() { return &matrix_[0]; } + const short *matrix() const { return &matrix_[0]; } + + bool openText(const char *filename); + bool open(const char *filename, const char *mode = "r"); + + bool is_valid(size_t lid, size_t rid) const { + return (lid >= 0 && lid < rsize_ && rid >= 0 && rid < lsize_); + } + + static bool compile(const char *, const char *); + + explicit Connector(): + cmmap_(new Mmap), matrix_(0), lsize_(0), rsize_(0) {} + + virtual ~Connector() { this->close(); } +}; +} +#endif // MECAB_CONNECTOR_H_ diff --git a/fts/third_party/mecab/src/context_id.cpp b/fts/third_party/mecab/src/context_id.cpp new file mode 100644 index 00000000..eeff28c4 --- /dev/null +++ b/fts/third_party/mecab/src/context_id.cpp @@ -0,0 +1,107 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include "context_id.h" +#include "iconv_utils.h" +#include "utils.h" + +namespace { + +using namespace MeCab; + +bool open_map(const char *filename, + std::map *cmap, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + cmap->clear(); + char *col[2]; + std::string line; + while (std::getline(ifs, line)) { + CHECK_DIE(2 == tokenize2(const_cast(line.c_str()), + " \t", col, 2)) + << "format error: " << line; + std::string pos = col[1]; + if (iconv) { + iconv->convert(&pos); + } + cmap->insert(std::pair + (pos, std::atoi(col[0]))); + } + return true; +} + +bool build(std::map *cmap, + const std::string &bos) { + int id = 1; // for BOS/EOS + for (std::map::iterator it = cmap->begin(); + it != cmap->end(); + ++it) it->second = id++; + cmap->insert(std::make_pair(bos, 0)); + return true; +} + +bool save(const char* filename, + std::map *cmap) { + std::ofstream ofs(WPATH(filename)); + CHECK_DIE(ofs) << "permission denied: " << filename; + for (std::map::const_iterator it = cmap->begin(); + it != cmap->end(); ++it) { + ofs << it->second << " " << it->first << std::endl; + } + return true; +} +} + +namespace MeCab { + +void ContextID::clear() { + left_.clear(); + right_.clear(); + left_bos_.clear(); + right_bos_.clear(); +} + +void ContextID::add(const char *l, const char *r) { + left_.insert(std::make_pair(std::string(l), 1)); + right_.insert(std::make_pair(std::string(r), 1)); +} + +void ContextID::addBOS(const char *l, const char *r) { + left_bos_ = l; + right_bos_ = r; +} + +bool ContextID::save(const char* lfile, + const char* rfile) { + return (::save(lfile, &left_) && ::save(rfile, &right_)); +} + +bool ContextID::open(const char *lfile, + const char *rfile, + Iconv *iconv) { + return (::open_map(lfile, &left_, iconv) && + ::open_map(rfile, &right_, iconv)); +} + +bool ContextID::build() { + return (::build(&left_, left_bos_) && ::build(&right_, right_bos_)); +} + +int ContextID::lid(const char *l) const { + std::map::const_iterator it = left_.find(l); + CHECK_DIE(it != left_.end()) + << "cannot find LEFT-ID for " << l; + return it->second; +} + +int ContextID::rid(const char *r) const { + std::map::const_iterator it = right_.find(r); + CHECK_DIE(it != right_.end()) + << "cannot find RIGHT-ID for " << r; + return it->second; +} +} diff --git a/fts/third_party/mecab/src/context_id.h b/fts/third_party/mecab/src/context_id.h new file mode 100644 index 00000000..28622610 --- /dev/null +++ b/fts/third_party/mecab/src/context_id.h @@ -0,0 +1,50 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_CONTEXT_ID_H +#define MECAB_CONTEXT_ID_H + +#include +#include +#include + +namespace MeCab { + +class Param; +class Iconv; + +class ContextID { + private: + std::map left_; + std::map right_; + std::string left_bos_; + std::string right_bos_; + + public: + void clear(); + void add(const char *l, const char *r); + void addBOS(const char *l, const char *r); + bool save(const char* lfile, + const char* rfile); + bool build(); + bool open(const char *lfile, + const char *rfile, + Iconv *iconv = 0); + int lid(const char *l) const; + int rid(const char *r) const; + + size_t left_size() const { return left_.size(); } + size_t right_size() const { return right_.size(); } + + const std::map& left_ids() const { return left_; } + const std::map& right_ids() const { return right_; } + + bool is_valid(size_t lid, size_t rid) { + return (lid >= 0 && lid < left_size() && + rid >= 0 && rid < right_size()); + } +}; +} +#endif diff --git a/fts/third_party/mecab/src/darts.h b/fts/third_party/mecab/src/darts.h new file mode 100644 index 00000000..49bd3edf --- /dev/null +++ b/fts/third_party/mecab/src/darts.h @@ -0,0 +1,518 @@ +/* + Darts -- Double-ARray Trie System + + + Copyright(C) 2001-2007 Taku Kudo +*/ +#ifndef DARTS_H_ +#define DARTS_H_ + +#define DARTS_VERSION "0.31" +#include +#include +#include + +#ifdef HAVE_ZLIB_H +namespace zlib { +#include +} +#define SH(p)((unsigned short)(unsigned char)((p)[0]) | ((unsigned short)(unsigned char)((p)[1]) << 8)) +#define LG(p)((unsigned long)(SH(p)) |((unsigned long)(SH((p)+2)) << 16)) +#endif + +namespace MeCab { + +namespace Darts { + +template inline T _max(T x, T y) { return(x > y) ? x : y; } +template inline T* _resize(T* ptr, size_t n, size_t l, T v) { + T *tmp = new T[l]; + for (size_t i = 0; i < n; ++i) tmp[i] = ptr[i]; + for (size_t i = n; i < l; ++i) tmp[i] = v; + delete [] ptr; + return tmp; +} + +template +class Length { + public: size_t operator()(const T *key) const + { size_t i; for (i = 0; key[i] != (T)0; ++i) {} return i; } +}; + +template <> class Length { + public: size_t operator()(const char *key) const + { return std::strlen(key); } +}; + +template > +class DoubleArrayImpl { + private: + + struct node_t { + array_u_type_ code; + size_t depth; + size_t left; + size_t right; + }; + + struct unit_t { + array_type_ base; + array_u_type_ check; + }; + + unit_t *array_; + unsigned char *used_; + size_t size_; + size_t alloc_size_; + node_type_ **key_; + size_t key_size_; + size_t *length_; + array_type_ *value_; + size_t progress_; + size_t next_check_pos_; + bool no_delete_; + int error_; + int (*progress_func_)(size_t, size_t); + + size_t resize(const size_t new_size) { + unit_t tmp; + tmp.base = 0; + tmp.check = 0; + array_ = _resize(array_, alloc_size_, new_size, tmp); + used_ = _resize(used_, alloc_size_, new_size, + static_cast(0)); + alloc_size_ = new_size; + return new_size; + } + + size_t fetch(const node_t &parent, std::vector &siblings) { + if (error_ < 0) return 0; + + array_u_type_ prev = 0; + + for (size_t i = parent.left; i < parent.right; ++i) { + if ((length_ ? length_[i] : length_func_()(key_[i])) < parent.depth) + continue; + + const node_u_type_ *tmp = reinterpret_cast(key_[i]); + + array_u_type_ cur = 0; + if ((length_ ? length_[i] : length_func_()(key_[i])) != parent.depth) + cur = (array_u_type_)tmp[parent.depth] + 1; + + if (prev > cur) { + error_ = -3; + return 0; + } + + if (cur != prev || siblings.empty()) { + node_t tmp_node; + tmp_node.depth = parent.depth + 1; + tmp_node.code = cur; + tmp_node.left = i; + if (!siblings.empty()) siblings[siblings.size()-1].right = i; + + siblings.push_back(tmp_node); + } + + prev = cur; + } + + if (!siblings.empty()) + siblings[siblings.size()-1].right = parent.right; + + return siblings.size(); + } + + size_t insert(const std::vector &siblings) { + if (error_ < 0) return 0; + + size_t begin = 0; + size_t pos = _max((size_t)siblings[0].code + 1, next_check_pos_) - 1; + size_t nonzero_num = 0; + int first = 0; + + if (alloc_size_ <= pos) resize(pos + 1); + + while (true) { + next: + ++pos; + + if (alloc_size_ <= pos) resize(pos + 1); + + if (array_[pos].check) { + ++nonzero_num; + continue; + } else if (!first) { + next_check_pos_ = pos; + first = 1; + } + + begin = pos - siblings[0].code; + if (alloc_size_ <= (begin + siblings[siblings.size()-1].code)) + resize(static_cast(alloc_size_ * + _max(1.05, 1.0 * key_size_ / progress_))); + + if (used_[begin]) continue; + + for (size_t i = 1; i < siblings.size(); ++i) + if (array_[begin + siblings[i].code].check != 0) goto next; + + break; + } + + // -- Simple heuristics -- + // if the percentage of non-empty contents in check between the index + // 'next_check_pos' and 'check' is greater than some constant + // value(e.g. 0.9), + // new 'next_check_pos' index is written by 'check'. + if (1.0 * nonzero_num/(pos - next_check_pos_ + 1) >= 0.95) + next_check_pos_ = pos; + + used_[begin] = 1; + size_ = _max(size_, + begin + + static_cast(siblings[siblings.size() - 1].code + 1)); + + for (size_t i = 0; i < siblings.size(); ++i) + array_[begin + siblings[i].code].check = begin; + + for (size_t i = 0; i < siblings.size(); ++i) { + std::vector new_siblings; + + if (!fetch(siblings[i], new_siblings)) { + array_[begin + siblings[i].code].base = + value_ ? + static_cast(-value_[siblings[i].left]-1) : + static_cast(-siblings[i].left-1); + + if (value_ && (array_type_)(-value_[siblings[i].left]-1) >= 0) { + error_ = -2; + return 0; + } + + ++progress_; + if (progress_func_)(*progress_func_)(progress_, key_size_); + + } else { + size_t h = insert(new_siblings); + array_[begin + siblings[i].code].base = h; + } + } + + return begin; + } + + public: + + typedef array_type_ value_type; + typedef node_type_ key_type; + typedef array_type_ result_type; // for compatibility + + struct result_pair_type { + value_type value; + size_t length; + }; + + explicit DoubleArrayImpl(): array_(0), used_(0), + size_(0), alloc_size_(0), + no_delete_(0), error_(0) {} + ~DoubleArrayImpl() { clear(); } + + void set_result(value_type& x, value_type r, size_t) const { + x = r; + } + + void set_result(result_pair_type& x, value_type r, size_t l) const { + x.value = r; + x.length = l; + } + + void set_array(void *ptr, size_t size = 0) { + clear(); + array_ = reinterpret_cast(ptr); + no_delete_ = true; + size_ = size; + } + + const void *array() const { + return const_cast(reinterpret_cast(array_)); + } + + void clear() { + if (!no_delete_) + delete [] array_; + delete [] used_; + array_ = 0; + used_ = 0; + alloc_size_ = 0; + size_ = 0; + no_delete_ = false; + } + + size_t unit_size() const { return sizeof(unit_t); } + size_t size() const { return size_; } + size_t total_size() const { return size_ * sizeof(unit_t); } + + size_t nonzero_size() const { + size_t result = 0; + for (size_t i = 0; i < size_; ++i) + if (array_[i].check) ++result; + return result; + } + + int build(size_t key_size, + key_type **key, + size_t *length = 0, + value_type *value = 0, + int (*progress_func)(size_t, size_t) = 0) { + if (!key_size || !key) return 0; + + progress_func_ = progress_func; + key_ = key; + length_ = length; + key_size_ = key_size; + value_ = value; + progress_ = 0; + + resize(8192); + + array_[0].base = 1; + next_check_pos_ = 0; + + node_t root_node; + root_node.left = 0; + root_node.right = key_size; + root_node.depth = 0; + + std::vector siblings; + fetch(root_node, siblings); + insert(siblings); + + size_ += (1 << 8 * sizeof(key_type)) + 1; + if (size_ >= alloc_size_) resize(size_); + + delete [] used_; + used_ = 0; + + return error_; + } + + int open(const char *file, + const char *mode = "rb", + size_t offset = 0, + size_t size = 0) { + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + if (std::fseek(fp, offset, SEEK_SET) != 0) return -1; + + if (!size) { + if (std::fseek(fp, 0L, SEEK_END) != 0) return -1; + size = std::ftell(fp); + if (std::fseek(fp, offset, SEEK_SET) != 0) return -1; + } + + clear(); + + size_ = size; + size_ /= sizeof(unit_t); + array_ = new unit_t[size_]; + if (size_ != std::fread(reinterpret_cast(array_), + sizeof(unit_t), size_, fp)) return -1; + std::fclose(fp); + + return 0; + } + + int save(const char *file, + const char *mode = "wb", + size_t offset = 0) { + if (!size_) return -1; + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + if (size_ != std::fwrite(reinterpret_cast(array_), + sizeof(unit_t), size_, fp)) + return -1; + std::fclose(fp); + return 0; + } + +#ifdef HAVE_ZLIB_H + int gzopen(const char *file, + const char *mode = "rb", + size_t offset = 0, + size_t size = 0) { + std::FILE *fp = std::fopen(file, mode); + if (!fp) return -1; + clear(); + + size_ = size; + if (!size_) { + if (-1L != static_cast(std::fseek(fp, -8, SEEK_END))) { + char buf[8]; + if (std::fread(static_cast(buf), + 1, 8, fp) != sizeof(buf)) { + std::fclose(fp); + return -1; + } + size_ = LG(buf+4); + size_ /= sizeof(unit_t); + } + } + std::fclose(fp); + + if (!size_) return -1; + + zlib::gzFile gzfp = zlib::gzopen(file, mode); + if (!gzfp) return -1; + array_ = new unit_t[size_]; + if (zlib::gzseek(gzfp, offset, SEEK_SET) != 0) return -1; + zlib::gzread(gzfp, reinterpret_cast(array_), + sizeof(unit_t) * size_); + zlib::gzclose(gzfp); + return 0; + } + + int gzsave(const char *file, const char *mode = "wb", + size_t offset = 0) { + zlib::gzFile gzfp = zlib::gzopen(file, mode); + if (!gzfp) return -1; + zlib::gzwrite(gzfp, reinterpret_cast(array_), + sizeof(unit_t) * size_); + zlib::gzclose(gzfp); + return 0; + } +#endif + + template + inline void exactMatchSearch(const key_type *key, + T & result, + size_t len = 0, + size_t node_pos = 0) const { + result = exactMatchSearch(key, len, node_pos); + return; + } + + template + inline T exactMatchSearch(const key_type *key, + size_t len = 0, + size_t node_pos = 0) const { + if (!len) len = length_func_()(key); + + T result; + set_result(result, -1, 0); + + array_type_ b = array_[node_pos].base; + array_u_type_ p; + + for (size_t i = 0; i < len; ++i) { + p = b +(node_u_type_)(key[i]) + 1; + if (static_cast(b) == array_[p].check) + b = array_[p].base; + else + return result; + } + + p = b; + array_type_ n = array_[p].base; + if (static_cast(b) == array_[p].check && n < 0) + set_result(result, -n-1, len); + + return result; + } + + template + size_t commonPrefixSearch(const key_type *key, + T* result, + size_t result_len, + size_t len = 0, + size_t node_pos = 0) const { + if (!len) len = length_func_()(key); + + array_type_ b = array_[node_pos].base; + size_t num = 0; + array_type_ n; + array_u_type_ p; + + for (size_t i = 0; i < len; ++i) { + p = b; // + 0; + n = array_[p].base; + if ((array_u_type_) b == array_[p].check && n < 0) { + // result[num] = -n-1; + if (num < result_len) set_result(result[num], -n-1, i); + ++num; + } + + p = b +(node_u_type_)(key[i]) + 1; + if ((array_u_type_) b == array_[p].check) + b = array_[p].base; + else + return num; + } + + p = b; + n = array_[p].base; + + if ((array_u_type_)b == array_[p].check && n < 0) { + if (num < result_len) set_result(result[num], -n-1, len); + ++num; + } + + return num; + } + + value_type traverse(const key_type *key, + size_t &node_pos, + size_t &key_pos, + size_t len = 0) const { + if (!len) len = length_func_()(key); + + array_type_ b = array_[node_pos].base; + array_u_type_ p; + + for (; key_pos < len; ++key_pos) { + p = b +(node_u_type_)(key[key_pos]) + 1; + if (static_cast(b) == array_[p].check) { + node_pos = p; + b = array_[p].base; + } else { + return -2; // no node + } + } + + p = b; + array_type_ n = array_[p].base; + if (static_cast(b) == array_[p].check && n < 0) + return -n-1; + + return -1; // found, but no value + } +}; + +#if 4 == 2 +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 4 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 4 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#define DARTS_ARRAY_SIZE_IS_DEFINED 1 +#endif + +#if 4 == 8 && !defined(DARTS_ARRAY_SIZE_IS_DEFINED) +typedef Darts::DoubleArrayImpl DoubleArray; +#endif +} +} +#endif diff --git a/fts/third_party/mecab/src/dictionary.cpp b/fts/third_party/mecab/src/dictionary.cpp new file mode 100644 index 00000000..0b9141fc --- /dev/null +++ b/fts/third_party/mecab/src/dictionary.cpp @@ -0,0 +1,535 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "connector.h" +#include "context_id.h" +#include "char_property.h" +#include "common.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "iconv_utils.h" +#include "mmap.h" +#include "param.h" +#include "scoped_ptr.h" +#include "utils.h" +#include "writer.h" + +namespace MeCab { +namespace { + +const unsigned int DictionaryMagicID = 0xef718f77u; + +int toInt(const char *str) { + if (!str || std::strlen(str) == 0) { + return INT_MAX; + } + return std::atoi(str); +} + +int calcCost(const std::string &w, const std::string &feature, + int factor, + DecoderFeatureIndex *fi, DictionaryRewriter *rewriter, + CharProperty *property) { + CHECK_DIE(fi); + CHECK_DIE(rewriter); + CHECK_DIE(property); + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + size_t mblen = 0; + const CharInfo cinfo = property->getCharInfo(w.c_str(), + w.c_str() + w.size(), + &mblen); + path.rnode->char_type = cinfo.default_type; + std::string ufeature, lfeature, rfeature; + rewriter->rewrite2(feature, &ufeature, &lfeature, &rfeature); + fi->buildUnigramFeature(&path, ufeature.c_str()); + fi->calcCost(&rnode); + return tocost(rnode.wcost, factor); +} + +int progress_bar_darts(size_t current, size_t total) { + return progress_bar("emitting double-array", current, total); +} + +// std::binary_function was removed in C++17; the functor works without it. +template +struct pair_1st_cmp { + bool operator()(const std::pair &x1, + const std::pair &x2) { + return x1.first < x2.first; + } +}; +} // namespace + +bool Dictionary::open(const char *file, const char *mode) { + close(); + filename_.assign(file); + CHECK_FALSE(dmmap_->open(file, mode)) + << "no such file or directory: " << file; + + CHECK_FALSE(dmmap_->size() >= 100) + << "dictionary file is broken: " << file; + + const char *ptr = dmmap_->begin(); + + unsigned int dsize; + unsigned int tsize; + unsigned int fsize; + unsigned int magic; + unsigned int dummy; + + read_static(&ptr, magic); + CHECK_FALSE((magic ^ DictionaryMagicID) == dmmap_->size()) + << "dictionary file is broken: " << file; + + read_static(&ptr, version_); + CHECK_FALSE(version_ == DIC_VERSION) + << "incompatible version: " << version_; + + read_static(&ptr, type_); + read_static(&ptr, lexsize_); + read_static(&ptr, lsize_); + read_static(&ptr, rsize_); + read_static(&ptr, dsize); + read_static(&ptr, tsize); + read_static(&ptr, fsize); + read_static(&ptr, dummy); + + charset_ = ptr; + ptr += 32; + da_.set_array(reinterpret_cast(const_cast(ptr))); + + ptr += dsize; + + token_ = reinterpret_cast(ptr); + ptr += tsize; + + feature_ = ptr; + ptr += fsize; + + CHECK_FALSE(ptr == dmmap_->end()) + << "dictionary file is broken: " << file; + + return true; +} + +void Dictionary::close() { + dmmap_->close(); +} + +#define DCONF(file) create_filename(dicdir, std::string(file)); + +bool Dictionary::assignUserDictionaryCosts( + const Param ¶m, + const std::vector &dics, + const char *output) { + Connector matrix; + DictionaryRewriter rewriter; + DecoderFeatureIndex fi; + ContextID cid; + CharProperty property; + + const std::string dicdir = param.get("dicdir"); + + const std::string matrix_file = DCONF(MATRIX_DEF_FILE); + const std::string matrix_bin_file = DCONF(MATRIX_FILE); + const std::string left_id_file = DCONF(LEFT_ID_FILE); + const std::string right_id_file = DCONF(RIGHT_ID_FILE); + const std::string rewrite_file = DCONF(REWRITE_FILE); + + const std::string from = param.get("dictionary-charset"); + + const int factor = param.get("cost-factor"); + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + + std::string config_charset = param.get("config-charset"); + if (config_charset.empty()) { + config_charset = from; + } + + CHECK_DIE(!from.empty()) << "input dictionary charset is empty"; + + Iconv config_iconv; + CHECK_DIE(config_iconv.open(config_charset.c_str(), from.c_str())) + << "iconv_open() failed with from=" << config_charset << " to=" << from; + + rewriter.open(rewrite_file.c_str(), &config_iconv); + CHECK_DIE(fi.open(param)) << "cannot open feature index"; + + CHECK_DIE(property.open(param)); + property.set_charset(from.c_str()); + + if (!matrix.openText(matrix_file.c_str()) && + !matrix.open(matrix_bin_file.c_str())) { + matrix.set_left_size(1); + matrix.set_right_size(1); + } + + cid.open(left_id_file.c_str(), + right_id_file.c_str(), &config_iconv); + CHECK_DIE(cid.left_size() == matrix.left_size() && + cid.right_size() == matrix.right_size()) + << "Context ID files(" + << left_id_file + << " or " + << right_id_file << " may be broken: " + << cid.left_size() << " " << matrix.left_size() << " " + << cid.right_size() << " " << matrix.right_size(); + + std::ofstream ofs(output); + CHECK_DIE(ofs) << "permission denied: " << output; + + for (size_t i = 0; i < dics.size(); ++i) { + std::ifstream ifs(WPATH(dics[i].c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << dics[i]; + std::cout << "reading " << dics[i] << " ... "; + scoped_fixed_array line; + while (ifs.getline(line.get(), line.size())) { + char *col[8]; + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + std::string w = col[0]; + const std::string feature = col[4]; + const int cost = calcCost(w, feature, factor, + &fi, &rewriter, &property); + std::string ufeature, lfeature, rfeature; + CHECK_DIE(rewriter.rewrite(feature, &ufeature, &lfeature, &rfeature)) + << "rewrite failed: " << feature; + const int lid = cid.lid(lfeature.c_str()); + const int rid = cid.rid(rfeature.c_str()); + CHECK_DIE(lid >= 0 && rid >= 0 && matrix.is_valid(lid, rid)) + << "invalid ids are found lid=" << lid << " rid=" << rid; + escape_csv_element(&w); + ofs << w << ',' << lid << ',' << rid << ',' + << cost << ',' << feature << '\n'; + } + } + + return true; +} + +bool Dictionary::compile(const Param ¶m, + const std::vector &dics, + const char *output) { + Connector matrix; + scoped_ptr rewrite; + scoped_ptr posid; + scoped_ptr fi; + scoped_ptr cid; + scoped_ptr writer; + scoped_ptr lattice; + scoped_ptr os; + scoped_ptr property; + Node node; + + const std::string dicdir = param.get("dicdir"); + + const std::string matrix_file = DCONF(MATRIX_DEF_FILE); + const std::string matrix_bin_file = DCONF(MATRIX_FILE); + const std::string left_id_file = DCONF(LEFT_ID_FILE); + const std::string right_id_file = DCONF(RIGHT_ID_FILE); + const std::string rewrite_file = DCONF(REWRITE_FILE); + const std::string pos_id_file = DCONF(POS_ID_FILE); + + std::vector > dic; + + size_t offset = 0; + unsigned int lexsize = 0; + std::string fbuf; + + const std::string from = param.get("dictionary-charset"); + const std::string to = param.get("charset"); + const bool wakati = param.get("wakati"); + const int type = param.get("type"); + const std::string node_format = param.get("node-format"); + const int factor = param.get("cost-factor"); + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + + // for backward compatibility + std::string config_charset = param.get("config-charset"); + if (config_charset.empty()) { + config_charset = from; + } + + CHECK_DIE(!from.empty()) << "input dictionary charset is empty"; + CHECK_DIE(!to.empty()) << "output dictionary charset is empty"; + + Iconv iconv; + CHECK_DIE(iconv.open(from.c_str(), to.c_str())) + << "iconv_open() failed with from=" << from << " to=" << to; + + Iconv config_iconv; + CHECK_DIE(config_iconv.open(config_charset.c_str(), from.c_str())) + << "iconv_open() failed with from=" << config_charset << " to=" << from; + + if (!node_format.empty()) { + writer.reset(new Writer); + lattice.reset(createLattice()); + os.reset(new StringBuffer); + memset(&node, 0, sizeof(node)); + } + + if (!matrix.openText(matrix_file.c_str()) && + !matrix.open(matrix_bin_file.c_str())) { + matrix.set_left_size(1); + matrix.set_right_size(1); + } + + posid.reset(new POSIDGenerator); + posid->open(pos_id_file.c_str(), &config_iconv); + + std::istringstream iss(UNK_DEF_DEFAULT); + + for (size_t i = 0; i < dics.size(); ++i) { + std::ifstream ifs(WPATH(dics[i].c_str())); + std::istream *is = &ifs; + if (!ifs) { + if (type == MECAB_UNK_DIC) { + std::cerr << dics[i] + << " is not found. minimum setting is used." << std::endl; + is = &iss; + } else { + CHECK_DIE(ifs) << "no such file or directory: " << dics[i]; + } + } + + std::cout << "reading " << dics[i] << " ... "; + + scoped_fixed_array line; + size_t num = 0; + + while (is->getline(line.get(), line.size())) { + char *col[8]; + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + + std::string w = col[0]; + int lid = toInt(col[1]); + int rid = toInt(col[2]); + int cost = toInt(col[3]); + std::string feature = col[4]; + const int pid = posid->id(feature.c_str()); + + if (cost == INT_MAX) { + CHECK_DIE(type == MECAB_USR_DIC) + << "cost field should not be empty in sys/unk dic."; + if (!rewrite.get()) { + rewrite.reset(new DictionaryRewriter); + rewrite->open(rewrite_file.c_str(), &config_iconv); + fi.reset(new DecoderFeatureIndex); + CHECK_DIE(fi->open(param)) << "cannot open feature index"; + property.reset(new CharProperty); + CHECK_DIE(property->open(param)); + property->set_charset(from.c_str()); + } + cost = calcCost(w, feature, factor, + fi.get(), rewrite.get(), property.get()); + } + + if (lid < 0 || rid < 0 || lid == INT_MAX || rid == INT_MAX) { + if (!rewrite.get()) { + rewrite.reset(new DictionaryRewriter); + rewrite->open(rewrite_file.c_str(), &config_iconv); + } + + std::string ufeature, lfeature, rfeature; + CHECK_DIE(rewrite->rewrite(feature, &ufeature, &lfeature, &rfeature)) + << "rewrite failed: " << feature; + + if (!cid.get()) { + cid.reset(new ContextID); + cid->open(left_id_file.c_str(), + right_id_file.c_str(), &config_iconv); + CHECK_DIE(cid->left_size() == matrix.left_size() && + cid->right_size() == matrix.right_size()) + << "Context ID files(" + << left_id_file + << " or " + << right_id_file << " may be broken"; + } + + lid = cid->lid(lfeature.c_str()); + rid = cid->rid(rfeature.c_str()); + } + + CHECK_DIE(lid >= 0 && rid >= 0 && matrix.is_valid(lid, rid)) + << "invalid ids are found lid=" << lid << " rid=" << rid; + + if (w.empty()) { + std::cerr << "empty word is found, discard this line" << std::endl; + continue; + } + + if (!iconv.convert(&feature)) { + std::cerr << "iconv conversion failed. skip this entry" + << std::endl; + continue; + } + + if (type != MECAB_UNK_DIC && !iconv.convert(&w)) { + std::cerr << "iconv conversion failed. skip this entry" + << std::endl; + continue; + } + + if (!node_format.empty()) { + node.surface = w.c_str(); + node.feature = feature.c_str(); + node.length = w.size(); + node.rlength = w.size(); + node.posid = pid; + node.stat = MECAB_NOR_NODE; + lattice->set_sentence(w.c_str()); + CHECK_DIE(os.get()); + CHECK_DIE(writer.get()); + os->clear(); + CHECK_DIE(writer->writeNode(lattice.get(), + node_format.c_str(), + &node, &*os)) << + "conversion error: " << feature << " with " << node_format; + *os << '\0'; + feature = os->str(); + } + + std::string key; + if (!wakati) { + key = feature + '\0'; + } + + Token* token = new Token; + token->lcAttr = lid; + token->rcAttr = rid; + token->posid = pid; + token->wcost = cost; + token->feature = offset; + token->compound = 0; + dic.push_back(std::pair(w, token)); + + // append to output buffer + if (!wakati) { + fbuf.append(key.data(), key.size()); + } + offset += key.size(); + + ++num; + ++lexsize; + } + + std::cout << num << std::endl; + } + + if (wakati) { + fbuf.append("\0", 1); + } + + std::stable_sort(dic.begin(), dic.end(), + pair_1st_cmp()); + + size_t bsize = 0; + size_t idx = 0; + std::string prev; + std::vector str; + std::vector len; + std::vector val; + + for (size_t i = 0; i < dic.size(); ++i) { + if (i != 0 && prev != dic[i].first) { + str.push_back(dic[idx].first.c_str()); + len.push_back(dic[idx].first.size()); + val.push_back(bsize +(idx << 8)); + bsize = 1; + idx = i; + } else { + ++bsize; + } + prev = dic[i].first; + } + str.push_back(dic[idx].first.c_str()); + len.push_back(dic[idx].first.size()); + val.push_back(bsize +(idx << 8)); + + CHECK_DIE(str.size() == len.size()); + CHECK_DIE(str.size() == val.size()); + + Darts::DoubleArray da; + CHECK_DIE(da.build(str.size(), const_cast(&str[0]), + &len[0], &val[0], &progress_bar_darts) == 0) + << "unknown error in building double-array"; + + std::string tbuf; + for (size_t i = 0; i < dic.size(); ++i) { + tbuf.append(reinterpret_cast(dic[i].second), + sizeof(Token)); + delete dic[i].second; + } + dic.clear(); + + // needs to be 8byte(64bit) aligned + while (tbuf.size() % 8 != 0) { + Token dummy; + memset(&dummy, 0, sizeof(Token)); + tbuf.append(reinterpret_cast(&dummy), sizeof(Token)); + } + + unsigned int dummy = 0; + unsigned int lsize = matrix.left_size(); + unsigned int rsize = matrix.right_size(); + unsigned int dsize = da.unit_size() * da.size(); + unsigned int tsize = tbuf.size(); + unsigned int fsize = fbuf.size(); + + unsigned int version = DIC_VERSION; + char charset[32]; + std::fill(charset, charset + sizeof(charset), '\0'); + std::strncpy(charset, to.c_str(), 31); + + std::ofstream bofs(WPATH(output), std::ios::binary|std::ios::out); + CHECK_DIE(bofs) << "permission denied: " << output; + + unsigned int magic = 0; + + // needs to be 64bit aligned + // 10*32 = 64*5 + bofs.write(reinterpret_cast(&magic), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&version), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&type), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&lexsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&lsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&rsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&dsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&tsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&fsize), sizeof(unsigned int)); + bofs.write(reinterpret_cast(&dummy), sizeof(unsigned int)); + + // 32 * 8 = 64 * 4 + bofs.write(reinterpret_cast(charset), sizeof(charset)); + + bofs.write(reinterpret_cast(da.array()), + da.unit_size() * da.size()); + bofs.write(const_cast(tbuf.data()), tbuf.size()); + bofs.write(const_cast(fbuf.data()), fbuf.size()); + + // save magic id + magic = static_cast(bofs.tellp()); + magic ^= DictionaryMagicID; + bofs.seekp(0); + bofs.write(reinterpret_cast(&magic), sizeof(unsigned int)); + + bofs.close(); + + return true; +} +} diff --git a/fts/third_party/mecab/src/dictionary.h b/fts/third_party/mecab/src/dictionary.h new file mode 100644 index 00000000..b170b764 --- /dev/null +++ b/fts/third_party/mecab/src/dictionary.h @@ -0,0 +1,99 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_DICTIONARY_H_ +#define MECAB_DICTIONARY_H_ + +#include "mecab.h" +#include "mmap.h" +#include "darts.h" +#include "char_property.h" + +namespace MeCab { + +class Param; + +struct Token { + unsigned short lcAttr; + unsigned short rcAttr; + unsigned short posid; + short wcost; + unsigned int feature; + unsigned int compound; +}; + +class Dictionary { + public: + typedef Darts::DoubleArray::result_pair_type result_type; + + bool open(const char *filename, const char *mode = "r"); + void close(); + + size_t commonPrefixSearch(const char* key, size_t len, + result_type *result, + size_t rlen) const { + return da_.commonPrefixSearch(key, result, rlen, len); + } + + result_type exactMatchSearch(const char* key) const { + result_type n; + da_.exactMatchSearch(key, n); + return n; + } + + bool isCompatible(const Dictionary &d) const { + return(version_ == d.version_ && + lsize_ == d.lsize_ && + rsize_ == d.rsize_ && + decode_charset(charset_) == + decode_charset(d.charset_)); + } + + const char *filename() const { return filename_.c_str(); } + const char *charset() const { return const_cast(charset_); } + unsigned short version() const { return version_; } + size_t size() const { return static_cast(lexsize_); } + int type() const { return static_cast(type_); } + size_t lsize() const { return static_cast(lsize_); } + size_t rsize() const { return static_cast(rsize_); } + + const Token *token(const result_type &n) const { + return token_ +(n.value >> 8); + } + size_t token_size(const result_type &n) const { return 0xff & n.value; } + const char *feature(const Token &t) const { return feature_ + t.feature; } + + static bool compile(const Param ¶m, + const std::vector &dics, + const char *output); // outputs + + static bool assignUserDictionaryCosts( + const Param ¶m, + const std::vector &dics, + const char *output); // outputs + + + const char *what() { return what_.str(); } + + explicit Dictionary(): dmmap_(new Mmap), token_(0), + feature_(0), charset_(0) {} + virtual ~Dictionary() { this->close(); } + + private: + scoped_ptr > dmmap_; + const Token *token_; + const char *feature_; + const char *charset_; + unsigned int version_; + unsigned int type_; + unsigned int lexsize_; + unsigned int lsize_; + unsigned int rsize_; + std::string filename_; + whatlog what_; + Darts::DoubleArray da_; +}; +} +#endif // MECAB_DICTIONARY_H_ diff --git a/fts/third_party/mecab/src/dictionary_compiler.cpp b/fts/third_party/mecab/src/dictionary_compiler.cpp new file mode 100644 index 00000000..38a25f77 --- /dev/null +++ b/fts/third_party/mecab/src/dictionary_compiler.cpp @@ -0,0 +1,156 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "char_property.h" +#include "connector.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "mecab.h" +#include "param.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace MeCab { + +class DictionaryComplier { + public: + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", "set DIR as dic dir (default \".\")" }, + { "outdir", 'o', ".", "DIR", + "set DIR as output dir (default \".\")" }, + { "model", 'm', 0, "FILE", "use FILE as model file" }, + { "userdic", 'u', 0, "FILE", "build user dictionary" }, + { "assign-user-dictionary-costs", 'a', 0, 0, + "only assign costs/ids to user dictionary" }, + { "build-unknown", 'U', 0, 0, + "build parameters for unknown words" }, + { "build-model", 'M', 0, 0, "build model file" }, + { "build-charcategory", 'C', 0, 0, "build character category maps" }, + { "build-sysdic", 's', 0, 0, "build system dictionary" }, + { "build-matrix", 'm', 0, 0, "build connection matrix" }, + { "charset", 'c', MECAB_DEFAULT_CHARSET, "ENC", + "make charset of binary dictionary ENC (default " + MECAB_DEFAULT_CHARSET ")" }, + { "charset", 't', MECAB_DEFAULT_CHARSET, "ENC", "alias of -c" }, + { "dictionary-charset", 'f', MECAB_DEFAULT_CHARSET, + "ENC", "assume charset of input CSVs as ENC (default " + MECAB_DEFAULT_CHARSET ")" }, + { "wakati", 'w', 0, 0, "build wakati-gaki only dictionary", }, + { "posid", 'p', 0, 0, "assign Part-of-speech id" }, + { "node-format", 'F', 0, "STR", + "use STR as the user defined node format" }, + { "version", 'v', 0, 0, "show the version and exit." }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + const std::string dicdir = param.get("dicdir"); + const std::string outdir = param.get("outdir"); + bool opt_unknown = param.get("build-unknown"); + bool opt_matrix = param.get("build-matrix"); + bool opt_charcategory = param.get("build-charcategory"); + bool opt_sysdic = param.get("build-sysdic"); + bool opt_model = param.get("build-model"); + bool opt_assign_user_dictionary_costs = param.get + ("assign-user-dictionary-costs"); + const std::string userdic = param.get("userdic"); + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() +#define OCONF(file) create_filename(outdir, std::string(file)).c_str() + + CHECK_DIE(param.load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + std::vector dic; + if (userdic.empty()) { + enum_csv_dictionaries(dicdir.c_str(), &dic); + } else { + dic = param.rest_args(); + } + + if (!userdic.empty()) { + CHECK_DIE(dic.size()) << "no dictionaries are specified"; + param.set("type", static_cast(MECAB_USR_DIC)); + if (opt_assign_user_dictionary_costs) { + Dictionary::assignUserDictionaryCosts(param, dic, + userdic.c_str()); + } else { + Dictionary::compile(param, dic, userdic.c_str()); + } + } else { + if (!opt_unknown && !opt_matrix && !opt_charcategory && + !opt_sysdic && !opt_model) { + opt_unknown = opt_matrix = opt_charcategory = + opt_sysdic = opt_model = true; + } + + if (opt_charcategory || opt_unknown) { + CharProperty::compile(DCONF(CHAR_PROPERTY_DEF_FILE), + DCONF(UNK_DEF_FILE), + OCONF(CHAR_PROPERTY_FILE)); + } + + if (opt_unknown) { + std::vector tmp; + tmp.push_back(DCONF(UNK_DEF_FILE)); + param.set("type", static_cast(MECAB_UNK_DIC)); + Dictionary::compile(param, tmp, OCONF(UNK_DIC_FILE)); + } + + if (opt_model) { + if (file_exists(DCONF(MODEL_DEF_FILE))) { + FeatureIndex::compile(param, + DCONF(MODEL_DEF_FILE), + OCONF(MODEL_FILE)); + } else { + std::cout << DCONF(MODEL_DEF_FILE) + << " is not found. skipped." << std::endl; + } + } + + if (opt_sysdic) { + CHECK_DIE(dic.size()) << "no dictionaries are specified"; + param.set("type", static_cast(MECAB_SYS_DIC)); + Dictionary::compile(param, dic, OCONF(SYS_DIC_FILE)); + } + + if (opt_matrix) { + Connector::compile(DCONF(MATRIX_DEF_FILE), + OCONF(MATRIX_FILE)); + } + } + + std::cout << "\ndone!\n"; + + return 0; + } +}; + +#undef DCONF +#undef OCONF +} + +int mecab_dict_index(int argc, char **argv) { + return MeCab::DictionaryComplier::run(argc, argv); +} diff --git a/fts/third_party/mecab/src/dictionary_generator.cpp b/fts/third_party/mecab/src/dictionary_generator.cpp new file mode 100644 index 00000000..38fef058 --- /dev/null +++ b/fts/third_party/mecab/src/dictionary_generator.cpp @@ -0,0 +1,295 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "char_property.h" +#include "common.h" +#include "context_id.h" +#include "dictionary.h" +#include "dictionary_rewriter.h" +#include "feature_index.h" +#include "mecab.h" +#include "mmap.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +void copy(const char *src, const char *dst) { + std::cout << "copying " << src << " to " << dst << std::endl; + Mmap mmap; + CHECK_DIE(mmap.open(src)) << mmap.what(); + std::ofstream ofs(WPATH(dst), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << dst; + ofs.write(reinterpret_cast(mmap.begin()), mmap.size()); + ofs.close(); +} + +class DictionaryGenerator { + private: + static void gencid_bos(const std::string &bos_feature, + DictionaryRewriter *rewrite, + ContextID *cid) { + std::string ufeature, lfeature, rfeature; + rewrite->rewrite2(bos_feature, &ufeature, &lfeature, &rfeature); + cid->addBOS(lfeature.c_str(), rfeature.c_str()); + } + + static void gencid(const char *filename, + DictionaryRewriter *rewrite, + ContextID *cid) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + scoped_fixed_array line; + std::cout << "reading " << filename << " ... " << std::flush; + size_t num = 0; + std::string feature, ufeature, lfeature, rfeature; + char *col[8]; + while (ifs.getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + feature = col[4]; + rewrite->rewrite2(feature, &ufeature, &lfeature, &rfeature); + cid->add(lfeature.c_str(), rfeature.c_str()); + ++num; + } + std::cout << num << std::endl; + ifs.close(); + } + + static bool genmatrix(const char *filename, + const ContextID &cid, + DecoderFeatureIndex *fi, + int factor) { + std::ofstream ofs(WPATH(filename)); + CHECK_DIE(ofs) << "permission denied: " << filename; + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + const std::map &left = cid.left_ids(); + const std::map &right = cid.right_ids(); + + CHECK_DIE(left.size() > 0) << "left id size is empty"; + CHECK_DIE(right.size() > 0) << "right id size is empty"; + + ofs << right.size() << ' ' << left.size() << std::endl; + + size_t l = 0; + for (std::map::const_iterator rit = right.begin(); + rit != right.end(); + ++rit) { + ++l; + progress_bar("emitting matrix ", l+1, right.size()); + for (std::map::const_iterator lit = left.begin(); + lit != left.end(); + ++lit) { + path.rnode->wcost = 0; + fi->buildBigramFeature(&path, rit->first.c_str(), lit->first.c_str()); + fi->calcCost(&path); + ofs << rit->second << ' ' << lit->second << ' ' + << tocost(path.cost, factor) << '\n'; + } + } + + return true; + } + + static void gendic(const char* ifile, + const char* ofile, + const CharProperty &property, + DictionaryRewriter *rewrite, + const ContextID &cid, + DecoderFeatureIndex *fi, + bool unk, + int factor) { + std::ifstream ifs(WPATH(ifile)); + CHECK_DIE(ifs) << "no such file or directory: " << ifile; + + std::ofstream ofs(WPATH(ofile)); + CHECK_DIE(ofs) << "permission denied: " << ofile; + + std::cout << "emitting " << ofile << " ... " << std::flush; + + LearnerPath path; + LearnerNode rnode; + LearnerNode lnode; + rnode.stat = lnode.stat = MECAB_NOR_NODE; + rnode.rpath = &path; + lnode.lpath = &path; + path.lnode = &lnode; + path.rnode = &rnode; + + scoped_fixed_array line; + char *col[8]; + size_t num = 0; + + while (ifs.getline(line.get(), line.size())) { + const size_t n = tokenizeCSV(line.get(), col, 5); + CHECK_DIE(n == 5) << "format error: " << line.get(); + + std::string w = std::string(col[0]); + const std::string feature = std::string(col[4]); + + std::string ufeature, lfeature, rfeature; + rewrite->rewrite2(feature, &ufeature, &lfeature, &rfeature); + const int lid = cid.lid(lfeature.c_str()); + const int rid = cid.rid(rfeature.c_str()); + + CHECK_DIE(lid > 0) << "CID is not found for " << lfeature; + CHECK_DIE(rid > 0) << "CID is not found for " << rfeature; + + if (unk) { + const int c = property.id(w.c_str()); + CHECK_DIE(c >= 0) << "unknown property [" << w << "]"; + path.rnode->char_type = static_cast(c); + } else { + size_t mblen = 0; + const CharInfo cinfo = property.getCharInfo(w.c_str(), + w.c_str() + w.size(), + &mblen); + path.rnode->char_type = cinfo.default_type; + } + + fi->buildUnigramFeature(&path, ufeature.c_str()); + fi->calcCost(&rnode); + CHECK_DIE(escape_csv_element(&w)) << "invalid character found: " << w; + + ofs << w << ',' << lid << ',' << rid << ',' + << tocost(rnode.wcost, factor) + << ',' << feature << '\n'; + ++num; + } + + std::cout << num << std::endl; + } + + public: + + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", "set DIR as dicdir(default \".\" )" }, + { "outdir", 'o', ".", "DIR", "set DIR as output dir" }, + { "model", 'm', 0, "FILE", "use FILE as model file" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) return 0; + + ContextID cid; + DecoderFeatureIndex fi; + DictionaryRewriter rewrite; + + const std::string dicdir = param.get("dicdir"); + const std::string outdir = param.get("outdir"); + const std::string model = param.get("model"); + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() +#define OCONF(file) create_filename(outdir, std::string(file)).c_str() + + CHECK_DIE(param.load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + std::string charset; + { + Dictionary dic; + CHECK_DIE(dic.open(DCONF(SYS_DIC_FILE), "r")); + charset = dic.charset(); + CHECK_DIE(!charset.empty()); + } + + CharProperty property; + CHECK_DIE(property.open(param)); + property.set_charset(charset.c_str()); + + const std::string bos = param.get("bos-feature"); + const int factor = param.get("cost-factor"); + + std::vector dic; + enum_csv_dictionaries(dicdir.c_str(), &dic); + + { + CHECK_DIE(dicdir != outdir) << + "output directory = dictionary directory! " + "Please specify different directory."; + CHECK_DIE(!outdir.empty()) << "output directory is empty"; + CHECK_DIE(!model.empty()) << "model file is empty"; + CHECK_DIE(fi.open(param)) << "cannot open feature index"; + CHECK_DIE(factor > 0) << "cost factor needs to be positive value"; + CHECK_DIE(!bos.empty()) << "bos-feature is empty"; + CHECK_DIE(dic.size()) << "no dictionary is found in " << dicdir; + CHECK_DIE(rewrite.open(DCONF(REWRITE_FILE))); + } + + gencid_bos(bos, &rewrite, &cid); + gencid(DCONF(UNK_DEF_FILE), &rewrite, &cid); + + for (std::vector::const_iterator it = dic.begin(); + it != dic.end(); + ++it) { + gencid(it->c_str(), &rewrite, &cid); + } + + std::cout << "emitting " + << OCONF(LEFT_ID_FILE) << "/ " + << OCONF(RIGHT_ID_FILE) << std::endl; + + cid.build(); + cid.save(OCONF(LEFT_ID_FILE), OCONF(RIGHT_ID_FILE)); + + gendic(DCONF(UNK_DEF_FILE), OCONF(UNK_DEF_FILE), property, + &rewrite, cid, &fi, true, factor); + + for (std::vector::const_iterator it = dic.begin(); + it != dic.end(); + ++it) { + std::string file = *it; + remove_pathname(&file); + gendic(it->c_str(), OCONF(file.c_str()), property, + &rewrite, cid, &fi, false, factor); + } + + genmatrix(OCONF(MATRIX_DEF_FILE), cid, &fi, factor); + + copy(DCONF(CHAR_PROPERTY_DEF_FILE), OCONF(CHAR_PROPERTY_DEF_FILE)); + copy(DCONF(REWRITE_FILE), OCONF(REWRITE_FILE)); + copy(DCONF(DICRC), OCONF(DICRC)); + copy(DCONF(FEATURE_FILE), OCONF(FEATURE_FILE)); + copy(model.c_str(), OCONF(MODEL_DEF_FILE)); + +#undef OCONF +#undef DCONF + + std::cout << "\ndone!\n"; + + return 0; + } +}; +} + +// export functions +int mecab_dict_gen(int argc, char **argv) { + return MeCab::DictionaryGenerator::run(argc, argv); +} diff --git a/fts/third_party/mecab/src/dictionary_rewriter.cpp b/fts/third_party/mecab/src/dictionary_rewriter.cpp new file mode 100644 index 00000000..ca6832fd --- /dev/null +++ b/fts/third_party/mecab/src/dictionary_rewriter.cpp @@ -0,0 +1,242 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "dictionary_rewriter.h" +#include "iconv_utils.h" +#include "scoped_ptr.h" +#include "utils.h" + +namespace { + +using namespace MeCab; + +void append_rewrite_rule(RewriteRules *r, char* str) { + char *col[3]; + const size_t n = tokenize2(str, " \t", col, 3); + CHECK_DIE(n >= 2) << "format error: " << str; + r->resize(r->size() + 1); + std::string tmp; + if (n >= 3) { + tmp = col[1]; + tmp += ' '; + tmp += col[2]; + col[1] = const_cast(tmp.c_str()); + } + r->back().set_pattern(col[0], col[1]); +} + +bool match_rewrite_pattern(const char *pat, + const char *str) { + if (pat[0] == '*' || std::strcmp(pat, str) == 0) + return true; + + size_t len = std::strlen(pat); + if (len >= 3 && pat[0] == '(' && pat[len-1] == ')') { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(len < buf.size() - 3) << "too long parameter"; + std::strncpy(buf.get(), pat + 1, buf.size()); + buf[len-2] = '\0'; + const size_t n = tokenize(buf.get(), "|", col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long OR nodes"; + for (size_t i = 0; i < n; ++i) { + if (std::strcmp(str, col[i]) == 0) return true; + } + } + return false; +} +} + +namespace MeCab { + +bool RewritePattern::set_pattern(const char *src, + const char *dst) { + scoped_fixed_array buf; + spat_.clear(); + dpat_.clear(); + + std::strncpy(buf.get(), src, buf.size()); + tokenizeCSV(buf.get(), back_inserter(spat_), 512); + + std::strncpy(buf.get(), dst, buf.size()); + tokenizeCSV(buf.get(), back_inserter(dpat_), 512); + + return (spat_.size() && dpat_.size()); +} + +bool RewritePattern::rewrite(size_t size, + const char **input, + std::string *output) const { + if (spat_.size() > size) return false; + for (size_t i = 0; i < spat_.size(); ++i) { + if (!match_rewrite_pattern(spat_[i].c_str(), input[i])) + return false; + } + + output->clear(); + for (size_t i = 0; i < dpat_.size(); ++i) { + std::string elm; + const char *begin = dpat_[i].c_str(); + const char *end = begin + dpat_[i].size(); + for (const char *p = begin; p < end; ++p) { + if (*p == '$') { + ++p; + size_t n = 0; + for (; p < end; ++p) { + switch (*p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n + (*p - '0'); + break; + default: + goto NEXT; + } + } + NEXT: + CHECK_DIE(n > 0 && (n - 1) < size) + << " out of range: [" << dpat_[i] << "] " << n; + elm += input[n - 1]; + if (p < end) elm += *p; + } else { + elm += *p; + } + } + CHECK_DIE(escape_csv_element(&elm)); + *output += elm; + if (i + 1 != dpat_.size()) *output += ","; + } + + return true; +} + +bool RewriteRules::rewrite(size_t size, + const char **input, + std::string *output) const { + for (size_t i = 0; i < this->size(); ++i) { + if ((*this)[i].rewrite(size, input, output)) + return true; + } + return false; +} + +void DictionaryRewriter::clear() { cache_.clear(); } + +bool DictionaryRewriter::open(const char *filename, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + int append_to = 0; + std::string line; + while (std::getline(ifs, line)) { + if (iconv) iconv->convert(&line); + if (line.empty() || line[0] == '#') continue; + if (line == "[unigram rewrite]") { + append_to = 1; + } else if (line == "[left rewrite]") { + append_to = 2; + } else if (line == "[right rewrite]") { + append_to = 3; + } else { + CHECK_DIE(append_to != 0) << "no sections found"; + char *str = const_cast(line.c_str()); + switch (append_to) { + case 1: append_rewrite_rule(&unigram_rewrite_, str); break; + case 2: append_rewrite_rule(&left_rewrite_, str); break; + case 3: append_rewrite_rule(&right_rewrite_, str); break; + } + } + } + return true; +} + +// without cache +bool DictionaryRewriter::rewrite(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) const { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(feature.size() < buf.size() - 1) << "too long feature"; + std::strncpy(buf.get(), feature.c_str(), buf.size() - 1); + const size_t n = tokenizeCSV(buf.get(), col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long CSV entities"; + return (unigram_rewrite_.rewrite(n, const_cast(col.get()), + ufeature) && + left_rewrite_.rewrite(n, const_cast(col.get()), + lfeature) && + right_rewrite_.rewrite(n, const_cast(col.get()), + rfeature)); +} + +// with cache +bool DictionaryRewriter::rewrite2(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) { + std::map::const_iterator it = cache_.find(feature); + if (it == cache_.end()) { + if (!rewrite(feature, ufeature, lfeature, rfeature)) return false; + FeatureSet f; + f.ufeature = *ufeature; + f.lfeature = *lfeature; + f.rfeature = *rfeature; + cache_.insert(std::pair(feature, f)); + } else { + *ufeature = it->second.ufeature; + *lfeature = it->second.lfeature; + *rfeature = it->second.rfeature; + } + + return true; +} + +bool POSIDGenerator::open(const char *filename, + Iconv *iconv) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + std::cerr << filename + << " is not found. minimum setting is used" << std::endl; + rewrite_.resize(1); + rewrite_.back().set_pattern("*", "1"); + return true; + } + + std::string line; + char *col[2]; + while (std::getline(ifs, line)) { + if (iconv) iconv->convert(&line); + const size_t n = tokenize2(const_cast(line.c_str()), + " \t", col, 2); + CHECK_DIE(n == 2) << "format error: " << line; + for (char *p = col[1]; *p; ++p) { + CHECK_DIE(*p >= '0' && *p <= '9') << "not a number: " << col[1]; + } + rewrite_.resize(rewrite_.size() + 1); + rewrite_.back().set_pattern(col[0], col[1]); + } + return true; +} + +int POSIDGenerator::id(const char *feature) const { + scoped_fixed_array buf; + scoped_fixed_array col; + CHECK_DIE(std::strlen(feature) < buf.size() - 1) << "too long feature"; + std::strncpy(buf.get(), feature, buf.size() - 1); + const size_t n = tokenizeCSV(buf.get(), col.get(), col.size()); + CHECK_DIE(n < col.size()) << "too long CSV entities"; + std::string tmp; + if (!rewrite_.rewrite(n, const_cast(col.get()), &tmp)) { + return -1; + } + return std::atoi(tmp.c_str()); +} +} diff --git a/fts/third_party/mecab/src/dictionary_rewriter.h b/fts/third_party/mecab/src/dictionary_rewriter.h new file mode 100644 index 00000000..9b06758b --- /dev/null +++ b/fts/third_party/mecab/src/dictionary_rewriter.h @@ -0,0 +1,75 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_DICTIONARY_REWRITER_H +#define MECAB_DICTIONARY_REWRITER_H + +#include +#include +#include +#include "common.h" +#include "mecab.h" +#include "freelist.h" + +namespace MeCab { + +class Iconv; + +class RewritePattern { + private: + std::vector spat_; + std::vector dpat_; + public: + bool set_pattern(const char *src, const char *dst); + bool rewrite(size_t size, + const char **input, + std::string *output) const; +}; + +class RewriteRules: public std::vector { + public: + bool rewrite(size_t size, const char **input, + std::string *output) const; +}; + +struct FeatureSet { + std::string ufeature; + std::string lfeature; + std::string rfeature; +}; + +class DictionaryRewriter { + private: + RewriteRules unigram_rewrite_; + RewriteRules left_rewrite_; + RewriteRules right_rewrite_; + std::map cache_; + + public: + bool open(const char *filename, + Iconv *iconv = 0); + void clear(); + bool rewrite(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature) const; + + bool rewrite2(const std::string &feature, + std::string *ufeature, + std::string *lfeature, + std::string *rfeature); +}; + +class POSIDGenerator { + private: + RewriteRules rewrite_; + public: + bool open(const char *filename, + Iconv *iconv = 0); + void clear() { rewrite_.clear(); } + int id(const char *key) const; +}; +} +#endif diff --git a/fts/third_party/mecab/src/eval.cpp b/fts/third_party/mecab/src/eval.cpp new file mode 100644 index 00000000..922266f1 --- /dev/null +++ b/fts/third_party/mecab/src/eval.cpp @@ -0,0 +1,268 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "common.h" +#include "mecab.h" +#include "param.h" +#include "stream_wrapper.h" +#include "scoped_ptr.h" +#include "utils.h" + +namespace MeCab { + +class Eval { + private: + static bool read(std::istream *is, + std::vector > *r, + const std::vector &level) { + if (!*is) { + return false; + } + + char *col[2]; + scoped_fixed_array buf; + scoped_fixed_array csv; + r->clear(); + while (is->getline(buf.get(), buf.size())) { + if (std::strcmp(buf.get(), "EOS") == 0) { + break; + } + CHECK_DIE(tokenize(buf.get(), "\t", col, 2) == 2) << "format error"; + csv[0] = col[0]; + size_t n = tokenizeCSV(col[1], csv.get() + 1, csv.size() - 1); + std::vector tmp; + for (size_t i = 0; i < level.size(); ++i) { + size_t m = level[i] < 0 ? n : level[i]; + CHECK_DIE(m <= n) << " out of range " << level[i]; + std::string output; + for (size_t j = 0; j <= m; ++j) { + output += csv[j]; + if (j != 0) { + output += "\t"; + } + } + tmp.push_back(output); + } + r->push_back(tmp); + } + + return true; + } + + static bool parseLevel(const char *level_str, + std::vector *level) { + scoped_fixed_array buf; + scoped_fixed_array col; + std::strncpy(buf.get(), level_str, buf.size()); + level->clear(); + size_t n = tokenize2(buf.get(), "\t ", col.get(), col.size()); + for (size_t i = 0; i < n; ++i) { + level->push_back(std::atoi(col[i])); + } + return true; + } + + static void printeval(std::ostream *os, size_t c, size_t p, size_t r) { + double pr = (p == 0) ? 0 : 100.0 * c/p; + double re = (r == 0) ? 0 : 100.0 * c/r; + double F = ((pr + re) == 0.0) ? 0 : 2 * pr * re /(pr + re); + scoped_fixed_array buf; + sprintf(buf.get(), "%4.4f(%d/%d) %4.4f(%d/%d) %4.4f\n", + pr, + static_cast(c), + static_cast(p), + re, + static_cast(c), + static_cast(r), + F); + *os << buf.get(); + } + + public: + static bool eval(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "level", 'l', "0 -1", "STR", "set level of evaluations" }, + { "output", 'o', 0, "FILE", "set the output file name" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + MeCab::Param param; + param.open(argc, argv, long_options); + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) return 0; + + const std::vector &files = param.rest_args(); + if (files.size() < 2) { + std::cout << "Usage: " << + param.program_name() << " output answer" << std::endl; + return -1; + } + + std::string output = param.get("output"); + if (output.empty()) output = "-"; + MeCab::ostream_wrapper ofs(output.c_str()); + CHECK_DIE(*ofs) << "no such file or directory: " << output; + + const std::string system = files[0]; + const std::string answer = files[1]; + + const std::string level_str = param.get("level"); + + std::ifstream ifs1(WPATH(files[0].c_str())); + std::ifstream ifs2(WPATH(files[1].c_str())); + + CHECK_DIE(ifs1) << "no such file or directory: " << files[0].c_str(); + CHECK_DIE(ifs2) << "no such file or directory: " << files[0].c_str(); + CHECK_DIE(!level_str.empty()) << "level_str is NULL"; + + std::vector level; + parseLevel(level_str.c_str(), &level); + CHECK_DIE(level.size()) << "level_str is empty: " << level_str; + std::vector result_tbl(level.size()); + std::fill(result_tbl.begin(), result_tbl.end(), 0); + + size_t prec = 0; + size_t recall = 0; + + std::vector > r1; + std::vector > r2; + + while (true) { + if (!read(&ifs1, &r1, level) || !read(&ifs2, &r2, level)) + break; + + size_t i1 = 0; + size_t i2 = 0; + size_t p1 = 0; + size_t p2 = 0; + + while (i1 < r1.size() && i2 < r2.size()) { + if (p1 == p2) { + for (size_t i = 0; i < result_tbl.size(); ++i) { + if (r1[i1][i] == r2[i2][i]) { + result_tbl[i]++; + } + } + p1 += r1[i1][0].size(); + p2 += r2[i2][0].size(); + ++i1; + ++i2; + ++prec; + ++recall; + } else if (p1 < p2) { + p1 += r1[i1][0].size(); + ++i1; + ++prec; + } else { + p2 += r2[i2][0].size(); + ++i2; + ++recall; + } + } + + while (i1 < r1.size()) { + ++prec; + ++i1; + } + + while (i2 < r2.size()) { + ++recall; + ++i2; + } + } + + *ofs << " precision recall F" + << std::endl; + for (size_t i = 0; i < result_tbl.size(); ++i) { + if (level[i] == -1) { + *ofs << "LEVEL ALL: "; + } else { + *ofs << "LEVEL " << level[i] << ": "; + } + printeval(&*ofs, result_tbl[i], prec, recall); + } + + return true; + } +}; + +class TestSentenceGenerator { + public: + static int run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "output", 'o', 0, "FILE", "set the output filename" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + MeCab::Param param; + param.open(argc, argv, long_options); + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + const std::vector &tmp = param.rest_args(); + std::vector files = tmp; + if (files.empty()) { + files.push_back("-"); + } + + std::string output = param.get("output"); + if (output.empty()) output = "-"; + MeCab::ostream_wrapper ofs(output.c_str()); + CHECK_DIE(*ofs) << "permission denied: " << output; + + scoped_fixed_array buf; + char *col[2]; + std::string str; + for (size_t i = 0; i < files.size(); ++i) { + MeCab::istream_wrapper ifs(files[i].c_str()); + CHECK_DIE(*ifs) << "no such file or directory: " << files[i]; + while (ifs->getline(buf.get(), buf.size())) { + const size_t n = tokenize(buf.get(), "\t ", col, 2); + CHECK_DIE(n <= 2) << "format error: " << buf.get(); + if (std::strcmp(col[0], "EOS") == 0 && !str.empty()) { + *ofs << str << std::endl; + str.clear(); + } else { + str += col[0]; + } + } + } + + return 0; + } +}; +} + +// exports +int mecab_system_eval(int argc, char **argv) { + return MeCab::Eval::eval(argc, argv); +} + +int mecab_test_gen(int argc, char **argv) { + return MeCab::TestSentenceGenerator::run(argc, argv); +} diff --git a/fts/third_party/mecab/src/feature_index.cpp b/fts/third_party/mecab/src/feature_index.cpp new file mode 100644 index 00000000..051bdf8b --- /dev/null +++ b/fts/third_party/mecab/src/feature_index.cpp @@ -0,0 +1,690 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "feature_index.h" +#include "param.h" +#include "iconv_utils.h" +#include "learner_node.h" +#include "scoped_ptr.h" +#include "string_buffer.h" +#include "utils.h" + +#define BUFSIZE (2048) +#define POSSIZE (64) + +#define ADDB(b) do { const int id = this->id((b)); \ + if (id != -1) feature_.push_back(id); } while (0) + +#define COPY_FEATURE(ptr) do { \ + feature_.push_back(-1); \ + (ptr) = feature_freelist_.alloc(feature_.size()); \ + std::copy(feature_.begin(), feature_.end(), const_cast(ptr)); \ + feature_.clear(); } while (0) + +namespace MeCab { + +const char* FeatureIndex::getIndex(char **p, char **column, size_t max) { + ++(*p); + + bool flg = false; + + if (**p == '?') { + flg = true; + ++(*p); + } // undef flg + + CHECK_DIE(**p =='[') << "getIndex(): unmatched '['"; + + size_t n = 0; + ++(*p); + + for (;; ++(*p)) { + switch (**p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n + (**p - '0'); + break; + case ']': + if (n >= max) { + return 0; + } + + if (flg == true && ((std::strcmp("*", column[n]) == 0) + || column[n][0] == '\0')) { + return 0; + } + return column[n]; // return; + break; + default: + CHECK_DIE(false) << "unmatched '['"; + } + } + + return 0; +} + +void FeatureIndex::set_alpha(const double *alpha) { + alpha_ = alpha; +} + +bool FeatureIndex::openTemplate(const Param ¶m) { + std::string filename = create_filename(param.get("dicdir"), + FEATURE_FILE); + std::ifstream ifs(WPATH(filename.c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << filename; + + scoped_fixed_array buf; + char *column[4]; + + unigram_templs_.clear(); + bigram_templs_.clear(); + + while (ifs.getline(buf.get(), buf.size())) { + if (buf[0] == '\0' || buf[0] == '#' || buf[0] == ' ') { + continue; + } + CHECK_DIE(tokenize2(buf.get(), "\t ", column, 2) == 2) + << "format error: " <strdup(column[1])); + } else if (std::strcmp(column[0], "BIGRAM") == 0) { + bigram_templs_.push_back(this->strdup(column[1])); + } else { + CHECK_DIE(false) << "format error: " << filename; + } + } + + // second, open rewrite rules + filename = create_filename(param.get("dicdir"), + REWRITE_FILE); + rewrite_.open(filename.c_str()); + + return true; +} + +bool EncoderFeatureIndex::open(const Param ¶m) { + return openTemplate(param); +} + +bool DecoderFeatureIndex::open(const Param ¶m) { + const std::string modelfile = param.get("model"); + // open the file as binary mode again and fallback to text file + if (!openBinaryModel(param)) { + std::cout << modelfile + << " is not a binary model. reopen it as text mode..." + << std::endl; + CHECK_DIE(openTextModel(param)) << + "no such file or directory: " << modelfile; + } + + if (!openTemplate(param)) { + close(); + return false; + } + + return true; +} + + +bool DecoderFeatureIndex::openFromArray(const char *begin, const char *end) { + const char *ptr = begin; + unsigned int maxid = 0; + read_static(&ptr, maxid); + maxid_ = static_cast(maxid); + const size_t file_size = static_cast(end - begin); + const size_t expected_file_size = + (sizeof(double) + sizeof(uint64_t)) * maxid_ + sizeof(maxid) + 32; + if (expected_file_size != file_size) { + return false; + } + charset_ = ptr; + ptr += 32; + alpha_ = reinterpret_cast(ptr); + ptr += (sizeof(alpha_[0]) * maxid_); + key_ = reinterpret_cast(ptr); + return true; +} + +bool DecoderFeatureIndex::openBinaryModel(const Param ¶m) { + const std::string modelfile = param.get("model"); + CHECK_DIE(mmap_.open(modelfile.c_str())) << mmap_.what(); + if (!openFromArray(mmap_.begin(), mmap_.end())) { + mmap_.close(); + return false; + } + const std::string to = param.get("charset"); + CHECK_DIE(decode_charset(charset_) == + decode_charset(to.c_str())) + << "model charset and dictionary charset are different. " + << "model_charset=" << charset_ + << " dictionary_charset=" << to; + + return true; +} + +bool DecoderFeatureIndex::openTextModel(const Param ¶m) { + const std::string modelfile = param.get("model"); + CHECK_DIE(FeatureIndex::convert(param, modelfile.c_str(), &model_buffer_)); + return openFromArray(model_buffer_.data(), + model_buffer_.data() + model_buffer_.size()); +} + +void DecoderFeatureIndex::clear() { + feature_freelist_.free(); +} + +void EncoderFeatureIndex::clear() {} + +void EncoderFeatureIndex::clearcache() { + feature_cache_.clear(); + rewrite_.clear(); +} + +void EncoderFeatureIndex::close() { + dic_.clear(); + feature_cache_.clear(); + maxid_ = 0; +} + +void DecoderFeatureIndex::close() { + mmap_.close(); + model_buffer_.clear(); + maxid_ = 0; +} + +void FeatureIndex::calcCost(LearnerNode *node) { + node->wcost = 0.0; + if (node->stat == MECAB_EOS_NODE) return; + for (const int *f = node->fvector; *f != -1; ++f) { + node->wcost += alpha_[*f]; + } +} + +void FeatureIndex::calcCost(LearnerPath *path) { + if (is_empty(path)) return; + path->cost = path->rnode->wcost; + for (const int *f = path->fvector; *f != -1; ++f) { + path->cost += alpha_[*f]; + } +} + +const char *FeatureIndex::strdup(const char *p) { + size_t len = std::strlen(p); + char *q = char_freelist_.alloc(len + 1); + std::strncpy(q, p, len + 1); + return q; +} + +bool DecoderFeatureIndex::buildFeature(LearnerPath *path) { + path->rnode->wcost = path->cost = 0.0; + + std::string ufeature1; + std::string lfeature1; + std::string rfeature1; + std::string ufeature2; + std::string lfeature2; + std::string rfeature2; + + CHECK_DIE(rewrite_.rewrite2(path->lnode->feature, + &ufeature1, + &lfeature1, + &rfeature1)) + << " cannot rewrite pattern: " + << path->lnode->feature; + + CHECK_DIE(rewrite_.rewrite2(path->rnode->feature, + &ufeature2, + &lfeature2, + &rfeature2)) + << " cannot rewrite pattern: " + << path->rnode->feature; + + if (!buildUnigramFeature(path, ufeature2.c_str())) { + return false; + } + + if (!buildBigramFeature(path, rfeature1.c_str(), lfeature2.c_str())) { + return false; + } + + return true; +} + +bool EncoderFeatureIndex::buildFeature(LearnerPath *path) { + path->rnode->wcost = path->cost = 0.0; + + std::string ufeature1; + std::string lfeature1; + std::string rfeature1; + std::string ufeature2; + std::string lfeature2; + std::string rfeature2; + + CHECK_DIE(rewrite_.rewrite2(path->lnode->feature, + &ufeature1, + &lfeature1, + &rfeature1)) + << " cannot rewrite pattern: " + << path->lnode->feature; + + CHECK_DIE(rewrite_.rewrite2(path->rnode->feature, + &ufeature2, + &lfeature2, + &rfeature2)) + << " cannot rewrite pattern: " + << path->rnode->feature; + + { + os_.clear(); + os_ << ufeature2 << ' ' << path->rnode->char_type << '\0'; + const std::string key(os_.str()); + std::map >::iterator + it = feature_cache_.find(key); + if (it != feature_cache_.end()) { + path->rnode->fvector = it->second.first; + it->second.second++; + } else { + if (!buildUnigramFeature(path, ufeature2.c_str())) { + return false; + } + feature_cache_.insert(std::pair + > + (key, + std::pair + (path->rnode->fvector, 1))); + } + } + + { + os_.clear(); + os_ << rfeature1 << ' ' << lfeature2 << '\0'; + std::string key(os_.str()); + std::map >::iterator + it = feature_cache_.find(key); + if (it != feature_cache_.end()) { + path->fvector = it->second.first; + it->second.second++; + } else { + if (!buildBigramFeature(path, rfeature1.c_str(), lfeature2.c_str())) + return false; + feature_cache_.insert(std::pair + > + (key, std::pair + (path->fvector, 1))); + } + } + + CHECK_DIE(path->fvector) << " fvector is NULL"; + CHECK_DIE(path->rnode->fvector) << "fevector is NULL"; + + return true; +} + +bool FeatureIndex::buildUnigramFeature(LearnerPath *path, + const char *ufeature) { + scoped_fixed_array ubuf; + scoped_fixed_array F; + + feature_.clear(); + std::strncpy(ubuf.get(), ufeature, ubuf.size()); + const size_t usize = tokenizeCSV(ubuf.get(), F.get(), F.size()); + + for (std::vector::const_iterator it = unigram_templs_.begin(); + it != unigram_templs_.end(); ++it) { + const char *p = *it; + os_.clear(); + + for (; *p; p++) { + switch (*p) { + default: os_ << *p; break; + case '\\': os_ << getEscapedChar(*++p); break; + case '%': { + switch (*++p) { + case 'F': { + const char *r = getIndex(const_cast(&p), F.get(), usize); + if (!r) goto NEXT; + os_ << r; + } break; + case 't': os_ << (size_t)path->rnode->char_type; break; + case 'u': os_ << ufeature; break; + case 'w': + if (path->rnode->stat == MECAB_NOR_NODE) { + os_.write(path->rnode->surface, path->rnode->length); + } + default: + CHECK_DIE(false) << "unknown meta char: " << *p; + } + } + } + } + + os_ << '\0'; + ADDB(os_.str()); + + NEXT: continue; + } + + COPY_FEATURE(path->rnode->fvector); + + return true; +} + +bool FeatureIndex::buildBigramFeature(LearnerPath *path, + const char *rfeature, + const char *lfeature) { + scoped_fixed_array rbuf; + scoped_fixed_array lbuf; + scoped_fixed_array R; + scoped_fixed_array L; + + feature_.clear(); + std::strncpy(lbuf.get(), rfeature, lbuf.size()); + std::strncpy(rbuf.get(), lfeature, rbuf.size()); + + const size_t lsize = tokenizeCSV(lbuf.get(), L.get(), L.size()); + const size_t rsize = tokenizeCSV(rbuf.get(), R.get(), R.size()); + + for (std::vector::const_iterator it = bigram_templs_.begin(); + it != bigram_templs_.end(); ++it) { + const char *p = *it; + os_.clear(); + + for (; *p; p++) { + switch (*p) { + default: os_ << *p; break; + case '\\': os_ << getEscapedChar(*++p); break; + case '%': { + switch (*++p) { + case 'L': { + const char *r = getIndex(const_cast(&p), L.get(), lsize); + if (!r) goto NEXT; + os_ << r; + } break; + case 'R': { + const char *r = getIndex(const_cast(&p), R.get(), rsize); + if (!r) goto NEXT; + os_ << r; + } break; + case 'l': os_ << lfeature; break; // use lfeature as it is + case 'r': os_ << rfeature; break; + default: + CHECK_DIE(false) << "unknown meta char: " << *p; + } + } + } + } + + os_ << '\0'; + + ADDB(os_.str()); + + NEXT: continue; + } + + COPY_FEATURE(path->fvector); + + return true; +} + +int DecoderFeatureIndex::id(const char *key) { + const uint64_t fp = fingerprint(key, std::strlen(key)); + const uint64_t *result = std::lower_bound(key_, + key_ + maxid_, + fp); + if (result == key_ + maxid_ || *result != fp) { + return -1; + } + const int n = static_cast(result - key_); + CHECK_DIE(key_[n] == fp); + return n; +} + +int EncoderFeatureIndex::id(const char *key) { + std::map::const_iterator it = dic_.find(key); + if (it == dic_.end()) { + dic_.insert(std::pair(std::string(key), maxid_)); + return maxid_++; + } else { + return it->second; + } + return -1; +} + +void EncoderFeatureIndex::shrink(size_t freq, + std::vector *observed) { + std::vector freqv; + // count fvector + freqv.resize(maxid_); + for (std::map >::const_iterator + it = feature_cache_.begin(); + it != feature_cache_.end(); ++it) { + for (const int *f = it->second.first; *f != -1; ++f) { + freqv[*f] += it->second.second; // freq + } + } + + if (freq <= 1) { + return; + } + + // make old2new map + maxid_ = 0; + std::map old2new; + for (size_t i = 0; i < freqv.size(); ++i) { + if (freqv[i] >= freq) { + old2new.insert(std::pair(i, maxid_++)); + } + } + + // update dic_ + for (std::map::iterator + it = dic_.begin(); it != dic_.end();) { + std::map::const_iterator it2 = old2new.find(it->second); + if (it2 != old2new.end()) { + it->second = it2->second; + ++it; + } else { + dic_.erase(it++); + } + } + + // update all fvector + for (std::map >::const_iterator + it = feature_cache_.begin(); it != feature_cache_.end(); ++it) { + int *to = const_cast(it->second.first); + for (const int *f = it->second.first; *f != -1; ++f) { + std::map::const_iterator it2 = old2new.find(*f); + if (it2 != old2new.end()) { + *to = it2->second; + ++to; + } + } + *to = -1; + } + + // update observed vector + std::vector observed_new(maxid_); + for (size_t i = 0; i < observed->size(); ++i) { + std::map::const_iterator it = old2new.find(static_cast(i)); + if (it != old2new.end()) { + observed_new[it->second] = (*observed)[i]; + } + } + + // copy + *observed = observed_new; + + return; +} + +bool FeatureIndex::compile(const Param ¶m, + const char* txtfile, const char *binfile) { + std::string buf; + FeatureIndex::convert(param, txtfile, &buf); + std::ofstream ofs(WPATH(binfile), std::ios::binary|std::ios::out); + CHECK_DIE(ofs) << "permission denied: " << binfile; + ofs.write(buf.data(), buf.size()); + return true; +} + +bool FeatureIndex::convert(const Param ¶m, + const char* txtfile, std::string *output) { + std::ifstream ifs(WPATH(txtfile)); + CHECK_DIE(ifs) << "no such file or directory: " << txtfile; + scoped_fixed_array buf; + char *column[4]; + std::vector > dic; + std::string model_charset; + + while (ifs.getline(buf.get(), buf.size())) { + if (std::strlen(buf.get()) == 0) { + break; + } + CHECK_DIE(tokenize2(buf.get(), ":", column, 2) == 2) + << "format error: " << buf.get(); + if (std::string(column[0]) == "charset") { + model_charset = column[1] + 1; + } + } + + std::string from = param.get("dictionary-charset"); + std::string to = param.get("charset"); + + if (!from.empty()) { + CHECK_DIE(decode_charset(model_charset.c_str()) + == decode_charset(from.c_str())) + << "dictionary charset and model charset are different. " + << "dictionary_charset=" << from + << " model_charset=" << model_charset; + } else { + from = model_charset; + } + + if (to.empty()) { + to = from; + } + + Iconv iconv; + CHECK_DIE(iconv.open(from.c_str(), to.c_str())) + << "cannot create model from=" << from + << " to=" << to; + + while (ifs.getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t", column, 2) == 2) + << "format error: " << buf.get(); + std::string feature = column[1]; + CHECK_DIE(iconv.convert(&feature)); + const uint64_t fp = fingerprint(feature); + const double alpha = atof(column[0]); + dic.push_back(std::pair(fp, alpha)); + } + + output->clear(); + unsigned int size = static_cast(dic.size()); + output->append(reinterpret_cast(&size), sizeof(size)); + + char charset_buf[32]; + std::fill(charset_buf, charset_buf + sizeof(charset_buf), '\0'); + std::strncpy(charset_buf, to.c_str(), 31); + output->append(reinterpret_cast(charset_buf), + sizeof(charset_buf)); + + std::sort(dic.begin(), dic.end()); + + for (size_t i = 0; i < dic.size(); ++i) { + const double alpha = dic[i].second; + output->append(reinterpret_cast(&alpha), sizeof(alpha)); + } + + for (size_t i = 0; i < dic.size(); ++i) { + const uint64_t fp = dic[i].first; + output->append(reinterpret_cast(&fp), sizeof(fp)); + } + + return true; +} + +// TODO(taku): consider charset +bool EncoderFeatureIndex::reopen(const char *filename, + const char *dic_charset, + std::vector *alpha, + Param *param) { + close(); + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + return false; + } + + scoped_fixed_array buf; + char *column[8]; + + std::string model_charset; + + while (ifs.getline(buf.get(), buf.size())) { + if (std::strlen(buf.get()) == 0) { + break; + } + CHECK_DIE(tokenize2(buf.get(), ":", column, 2) == 2) + << "format error: " << buf.get(); + if (std::string(column[0]) == "charset") { + model_charset = column[1] + 1; + } else { + param->set(column[0], column[1] + 1, true); + } + } + + CHECK_DIE(dic_charset); + CHECK_DIE(!model_charset.empty()) << "charset is empty"; + + Iconv iconv; + CHECK_DIE(iconv.open(model_charset.c_str(), dic_charset)) + << "cannot create model from=" << model_charset + << " to=" << dic_charset; + + alpha->clear(); + CHECK_DIE(maxid_ == 0); + CHECK_DIE(dic_.empty()); + + while (ifs.getline(buf.get(), buf.size())) { + CHECK_DIE(tokenize2(buf.get(), "\t", column, 2) == 2) + << "format error: " << buf.get(); + std::string feature = column[1]; + CHECK_DIE(iconv.convert(&feature)); + dic_.insert(std::make_pair(feature, maxid_++)); + alpha->push_back(atof(column[0])); + } + + return true; +} + +bool EncoderFeatureIndex::save(const char *filename, const char *header) const { + CHECK_DIE(header); + CHECK_DIE(alpha_); + + std::ofstream ofs(WPATH(filename)); + if (!ofs) { + return false; + } + + ofs.setf(std::ios::fixed, std::ios::floatfield); + ofs.precision(16); + + ofs << header; + ofs << std::endl; + + for (std::map::const_iterator it = dic_.begin(); + it != dic_.end(); ++it) { + ofs << alpha_[it->second] << '\t' << it->first << '\n'; + } + + return true; +} +} diff --git a/fts/third_party/mecab/src/feature_index.h b/fts/third_party/mecab/src/feature_index.h new file mode 100644 index 00000000..9e08caab --- /dev/null +++ b/fts/third_party/mecab/src/feature_index.h @@ -0,0 +1,115 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_FEATUREINDEX_H_ +#define MECAB_FEATUREINDEX_H_ + +#include +#include +#include "mecab.h" +#include "mmap.h" +#include "darts.h" +#include "freelist.h" +#include "common.h" +#include "learner_node.h" +#include "string_buffer.h" +#include "dictionary_rewriter.h" + +namespace MeCab { + +class Param; + +class FeatureIndex { + public: + virtual bool open(const Param ¶m) = 0; + virtual void clear() = 0; + virtual void close() = 0; + virtual bool buildFeature(LearnerPath *path) = 0; + + void set_alpha(const double *alpha); + + size_t size() const { return maxid_; } + + bool buildUnigramFeature(LearnerPath *, const char *); + bool buildBigramFeature(LearnerPath *, const char *, const char*); + + void calcCost(LearnerPath *path); + void calcCost(LearnerNode *node); + + const char *strdup(const char *str); + + static bool convert(const Param ¶m, + const char *text_filename, std::string *output); + static bool compile(const Param ¶m, + const char *text_filename, const char *binary_filename); + + explicit FeatureIndex(): feature_freelist_(8192 * 32), + char_freelist_(8192 * 32), + maxid_(0), alpha_(0) {} + virtual ~FeatureIndex() {} + + protected: + std::vector feature_; + ChunkFreeList feature_freelist_; + ChunkFreeList char_freelist_; + std::vector unigram_templs_; + std::vector bigram_templs_; + DictionaryRewriter rewrite_; + StringBuffer os_; + size_t maxid_; + const double *alpha_; + + virtual int id(const char *key) = 0; + const char* getIndex(char **, char **, size_t); + bool openTemplate(const Param ¶m); +}; + +class EncoderFeatureIndex: public FeatureIndex { + public: + bool open(const Param ¶m); + void close(); + void clear(); + + bool reopen(const char *filename, + const char *charset, + std::vector *alpha, + Param *param); + + bool save(const char *filename, const char *header) const; + void shrink(size_t freq, + std::vector *observed); + bool buildFeature(LearnerPath *path); + void clearcache(); + + private: + std::map dic_; + std::map > feature_cache_; + int id(const char *key); +}; + +class DecoderFeatureIndex: public FeatureIndex { + public: + bool open(const Param ¶m); + void clear(); + void close(); + bool buildFeature(LearnerPath *path); + + const char *charset() const { + return charset_; + } + + private: + bool openFromArray(const char *begin, const char *end); + bool openBinaryModel(const Param ¶m); + bool openTextModel(const Param ¶m); + int id(const char *key); + + Mmap mmap_; + std::string model_buffer_; + const uint64_t *key_; + const char *charset_; +}; +} +#endif diff --git a/fts/third_party/mecab/src/freelist.h b/fts/third_party/mecab/src/freelist.h new file mode 100644 index 00000000..85de6344 --- /dev/null +++ b/fts/third_party/mecab/src/freelist.h @@ -0,0 +1,85 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_FREELIST_H +#define MECAB_FREELIST_H + +#include +#include +#include "utils.h" +#include "common.h" + +namespace MeCab { + +template class FreeList { + private: + std::vector freeList; + size_t pi_; + size_t li_; + size_t size; + + public: + void free() { li_ = pi_ = 0; } + + T* alloc() { + if (pi_ == size) { + li_++; + pi_ = 0; + } + if (li_ == freeList.size()) freeList.push_back(new T[size]); + return freeList[li_] + (pi_++); + } + + explicit FreeList(size_t _size): pi_(0), li_(0), size(_size) {} + + virtual ~FreeList() { + for (li_ = 0; li_ < freeList.size(); li_++) + delete [] freeList[li_]; + } +}; + +template class ChunkFreeList { + private: + std::vector > freelist_; + size_t pi_; + size_t li_; + size_t default_size; + + public: + void free() { li_ = pi_ = 0; } + + T* alloc(T *src) { + T* n = alloc(1); + *n = *src; + return n; + } + + T* alloc(size_t req = 1) { + while (li_ < freelist_.size()) { + if ((pi_ + req) < freelist_[li_].first) { + T *r = freelist_[li_].second + pi_; + pi_ += req; + return r; + } + li_++; + pi_ = 0; + } + size_t _size = std::max(req, default_size); + freelist_.push_back(std::make_pair(_size, new T[_size])); + li_ = freelist_.size() - 1; + pi_ += req; + return freelist_[li_].second; + } + + explicit ChunkFreeList(size_t _size): + pi_(0), li_(0), default_size(_size) {} + + virtual ~ChunkFreeList() { + for (li_ = 0; li_ < freelist_.size(); li_++) + delete [] freelist_[li_].second; + } +}; +} +#endif diff --git a/fts/third_party/mecab/src/iconv_utils.cpp b/fts/third_party/mecab/src/iconv_utils.cpp new file mode 100644 index 00000000..1a815664 --- /dev/null +++ b/fts/third_party/mecab/src/iconv_utils.cpp @@ -0,0 +1,203 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "scoped_ptr.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "char_property.h" +#include "iconv_utils.h" + +#if defined(_WIN32) && !defined(__CYGWIN__) +#include "windows.h" +#endif + +namespace { + +#ifdef HAVE_ICONV +const char * decode_charset_iconv(const char *str) { + const int charset = MeCab::decode_charset(str); + switch (charset) { + case MeCab::UTF8: + return "UTF-8"; + case MeCab::EUC_JP: + return "EUC-JP"; + case MeCab::CP932: + return "SHIFT-JIS"; + case MeCab::UTF16: + return "UTF-16"; + case MeCab::UTF16LE: + return "UTF-16LE"; + case MeCab::UTF16BE: + return "UTF-16BE"; + default: + std::cerr << "charset " << str + << " is not defined, use " MECAB_DEFAULT_CHARSET; + return MECAB_DEFAULT_CHARSET; + } + return MECAB_DEFAULT_CHARSET; +} +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +DWORD decode_charset_win32(const char *str) { + const int charset = MeCab::decode_charset(str); + switch (charset) { + case MeCab::UTF8: + return CP_UTF8; + case MeCab::UTF16: + return 1200; + case MeCab::UTF16LE: + return 1200; + case MeCab::UTF16BE: + return 1201; + case MeCab::EUC_JP: + // return 51932; + return 20932; + case MeCab::CP932: + return 932; + default: + std::cerr << "charset " << str + << " is not defined, use 'CP_THREAD_ACP'"; + return CP_THREAD_ACP; + } + return 0; +} +#endif +} // namespace + +namespace MeCab { +bool Iconv::open(const char* from, const char* to) { + ic_ = 0; +#if defined HAVE_ICONV + const char *from2 = decode_charset_iconv(from); + const char *to2 = decode_charset_iconv(to); + if (std::strcmp(from2, to2) == 0) { + return true; + } + ic_ = 0; + ic_ = iconv_open(to2, from2); + if (ic_ == (iconv_t)(-1)) { + ic_ = 0; + return false; + } +#else +#if defined(_WIN32) && !defined(__CYGWIN__) + from_cp_ = decode_charset_win32(from); + to_cp_ = decode_charset_win32(to); + if (from_cp_ == to_cp_) { + return true; + } + ic_ = from_cp_; +#else + std::cerr << "iconv_open is not supported" << std::endl; +#endif +#endif + + return true; +} + +bool Iconv::convert(std::string *str) { + if (str->empty()) { + return true; + } + if (ic_ == 0) { + return true; + } + +#if defined HAVE_ICONV + size_t ilen = 0; + size_t olen = 0; + ilen = str->size(); + olen = ilen * 4; + std::string tmp; + tmp.reserve(olen); + char *ibuf = const_cast(str->data()); + char *obuf_org = const_cast(tmp.data()); + char *obuf = obuf_org; + std::fill(obuf, obuf + olen, 0); + size_t olen_org = olen; + iconv(ic_, 0, &ilen, 0, &olen); // reset iconv state + while (ilen != 0) { + if (iconv(ic_, (ICONV_CONST char **)&ibuf, &ilen, &obuf, &olen) + == (size_t) -1) { + return false; + } + } + str->assign(obuf_org, olen_org - olen); +#else +#if defined(_WIN32) && !defined(__CYGWIN__) + // covert it to wide character first + const size_t wide_len = ::MultiByteToWideChar(from_cp_, 0, + str->c_str(), + -1, NULL, 0); + if (wide_len == 0) { + return false; + } + + scoped_array wide_str(new wchar_t[wide_len + 1]); + + if (!wide_str.get()) { + return false; + }; + + if (::MultiByteToWideChar(from_cp_, 0, str->c_str(), -1, + wide_str.get(), wide_len + 1) == 0) { + return false; + } + + if (to_cp_ == 1200 || to_cp_ == 1201) { + str->resize(2 * wide_len); + std::memcpy(const_cast(str->data()), + reinterpret_cast(wide_str.get()), wide_len * 2); + if (to_cp_ == 1201) { + char *buf = const_cast(str->data()); + for (size_t i = 0; i < 2 * wide_len; i += 2) { + std::swap(buf[i], buf[i+1]); + } + } + return true; + } + + const size_t output_len = ::WideCharToMultiByte(to_cp_, 0, + wide_str.get(), + -1, + NULL, 0, NULL, NULL); + + if (output_len == 0) { + return false; + } + + scoped_array encoded(new char[output_len + 1]); + if (::WideCharToMultiByte(to_cp_, 0, wide_str.get(), wide_len, + encoded.get(), output_len + 1, + NULL, NULL) == 0) { + return false; + } + + str->assign(encoded.get()); + +#endif +#endif + + return true; +} + +Iconv::Iconv() : ic_(0) {} + +Iconv::~Iconv() { +#if defined HAVE_ICONV + if (ic_ != 0) iconv_close(ic_); +#endif +} +} diff --git a/fts/third_party/mecab/src/iconv_utils.h b/fts/third_party/mecab/src/iconv_utils.h new file mode 100644 index 00000000..69b5a029 --- /dev/null +++ b/fts/third_party/mecab/src/iconv_utils.h @@ -0,0 +1,40 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_ICONV_H +#define MECAB_ICONV_H + +#if defined HAVE_ICONV +#include +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +#include "windows.h" +#endif + +namespace MeCab { + +class Iconv { + private: +#ifdef HAVE_ICONV + iconv_t ic_; +#else + int ic_; +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) + DWORD from_cp_; + DWORD to_cp_; +#endif + + public: + explicit Iconv(); + virtual ~Iconv(); + bool open(const char *from, const char *to); + bool convert(std::string *); +}; +} + +#endif diff --git a/fts/third_party/mecab/src/lbfgs.cpp b/fts/third_party/mecab/src/lbfgs.cpp new file mode 100644 index 00000000..b9ed45f2 --- /dev/null +++ b/fts/third_party/mecab/src/lbfgs.cpp @@ -0,0 +1,572 @@ +// MeCab: Yet Another Part-of-Speech and Morphological Analyzer +// +// +// lbfgs.c was ported from the FORTRAN code of lbfgs.m to C +// using f2c converter +// +// http://www.ece.northwestern.edu/~nocedal/lbfgs.html +// +// Software for Large-scale Unconstrained Optimization +// L-BFGS is a limited-memory quasi-Newton code for unconstrained +// optimization. +// The code has been developed at the Optimization Technology Center, +// a joint venture of Argonne National Laboratory and Northwestern University. +// +// Authors +// Jorge Nocedal +// +// References +// - J. Nocedal. Updating Quasi-Newton Matrices with Limited Storage(1980), +// Mathematics of Computation 35, pp. 773-782. +// - D.C. Liu and J. Nocedal. On the Limited Memory Method for +// Large Scale Optimization(1989), +// Mathematical Programming B, 45, 3, pp. 503-528. +#include +#include +#include +#include "lbfgs.h" +#include "common.h" + +namespace { +static const double ftol = 1e-4; +static const double xtol = 1e-16; +static const double eps = 1e-7; +static const double lb3_1_gtol = 0.9; +static const double lb3_1_stpmin = 1e-20; +static const double lb3_1_stpmax = 1e20; +static const int lb3_1_mp = 6; +static const int lb3_1_lp = 6; + +inline double sigma(double x) { + if (x > 0) { + return 1.0; + } else if (x < 0) { + return -1.0; + } + return 0.0; +} + +inline double pi(double x, double y) { + return sigma(x) == sigma(y) ? x : 0.0; +} + +inline void daxpy_(int n, double da, const double *dx, double *dy) { + for (int i = 0; i < n; ++i) { + dy[i] += da * dx[i]; + } +} + +inline double ddot_(int size, const double *dx, const double *dy) { + return std::inner_product(dx, dx + size, dy, 0.0); +} + +void mcstep(double *stx, double *fx, double *dx, + double *sty, double *fy, double *dy, + double *stp, double fp, double dp, + int *brackt, + double stpmin, double stpmax, + int *info) { + bool bound = true; + double p, q, s, d1, d2, d3, r, gamma, theta, stpq, stpc, stpf; + *info = 0; + + if (*brackt && ((*stp <= std::min(*stx, *sty) || + *stp >= std::max(*stx, *sty)) || + *dx * (*stp - *stx) >= 0.0 || stpmax < stpmin)) { + return; + } + + double sgnd = dp * (*dx / std::abs(*dx)); + + if (fp > *fx) { + *info = 1; + bound = true; + theta =(*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dx / s *(dp / s)); + if (*stp < *stx) { + gamma = -gamma; + } + p = gamma - *dx + theta; + q = gamma - *dx + gamma + dp; + r = p / q; + stpc = *stx + r * (*stp - *stx); + stpq = *stx + *dx / ((*fx - fp) / + (*stp - *stx) + *dx) / 2 * (*stp - *stx); + if ((d1 = stpc - *stx, std::abs(d1)) < (d2 = stpq - *stx, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpc + (stpq - stpc) / 2; + } + *brackt = true; + } else if (sgnd < 0.0) { + *info = 2; + bound = false; + theta = (*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dx / s * (dp / s)); + if (*stp > *stx) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma - dp + gamma + *dx; + r = p / q; + stpc = *stp + r *(*stx - *stp); + stpq = *stp + dp /(dp - *dx) * (*stx - *stp); + if ((d1 = stpc - *stp, std::abs(d1)) > (d2 = stpq - *stp, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + *brackt = true; + } else if (std::abs(dp) < std::abs(*dx)) { + *info = 3; + bound = true; + theta = (*fx - fp) * 3 / (*stp - *stx) + *dx + dp; + d1 = std::abs(theta); + d2 = std::abs(*dx); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d3 = theta / s; + d1 = 0.0; + d2 = d3 * d3 - *dx / s *(dp / s); + gamma = s * std::sqrt((std::max(d1, d2))); + if (*stp > *stx) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma + (*dx - dp) + gamma; + r = p / q; + if (r < 0.0 && gamma != 0.0) { + stpc = *stp + r *(*stx - *stp); + } else if (*stp > *stx) { + stpc = stpmax; + } else { + stpc = stpmin; + } + stpq = *stp + dp /(dp - *dx) * (*stx - *stp); + if (*brackt) { + if ((d1 = *stp - stpc, std::abs(d1)) < + (d2 = *stp - stpq, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + } else { + if ((d1 = *stp - stpc, std::abs(d1)) > + (d2 = *stp - stpq, std::abs(d2))) { + stpf = stpc; + } else { + stpf = stpq; + } + } + } else { + *info = 4; + bound = false; + if (*brackt) { + theta =(fp - *fy) * 3 / (*sty - *stp) + *dy + dp; + d1 = std::abs(theta); + d2 = std::abs(*dy); + d1 = std::max(d1, d2); + d2 = std::abs(dp); + s = std::max(d1, d2); + d1 = theta / s; + gamma = s * std::sqrt(d1 * d1 - *dy / s * (dp / s)); + if (*stp > *sty) { + gamma = -gamma; + } + p = gamma - dp + theta; + q = gamma - dp + gamma + *dy; + r = p / q; + stpc = *stp + r * (*sty - *stp); + stpf = stpc; + } else if (*stp > *stx) { + stpf = stpmax; + } else { + stpf = stpmin; + } + } + + if (fp > *fx) { + *sty = *stp; + *fy = fp; + *dy = dp; + } else { + if (sgnd < 0.0) { + *sty = *stx; + *fy = *fx; + *dy = *dx; + } + *stx = *stp; + *fx = fp; + *dx = dp; + } + + stpf = std::min(stpmax, stpf); + stpf = std::max(stpmin, stpf); + *stp = stpf; + if (*brackt && bound) { + if (*sty > *stx) { + d1 = *stx + (*sty - *stx) * 0.66; + *stp = std::min(d1, *stp); + } else { + d1 = *stx + (*sty - *stx) * 0.66; + *stp = std::max(d1, *stp); + } + } + + return; +} +} + +namespace MeCab { + +class LBFGS::Mcsrch { + private: + int infoc, stage1, brackt; + double finit, dginit, dgtest, width, width1; + double stx, fx, dgx, sty, fy, dgy, stmin, stmax; + + public: + Mcsrch(): + infoc(0), + stage1(0), + brackt(0), + finit(0.0), dginit(0.0), dgtest(0.0), width(0.0), width1(0.0), + stx(0.0), fx(0.0), dgx(0.0), sty(0.0), fy(0.0), dgy(0.0), + stmin(0.0), stmax(0.0) {} + + void mcsrch(int size, + double *x, + double f, const double *g, double *s, + double *stp, + int *info, int *nfev, double *wa, bool orthant, double C) { + const double p5 = 0.5; + const double p66 = 0.66; + const double xtrapf = 4.0; + const int maxfev = 20; + + /* Parameter adjustments */ + --wa; + --s; + --g; + --x; + + if (*info == -1) { + goto L45; + } + infoc = 1; + + if (size <= 0 || *stp <= 0.0) { + return; + } + + dginit = ddot_(size, &g[1], &s[1]); + if (dginit >= 0.0) { + return; + } + + brackt = false; + stage1 = true; + *nfev = 0; + finit = f; + dgtest = ftol * dginit; + width = lb3_1_stpmax - lb3_1_stpmin; + width1 = width / p5; + for (int j = 1; j <= size; ++j) { + wa[j] = x[j]; + } + + stx = 0.0; + fx = finit; + dgx = dginit; + sty = 0.0; + fy = finit; + dgy = dginit; + + while (true) { + if (brackt) { + stmin = std::min(stx, sty); + stmax = std::max(stx, sty); + } else { + stmin = stx; + stmax = *stp + xtrapf * (*stp - stx); + } + + *stp = std::max(*stp, lb3_1_stpmin); + *stp = std::min(*stp, lb3_1_stpmax); + + if ((brackt && ((*stp <= stmin || *stp >= stmax) || + *nfev >= maxfev - 1 || infoc == 0)) || + (brackt && (stmax - stmin <= xtol * stmax))) { + *stp = stx; + } + + if (orthant) { + for (int j = 1; j <= size; ++j) { + double grad_neg = 0.0; + double grad_pos = 0.0; + double grad = 0.0; + if (wa[j] == 0.0) { + grad_neg = g[j] - 1.0 / C; + grad_pos = g[j] + 1.0 / C; + } else { + grad_pos = grad_neg = g[j] + 1.0 * sigma(wa[j]) / C; + } + if (grad_neg > 0.0) { + grad = grad_neg; + } else if (grad_pos < 0.0) { + grad = grad_pos; + } else { + grad = 0.0; + } + const double p = pi(s[j], -grad); + const double xi = wa[j] == 0.0 ? sigma(-grad) : sigma(wa[j]); + x[j] = pi(wa[j] + *stp * p, xi); + } + } else { + for (int j = 1; j <= size; ++j) { + x[j] = wa[j] + *stp * s[j]; + } + } + *info = -1; + return; + + L45: + *info = 0; + ++(*nfev); + double dg = ddot_(size, &g[1], &s[1]); + double ftest1 = finit + *stp * dgtest; + + if (brackt && ((*stp <= stmin || *stp >= stmax) || infoc == 0)) { + *info = 6; + } + if (*stp == lb3_1_stpmax && f <= ftest1 && dg <= dgtest) { + *info = 5; + } + if (*stp == lb3_1_stpmin && (f > ftest1 || dg >= dgtest)) { + *info = 4; + } + if (*nfev >= maxfev) { + *info = 3; + } + if (brackt && stmax - stmin <= xtol * stmax) { + *info = 2; + } + if (f <= ftest1 && std::abs(dg) <= lb3_1_gtol * (-dginit)) { + *info = 1; + } + + if (*info != 0) { + return; + } + + if (stage1 && f <= ftest1 && dg >= std::min(ftol, lb3_1_gtol) * dginit) { + stage1 = false; + } + + if (stage1 && f <= fx && f > ftest1) { + double fm = f - *stp * dgtest; + double fxm = fx - stx * dgtest; + double fym = fy - sty * dgtest; + double dgm = dg - dgtest; + double dgxm = dgx - dgtest; + double dgym = dgy - dgtest; + mcstep(&stx, &fxm, &dgxm, &sty, &fym, &dgym, stp, fm, dgm, &brackt, + stmin, stmax, &infoc); + fx = fxm + stx * dgtest; + fy = fym + sty * dgtest; + dgx = dgxm + dgtest; + dgy = dgym + dgtest; + } else { + mcstep(&stx, &fx, &dgx, &sty, &fy, &dgy, stp, f, dg, &brackt, + stmin, stmax, &infoc); + } + + if (brackt) { + double d1 = 0.0; + if ((d1 = sty - stx, std::abs(d1)) >= p66 * width1) { + *stp = stx + p5 * (sty - stx); + } + width1 = width; + width = (d1 = sty - stx, std::abs(d1)); + } + } + + return; + } +}; + +void LBFGS::clear() { + iflag_ = iscn = nfev = iycn = point = npt = + iter = info = ispt = isyt = iypt = 0; + stp = stp1 = 0.0; + diag_.clear(); + w_.clear(); + delete mcsrch_; + mcsrch_ = 0; +} + +void LBFGS::lbfgs_optimize(int size, + int msize, + double *x, + double f, + const double *g, + double *diag, + double *w, + bool orthant, + double C, + int *iflag) { + double yy = 0.0; + double ys = 0.0; + int bound = 0; + int cp = 0; + + --diag; + --g; + --x; + --w; + + if (!mcsrch_) { + mcsrch_ = new Mcsrch; + } + + if (*iflag == 1) { + goto L172; + } + if (*iflag == 2) { + goto L100; + } + + // initialization + if (*iflag == 0) { + point = 0; + for (int i = 1; i <= size; ++i) { + diag[i] = 1.0; + } + ispt = size + (msize << 1); + iypt = ispt + size * msize; + for (int i = 1; i <= size; ++i) { + w[ispt + i] = -g[i] * diag[i]; + } + stp1 = 1.0 / std::sqrt(ddot_(size, &g[1], &g[1])); + } + + // MAIN ITERATION LOOP + while (true) { + ++iter; + info = 0; + if (iter == 1) goto L165; + if (iter > size) bound = size; + + // COMPUTE -H*G USING THE FORMULA GIVEN IN: Nocedal, J. 1980, + // "Updating quasi-Newton matrices with limited storage", + // Mathematics of Computation, Vol.24, No.151, pp. 773-782. + ys = ddot_(size, &w[iypt + npt + 1], &w[ispt + npt + 1]); + yy = ddot_(size, &w[iypt + npt + 1], &w[iypt + npt + 1]); + for (int i = 1; i <= size; ++i) { + diag[i] = ys / yy; + } + + L100: + cp = point; + if (point == 0) cp = msize; + w[size + cp] = 1.0 / ys; + + for (int i = 1; i <= size; ++i) { + w[i] = -g[i]; + } + + bound = std::min(iter - 1, msize); + + cp = point; + for (int i = 1; i <= bound; ++i) { + --cp; + if (cp == -1) cp = msize - 1; + double sq = ddot_(size, &w[ispt + cp * size + 1], &w[1]); + int inmc = size + msize + cp + 1; + iycn = iypt + cp * size; + w[inmc] = w[size + cp + 1] * sq; + double d = -w[inmc]; + daxpy_(size, d, &w[iycn + 1], &w[1]); + } + + for (int i = 1; i <= size; ++i) { + w[i] = diag[i] * w[i]; + } + + for (int i = 1; i <= bound; ++i) { + double yr = ddot_(size, &w[iypt + cp * size + 1], &w[1]); + double beta = w[size + cp + 1] * yr; + int inmc = size + msize + cp + 1; + beta = w[inmc] - beta; + iscn = ispt + cp * size; + daxpy_(size, beta, &w[iscn + 1], &w[1]); + ++cp; + if (cp == msize) { + cp = 0; + } + } + + // STORE THE NEW SEARCH DIRECTION + for (int i = 1; i <= size; ++i) { + w[ispt + point * size + i] = w[i]; + } + + L165: + // OBTAIN THE ONE-DIMENSIONAL MINIMIZER OF THE FUNCTION + // BY USING THE LINE SEARCH ROUTINE MCSRCH + nfev = 0; + stp = 1.0; + if (iter == 1) { + stp = stp1; + } + for (int i = 1; i <= size; ++i) { + w[i] = g[i]; + } + + L172: + mcsrch_->mcsrch(size, &x[1], f, &g[1], &w[ispt + point * size + 1], + &stp, &info, &nfev, &diag[1], orthant, C); + if (info == -1) { + *iflag = 1; // next value + return; + } + if (info != 1) { + std::cerr << "The line search routine mcsrch failed: error code:" + << info << std::endl; + *iflag = -1; + return; + } + + // COMPUTE THE NEW STEP AND GRADIENT CHANGE + npt = point * size; + for (int i = 1; i <= size; ++i) { + w[ispt + npt + i] = stp * w[ispt + npt + i]; + w[iypt + npt + i] = g[i] - w[i]; + } + ++point; + if (point == msize) { + point = 0; + } + + double gnorm = std::sqrt(ddot_(size, &g[1], &g[1])); + double xnorm = std::max(1.0, std::sqrt(ddot_(size, &x[1], &x[1]))); + if (gnorm / xnorm <= eps) { + *iflag = 0; // OK terminated + return; + } + } +} +} diff --git a/fts/third_party/mecab/src/lbfgs.h b/fts/third_party/mecab/src/lbfgs.h new file mode 100644 index 00000000..64eb0f2b --- /dev/null +++ b/fts/third_party/mecab/src/lbfgs.h @@ -0,0 +1,71 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_LBFGS_H_ +#define MECAB_LBFGS_H_ + +#include +#include + +namespace MeCab { + +class LBFGS { + public: + explicit LBFGS(): iflag_(0), iscn(0), nfev(0), iycn(0), + point(0), npt(0), iter(0), info(0), + ispt(0), isyt(0), iypt(0), maxfev(0), + stp(0.0), stp1(0.0), mcsrch_(0) {} + virtual ~LBFGS() { clear(); } + + void clear(); + + int optimize(size_t size, double *x, double f, double *g, + bool orthant, double C) { + static const int msize = 5; + if (w_.empty()) { + iflag_ = 0; + w_.resize(size * (2 * msize + 1) + 2 * msize); + diag_.resize(size); + } else if (diag_.size() != size) { + std::cerr << "size of array is different" << std::endl; + return -1; + } + + lbfgs_optimize(static_cast(size), + msize, x, f, g, &diag_[0], &w_[0], orthant, C, &iflag_); + + if (iflag_ < 0) { + std::cerr << "routine stops with unexpected error" << std::endl; + return -1; + } + + if (iflag_ == 0) { + clear(); + return 0; // terminate + } + + return 1; // evaluate next f and g + } + + private: + class Mcsrch; + int iflag_, iscn, nfev, iycn, point, npt; + int iter, info, ispt, isyt, iypt, maxfev; + double stp, stp1; + std::vector diag_; + std::vector w_; + Mcsrch *mcsrch_; + + void lbfgs_optimize(int size, + int msize, + double *x, + double f, + const double *g, + double *diag, + double *w, bool orthant, double C, int *iflag); +}; +} + +#endif diff --git a/fts/third_party/mecab/src/learner.cpp b/fts/third_party/mecab/src/learner.cpp new file mode 100644 index 00000000..a04a6727 --- /dev/null +++ b/fts/third_party/mecab/src/learner.cpp @@ -0,0 +1,320 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include "common.h" +#include "feature_index.h" +#include "freelist.h" +#include "lbfgs.h" +#include "learner_tagger.h" +#include "param.h" +#include "string_buffer.h" +#include "thread.h" +#include "utils.h" + +namespace MeCab { +namespace { + +#define DCONF(file) create_filename(dicdir, std::string(file)).c_str() + +#ifdef MECAB_USE_THREAD +class learner_thread: public thread { + public: + unsigned short start_i; + unsigned short thread_num; + size_t size; + size_t micro_p; + size_t micro_r; + size_t micro_c; + size_t err; + double f; + EncoderLearnerTagger **x; + std::vector expected; + void run() { + micro_p = micro_r = micro_c = err = 0; + f = 0.0; + std::fill(expected.begin(), expected.end(), 0.0); + for (size_t i = start_i; i < size; i += thread_num) { + f += x[i]->gradient(&expected[0]); + err += x[i]->eval(µ_c, µ_p, µ_r); + } + } +}; +#endif + +class CRFLearner { + public: + static int run(Param *param) { + const std::string dicdir = param->get("dicdir"); + CHECK_DIE(param->load(DCONF(DICRC))) + << "no such file or directory: " << DCONF(DICRC); + + const std::vector &files = param->rest_args(); + if (files.size() != 2) { + std::cout << "Usage: " << + param->program_name() << " corpus model" << std::endl; + return -1; + } + + const std::string ifile = files[0]; + const std::string model = files[1]; + const std::string old_model = param->get("old-model"); + + EncoderFeatureIndex feature_index; + std::vector expected; + std::vector observed; + std::vector alpha; + std::vector old_alpha; + std::vector x; + Tokenizer tokenizer; + Allocator allocator; + + CHECK_DIE(tokenizer.open(*param)) << "cannot open tokenizer"; + CHECK_DIE(feature_index.open(*param)) << "cannot open feature index"; + + if (!old_model.empty()) { + std::cout << "Using previous model: " << old_model << std::endl; + std::cout << "--cost --freq and --eta options are overwritten." + << std::endl; + CHECK_DIE(tokenizer.dictionary_info()); + const char *dic_charset = tokenizer.dictionary_info()->charset; + feature_index.reopen(old_model.c_str(), + dic_charset, &old_alpha, param); + } + + const double C = param->get("cost"); + const double eta = param->get("eta"); + const size_t eval_size = param->get("eval-size"); + const size_t unk_eval_size = param->get("unk-eval-size"); + const size_t thread_num = param->get("thread"); + const size_t freq = param->get("freq"); + + CHECK_DIE(C > 0) << "cost parameter is out of range: " << C; + CHECK_DIE(eta > 0) "eta is out of range: " << eta; + CHECK_DIE(eval_size > 0) << "eval-size is out of range: " << eval_size; + CHECK_DIE(unk_eval_size > 0) << + "unk-eval-size is out of range: " << unk_eval_size; + CHECK_DIE(freq > 0) << + "freq is out of range: " << unk_eval_size; + CHECK_DIE(thread_num > 0 && thread_num <= 512) + << "# thread is invalid: " << thread_num; + + std::cout.setf(std::ios::fixed, std::ios::floatfield); + std::cout.precision(5); + + std::cout << "reading corpus ..." << std::flush; + + std::ifstream ifs(WPATH(ifile.c_str())); + CHECK_DIE(ifs) << "no such file or directory: " << ifile; + + while (ifs) { + EncoderLearnerTagger *tagger = new EncoderLearnerTagger(); + + CHECK_DIE(tagger->open(&tokenizer, + &allocator, + &feature_index, + eval_size, + unk_eval_size)); + + CHECK_DIE(tagger->read(&ifs, &observed)); + + if (!tagger->empty()) { + x.push_back(tagger); + } else { + delete tagger; + } + + if (x.size() % 100 == 0) { + std::cout << x.size() << "... " << std::flush; + } + } + + feature_index.shrink(freq, &observed); + feature_index.clearcache(); + + const size_t psize = feature_index.size(); + observed.resize(psize); + expected.resize(psize); + alpha.resize(psize); + old_alpha.resize(psize); + alpha = old_alpha; + + feature_index.set_alpha(&alpha[0]); + + std::cout << std::endl; + std::cout << "Number of sentences: " << x.size() << std::endl; + std::cout << "Number of features: " << psize << std::endl; + std::cout << "eta: " << eta << std::endl; + std::cout << "freq: " << freq << std::endl; + std::cout << "eval-size: " << eval_size << std::endl; + std::cout << "unk-eval-size: " << unk_eval_size << std::endl; +#ifdef MECAB_USE_THREAD + std::cout << "threads: " << thread_num << std::endl; +#endif + std::cout << "charset: " << + tokenizer.dictionary_info()->charset << std::endl; + std::cout << "C(sigma^2): " << C << std::endl + << std::endl; + +#ifdef MECAB_USE_THREAD + std::vector thread; + if (thread_num > 1) { + thread.resize(thread_num); + for (size_t i = 0; i < thread_num; ++i) { + thread[i].start_i = i; + thread[i].size = x.size(); + thread[i].thread_num = thread_num; + thread[i].x = &x[0]; + thread[i].expected.resize(expected.size()); + } + } +#endif + + int converge = 0; + double prev_obj = 0.0; + LBFGS lbfgs; + + for (size_t itr = 0; ; ++itr) { + std::fill(expected.begin(), expected.end(), 0.0); + double obj = 0.0; + size_t err = 0; + size_t micro_p = 0; + size_t micro_r = 0; + size_t micro_c = 0; + +#ifdef MECAB_USE_THREAD + if (thread_num > 1) { + for (size_t i = 0; i < thread_num; ++i) { + thread[i].start(); + } + + for (size_t i = 0; i < thread_num; ++i) { + thread[i].join(); + } + + for (size_t i = 0; i < thread_num; ++i) { + obj += thread[i].f; + err += thread[i].err; + micro_r += thread[i].micro_r; + micro_p += thread[i].micro_p; + micro_c += thread[i].micro_c; + for (size_t k = 0; k < psize; ++k) { + expected[k] += thread[i].expected[k]; + } + } + } else +#endif + { + for (size_t i = 0; i < x.size(); ++i) { + obj += x[i]->gradient(&expected[0]); + err += x[i]->eval(µ_c, µ_p, µ_r); + } + } + + const double p = 1.0 * micro_c / micro_p; + const double r = 1.0 * micro_c / micro_r; + const double micro_f = 2 * p * r / (p + r); + + for (size_t i = 0; i < psize; ++i) { + const double penalty = (alpha[i] - old_alpha[i]); + obj += (penalty * penalty / (2.0 * C)); + expected[i] = expected[i] - observed[i] + penalty / C; + } + + const double diff = (itr == 0 ? 1.0 : + std::fabs(1.0 * (prev_obj - obj)) / prev_obj); + std::cout << "iter=" << itr + << " err=" << 1.0 * err/x.size() + << " F=" << micro_f + << " target=" << obj + << " diff=" << diff << std::endl; + prev_obj = obj; + + if (diff < eta) { + converge++; + } else { + converge = 0; + } + + if (converge == 3) { + break; // 3 is ad-hoc + } + + const int ret = lbfgs.optimize(psize, + &alpha[0], obj, + &expected[0], false, C); + + CHECK_DIE(ret >= 0) << "unexpected error in LBFGS routin"; + + if (ret == 0) { + break; + } + } + + std::cout << "\nDone! writing model file ... " << std::endl; + + std::ostringstream oss; + + oss << "eta: " << eta << std::endl; + oss << "freq: " << freq << std::endl; + oss << "C: " << C << std::endl; + oss.setf(std::ios::fixed, std::ios::floatfield); + oss.precision(16); + oss << "eval-size: " << eval_size << std::endl; + oss << "unk-eval-size: " << unk_eval_size << std::endl; + oss << "charset: " << tokenizer.dictionary_info()->charset << std::endl; + + const std::string header = oss.str(); + + CHECK_DIE(feature_index.save(model.c_str(), header.c_str())) + << "permission denied: " << model; + + return 0; + } +}; + +class Learner { + public: + static bool run(int argc, char **argv) { + static const MeCab::Option long_options[] = { + { "dicdir", 'd', ".", "DIR", + "set DIR as dicdir(default \".\" )" }, + { "old-model", 'M', 0, "FILE", + "set FILE as old CRF model file" }, + { "cost", 'c', "1.0", "FLOAT", + "set FLOAT for cost C for constraints violatoin" }, + { "freq", 'f', "1", "INT", + "set the frequency cut-off (default 1)" }, + { "eta", 'e', "0.00005", "DIR", + "set FLOAT for tolerance of termination criterion" }, + { "thread", 'p', "1", "INT", "number of threads(default 1)" }, + { "version", 'v', 0, 0, "show the version and exit" }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } + }; + + Param param; + + if (!param.open(argc, argv, long_options)) { + std::cout << param.what() << "\n\n" << COPYRIGHT + << "\ntry '--help' for more information." << std::endl; + return -1; + } + + if (!param.help_version()) { + return 0; + } + + return CRFLearner::run(¶m); + } +}; +} +} + +int mecab_cost_train(int argc, char **argv) { + return MeCab::Learner::run(argc, argv); +} diff --git a/fts/third_party/mecab/src/learner_node.h b/fts/third_party/mecab/src/learner_node.h new file mode 100644 index 00000000..db0ac329 --- /dev/null +++ b/fts/third_party/mecab/src/learner_node.h @@ -0,0 +1,134 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_LEARNER_NODE_H_ +#define MECAB_LEARNER_NODE_H_ + +#include +#include "mecab.h" +#include "common.h" +#include "utils.h" + +struct mecab_learner_path_t { + struct mecab_learner_node_t* rnode; + struct mecab_learner_path_t* rnext; + struct mecab_learner_node_t* lnode; + struct mecab_learner_path_t* lnext; + double cost; + const int *fvector; +}; + +struct mecab_learner_node_t { + struct mecab_learner_node_t *prev; + struct mecab_learner_node_t *next; + struct mecab_learner_node_t *enext; + struct mecab_learner_node_t *bnext; + struct mecab_learner_path_t *rpath; + struct mecab_learner_path_t *lpath; + struct mecab_learner_node_t *anext; + const char *surface; + const char *feature; + unsigned int id; + unsigned short length; + unsigned short rlength; + unsigned short rcAttr; + unsigned short lcAttr; + unsigned short posid; + unsigned char char_type; + unsigned char stat; + unsigned char isbest; + double alpha; + double beta; + short wcost2; + double wcost; + double cost; + const int *fvector; + struct mecab_token_t *token; +}; + +namespace MeCab { + +typedef struct mecab_learner_path_t LearnerPath; +typedef struct mecab_learner_node_t LearnerNode; + +template T1 repeat_find_if(T1 b, T1 e, + const T2& v, size_t n) { + T1 r = b; + for (size_t i = 0; i < n; ++i) { + r = std::find(b, e, v); + if (r == e) return e; + b = r + 1; + } + return r; +} + +// NOTE: first argment: answer, +// second argment: system output +inline bool node_cmp_eq(const LearnerNode &node1, + const LearnerNode &node2, + size_t size, size_t unk_size) { + if (node1.length == node2.length && + strncmp(node1.surface, node2.surface, node1.length) == 0) { + const char *p1 = node1.feature; + const char *p2 = node2.feature; + // There is NO case when node1 becomes MECAB_UNK_NODE + if (node2.stat == MECAB_UNK_NODE) + size = unk_size; // system cannot output other extra information + const char *r1 = repeat_find_if(p1, p1 + std::strlen(p1), ',', size); + const char *r2 = repeat_find_if(p2, p2 + std::strlen(p2), ',', size); + if (static_cast(r1 - p1) == static_cast(r2 - p2) && + std::strncmp(p1, p2, static_cast(r1 - p1)) == 0) { + return true; + } + } + + return false; +} + +inline bool is_empty(LearnerPath *path) { + return ((!path->rnode->rpath && path->rnode->stat != MECAB_EOS_NODE) || + (!path->lnode->lpath && path->lnode->stat != MECAB_BOS_NODE) ); +} + +inline void calc_expectation(LearnerPath *path, double *expected, double Z) { + if (is_empty(path)) { + return; + } + + const double c = std::exp(path->lnode->alpha + + path->cost + + path->rnode->beta - Z); + + for (const int *f = path->fvector; *f != -1; ++f) { + expected[*f] += c; + } + + if (path->rnode->stat != MECAB_EOS_NODE) { + for (const int *f = path->rnode->fvector; *f != -1; ++f) { + expected[*f] += c; + } + } +} + +inline void calc_alpha(LearnerNode *n) { + n->alpha = 0.0; + for (LearnerPath *path = n->lpath; path; path = path->lnext) { + n->alpha = logsumexp(n->alpha, + path->cost + path->lnode->alpha, + path == n->lpath); + } +} + +inline void calc_beta(LearnerNode *n) { + n->beta = 0.0; + for (LearnerPath *path = n->rpath; path; path = path->rnext) { + n->beta = logsumexp(n->beta, + path->cost + path->rnode->beta, + path == n->rpath); + } +} +} + +#endif // MECAB_LEARNER_NODE_H_ diff --git a/fts/third_party/mecab/src/learner_tagger.cpp b/fts/third_party/mecab/src/learner_tagger.cpp new file mode 100644 index 00000000..ff4228a5 --- /dev/null +++ b/fts/third_party/mecab/src/learner_tagger.cpp @@ -0,0 +1,418 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include +#include "common.h" +#include "learner_node.h" +#include "learner_tagger.h" +#include "utils.h" + +namespace MeCab { +namespace { +char *mystrdup(const char *str) { + const size_t l = std::strlen(str); + char *r = new char[l + 1]; + std::strncpy(r, str, l+1); + return r; +} + +char *mystrdup(const std::string &str) { + return mystrdup(str.c_str()); +} +} // namespace + +bool EncoderLearnerTagger::open(Tokenizer *tokenizer, + Allocator *allocator, + FeatureIndex *feature_index, + size_t eval_size, + size_t unk_eval_size) { + close(); + tokenizer_ = tokenizer; + allocator_ = allocator; + feature_index_ = feature_index; + eval_size_ = eval_size; + unk_eval_size_ = unk_eval_size; + return true; +} + +bool DecoderLearnerTagger::open(const Param ¶m) { + close(); + allocator_data_.reset(new Allocator()); + tokenizer_data_.reset(new Tokenizer()); + feature_index_data_.reset(new DecoderFeatureIndex); + allocator_ = allocator_data_.get(); + tokenizer_ = tokenizer_data_.get(); + feature_index_ = feature_index_data_.get(); + + CHECK_DIE(tokenizer_->open(param)) << tokenizer_->what(); + CHECK_DIE(feature_index_->open(param)); + + return true; +} + +bool EncoderLearnerTagger::read(std::istream *is, + std::vector *observed) { + scoped_fixed_array line; + char *column[8]; + std::string sentence; + std::vector corpus; + ans_path_list_.clear(); + + bool eos = false; + + for (;;) { + if (!is->getline(line.get(), line.size())) { + is->clear(std::ios::eofbit|std::ios::badbit); + return true; + } + + eos = (std::strcmp(line.get(), "EOS") == 0 || line[0] == '\0'); + + LearnerNode *m = new LearnerNode; + std::memset(m, 0, sizeof(LearnerNode)); + + if (eos) { + m->stat = MECAB_EOS_NODE; + } else { + const size_t size = tokenize(line.get(), "\t", column, 2); + CHECK_DIE(size == 2) << "format error: " << line.get(); + m->stat = MECAB_NOR_NODE; + m->surface = mystrdup(column[0]); + m->feature = mystrdup(column[1]); + m->length = m->rlength = std::strlen(column[0]); + } + + corpus.push_back(m); + + if (eos) { + break; + } + + sentence.append(column[0]); + } + + CHECK_DIE(!sentence.empty()) << "empty sentence"; + + CHECK_DIE(eos) << "\"EOS\" is not found"; + + begin_data_.reset_string(sentence); + begin_ = begin_data_.get(); + + initList(); + + size_t pos = 0; + for (size_t i = 0; corpus[i]->stat != MECAB_EOS_NODE; ++i) { + LearnerNode *found = 0; + for (LearnerNode *node = lookup(pos); node; node = node->bnext) { + if (node_cmp_eq(*(corpus[i]), *node, eval_size_, unk_eval_size_)) { + found = node; + break; + } + } + + // cannot find node even using UNKNOWN WORD PROSESSING + if (!found) { + LearnerNode *node = allocator_->newNode(); + node->surface = begin_ + pos; + node->length = node->rlength = std::strlen(corpus[i]->surface); + node->feature = feature_index_->strdup(corpus[i]->feature); + node->stat = MECAB_NOR_NODE; + node->fvector = 0; + node->wcost = 0.0; + node->bnext = begin_node_list_[pos]; + begin_node_list_[pos] = node; + std::cout << "adding virtual node: " << node->feature << std::endl; + } + + pos += corpus[i]->length; + } + + buildLattice(); + + LearnerNode* prev = end_node_list_[0]; // BOS + prev->anext = 0; + pos = 0; + + for (size_t i = 0; i < corpus.size(); ++i) { + LearnerNode *rNode = 0; + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + if (corpus[i]->stat == MECAB_EOS_NODE || + node_cmp_eq(*(corpus[i]), *node, eval_size_, unk_eval_size_)) { + rNode = node; // take last node + } + } + + LearnerPath *lpath = 0; + for (LearnerPath *path = rNode->lpath; path; path = path->lnext) { + if (prev == path->lnode) { + lpath = path; + break; + } + } + + CHECK_DIE(lpath->fvector) << "lpath is NULL"; + for (const int *f = lpath->fvector; *f != -1; ++f) { + if (*f >= static_cast(observed->size())) { + observed->resize(*f + 1); + } + ++(*observed)[*f]; + } + + if (lpath->rnode->stat != MECAB_EOS_NODE) { + for (const int *f = lpath->rnode->fvector; *f != -1; ++f) { + if (*f >= static_cast(observed->size())) { + observed->resize(*f + 1); + } + ++(*observed)[*f]; + } + } + + ans_path_list_.push_back(lpath); + + prev->anext = rNode; + prev = rNode; + + if (corpus[i]->stat == MECAB_EOS_NODE) { + break; + } + + pos += std::strlen(corpus[i]->surface); + } + + prev->anext = begin_node_list_[len_]; // connect to EOS + begin_node_list_[len_]->anext = 0; + + for (size_t i = 0 ; i < corpus.size(); ++i) { + delete [] corpus[i]->surface; + delete [] corpus[i]->feature; + delete corpus[i]; + } + + return true; +} + +int EncoderLearnerTagger::eval(size_t *crr, + size_t *prec, size_t *recall) const { + int zeroone = 0; + + LearnerNode *res = end_node_list_[0]->next; + LearnerNode *ans = end_node_list_[0]->anext; + + size_t resp = 0; + size_t ansp = 0; + + while (ans->anext && res->next) { + if (resp == ansp) { + if (node_cmp_eq(*ans, *res, eval_size_, unk_eval_size_)) { + ++(*crr); // same + } else { + zeroone = 1; + } + ++(*prec); + ++(*recall); + res = res->next; + ans = ans->anext; + resp += res->rlength; + ansp += ans->rlength; + } else if (resp < ansp) { + res = res->next; + resp += res->rlength; + zeroone = 1; + ++(*recall); + } else { + ans = ans->anext; + ansp += ans->rlength; + zeroone = 1; + ++(*prec); + } + } + + while (ans->anext) { + ++(*prec); + ans = ans->anext; + } + + while (res->next) { + ++(*recall); + res = res->next; + } + + return zeroone; +} + +bool DecoderLearnerTagger::parse(std::istream* is, std::ostream *os) { + allocator_->free(); + feature_index_->clear(); + + if (!begin_) { + begin_data_.reset(new char[BUF_SIZE * 16]); + begin_ = begin_data_.get(); + } + + if (!is->getline(const_cast(begin_), BUF_SIZE * 16)) { + is->clear(std::ios::eofbit|std::ios::badbit); + return false; + } + + initList(); + buildLattice(); + viterbi(); + + for (LearnerNode *node = end_node_list_[0]->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature << '\n'; + } + *os << "EOS\n"; + + return true; +} + +LearnerNode *LearnerTagger::lookup(size_t pos) { + if (begin_node_list_[pos]) { + return begin_node_list_[pos]; + } + LearnerNode *m = tokenizer_->lookup(begin_ + pos, end_, allocator_, 0); + begin_node_list_[pos] = m; + return m; +} + +bool LearnerTagger::connect(size_t pos, LearnerNode *_rNode) { + for (LearnerNode *rNode = _rNode ; rNode; rNode = rNode->bnext) { + for (LearnerNode *lNode = end_node_list_[pos]; lNode; + lNode = lNode->enext) { + LearnerPath *path = allocator_->newPath(); + std::memset(path, 0, sizeof(Path)); + path->rnode = rNode; + path->lnode = lNode; + path->fvector = 0; + path->cost = 0.0; + path->rnode = rNode; + path->lnode = lNode; + path->lnext = rNode->lpath; + rNode->lpath = path; + path->rnext = lNode->rpath; + lNode->rpath = path; + CHECK_DIE(feature_index_->buildFeature(path)); + CHECK_DIE(path->fvector); + } + const size_t x = rNode->rlength + pos; + rNode->enext = end_node_list_[x]; + end_node_list_[x] = rNode; + } + + return true; +} + +bool LearnerTagger::initList() { + if (!begin_) { + return false; + } + + len_ = std::strlen(begin_); + end_ = begin_ + len_; + + end_node_list_.resize(len_ + 2); + std::fill(end_node_list_.begin(), end_node_list_.end(), + reinterpret_cast(0)); + + begin_node_list_.resize(len_ + 2); + std::fill(begin_node_list_.begin(), begin_node_list_.end(), + reinterpret_cast(0)); + + end_node_list_[0] = tokenizer_->getBOSNode(allocator_); + end_node_list_[0]->surface = begin_; + begin_node_list_[len_] = tokenizer_->getEOSNode(allocator_); + + return true; +} + +bool LearnerTagger::buildLattice() { + for (int pos = 0; pos <= static_cast(len_); pos++) { + if (!end_node_list_[pos]) { + continue; + } + connect(pos, lookup(pos)); + } + + if (!end_node_list_[len_]) { + begin_node_list_[len_] = lookup(len_); + for (size_t pos = len_; static_cast(pos) >= 0; pos--) { + if (end_node_list_[pos]) { + connect(pos, begin_node_list_[len_]); + break; + } + } + } + + return true; +} + +bool LearnerTagger::viterbi() { + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + double bestc = -1e37; + LearnerNode *best = 0; + feature_index_->calcCost(node); + for (LearnerPath *path = node->lpath; path; path = path->lnext) { + feature_index_->calcCost(path); + double cost = path->cost + path->lnode->cost; + if (cost > bestc) { + bestc = cost; + best = path->lnode; + } + } + + node->prev = best; + node->cost = bestc; + } + } + + LearnerNode *node = begin_node_list_[len_]; // EOS + for (LearnerNode *prev; node->prev;) { + prev = node->prev; + prev->next = node; + node = prev; + } + + return true; +} + +double EncoderLearnerTagger::gradient(double *expected) { + viterbi(); + + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + calc_alpha(node); + } + } + + for (int pos = static_cast(len_); pos >=0; --pos) { + for (LearnerNode *node = end_node_list_[pos]; node; node = node->enext) { + calc_beta(node); + } + } + + double Z = begin_node_list_[len_]->alpha; // alpha of EOS + + for (int pos = 0; pos <= static_cast(len_); ++pos) { + for (LearnerNode *node = begin_node_list_[pos]; node; node = node->bnext) { + for (LearnerPath *path = node->lpath; path; path = path->lnext) { + calc_expectation(path, expected, Z); + } + } + } + + for (size_t i = 0; i < ans_path_list_.size(); ++i) { + Z -= ans_path_list_[i]->cost; + } + + return Z; +} +} diff --git a/fts/third_party/mecab/src/learner_tagger.h b/fts/third_party/mecab/src/learner_tagger.h new file mode 100644 index 00000000..abea4594 --- /dev/null +++ b/fts/third_party/mecab/src/learner_tagger.h @@ -0,0 +1,80 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_TAGGER_H +#define MECAB_TAGGER_H + +#include +#include "mecab.h" +#include "freelist.h" +#include "feature_index.h" +#include "tokenizer.h" +#include "scoped_ptr.h" + +namespace MeCab { + +class FeatureIndex; + +class LearnerTagger { + public: + bool empty() const { return (len_ == 0); } + void close() {} + void clear() {} + + explicit LearnerTagger(): tokenizer_(0), path_allocator_(0), + feature_index_(0), begin_(0), end_(0), len_(0) {} + virtual ~LearnerTagger() {} + + protected: + Tokenizer *tokenizer_; + Allocator *allocator_; + FreeList *path_allocator_; + FeatureIndex *feature_index_; + scoped_string begin_data_; + const char *begin_; + const char *end_; + size_t len_; + std::vector begin_node_list_; + std::vector end_node_list_; + + LearnerNode *lookup(size_t); + bool connect(size_t, LearnerNode *); + bool viterbi(); + bool buildLattice(); + bool initList(); +}; + +class EncoderLearnerTagger: public LearnerTagger { + public: + bool open(Tokenizer *tokenzier, + Allocator *allocator, + FeatureIndex *feature_index, + size_t eval_size, size_t unk_eval_size); + bool read(std::istream *, std::vector *); + int eval(size_t *, size_t *, size_t *) const; + double gradient(double *expected); + explicit EncoderLearnerTagger(): eval_size_(1024), unk_eval_size_(1024) {} + virtual ~EncoderLearnerTagger() { close(); } + + private: + size_t eval_size_; + size_t unk_eval_size_; + std::vector ans_path_list_; +}; + +class DecoderLearnerTagger: public LearnerTagger { + public: + bool open(const Param &); + bool parse(std::istream *, std::ostream *); + virtual ~DecoderLearnerTagger() { close(); } + + private: + scoped_ptr > tokenizer_data_; + scoped_ptr > allocator_data_; + scoped_ptr feature_index_data_; +}; +} + +#endif diff --git a/fts/third_party/mecab/src/libmecab.cpp b/fts/third_party/mecab/src/libmecab.cpp new file mode 100644 index 00000000..413c4754 --- /dev/null +++ b/fts/third_party/mecab/src/libmecab.cpp @@ -0,0 +1,496 @@ +// +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#if defined(_WIN32) && !defined(__CYGWIN__) +#include +#endif + +#include "mecab.h" +#include "tokenizer.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace { +const char kUnknownError[] = "Unknown Error"; +const size_t kErrorBufferSize = 256; +} + +#if defined(_WIN32) && !defined(__CYGWIN__) +namespace { +DWORD g_tls_index = TLS_OUT_OF_INDEXES; +} + +const char *getGlobalError() { + LPVOID data = ::TlsGetValue(g_tls_index); + return data == NULL ? kUnknownError : reinterpret_cast(data); +} + +void setGlobalError(const char *str) { + char *data = reinterpret_cast(::TlsGetValue(g_tls_index)); + if (data == NULL) { + return; + } + strncpy(data, str, kErrorBufferSize - 1); + data[kErrorBufferSize - 1] = '\0'; +} + +HINSTANCE DllInstance = 0; + +extern "C" { + BOOL WINAPI DllMain(HINSTANCE hinst, DWORD dwReason, LPVOID) { + LPVOID data = 0; + if (!DllInstance) { + DllInstance = hinst; + } + switch (dwReason) { + case DLL_PROCESS_ATTACH: + if ((g_tls_index = ::TlsAlloc()) == TLS_OUT_OF_INDEXES) { + return FALSE; + } + // Not break in order to initialize the TLS. + case DLL_THREAD_ATTACH: + data = (LPVOID)::LocalAlloc(LPTR, kErrorBufferSize); + if (data) { + ::TlsSetValue(g_tls_index, data); + } + break; + case DLL_THREAD_DETACH: + data = ::TlsGetValue(g_tls_index); + if (data) { + ::LocalFree((HLOCAL)data); + } + break; + case DLL_PROCESS_DETACH: + data = ::TlsGetValue(g_tls_index); + if (data) { + ::LocalFree((HLOCAL)data); + } + ::TlsFree(g_tls_index); + g_tls_index = TLS_OUT_OF_INDEXES; + break; + default: + break; + } + return TRUE; + } +} +#else // _WIN32 +namespace { +#ifdef HAVE_TLS_KEYWORD +__thread char kErrorBuffer[kErrorBufferSize]; +#else +char kErrorBuffer[kErrorBufferSize]; +#endif +} + +const char *getGlobalError() { + return kErrorBuffer; +} + +void setGlobalError(const char *str) { + strncpy(kErrorBuffer, str, kErrorBufferSize - 1); + kErrorBuffer[kErrorBufferSize - 1] = '\0'; +} +#endif + +mecab_t* mecab_new(int argc, char **argv) { + MeCab::Tagger *tagger = MeCab::createTagger(argc, argv); + if (!tagger) { + MeCab::deleteTagger(tagger); + return 0; + } + return reinterpret_cast(tagger); +} + +mecab_t* mecab_new2(const char *arg) { + MeCab::Tagger *tagger = MeCab::createTagger(arg); + if (!tagger) { + MeCab::deleteTagger(tagger); + return 0; + } + return reinterpret_cast(tagger); +} + +const char *mecab_version() { + return MeCab::Tagger::version(); +} + +const char* mecab_strerror(mecab_t *tagger) { + if (!tagger) { + return MeCab::getLastError(); + } + return reinterpret_cast(tagger)->what(); +} + +void mecab_destroy(mecab_t *tagger) { + MeCab::Tagger *ptr = reinterpret_cast(tagger); + MeCab::deleteTagger(ptr); + ptr = 0; +} + +int mecab_get_partial(mecab_t *tagger) { + return reinterpret_cast(tagger)->partial(); +} + +void mecab_set_partial(mecab_t *tagger, int partial) { + reinterpret_cast(tagger)->set_partial(partial); +} + +float mecab_get_theta(mecab_t *tagger) { + return reinterpret_cast(tagger)->theta(); +} + +void mecab_set_theta(mecab_t *tagger, float theta) { + reinterpret_cast(tagger)->set_theta(theta); +} + +int mecab_get_lattice_level(mecab_t *tagger) { + return reinterpret_cast(tagger)->lattice_level(); +} + +void mecab_set_lattice_level(mecab_t *tagger, int level) { + reinterpret_cast(tagger)->set_lattice_level(level); +} + +int mecab_get_all_morphs(mecab_t *tagger) { + return static_cast( + reinterpret_cast(tagger)->all_morphs()); +} + +void mecab_set_all_morphs(mecab_t *tagger, int all_morphs) { + reinterpret_cast(tagger)->set_all_morphs(all_morphs); +} + +const char* mecab_sparse_tostr(mecab_t *tagger, const char *str) { + return reinterpret_cast(tagger)->parse(str); +} + +const char* mecab_sparse_tostr2(mecab_t *tagger, const char *str, size_t len) { + return reinterpret_cast(tagger)->parse(str, len); +} + +char* mecab_sparse_tostr3(mecab_t *tagger, const char *str, size_t len, + char *out, size_t len2) { + return const_cast( + reinterpret_cast(tagger)->parse( + str, len, out, len2)); +} + +const mecab_node_t* mecab_sparse_tonode(mecab_t *tagger, const char *str) { + return reinterpret_cast( + reinterpret_cast(tagger)->parseToNode(str)); +} + +const mecab_node_t* mecab_sparse_tonode2(mecab_t *tagger, + const char *str, size_t len) { + return reinterpret_cast( + reinterpret_cast(tagger)->parseToNode(str, len)); +} + +const char* mecab_nbest_sparse_tostr(mecab_t *tagger, size_t N, + const char *str) { + return reinterpret_cast(tagger)->parseNBest(N, str); +} + +const char* mecab_nbest_sparse_tostr2(mecab_t *tagger, size_t N, + const char* str, size_t len) { + return reinterpret_cast( + tagger)->parseNBest(N, str, len); +} + +char* mecab_nbest_sparse_tostr3(mecab_t *tagger, size_t N, + const char *str, size_t len, + char *out, size_t len2) { + return const_cast( + reinterpret_cast( + tagger)->parseNBest(N, str, len, out, len2)); +} + +int mecab_nbest_init(mecab_t *tagger, const char *str) { + return reinterpret_cast< + MeCab::Tagger *>(tagger)->parseNBestInit(str); +} + +int mecab_nbest_init2(mecab_t *tagger, const char *str, size_t len) { + return reinterpret_cast< + MeCab::Tagger *>(tagger)->parseNBestInit(str, len); +} + +const char* mecab_nbest_next_tostr(mecab_t *tagger) { + return reinterpret_cast(tagger)->next(); +} + +char* mecab_nbest_next_tostr2(mecab_t *tagger, char *out, size_t len2) { + return const_cast( + reinterpret_cast(tagger)->next(out, len2)); +} + +const mecab_node_t* mecab_nbest_next_tonode(mecab_t *tagger) { + return reinterpret_cast( + reinterpret_cast(tagger)->nextNode()); +} + +const char* mecab_format_node(mecab_t *tagger, const mecab_node_t* n) { + return reinterpret_cast(tagger)->formatNode(n); +} + +const mecab_dictionary_info_t *mecab_dictionary_info(mecab_t *tagger) { + return reinterpret_cast( + reinterpret_cast(tagger)->dictionary_info()); +} + +int mecab_parse_lattice(mecab_t *mecab, mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(mecab)->parse( + reinterpret_cast(lattice))); +} + +mecab_lattice_t *mecab_lattice_new() { + return reinterpret_cast(MeCab::createLattice()); +} + +void mecab_lattice_destroy(mecab_lattice_t *lattice) { + MeCab::Lattice *ptr = reinterpret_cast(lattice); + MeCab::deleteLattice(ptr); + ptr = 0; +} + +void mecab_lattice_clear(mecab_lattice_t *lattice) { + reinterpret_cast(lattice)->clear(); +} + +int mecab_lattice_is_available(mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(lattice)->is_available()); +} +mecab_node_t *mecab_lattice_get_bos_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->bos_node()); +} + +mecab_node_t *mecab_lattice_get_eos_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->eos_node()); +} + +mecab_node_t **mecab_lattice_get_all_begin_nodes(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->begin_nodes()); +} + +mecab_node_t **mecab_lattice_get_all_end_nodes(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->end_nodes()); +} + +mecab_node_t *mecab_lattice_get_begin_nodes(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + reinterpret_cast(lattice)->begin_nodes(pos)); +} + +mecab_node_t *mecab_lattice_get_end_nodes(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + reinterpret_cast(lattice)->end_nodes(pos)); +} + +const char *mecab_lattice_get_sentence(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->sentence(); +} + +void mecab_lattice_set_sentence(mecab_lattice_t *lattice, + const char *sentence) { + reinterpret_cast(lattice)->set_sentence(sentence); +} + +void mecab_lattice_set_sentence2(mecab_lattice_t *lattice, + const char *sentence, size_t len) { + reinterpret_cast(lattice)->set_sentence( + sentence, len); +} + +size_t mecab_lattice_get_size(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->size(); +} + +double mecab_lattice_get_z(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->Z(); +} + +void mecab_lattice_set_z(mecab_lattice_t *lattice, double Z) { + reinterpret_cast(lattice)->set_Z(Z); +} + +double mecab_lattice_get_theta(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->theta(); +} + +void mecab_lattice_set_theta(mecab_lattice_t *lattice, double theta) { + reinterpret_cast(lattice)->set_theta(theta); +} + +int mecab_lattice_next(mecab_lattice_t *lattice) { + return static_cast( + reinterpret_cast(lattice)->next()); +} + +int mecab_lattice_get_request_type(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->request_type(); +} + +int mecab_lattice_has_request_type(mecab_lattice_t *lattice, + int request_type) { + return reinterpret_cast( + lattice)->has_request_type(request_type); +} + +void mecab_lattice_set_request_type(mecab_lattice_t *lattice, + int request_type) { + reinterpret_cast( + lattice)->set_request_type(request_type); +} + +void mecab_lattice_add_request_type(mecab_lattice_t *lattice, + int request_type) { + reinterpret_cast( + lattice)->add_request_type(request_type); +} + +void mecab_lattice_remove_request_type(mecab_lattice_t *lattice, + int request_type) { + return reinterpret_cast( + lattice)->remove_request_type(request_type); +} + +mecab_node_t *mecab_lattice_new_node(mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(lattice)->newNode()); +} + +const char *mecab_lattice_tostr(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->toString(); +} + +const char *mecab_lattice_tostr2(mecab_lattice_t *lattice, + char *buf, size_t size) { + return reinterpret_cast( + lattice)->toString(buf, size); +} +const char *mecab_lattice_nbest_tostr(mecab_lattice_t *lattice, + size_t N) { + return reinterpret_cast( + lattice)->enumNBestAsString(N); +} +const char *mecab_lattice_nbest_tostr2(mecab_lattice_t *lattice, + size_t N, char *buf, size_t size) { + return reinterpret_cast( + lattice)->enumNBestAsString(N, buf, size); +} + +int mecab_lattice_has_constraint(mecab_lattice_t *lattice) { + return static_cast(reinterpret_cast( + lattice)->has_constraint()); +} + +int mecab_lattice_get_boundary_constraint(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + lattice)->boundary_constraint(pos); +} + +const char *mecab_lattice_get_feature_constraint(mecab_lattice_t *lattice, + size_t pos) { + return reinterpret_cast( + lattice)->feature_constraint(pos); +} + +void mecab_lattice_set_boundary_constraint(mecab_lattice_t *lattice, + size_t pos, int boundary_type) { + return reinterpret_cast( + lattice)->set_boundary_constraint(pos, boundary_type); +} + +void mecab_lattice_set_feature_constraint(mecab_lattice_t *lattice, + size_t begin_pos, size_t end_pos, + const char *feature) { + return reinterpret_cast( + lattice)->set_feature_constraint(begin_pos, end_pos, feature); +} + +void mecab_lattice_set_result(mecab_lattice_t *lattice, + const char *result) { + return reinterpret_cast(lattice)->set_result(result); +} + +const char *mecab_lattice_strerror(mecab_lattice_t *lattice) { + return reinterpret_cast(lattice)->what(); +} + +mecab_model_t *mecab_model_new(int argc, char **argv) { + MeCab::Model *model = MeCab::createModel(argc, argv); + if (!model) { + MeCab::deleteModel(model); + return 0; + } + return reinterpret_cast(model); +} + +mecab_model_t *mecab_model_new2(const char *arg) { + MeCab::Model *model = MeCab::createModel(arg); + if (!model) { + MeCab::deleteModel(model); + return 0; + } + return reinterpret_cast(model); +} + +void mecab_model_destroy(mecab_model_t *model) { + MeCab::Model *ptr = reinterpret_cast(model); + MeCab::deleteModel(ptr); + ptr = 0; +} + +mecab_t *mecab_model_new_tagger(mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->createTagger()); +} + +mecab_lattice_t *mecab_model_new_lattice(mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->createLattice()); +} + +int mecab_model_swap(mecab_model_t *model, mecab_model_t *new_model) { + return static_cast( + reinterpret_cast(model)->swap( + reinterpret_cast(new_model))); +} + +const mecab_dictionary_info_t* mecab_model_dictionary_info( + mecab_model_t *model) { + return reinterpret_cast( + reinterpret_cast(model)->dictionary_info()); +} + +int mecab_model_transition_cost(mecab_model_t *model, + unsigned short rcAttr, + unsigned short lcAttr) { + return reinterpret_cast(model)->transition_cost( + rcAttr, lcAttr); +} + +mecab_node_t *mecab_model_lookup(mecab_model_t *model, + const char *begin, + const char *end, + mecab_lattice_t *lattice) { + return reinterpret_cast( + reinterpret_cast(model)->lookup( + begin, end, + reinterpret_cast(lattice))); +} diff --git a/fts/third_party/mecab/src/make.bat b/fts/third_party/mecab/src/make.bat new file mode 100644 index 00000000..f782e521 --- /dev/null +++ b/fts/third_party/mecab/src/make.bat @@ -0,0 +1,8 @@ +Set PATH=c:\Program Files\Microsoft Visual Studio 8\VC\bin;%PATH% +Set INCLUDE=c:\Program Files\Microsoft Visual Studio 8\VC\include;c:\Program Files\Microsoft Platform SDK\Include;%INCLUDE% +Set LIB=c:\Program Files\Microsoft Visual Studio 8\VC\lib;c:\Program Files\Microsoft Platform SDK\Lib;%LIB% +Set COMSPEC=cmd.exe +rem nmake -f Makefile.msvc clean +nmake -f Makefile.msvc + + diff --git a/fts/third_party/mecab/src/mecab-cost-train.cpp b/fts/third_party/mecab/src/mecab-cost-train.cpp new file mode 100644 index 00000000..fca181aa --- /dev/null +++ b/fts/third_party/mecab/src/mecab-cost-train.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_cost_train(argc, argv); +} + diff --git a/fts/third_party/mecab/src/mecab-dict-gen.cpp b/fts/third_party/mecab/src/mecab-dict-gen.cpp new file mode 100644 index 00000000..c354b82d --- /dev/null +++ b/fts/third_party/mecab/src/mecab-dict-gen.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_dict_gen(argc, argv); +} + diff --git a/fts/third_party/mecab/src/mecab-dict-index.cpp b/fts/third_party/mecab/src/mecab-dict-index.cpp new file mode 100644 index 00000000..6ace3eda --- /dev/null +++ b/fts/third_party/mecab/src/mecab-dict-index.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_dict_index(argc, argv); +} + diff --git a/fts/third_party/mecab/src/mecab-system-eval.cpp b/fts/third_party/mecab/src/mecab-system-eval.cpp new file mode 100644 index 00000000..3b650573 --- /dev/null +++ b/fts/third_party/mecab/src/mecab-system-eval.cpp @@ -0,0 +1,12 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_system_eval(argc, argv); +} + diff --git a/fts/third_party/mecab/src/mecab-test-gen.cpp b/fts/third_party/mecab/src/mecab-test-gen.cpp new file mode 100644 index 00000000..e8c3f189 --- /dev/null +++ b/fts/third_party/mecab/src/mecab-test-gen.cpp @@ -0,0 +1,11 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_test_gen(argc, argv); +} diff --git a/fts/third_party/mecab/src/mecab.cpp b/fts/third_party/mecab/src/mecab.cpp new file mode 100644 index 00000000..2ec2c23c --- /dev/null +++ b/fts/third_party/mecab/src/mecab.cpp @@ -0,0 +1,11 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "mecab.h" +#include "winmain.h" + +int main(int argc, char **argv) { + return mecab_do (argc, argv); +} diff --git a/fts/third_party/mecab/src/mecab.h b/fts/third_party/mecab/src/mecab.h new file mode 100644 index 00000000..ce7f265c --- /dev/null +++ b/fts/third_party/mecab/src/mecab.h @@ -0,0 +1,1509 @@ +/* + MeCab -- Yet Another Part-of-Speech and Morphological Analyzer + + Copyright(C) 2001-2011 Taku Kudo + Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +*/ +#ifndef MECAB_MECAB_H_ +#define MECAB_MECAB_H_ + +/* C/C++ common data structures */ + +/** + * DictionaryInfo structure + */ +struct mecab_dictionary_info_t { + /** + * filename of dictionary + * On Windows, filename is stored in UTF-8 encoding + */ + const char *filename; + + /** + * character set of the dictionary. e.g., "SHIFT-JIS", "UTF-8" + */ + const char *charset; + + /** + * How many words are registered in this dictionary. + */ + unsigned int size; + + /** + * dictionary type + * this value should be MECAB_USR_DIC, MECAB_SYS_DIC, or MECAB_UNK_DIC. + */ + int type; + + /** + * left attributes size + */ + unsigned int lsize; + + /** + * right attributes size + */ + unsigned int rsize; + + /** + * version of this dictionary + */ + unsigned short version; + + /** + * pointer to the next dictionary info. + */ + struct mecab_dictionary_info_t *next; +}; + +/** + * Path structure + */ +struct mecab_path_t { + /** + * pointer to the right node + */ + struct mecab_node_t* rnode; + + /** + * pointer to the next right path + */ + struct mecab_path_t* rnext; + + /** + * pointer to the left node + */ + struct mecab_node_t* lnode; + + /** + * pointer to the next left path + */ + + struct mecab_path_t* lnext; + + /** + * local cost + */ + int cost; + + /** + * marginal probability + */ + float prob; +}; + +/** + * Node structure + */ +struct mecab_node_t { + /** + * pointer to the previous node. + */ + struct mecab_node_t *prev; + + /** + * pointer to the next node. + */ + struct mecab_node_t *next; + + /** + * pointer to the node which ends at the same position. + */ + struct mecab_node_t *enext; + + /** + * pointer to the node which starts at the same position. + */ + struct mecab_node_t *bnext; + + /** + * pointer to the right path. + * this value is NULL if MECAB_ONE_BEST mode. + */ + struct mecab_path_t *rpath; + + /** + * pointer to the right path. + * this value is NULL if MECAB_ONE_BEST mode. + */ + struct mecab_path_t *lpath; + + /** + * surface string. + * this value is not 0 terminated. + * You can get the length with length/rlength members. + */ + const char *surface; + + /** + * feature string + */ + const char *feature; + + /** + * unique node id + */ + unsigned int id; + + /** + * length of the surface form. + */ + unsigned short length; + + /** + * length of the surface form including white space before the morph. + */ + unsigned short rlength; + + /** + * right attribute id + */ + unsigned short rcAttr; + + /** + * left attribute id + */ + unsigned short lcAttr; + + /** + * unique part of speech id. This value is defined in "pos.def" file. + */ + unsigned short posid; + + /** + * character type + */ + unsigned char char_type; + + /** + * status of this model. + * This value is MECAB_NOR_NODE, MECAB_UNK_NODE, MECAB_BOS_NODE, MECAB_EOS_NODE, or MECAB_EON_NODE. + */ + unsigned char stat; + + /** + * set 1 if this node is best node. + */ + unsigned char isbest; + + /** + * forward accumulative log summation. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float alpha; + + /** + * backward accumulative log summation. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float beta; + + /** + * marginal probability. + * This value is only available when MECAB_MARGINAL_PROB is passed. + */ + float prob; + + /** + * word cost. + */ + short wcost; + + /** + * best accumulative cost from bos node to this node. + */ + long cost; +}; + +/** + * Parameters for MeCab::Node::stat + */ +enum { + /** + * Normal node defined in the dictionary. + */ + MECAB_NOR_NODE = 0, + /** + * Unknown node not defined in the dictionary. + */ + MECAB_UNK_NODE = 1, + /** + * Virtual node representing a beginning of the sentence. + */ + MECAB_BOS_NODE = 2, + /** + * Virtual node representing a end of the sentence. + */ + MECAB_EOS_NODE = 3, + + /** + * Virtual node representing a end of the N-best enumeration. + */ + MECAB_EON_NODE = 4 +}; + +/** + * Parameters for MeCab::DictionaryInfo::type + */ +enum { + /** + * This is a system dictionary. + */ + MECAB_SYS_DIC = 0, + + /** + * This is a user dictionary. + */ + MECAB_USR_DIC = 1, + + /** + * This is a unknown word dictionary. + */ + MECAB_UNK_DIC = 2 +}; + +/** + * Parameters for MeCab::Lattice::request_type + */ +enum { + /** + * One best result is obtained (default mode) + */ + MECAB_ONE_BEST = 1, + /** + * Set this flag if you want to obtain N best results. + */ + MECAB_NBEST = 2, + /** + * Set this flag if you want to enable a partial parsing mode. + * When this flag is set, the input |sentence| needs to be written + * in partial parsing format. + */ + MECAB_PARTIAL = 4, + /** + * Set this flag if you want to obtain marginal probabilities. + * Marginal probability is set in MeCab::Node::prob. + * The parsing speed will get 3-5 times slower than the default mode. + */ + MECAB_MARGINAL_PROB = 8, + /** + * Set this flag if you want to obtain alternative results. + * Not implemented. + */ + MECAB_ALTERNATIVE = 16, + /** + * When this flag is set, the result linked-list (Node::next/prev) + * traverses all nodes in the lattice. + */ + MECAB_ALL_MORPHS = 32, + + /** + * When this flag is set, tagger internally copies the body of passed + * sentence into internal buffer. + */ + MECAB_ALLOCATE_SENTENCE = 64 +}; + +/** + * Parameters for MeCab::Lattice::boundary_constraint_type + */ +enum { + /** + * The token boundary is not specified. + */ + MECAB_ANY_BOUNDARY = 0, + + /** + * The position is a strong token boundary. + */ + MECAB_TOKEN_BOUNDARY = 1, + + /** + * The position is not a token boundary. + */ + MECAB_INSIDE_TOKEN = 2 +}; + +/* C interface */ +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _WIN32 +#include +# ifdef DLL_EXPORT +# define MECAB_DLL_EXTERN __declspec(dllexport) +# define MECAB_DLL_CLASS_EXTERN __declspec(dllexport) +# else +# define MECAB_DLL_EXTERN __declspec(dllimport) +# endif +#endif + +#ifndef MECAB_DLL_EXTERN +# define MECAB_DLL_EXTERN extern +#endif + +#ifndef MECAB_DLL_CLASS_EXTERN +# define MECAB_DLL_CLASS_EXTERN +#endif + + typedef struct mecab_t mecab_t; + typedef struct mecab_model_t mecab_model_t; + typedef struct mecab_lattice_t mecab_lattice_t; + typedef struct mecab_dictionary_info_t mecab_dictionary_info_t; + typedef struct mecab_node_t mecab_node_t; + typedef struct mecab_path_t mecab_path_t; + +#ifndef SWIG + /* C interface */ + + /* old mecab interface */ + /** + * C wrapper of MeCab::Tagger::create(argc, argv) + */ + MECAB_DLL_EXTERN mecab_t* mecab_new(int argc, char **argv); + + /** + * C wrapper of MeCab::Tagger::create(arg) + */ + MECAB_DLL_EXTERN mecab_t* mecab_new2(const char *arg); + + /** + * C wrapper of MeCab::Tagger::version() + */ + MECAB_DLL_EXTERN const char* mecab_version(); + + /** + * C wrapper of MeCab::getLastError() + */ + MECAB_DLL_EXTERN const char* mecab_strerror(mecab_t *mecab); + + /** + * C wrapper of MeCab::deleteTagger(tagger) + */ + MECAB_DLL_EXTERN void mecab_destroy(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger:set_partial() + */ + MECAB_DLL_EXTERN int mecab_get_partial(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::partial() + */ + MECAB_DLL_EXTERN void mecab_set_partial(mecab_t *mecab, int partial); + + /** + * C wrapper of MeCab::Tagger::theta() + */ + MECAB_DLL_EXTERN float mecab_get_theta(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_theta() + */ + MECAB_DLL_EXTERN void mecab_set_theta(mecab_t *mecab, float theta); + + /** + * C wrapper of MeCab::Tagger::lattice_level() + */ + MECAB_DLL_EXTERN int mecab_get_lattice_level(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_lattice_level() + */ + MECAB_DLL_EXTERN void mecab_set_lattice_level(mecab_t *mecab, int level); + + /** + * C wrapper of MeCab::Tagger::all_morphs() + */ + MECAB_DLL_EXTERN int mecab_get_all_morphs(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::set_all_moprhs() + */ + MECAB_DLL_EXTERN void mecab_set_all_morphs(mecab_t *mecab, int all_morphs); + + /** + * C wrapper of MeCab::Tagger::parse(MeCab::Lattice *lattice) + */ + MECAB_DLL_EXTERN int mecab_parse_lattice(mecab_t *mecab, mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str) + */ + MECAB_DLL_EXTERN const char* mecab_sparse_tostr(mecab_t *mecab, const char *str); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str, size_t len) + */ + MECAB_DLL_EXTERN const char* mecab_sparse_tostr2(mecab_t *mecab, const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::parse(const char *str, char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_sparse_tostr3(mecab_t *mecab, const char *str, size_t len, + char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::parseToNode(const char *str) + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_sparse_tonode(mecab_t *mecab, const char*); + + /** + * C wrapper of MeCab::Tagger::parseToNode(const char *str, size_t len) + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_sparse_tonode2(mecab_t *mecab, const char*, size_t); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str) + */ + MECAB_DLL_EXTERN const char* mecab_nbest_sparse_tostr(mecab_t *mecab, size_t N, const char *str); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str, size_t len) + */ + MECAB_DLL_EXTERN const char* mecab_nbest_sparse_tostr2(mecab_t *mecab, size_t N, + const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::parseNBest(size_t N, const char *str, char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_nbest_sparse_tostr3(mecab_t *mecab, size_t N, + const char *str, size_t len, + char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::parseNBestInit(const char *str) + */ + MECAB_DLL_EXTERN int mecab_nbest_init(mecab_t *mecab, const char *str); + + /** + * C wrapper of MeCab::Tagger::parseNBestInit(const char *str, size_t len) + */ + MECAB_DLL_EXTERN int mecab_nbest_init2(mecab_t *mecab, const char *str, size_t len); + + /** + * C wrapper of MeCab::Tagger::next() + */ + MECAB_DLL_EXTERN const char* mecab_nbest_next_tostr(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::next(char *ostr, size_t olen) + */ + MECAB_DLL_EXTERN char* mecab_nbest_next_tostr2(mecab_t *mecab, char *ostr, size_t olen); + + /** + * C wrapper of MeCab::Tagger::nextNode() + */ + MECAB_DLL_EXTERN const mecab_node_t* mecab_nbest_next_tonode(mecab_t *mecab); + + /** + * C wrapper of MeCab::Tagger::formatNode(const Node *node) + */ + MECAB_DLL_EXTERN const char* mecab_format_node(mecab_t *mecab, const mecab_node_t *node); + + /** + * C wrapper of MeCab::Tagger::dictionary_info() + */ + MECAB_DLL_EXTERN const mecab_dictionary_info_t* mecab_dictionary_info(mecab_t *mecab); + + /* lattice interface */ + /** + * C wrapper of MeCab::createLattice() + */ + MECAB_DLL_EXTERN mecab_lattice_t *mecab_lattice_new(); + + /** + * C wrapper of MeCab::deleteLattice(lattice) + */ + MECAB_DLL_EXTERN void mecab_lattice_destroy(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::clear() + */ + MECAB_DLL_EXTERN void mecab_lattice_clear(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::is_available() + */ + + MECAB_DLL_EXTERN int mecab_lattice_is_available(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::bos_node() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_bos_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::eos_node() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_eos_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::begin_nodes() + */ + + MECAB_DLL_EXTERN mecab_node_t **mecab_lattice_get_all_begin_nodes(mecab_lattice_t *lattice); + /** + * C wrapper of MeCab::Lattice::end_nodes() + */ + MECAB_DLL_EXTERN mecab_node_t **mecab_lattice_get_all_end_nodes(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::begin_nodes(pos) + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_begin_nodes(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::end_nodes(pos) + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_get_end_nodes(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::sentence() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_get_sentence(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_sentence(sentence) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_sentence(mecab_lattice_t *lattice, const char *sentence); + + /** + * C wrapper of MeCab::Lattice::set_sentence(sentence, len) + */ + + MECAB_DLL_EXTERN void mecab_lattice_set_sentence2(mecab_lattice_t *lattice, const char *sentence, size_t len); + + /** + * C wrapper of MeCab::Lattice::size() + */ + MECAB_DLL_EXTERN size_t mecab_lattice_get_size(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::Z() + */ + MECAB_DLL_EXTERN double mecab_lattice_get_z(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_Z() + */ + MECAB_DLL_EXTERN void mecab_lattice_set_z(mecab_lattice_t *lattice, double Z); + + /** + * C wrapper of MeCab::Lattice::theta() + */ + MECAB_DLL_EXTERN double mecab_lattice_get_theta(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::set_theta() + */ + + MECAB_DLL_EXTERN void mecab_lattice_set_theta(mecab_lattice_t *lattice, double theta); + + /** + * C wrapper of MeCab::Lattice::next() + */ + MECAB_DLL_EXTERN int mecab_lattice_next(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::request_type() + */ + MECAB_DLL_EXTERN int mecab_lattice_get_request_type(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::has_request_type() + */ + MECAB_DLL_EXTERN int mecab_lattice_has_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::set_request_type() + */ + MECAB_DLL_EXTERN void mecab_lattice_set_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::add_request_type() + */ + + MECAB_DLL_EXTERN void mecab_lattice_add_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::remove_request_type() + */ + MECAB_DLL_EXTERN void mecab_lattice_remove_request_type(mecab_lattice_t *lattice, int request_type); + + /** + * C wrapper of MeCab::Lattice::newNode(); + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_lattice_new_node(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::toString() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_tostr(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::toString(buf, size) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_tostr2(mecab_lattice_t *lattice, char *buf, size_t size); + + /** + * C wrapper of MeCab::Lattice::enumNBestAsString(N) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_nbest_tostr(mecab_lattice_t *lattice, size_t N); + + /** + * C wrapper of MeCab::Lattice::enumNBestAsString(N, buf, size) + */ + + MECAB_DLL_EXTERN const char *mecab_lattice_nbest_tostr2(mecab_lattice_t *lattice, size_t N, char *buf, size_t size); + + /** + * C wrapper of MeCab::Lattice::has_constraint() + */ + MECAB_DLL_EXTERN int mecab_lattice_has_constraint(mecab_lattice_t *lattice); + + /** + * C wrapper of MeCab::Lattice::boundary_constraint(pos) + */ + MECAB_DLL_EXTERN int mecab_lattice_get_boundary_constraint(mecab_lattice_t *lattice, size_t pos); + + + /** + * C wrapper of MeCab::Lattice::feature_constraint(pos) + */ + MECAB_DLL_EXTERN const char *mecab_lattice_get_feature_constraint(mecab_lattice_t *lattice, size_t pos); + + /** + * C wrapper of MeCab::Lattice::boundary_constraint(pos, type) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_boundary_constraint(mecab_lattice_t *lattice, size_t pos, int boundary_type); + + /** + * C wrapper of MeCab::Lattice::set_feature_constraint(begin_pos, end_pos, feature) + */ + MECAB_DLL_EXTERN void mecab_lattice_set_feature_constraint(mecab_lattice_t *lattice, size_t begin_pos, size_t end_pos, const char *feature); + + /** + * C wrapper of MeCab::Lattice::set_result(result); + */ + MECAB_DLL_EXTERN void mecab_lattice_set_result(mecab_lattice_t *lattice, const char *result); + + /** + * C wrapper of MeCab::Lattice::what() + */ + MECAB_DLL_EXTERN const char *mecab_lattice_strerror(mecab_lattice_t *lattice); + + + /* model interface */ + /** + * C wapper of MeCab::Model::create(argc, argv) + */ + MECAB_DLL_EXTERN mecab_model_t *mecab_model_new(int argc, char **argv); + + /** + * C wapper of MeCab::Model::create(arg) + */ + MECAB_DLL_EXTERN mecab_model_t *mecab_model_new2(const char *arg); + + /** + * C wapper of MeCab::deleteModel(model) + */ + + MECAB_DLL_EXTERN void mecab_model_destroy(mecab_model_t *model); + + /** + * C wapper of MeCab::Model::createTagger() + */ + MECAB_DLL_EXTERN mecab_t *mecab_model_new_tagger(mecab_model_t *model); + + /** + * C wapper of MeCab::Model::createLattice() + */ + MECAB_DLL_EXTERN mecab_lattice_t *mecab_model_new_lattice(mecab_model_t *model); + + /** + * C wrapper of MeCab::Model::swap() + */ + MECAB_DLL_EXTERN int mecab_model_swap(mecab_model_t *model, mecab_model_t *new_model); + + /** + * C wapper of MeCab::Model::dictionary_info() + */ + MECAB_DLL_EXTERN const mecab_dictionary_info_t* mecab_model_dictionary_info(mecab_model_t *model); + + /** + * C wrapper of MeCab::Model::transition_cost() + */ + MECAB_DLL_EXTERN int mecab_model_transition_cost(mecab_model_t *model, + unsigned short rcAttr, + unsigned short lcAttr); + + /** + * C wrapper of MeCab::Model::lookup() + */ + MECAB_DLL_EXTERN mecab_node_t *mecab_model_lookup(mecab_model_t *model, + const char *begin, + const char *end, + mecab_lattice_t *lattice); + + /* static functions */ + MECAB_DLL_EXTERN int mecab_do(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_dict_index(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_dict_gen(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_cost_train(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_system_eval(int argc, char **argv); + MECAB_DLL_EXTERN int mecab_test_gen(int argc, char **argv); +#endif + +#ifdef __cplusplus +} +#endif + +/* C++ interface */ +#ifdef __cplusplus + +namespace MeCab { +typedef struct mecab_dictionary_info_t DictionaryInfo; +typedef struct mecab_path_t Path; +typedef struct mecab_node_t Node; + +template class Allocator; +class Tagger; + +/** + * Lattice class + */ +class MECAB_DLL_CLASS_EXTERN Lattice { +public: + /** + * Clear all internal lattice data. + */ + virtual void clear() = 0; + + /** + * Return true if result object is available. + * @return boolean + */ + virtual bool is_available() const = 0; + + /** + * Return bos (begin of sentence) node. + * You can obtain all nodes via "for (const Node *node = lattice->bos_node(); node; node = node->next) {}" + * @return bos node object + */ + virtual Node *bos_node() const = 0; + + /** + * Return eos (end of sentence) node. + * @return eos node object + */ + virtual Node *eos_node() const = 0; + +#ifndef SWIG + /** + * This method is used internally. + */ + virtual Node **begin_nodes() const = 0; + + /** + * This method is used internally. + */ + virtual Node **end_nodes() const = 0; +#endif + + /** + * Return node linked list ending at |pos|. + * You can obtain all nodes via "for (const Node *node = lattice->end_nodes(pos); node; node = node->enext) {}" + * @param pos position of nodes. 0 <= pos < size() + * @return node linked list + */ + virtual Node *end_nodes(size_t pos) const = 0; + + /** + * Return node linked list starting at |pos|. + * You can obtain all nodes via "for (const Node *node = lattice->begin_nodes(pos); node; node = node->bnext) {}" + * @param pos position of nodes. 0 <= pos < size() + * @return node linked list + */ + virtual Node *begin_nodes(size_t pos) const = 0; + + /** + * Return sentence. + * If MECAB_NBEST or MECAB_PARTIAL mode is off, the returned poiner is the same as the one set by set_sentence(). + * @return sentence + */ + virtual const char *sentence() const = 0; + + /** + * Set sentence. This method does not take the ownership of the object. + * @param sentence sentence + */ + virtual void set_sentence(const char *sentence) = 0; + +#ifndef SWIG + /** + * Set sentence. This method does not take the ownership of the object. + * @param sentence sentence + * @param len length of the sentence + */ + virtual void set_sentence(const char *sentence, size_t len) = 0; +#endif + + /** + * Return sentence size. + * @return sentence size + */ + virtual size_t size() const = 0; + + /** + * Set normalization factor of CRF. + * @param Z new normalization factor. + */ + virtual void set_Z(double Z) = 0; + + /** + * return normalization factor of CRF. + * @return normalization factor. + */ + virtual double Z() const = 0; + + /** + * Set temparature parameter theta. + * @param theta temparature parameter. + */ + virtual void set_theta(float theta) = 0; + + /** + * Return temparature parameter theta. + * @return temparature parameter. + */ + virtual float theta() const = 0; + + /** + * Obtain next-best result. The internal linked list structure is updated. + * You should set MECAB_NBEST reques_type in advance. + * Return false if no more results are available or request_type is invalid. + * @return boolean + */ + virtual bool next() = 0; + + /** + * Return the current request type. + * @return request type + */ + virtual int request_type() const = 0; + + /** + * Return true if the object has a specified request type. + * @return boolean + */ + virtual bool has_request_type(int request_type) const = 0; + + /** + * Set request type. + * @param request_type new request type assigned + */ + virtual void set_request_type(int request_type) = 0; + + /** + * Add request type. + * @param request_type new request type added + */ + virtual void add_request_type(int request_type) = 0; + + /** + * Remove request type. + * @param request_type new request type removed + */ + virtual void remove_request_type(int request_type) = 0; + +#ifndef SWIG + /** + * This method is used internally. + */ + virtual Allocator *allocator() const = 0; +#endif + + /** + * Return new node. Lattice objects has the ownership of the node. + * @return new node object + */ + virtual Node *newNode() = 0; + + /** + * Return string representation of the lattice. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the lattice + */ + virtual const char *toString() = 0; + + /** + * Return string representation of the node. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the node + * @param node node object + */ + virtual const char *toString(const Node *node) = 0; + + /** + * Return string representation of the N-best results. + * Returned object is managed by this instance. When clear/set_sentence() method + * is called, the returned buffer is initialized. + * @return string representation of the node + * @param N how many results you want to obtain + */ + virtual const char *enumNBestAsString(size_t N) = 0; + +#ifndef SWIG + /** + * Return string representation of the lattice. + * Result is saved in the specified buffer. + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *toString(char *buf, size_t size) = 0; + + /** + * Return string representation of the node. + * Result is saved in the specified buffer. + * @param node node object + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *toString(const Node *node, + char *buf, size_t size) = 0; + + /** + * Return string representation of the N-best result. + * Result is saved in the specified. + * @param N how many results you want to obtain + * @param buf output buffer + * @param size output buffer size + * @return string representation of the lattice + */ + virtual const char *enumNBestAsString(size_t N, char *buf, size_t size) = 0; +#endif + + /** + * Returns true if any parsing constraint is set + */ + virtual bool has_constraint() const = 0; + + /** + * Returns the boundary constraint at the position. + * @param pos the position of constraint + * @return boundary constraint type + */ + virtual int boundary_constraint(size_t pos) const = 0; + + /** + * Returns the token constraint at the position. + * @param pos the beginning position of constraint. + * @return constrained node starting at the position. + */ + virtual const char *feature_constraint(size_t pos) const = 0; + + /** + * Set parsing constraint for partial parsing mode. + * @param pos the position of the boundary + * @param boundary_constraint_type the type of boundary + */ + virtual void set_boundary_constraint(size_t pos, + int boundary_constraint_type) = 0; + + /** + * Set parsing constraint for partial parsing mode. + * @param begin_pos the starting position of the constrained token. + * @param end_pos the the ending position of the constrained token. + * @param feature the feature of the constrained token. + */ + virtual void set_feature_constraint( + size_t begin_pos, size_t end_pos, + const char *feature) = 0; + + /** + * Set golden parsing results for unittesting. + * @param result the parsing result written in the standard mecab output. + */ + virtual void set_result(const char *result) = 0; + + /** + * Return error string. + * @return error string + */ + virtual const char *what() const = 0; + + /** + * Set error string. given string is copied to the internal buffer. + * @param str new error string + */ + virtual void set_what(const char *str) = 0; + +#ifndef SWIG + /** + * Create new Lattice object + * @return new Lattice object + */ + static Lattice *create(); +#endif + + virtual ~Lattice() {} +}; + +/** + * Model class + */ +class MECAB_DLL_CLASS_EXTERN Model { +public: + /** + * Return DictionaryInfo linked list. + * @return DictionaryInfo linked list + */ + virtual const DictionaryInfo *dictionary_info() const = 0; + + /** + * Return transtion cost from rcAttr to lcAttr. + * @return transtion cost + */ + virtual int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const = 0; + + /** + * perform common prefix search from the range [begin, end). + * |lattice| takes the ownership of return value. + * @return node linked list. + */ + virtual Node *lookup(const char *begin, const char *end, + Lattice *lattice) const = 0; + + /** + * Create a new Tagger object. + * All returned tagger object shares this model object as a parsing model. + * Never delete this model object before deleting tagger object. + * @return new Tagger object + */ + virtual Tagger *createTagger() const = 0; + + /** + * Create a new Lattice object. + * @return new Lattice object + */ + virtual Lattice *createLattice() const = 0; + + /** + * Swap the instance with |model|. + * The ownership of |model| always moves to this instance, + * meaning that passed |model| will no longer be accessible after calling this method. + * return true if new model is swapped successfully. + * This method is thread safe. All taggers created by + * Model::createTagger() method will also be updated asynchronously. + * No need to stop the parsing thread excplicitly before swapping model object. + * @return boolean + * @param model new model which is going to be swapped with the current model. + */ + virtual bool swap(Model *model) = 0; + + /** + * Return a version string + * @return version string + */ + static const char *version(); + + virtual ~Model() {} + +#ifndef SWIG + /** + * Factory method to create a new Model with a specified main's argc/argv-style parameters. + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param argc number of parameters + * @param argv parameter list + */ + static Model* create(int argc, char **argv); + + /** + * Factory method to create a new Model with a string parameter representation, i.e., + * "-d /user/local/mecab/dic/ipadic -Ochasen". + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param arg single string representation of the argment. + */ + static Model* create(const char *arg); +#endif +}; + +/** + * Tagger class + */ +class MECAB_DLL_CLASS_EXTERN Tagger { +public: + /** + * Handy static method. + * Return true if lattice is parsed successfully. + * This function is equivalent to + * { + * Tagger *tagger = model.createModel(); + * cosnt bool result = tagger->parse(lattice); + * delete tagger; + * return result; + * } + * @return boolean + */ + static bool parse(const Model &model, Lattice *lattice); + + /** + * Parse lattice object. + * Return true if lattice is parsed successfully. + * A sentence must be set to the lattice with Lattice:set_sentence object before calling this method. + * Parsed node object can be obtained with Lattice:bos_node. + * This method is thread safe. + * @return lattice lattice object + * @return boolean + */ + virtual bool parse(Lattice *lattice) const = 0; + + /** + * Parse given sentence and return parsed result as string. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is NOT thread safe. + * @param str sentence + * @return parsed result + */ + virtual const char* parse(const char *str) = 0; + + /** + * Parse given sentence and return Node object. + * You should not delete the returned node object. The returned buffer + * is overwritten when parse method is called again. + * You can traverse all nodes via Node::next member. + * This method is NOT thread safe. + * @param str sentence + * @return bos node object + */ + virtual const Node* parseToNode(const char *str) = 0; + + /** + * Parse given sentence and obtain N-best results as a string format. + * Currently, N must be 1 <= N <= 512 due to the limitation of the buffer size. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is DEPRECATED. Use Lattice class. + * @param N how many results you want to obtain + * @param str sentence + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str) = 0; + + /** + * Initialize N-best enumeration with a sentence. + * Return true if initialization finishes successfully. + * N-best result is obtained by calling next() or nextNode() in sequence. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @param str sentence + * @return boolean + */ + virtual bool parseNBestInit(const char *str) = 0; + + /** + * Return next-best parsed result. You must call parseNBestInit() in advance. + * Return NULL if no more reuslt is available. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @return node object + */ + virtual const Node* nextNode() = 0; + + /** + * Return next-best parsed result. You must call parseNBestInit() in advance. + * Return NULL if no more reuslt is available. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @return parsed result + */ + virtual const char* next() = 0; + + /** + * Return formatted node object. The format is specified with + * --unk-format, --bos-format, --eos-format, and --eon-format respectively. + * You should not delete the returned string. The returned buffer + * is overwritten when parse method is called again. + * This method is NOT thread safe. + * This method is DEPRECATED. Use Lattice class. + * @param node node object. + * @return parsed result + */ + virtual const char* formatNode(const Node *node) = 0; + +#ifndef SWIG + /** + * The same as parse() method, but input length and output buffer are passed. + * Return parsed result as string. The result pointer is the same as |ostr|. + * Return NULL, if parsed result string cannot be stored within |olen| bytes. + * @param str sentence + * @param len sentence length + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* parse(const char *str, size_t len, char *ostr, size_t olen) = 0; + + /** + * The same as parse() method, but input length can be passed. + * @param str sentence + * @param len sentence length + * @return parsed result + */ + virtual const char* parse(const char *str, size_t len) = 0; + + /** + * The same as parseToNode(), but input lenth can be passed. + * @param str sentence + * @param len sentence length + * @return node object + */ + virtual const Node* parseToNode(const char *str, size_t len) = 0; + + /** + * The same as parseNBest(), but input length can be passed. + * @param N how many results you want to obtain + * @param str sentence + * @param len sentence length + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str, size_t len) = 0; + + /** + * The same as parseNBestInit(), but input length can be passed. + * @param str sentence + * @param len sentence length + * @return boolean + * @return parsed result + */ + virtual bool parseNBestInit(const char *str, size_t len) = 0; + + /** + * The same as next(), but output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* next(char *ostr , size_t olen) = 0; + + /** + * The same as parseNBest(), but input length and output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param N how many results you want to obtain + * @param str input sentence + * @param len input sentence length + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* parseNBest(size_t N, const char *str, + size_t len, char *ostr, size_t olen) = 0; + + /** + * The same as formatNode(), but output buffer can be passed. + * Return NULL if more than |olen| buffer is required to store output string. + * @param node node object + * @param ostr output buffer + * @param olen output buffer length + * @return parsed result + */ + virtual const char* formatNode(const Node *node, char *ostr, size_t olen) = 0; +#endif + + /** + * Set request type. + * This method is DEPRECATED. Use Lattice::set_request_type(MECAB_PARTIAL). + * @param request_type new request type assigned + */ + virtual void set_request_type(int request_type) = 0; + + /** + * Return the current request type. + * This method is DEPRECATED. Use Lattice class. + * @return request type + */ + virtual int request_type() const = 0; + + /** + * Return true if partial parsing mode is on. + * This method is DEPRECATED. Use Lattice::has_request_type(MECAB_PARTIAL). + * @return boolean + */ + virtual bool partial() const = 0; + + /** + * set partial parsing mode. + * This method is DEPRECATED. Use Lattice::add_request_type(MECAB_PARTIAL) or Lattice::remove_request_type(MECAB_PARTIAL) + * @param partial partial mode + */ + virtual void set_partial(bool partial) = 0; + + /** + * Return lattice level. + * This method is DEPRECATED. Use Lattice::*_request_type() + * @return int lattice level + */ + virtual int lattice_level() const = 0; + + /** + * Set lattice level. + * This method is DEPRECATED. Use Lattice::*_request_type() + * @param level lattice level + */ + virtual void set_lattice_level(int level) = 0; + + /** + * Return true if all morphs output mode is on. + * This method is DEPRECATED. Use Lattice::has_request_type(MECAB_ALL_MORPHS). + * @return boolean + */ + virtual bool all_morphs() const = 0; + + /** + * set all-morphs output mode. + * This method is DEPRECATED. Use Lattice::add_request_type(MECAB_ALL_MORPHS) or Lattice::remove_request_type(MECAB_ALL_MORPHS) + * @param all_morphs + */ + virtual void set_all_morphs(bool all_morphs) = 0; + + /** + * Set temparature parameter theta. + * @param theta temparature parameter. + */ + virtual void set_theta(float theta) = 0; + + /** + * Return temparature parameter theta. + * @return temparature parameter. + */ + virtual float theta() const = 0; + + /** + * Return DictionaryInfo linked list. + * @return DictionaryInfo linked list + */ + virtual const DictionaryInfo* dictionary_info() const = 0; + + /** + * Return error string. + * @return error string + */ + virtual const char* what() const = 0; + + virtual ~Tagger() {} + +#ifndef SWIG + /** + * Factory method to create a new Tagger with a specified main's argc/argv-style parameters. + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Tagger object + * @param argc number of parameters + * @param argv parameter list + */ + static Tagger *create(int argc, char **argv); + + /** + * Factory method to create a new Tagger with a string parameter representation, i.e., + * "-d /user/local/mecab/dic/ipadic -Ochasen". + * Return NULL if new model cannot be initialized. Use MeCab::getLastError() to obtain the + * cause of the errors. + * @return new Model object + * @param arg single string representation of the argment. + */ + static Tagger *create(const char *arg); +#endif + + /** + * Return a version string + * @return version string + */ + static const char *version(); +}; + +#ifndef SWIG +/** + * Alias of Lattice::create() + */ +MECAB_DLL_EXTERN Lattice *createLattice(); + +/** + * Alias of Mode::create(argc, argv) + */ +MECAB_DLL_EXTERN Model *createModel(int argc, char **argv); + +/** + * Alias of Mode::create(arg) + */ +MECAB_DLL_EXTERN Model *createModel(const char *arg); + +/** + * Alias of Tagger::create(argc, argv) + */ +MECAB_DLL_EXTERN Tagger *createTagger(int argc, char **argv); + +/** + * Alias of Tagger::create(arg) + */ +MECAB_DLL_EXTERN Tagger *createTagger(const char *arg); + +/** + * delete Lattice object. + * This method calles "delete lattice". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param lattice lattice object + */ +MECAB_DLL_EXTERN void deleteLattice(Lattice *lattice); + + +/** + * delete Model object. + * This method calles "delete model". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param model model object + */ +MECAB_DLL_EXTERN void deleteModel(Model *model); + +/** + * delete Tagger object. + * This method calles "delete tagger". + * In some environment, e.g., MS-Windows, an object allocated inside a DLL must be deleted in the same DLL too. + * @param tagger tagger object + */ +MECAB_DLL_EXTERN void deleteTagger(Tagger *tagger); + +/** + * Return last error string. + * @return error string + */ +MECAB_DLL_EXTERN const char* getLastError(); + +/** + * An alias of getLastError. + * It is kept for backward compatibility. + * @return error string + */ +MECAB_DLL_EXTERN const char* getTaggerError(); +#endif +} +#endif +#endif /* MECAB_MECAB_H_ */ diff --git a/fts/third_party/mecab/src/mmap.h b/fts/third_party/mecab/src/mmap.h new file mode 100644 index 00000000..3a174f59 --- /dev/null +++ b/fts/third_party/mecab/src/mmap.h @@ -0,0 +1,212 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_MMAP_H +#define MECAB_MMAP_H + +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +extern "C" { + +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#ifdef HAVE_STRING_H +#include +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) +#ifdef HAVE_WINDOWS_H +#include +#endif +#else + +#ifdef HAVE_SYS_MMAN_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif +#endif +} + +#include "common.h" +#include "utils.h" + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +namespace MeCab { + +template class Mmap { + private: + T *text; + size_t length; + std::string fileName; + whatlog what_; + +#if defined(_WIN32) && !defined(__CYGWIN__) + HANDLE hFile; + HANDLE hMap; +#else + int fd; + int flag; +#endif + + public: + T& operator[](size_t n) { return *(text + n); } + const T& operator[](size_t n) const { return *(text + n); } + T* begin() { return text; } + const T* begin() const { return text; } + T* end() { return text + size(); } + const T* end() const { return text + size(); } + size_t size() { return length/sizeof(T); } + const char *what() { return what_.str(); } + const char *file_name() { return fileName.c_str(); } + size_t file_size() { return length; } + bool empty() { return(length == 0); } + + // This code is imported from sufary, develoved by + // TATUO Yamashita Thanks! +#if defined(_WIN32) && !defined(__CYGWIN__) + bool open(const char *filename, const char *mode = "r") { + this->close(); + unsigned long mode1, mode2, mode3; + fileName = std::string(filename); + + if (std::strcmp(mode, "r") == 0) { + mode1 = GENERIC_READ; + mode2 = PAGE_READONLY; + mode3 = FILE_MAP_READ; + } else if (std::strcmp(mode, "r+") == 0) { + mode1 = GENERIC_READ | GENERIC_WRITE; + mode2 = PAGE_READWRITE; + mode3 = FILE_MAP_ALL_ACCESS; + } else { + CHECK_FALSE(false) << "unknown open mode:" << filename; + } + + hFile = ::CreateFileW(WPATH_FORCE(filename), mode1, FILE_SHARE_READ, 0, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + CHECK_FALSE(hFile != INVALID_HANDLE_VALUE) + << "CreateFile() failed: " << filename; + + length = ::GetFileSize(hFile, 0); + + hMap = ::CreateFileMapping(hFile, 0, mode2, 0, 0, 0); + CHECK_FALSE(hMap) << "CreateFileMapping() failed: " << filename; + + text = reinterpret_cast(::MapViewOfFile(hMap, mode3, 0, 0, 0)); + CHECK_FALSE(text) << "MapViewOfFile() failed: " << filename; + + return true; + } + + void close() { + if (text) { ::UnmapViewOfFile(text); } + if (hFile != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile); + hFile = INVALID_HANDLE_VALUE; + } + if (hMap) { + ::CloseHandle(hMap); + hMap = 0; + } + text = 0; + } + + Mmap(): text(0), hFile(INVALID_HANDLE_VALUE), hMap(0) {} + +#else + + bool open(const char *filename, const char *mode = "r") { + this->close(); + struct stat st; + fileName = std::string(filename); + + if (std::strcmp(mode, "r") == 0) + flag = O_RDONLY; + else if (std::strcmp(mode, "r+") == 0) + flag = O_RDWR; + else + CHECK_FALSE(false) << "unknown open mode: " << filename; + + CHECK_FALSE((fd = ::open(filename, flag | O_BINARY)) >= 0) + << "open failed: " << filename; + + CHECK_FALSE(::fstat(fd, &st) >= 0) + << "failed to get file size: " << filename; + + length = st.st_size; + +#ifdef HAVE_MMAP + int prot = PROT_READ; + if (flag == O_RDWR) prot |= PROT_WRITE; + char *p; + CHECK_FALSE((p = reinterpret_cast + (::mmap(0, length, prot, MAP_SHARED, fd, 0))) + != MAP_FAILED) + << "mmap() failed: " << filename; + + text = reinterpret_cast(p); +#else + text = new T[length]; + CHECK_FALSE(::read(fd, text, length) >= 0) + << "read() failed: " << filename; +#endif + ::close(fd); + fd = -1; + + return true; + } + + void close() { + if (fd >= 0) { + ::close(fd); + fd = -1; + } + + if (text) { +#ifdef HAVE_MMAP + ::munmap(reinterpret_cast(text), length); + text = 0; +#else + if (flag == O_RDWR) { + int fd2; + if ((fd2 = ::open(fileName.c_str(), O_RDWR)) >= 0) { + ::write(fd2, text, length); + ::close(fd2); + } + } + delete [] text; +#endif + } + + text = 0; + } + + Mmap() : text(0), fd(-1) {} +#endif + + virtual ~Mmap() { this->close(); } +}; +} +#endif diff --git a/fts/third_party/mecab/src/nbest_generator.cpp b/fts/third_party/mecab/src/nbest_generator.cpp new file mode 100644 index 00000000..d30796ec --- /dev/null +++ b/fts/third_party/mecab/src/nbest_generator.cpp @@ -0,0 +1,52 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include "mecab.h" +#include "nbest_generator.h" + +namespace MeCab { + +bool NBestGenerator::set(Lattice *lattice) { + freelist_.free(); + while (!agenda_.empty()) { + agenda_.pop(); // make empty + } + QueueElement *eos = freelist_.alloc(); + eos->node = lattice->eos_node(); + eos->next = 0; + eos->fx = eos->gx = 0; + agenda_.push(eos); + return true; +} + +bool NBestGenerator::next() { + while (!agenda_.empty()) { + QueueElement *top = agenda_.top(); + agenda_.pop(); + Node *rnode = top->node; + + if (rnode->stat == MECAB_BOS_NODE) { // BOS + for (QueueElement *n = top; n->next; n = n->next) { + n->node->next = n->next->node; // change next & prev + n->next->node->prev = n->node; + // TODO: rewrite costs; + } + return true; + } + + for (Path *path = rnode->lpath; path; path = path->lnext) { + QueueElement *n = freelist_.alloc(); + n->node = path->lnode; + n->gx = path->cost + top->gx; + n->fx = path->lnode->cost + path->cost + top->gx; + n->next = top; + agenda_.push(n); + } + } + + return false; +} +} diff --git a/fts/third_party/mecab/src/nbest_generator.h b/fts/third_party/mecab/src/nbest_generator.h new file mode 100644 index 00000000..e09e8ac7 --- /dev/null +++ b/fts/third_party/mecab/src/nbest_generator.h @@ -0,0 +1,43 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_NBEST_GENERATOR_H_ +#define MECAB_NBEST_GENERATOR_H_ + +#include +#include "mecab.h" +#include "freelist.h" + +namespace MeCab { + +class NBestGenerator { + private: + struct QueueElement { + Node *node; + QueueElement *next; + long fx; // f(x) = h(x) + g(x): cost function for A* search + long gx; // g(x) + }; + + class QueueElementComp { + public: + const bool operator()(QueueElement *q1, QueueElement *q2) { + return (q1->fx > q2->fx); + } + }; + + std::priority_queue, + QueueElementComp> agenda_; + FreeList freelist_; + + public: + explicit NBestGenerator() : freelist_(512) {} + virtual ~NBestGenerator() {} + bool set(Lattice *lattice); + bool next(); +}; +} + +#endif // MECAB_NBEST_GENERATOR_H_ diff --git a/fts/third_party/mecab/src/param.cpp b/fts/third_party/mecab/src/param.cpp new file mode 100644 index 00000000..322a74f5 --- /dev/null +++ b/fts/third_party/mecab/src/param.cpp @@ -0,0 +1,223 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "param.h" +#include "string_buffer.h" +#include "utils.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +namespace MeCab { +namespace { +void init_param(std::string *help, + std::string *version, + const std::string &system_name, + const Option *opts) { + *help = std::string(COPYRIGHT) + "\nUsage: " + + system_name + " [options] files\n"; + + *version = std::string(PACKAGE) + " of " + VERSION + '\n'; + + size_t max = 0; + for (size_t i = 0; opts[i].name; ++i) { + size_t l = 1 + std::strlen(opts[i].name); + if (opts[i].arg_description) + l += (1 + std::strlen(opts[i].arg_description)); + max = std::max(l, max); + } + + for (size_t i = 0; opts[i].name; ++i) { + size_t l = std::strlen(opts[i].name); + if (opts[i].arg_description) + l += (1 + std::strlen(opts[i].arg_description)); + *help += " -"; + *help += opts[i].short_name; + *help += ", --"; + *help += opts[i].name; + if (opts[i].arg_description) { + *help += '='; + *help += opts[i].arg_description; + } + for (; l <= max; l++) *help += ' '; + *help += opts[i].description; + *help += '\n'; + } + + *help += '\n'; + return; +} +} // namespace + +void Param::dump_config(std::ostream *os) const { + for (std::map::const_iterator it = conf_.begin(); + it != conf_.end(); + ++it) { + *os << it->first << ": " << it->second << std::endl; + } +} + +bool Param::load(const char *filename) { + std::ifstream ifs(WPATH(filename)); + + CHECK_FALSE(ifs) << "no such file or directory: " << filename; + + std::string line; + while (std::getline(ifs, line)) { + if (!line.size() || + (line.size() && (line[0] == ';' || line[0] == '#'))) continue; + + size_t pos = line.find('='); + CHECK_FALSE(pos != std::string::npos) << "format error: " << line; + + size_t s1, s2; + for (s1 = pos+1; s1 < line.size() && isspace(line[s1]); s1++); + for (s2 = pos-1; static_cast(s2) >= 0 && isspace(line[s2]); s2--); + const std::string value = line.substr(s1, line.size() - s1); + const std::string key = line.substr(0, s2 + 1); + set(key.c_str(), value, false); + } + + return true; +} + +bool Param::open(int argc, char **argv, const Option *opts) { + int ind = 0; + int _errno = 0; + +#define GOTO_ERROR(n) { \ + _errno = n; \ + goto ERROR; } while (0) + + if (argc <= 0) { + system_name_ = "unknown"; + return true; // this is not error + } + + system_name_ = std::string(argv[0]); + + init_param(&help_, &version_, system_name_, opts); + + for (size_t i = 0; opts[i].name; ++i) { + if (opts[i].default_value) set + (opts[i].name, opts[i].default_value); + } + + for (ind = 1; ind < argc; ind++) { + if (argv[ind][0] == '-') { + // long options + if (argv[ind][1] == '-') { + char *s; + for (s = &argv[ind][2]; *s != '\0' && *s != '='; s++); + size_t len = (size_t)(s - &argv[ind][2]); + if (len == 0) return true; // stop the scanning + + bool hit = false; + size_t i = 0; + for (i = 0; opts[i].name; ++i) { + size_t nlen = std::strlen(opts[i].name); + if (nlen == len && std::strncmp(&argv[ind][2], + opts[i].name, len) == 0) { + hit = true; + break; + } + } + + if (!hit) GOTO_ERROR(0); + + if (opts[i].arg_description) { + if (*s == '=') { + set(opts[i].name, s+1); + } else { + if (argc == (ind+1)) GOTO_ERROR(1); + set(opts[i].name, argv[++ind]); + } + } else { + if (*s == '=') GOTO_ERROR(2); + set(opts[i].name, 1); + } + + // short options + } else if (argv[ind][1] != '\0') { + size_t i = 0; + bool hit = false; + for (i = 0; opts[i].name; ++i) { + if (opts[i].short_name == argv[ind][1]) { + hit = true; + break; + } + } + + if (!hit) GOTO_ERROR(0); + + if (opts[i].arg_description) { + if (argv[ind][2] != '\0') { + set(opts[i].name, &argv[ind][2]); + } else { + if (argc == (ind+1)) GOTO_ERROR(1); + set(opts[i].name, argv[++ind]); + } + } else { + if (argv[ind][2] != '\0') GOTO_ERROR(2); + set(opts[i].name, 1); + } + } + } else { + rest_.push_back(std::string(argv[ind])); // others + } + } + + return true; + +ERROR: + switch (_errno) { + case 0: WHAT << "unrecognized option `" << argv[ind] << "`"; break; + case 1: WHAT << "`" << argv[ind] << "` requires an argument"; break; + case 2: WHAT << "`" << argv[ind] << "` doesn't allow an argument"; break; + } + return false; +} + +void Param::clear() { + conf_.clear(); + rest_.clear(); +} + +bool Param::open(const char *arg, const Option *opts) { + scoped_fixed_array str; + std::strncpy(str.get(), arg, str.size()); + char* ptr[64]; + unsigned int size = 1; + ptr[0] = const_cast(PACKAGE); + + for (char *p = str.get(); *p;) { + while (isspace(*p)) *p++ = '\0'; + if (*p == '\0') break; + ptr[size++] = p; + if (size == sizeof(ptr)) break; + while (*p && !isspace(*p)) p++; + } + + return open(size, ptr, opts); +} + +int Param::help_version() const { + if (get("help")) { + std::cout << help(); + return 0; + } + + if (get("version")) { + std::cout << version(); + return 0; + } + + return 1; +} +} diff --git a/fts/third_party/mecab/src/param.h b/fts/third_party/mecab/src/param.h new file mode 100644 index 00000000..89a449fc --- /dev/null +++ b/fts/third_party/mecab/src/param.h @@ -0,0 +1,92 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_PARAM_H +#define MECAB_PARAM_H + +#include +#include +#include +#include +#include "scoped_ptr.h" +#include "common.h" + +namespace { +template +Target lexical_cast(Source arg) { + std::stringstream interpreter; + Target result; + if (!(interpreter << arg) || !(interpreter >> result) || + !(interpreter >> std::ws).eof()) { + MeCab::scoped_ptr r(new Target()); // return default value + return *r; + } + return result; +} + +template <> +std::string lexical_cast(std::string arg) { + return arg; +} +} + +namespace MeCab { + +struct Option { + const char *name; + char short_name; + const char *default_value; + const char *arg_description; + const char *description; +}; + +class Param { + private: + std::map conf_; + std::vector rest_; + std::string system_name_; + std::string help_; + std::string version_; + whatlog what_; + + public: + bool open(int argc, char **argv, const Option *opt); + bool open(const char *arg, const Option *opt); + bool load(const char *filename); + void clear(); + const std::vector& rest_args() const { return rest_; } + + const char* program_name() const { return system_name_.c_str(); } + const char *what() { return what_.str(); } + const char* help() const { return help_.c_str(); } + const char* version() const { return version_.c_str(); } + int help_version() const; + + template + T get(const char *key) const { + std::map::const_iterator it = conf_.find(key); + if (it == conf_.end()) { + scoped_ptr r(new T()); + return *r; + } + return lexical_cast(it->second); + } + + template + void set(const char* key, const T &value, + bool rewrite = true) { + std::string key2 = std::string(key); + if (rewrite || (!rewrite && conf_.find(key2) == conf_.end())) + conf_[key2] = lexical_cast(value); + } + + void dump_config(std::ostream *os) const; + + explicit Param() {} + virtual ~Param() {} +}; +} + +#endif diff --git a/fts/third_party/mecab/src/scoped_ptr.h b/fts/third_party/mecab/src/scoped_ptr.h new file mode 100644 index 00000000..325f5a7e --- /dev/null +++ b/fts/third_party/mecab/src/scoped_ptr.h @@ -0,0 +1,95 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_SCOPED_PTR_H +#define MECAB_SCOPED_PTR_H + +#include +#include + +namespace MeCab { + +template class scoped_ptr { + private: + T * ptr_; + scoped_ptr(scoped_ptr const &); + scoped_ptr & operator= (scoped_ptr const &); + typedef scoped_ptr this_type; + + public: + typedef T element_type; + explicit scoped_ptr(T * p = 0): ptr_(p) {} + virtual ~scoped_ptr() { delete ptr_; } + void reset(T * p = 0) { + delete ptr_; + ptr_ = p; + } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } +}; + +template class scoped_array { + private: + T * ptr_; + scoped_array(scoped_array const &); + scoped_array & operator= (scoped_array const &); + typedef scoped_array this_type; + + public: + typedef T element_type; + explicit scoped_array(T * p = 0): ptr_(p) {} + virtual ~scoped_array() { delete [] ptr_; } + void reset(T * p = 0) { + delete [] ptr_; + ptr_ = p; + } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } + T & operator[](size_t i) const { return ptr_[i]; } +}; + +template class scoped_fixed_array { + private: + T * ptr_; + size_t size_; + scoped_fixed_array(scoped_fixed_array const &); + scoped_fixed_array & operator= (scoped_fixed_array const &); + typedef scoped_fixed_array this_type; + + public: + typedef T element_type; + explicit scoped_fixed_array() + : ptr_(new T[N]), size_(N) {} + virtual ~scoped_fixed_array() { delete [] ptr_; } + size_t size() const { return size_; } + T & operator*() const { return *ptr_; } + T * operator->() const { return ptr_; } + T * get() const { return ptr_; } + T & operator[](size_t i) const { return ptr_[i]; } +}; + +class scoped_string: public scoped_array { + public: + explicit scoped_string() { reset_string(""); } + explicit scoped_string(const std::string &str) { + reset_string(str); + } + + void reset_string(const std::string &str) { + char *p = new char[str.size() + 1]; + std::strcpy(p, str.c_str()); + reset(p); + } + + void reset_string(const char *str) { + char *p = new char[std::strlen(str) + 1]; + std::strcpy(p, str); + reset(p); + } +}; +} +#endif diff --git a/fts/third_party/mecab/src/stream_wrapper.h b/fts/third_party/mecab/src/stream_wrapper.h new file mode 100644 index 00000000..f4ce1a19 --- /dev/null +++ b/fts/third_party/mecab/src/stream_wrapper.h @@ -0,0 +1,55 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_STREAM_WRAPPER_H_ +#define MECAB_STREAM_WRAPPER_H_ + +#include +#include +#include +#include "utils.h" + +namespace MeCab { + +class istream_wrapper { + private: + std::istream* is_; + public: + std::istream &operator*() const { return *is_; } + std::istream *operator->() const { return is_; } + explicit istream_wrapper(const char* filename): is_(0) { + if (std::strcmp(filename, "-") == 0) { + is_ = &std::cin; + } else { + is_ = new std::ifstream(WPATH(filename)); + } + } + + virtual ~istream_wrapper() { + if (is_ != &std::cin) delete is_; + } +}; + +class ostream_wrapper { + private: + std::ostream* os_; + public: + std::ostream &operator*() const { return *os_; } + std::ostream *operator->() const { return os_; } + explicit ostream_wrapper(const char* filename): os_(0) { + if (std::strcmp(filename, "-") == 0) { + os_ = &std::cout; + } else { + os_ = new std::ofstream(WPATH(filename)); + } + } + + virtual ~ostream_wrapper() { + if (os_ != &std::cout) delete os_; + } +}; +} + +#endif // MECAB_STREAM_WRAPPER_H_ diff --git a/fts/third_party/mecab/src/string_buffer.cpp b/fts/third_party/mecab/src/string_buffer.cpp new file mode 100644 index 00000000..6a65b310 --- /dev/null +++ b/fts/third_party/mecab/src/string_buffer.cpp @@ -0,0 +1,65 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include "common.h" +#include "string_buffer.h" + +#define DEFAULT_ALLOC_SIZE BUF_SIZE + +namespace MeCab { + +bool StringBuffer::reserve(size_t length) { + if (!is_delete_) { + error_ = (size_ + length >= alloc_size_); + return (!error_); + } + + if (size_ + length >= alloc_size_) { + if (alloc_size_ == 0) { + alloc_size_ = DEFAULT_ALLOC_SIZE; + ptr_ = new char[alloc_size_]; + } + size_t len = size_ + length; + do { + alloc_size_ *= 2; + } while (len >= alloc_size_); + char *new_ptr = new char[alloc_size_]; + std::memcpy(new_ptr, ptr_, size_); + delete [] ptr_; + ptr_ = new_ptr; + } + + return true; +} + +StringBuffer::~StringBuffer() { + if (is_delete_) { + delete [] ptr_; + ptr_ = 0; + } +} + +StringBuffer& StringBuffer::write(char str) { + if (reserve(1)) { + ptr_[size_] = str; + ++size_; + } + return *this; +} + +StringBuffer& StringBuffer::write(const char* str) { + return this->write(str, std::strlen(str)); +} + +StringBuffer& StringBuffer::write(const char* str, size_t length) { + if (reserve(length)) { + std::memcpy(ptr_ + size_ , str, length); + size_ += length; + } + return *this; +} +} diff --git a/fts/third_party/mecab/src/string_buffer.h b/fts/third_party/mecab/src/string_buffer.h new file mode 100644 index 00000000..8fc8a682 --- /dev/null +++ b/fts/third_party/mecab/src/string_buffer.h @@ -0,0 +1,74 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_STRINGBUFFER_H +#define MECAB_STRINGBUFFER_H + +#include +#include "common.h" +#include "utils.h" + +namespace MeCab { + +#define _ITOA(n) do { char fbuf[64]; itoa(n, fbuf); return this->write(fbuf); } while (0) +#define _UITOA(n) do { char fbuf[64]; uitoa(n, fbuf); return this->write(fbuf);} while (0) +#define _DTOA(n) do { char fbuf[64]; dtoa(n, fbuf); return this->write(fbuf); } while (0) + +class StringBuffer { + private: + size_t size_; + size_t alloc_size_; + char *ptr_; + bool is_delete_; + bool error_; + bool reserve(size_t); + + public: + explicit StringBuffer(): size_(0), alloc_size_(0), + ptr_(0), is_delete_(true), error_(false) {} + explicit StringBuffer(char *_s, size_t _l): + size_(0), alloc_size_(_l), ptr_(_s), + is_delete_(false), error_(false) {} + + virtual ~StringBuffer(); + + StringBuffer& write(char); + StringBuffer& write(const char*, size_t); + StringBuffer& write(const char*); + StringBuffer& operator<<(double n) { _DTOA(n); } + StringBuffer& operator<<(short int n) { _ITOA(n); } + StringBuffer& operator<<(int n) { _ITOA(n); } + StringBuffer& operator<<(long int n) { _ITOA(n); } + StringBuffer& operator<<(unsigned short int n) { _UITOA(n); } + StringBuffer& operator<<(unsigned int n) { _UITOA(n); } + StringBuffer& operator<<(unsigned long int n) { _UITOA(n); } +#ifdef HAVE_UNSIGNED_LONG_LONG_INT + StringBuffer& operator<<(unsigned long long int n) { _UITOA(n); } +#endif + + StringBuffer& operator<< (char n) { + return this->write(n); + } + + StringBuffer& operator<< (unsigned char n) { + return this->write(n); + } + + StringBuffer& operator<< (const char* n) { + return this->write(n); + } + + StringBuffer& operator<< (const std::string& n) { + return this->write(n.c_str()); + } + + void clear() { size_ = 0; } + const char *str() const { + return error_ ? 0 : const_cast(ptr_); + } +}; +} + +#endif diff --git a/fts/third_party/mecab/src/tagger.cpp b/fts/third_party/mecab/src/tagger.cpp new file mode 100644 index 00000000..2780dbb2 --- /dev/null +++ b/fts/third_party/mecab/src/tagger.cpp @@ -0,0 +1,1277 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include "common.h" +#include "connector.h" +#include "mecab.h" +#include "nbest_generator.h" +#include "param.h" +#include "scoped_ptr.h" +#include "stream_wrapper.h" +#include "string_buffer.h" +#include "thread.h" +#include "tokenizer.h" +#include "viterbi.h" +#include "writer.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +const char *getGlobalError(); +void setGlobalError(const char *str); + +namespace MeCab { +namespace { + +const float kDefaultTheta = 0.75; + +const MeCab::Option long_options[] = { + { "rcfile", 'r', 0, "FILE", "use FILE as resource file" }, + { "dicdir", 'd', 0, "DIR", "set DIR as a system dicdir" }, + { "userdic", 'u', 0, "FILE", "use FILE as a user dictionary" }, + { "lattice-level", 'l', "0", "INT", + "lattice information level (DEPRECATED)" }, + { "dictionary-info", 'D', 0, 0, "show dictionary information and exit" }, + { "output-format-type", 'O', 0, "TYPE", + "set output format type (wakati,none,...)" }, + { "all-morphs", 'a', 0, 0, "output all morphs(default false)" }, + { "nbest", 'N', "1", + "INT", "output N best results (default 1)" }, + { "partial", 'p', 0, 0, + "partial parsing mode (default false)" }, + { "marginal", 'm', 0, 0, + "output marginal probability (default false)" }, + { "max-grouping-size", 'M', "24", + "INT", "maximum grouping size for unknown words (default 24)" }, + { "node-format", 'F', "%m\\t%H\\n", "STR", + "use STR as the user-defined node format" }, + { "unk-format", 'U', "%m\\t%H\\n", "STR", + "use STR as the user-defined unknown node format" }, + { "bos-format", 'B', "", "STR", + "use STR as the user-defined beginning-of-sentence format" }, + { "eos-format", 'E', "EOS\\n", "STR", + "use STR as the user-defined end-of-sentence format" }, + { "eon-format", 'S', "", "STR", + "use STR as the user-defined end-of-NBest format" }, + { "unk-feature", 'x', 0, "STR", + "use STR as the feature for unknown word" }, + { "input-buffer-size", 'b', 0, "INT", + "set input buffer size (default 8192)" }, + { "dump-config", 'P', 0, 0, "dump MeCab parameters" }, + { "allocate-sentence", 'C', 0, 0, + "allocate new memory for input sentence" }, + { "theta", 't', "0.75", "FLOAT", + "set temparature parameter theta (default 0.75)" }, + { "cost-factor", 'c', "700", "INT", + "set cost factor (default 700)" }, + { "output", 'o', 0, "FILE", "set the output file name" }, + { "version", 'v', 0, 0, "show the version and exit." }, + { "help", 'h', 0, 0, "show this help and exit." }, + { 0, 0, 0, 0 } +}; + +class ModelImpl: public Model { + public: + ModelImpl(); + virtual ~ModelImpl(); + + bool open(int argc, char **argv); + bool open(const char *arg); + bool open(const Param ¶m); + + bool swap(Model *model); + + bool is_available() const { + return (viterbi_ && writer_.get()); + } + + int request_type() const { + return request_type_; + } + + double theta() const { + return theta_; + } + + const DictionaryInfo *dictionary_info() const { + return viterbi_->tokenizer() ? + viterbi_->tokenizer()->dictionary_info() : 0; + } + + int transition_cost(unsigned short rcAttr, + unsigned short lcAttr) const { + return viterbi_->connector()->transition_cost(rcAttr, lcAttr); + } + + Node *lookup(const char *begin, const char *end, + Lattice *lattice) const { + return viterbi_->tokenizer()->lookup( + begin, end, + lattice->allocator(), lattice); + } + + Tagger *createTagger() const; + + Lattice *createLattice() const; + + const Viterbi *viterbi() const { + return viterbi_; + } + + // moves the owership. + Viterbi *take_viterbi() { + Viterbi *result = viterbi_; + viterbi_ = 0; + return result; + } + + const Writer *writer() const { + return writer_.get(); + } + +#ifdef HAVE_ATOMIC_OPS + read_write_mutex *mutex() const { + return &mutex_; + } +#endif + + private: + Viterbi *viterbi_; + scoped_ptr writer_; + int request_type_; + double theta_; + +#ifdef HAVE_ATOMIC_OPS + mutable read_write_mutex mutex_; +#endif +}; + +class TaggerImpl: public Tagger { + public: + bool open(int argc, char **argv); + bool open(const char *arg); + bool open(const ModelImpl &model); + + bool parse(Lattice *lattice) const; + + void set_request_type(int request_type); + int request_type() const; + + const char* parse(const char*); + const char* parse(const char*, size_t); + const char* parse(const char*, size_t, char*, size_t); + const Node* parseToNode(const char*); + const Node* parseToNode(const char*, size_t = 0); + const char* parseNBest(size_t, const char*); + const char* parseNBest(size_t, const char*, size_t); + const char* parseNBest(size_t, const char*, + size_t, char *, size_t); + bool parseNBestInit(const char*); + bool parseNBestInit(const char*, size_t); + const Node* nextNode(); + const char* next(); + const char* next(char*, size_t); + + const char *formatNode(const Node *); + const char *formatNode(const Node *, char *, size_t); + + const DictionaryInfo *dictionary_info() const; + + void set_partial(bool partial); + bool partial() const; + void set_theta(float theta); + float theta() const; + void set_lattice_level(int level); + int lattice_level() const; + void set_all_morphs(bool all_morphs); + bool all_morphs() const; + + const char* what() const; + + TaggerImpl(); + virtual ~TaggerImpl(); + + private: + const ModelImpl *model() const { return current_model_; } + + void set_what(const char *str) { + what_.assign(str); + } + + void initRequestType() { + mutable_lattice()->set_request_type(request_type_); + mutable_lattice()->set_theta(theta_); + } + + Lattice *mutable_lattice() { + if (!lattice_.get()) { + lattice_.reset(model()->createLattice()); + } + return lattice_.get(); + } + + const ModelImpl *current_model_; + scoped_ptr model_; + scoped_ptr lattice_; + int request_type_; + double theta_; + std::string what_; +}; + +class LatticeImpl : public Lattice { + public: + explicit LatticeImpl(const Writer *writer = 0); + ~LatticeImpl(); + + // clear internal lattice + void clear(); + + bool is_available() const { + return (sentence_ && + !begin_nodes_.empty() && + !end_nodes_.empty()); + } + + // nbest; + bool next(); + + // return bos/eos node + Node *bos_node() const { return end_nodes_[0]; } + Node *eos_node() const { return begin_nodes_[size()]; } + Node **begin_nodes() const { return const_cast(&begin_nodes_[0]); } + Node **end_nodes() const { return const_cast(&end_nodes_[0]); } + Node *begin_nodes(size_t pos) const { return begin_nodes_[pos]; } + Node *end_nodes(size_t pos) const { return end_nodes_[pos]; } + + const char *sentence() const { return sentence_; } + void set_sentence(const char *sentence); + void set_sentence(const char *sentence, size_t len); + size_t size() const { return size_; } + + void set_Z(double Z) { Z_ = Z; } + double Z() const { return Z_; } + + float theta() const { return theta_; } + void set_theta(float theta) { theta_ = theta; } + + int request_type() const { return request_type_; } + + void set_request_type(int request_type) { + request_type_ = request_type; + } + bool has_request_type(int request_type) const { + return request_type & request_type_; + } + void add_request_type(int request_type) { + request_type_ |= request_type; + } + void remove_request_type(int request_type) { + request_type_ &= ~request_type; + } + + Allocator *allocator() const { + return allocator_.get(); + } + + Node *newNode() { + return allocator_->newNode(); + } + + bool has_constraint() const; + int boundary_constraint(size_t pos) const; + const char *feature_constraint(size_t begin_pos) const; + + void set_boundary_constraint(size_t pos, + int boundary_constraint_type); + + void set_feature_constraint(size_t begin_pos, size_t end_pos, + const char *feature); + + void set_result(const char *result); + + const char *what() const { return what_.c_str(); } + + void set_what(const char *str) { + what_.assign(str); + } + + const char *toString(); + const char *toString(char *buf, size_t size); + const char *toString(const Node *node); + const char *toString(const Node *node, + char *buf, size_t size); + const char *enumNBestAsString(size_t N); + const char *enumNBestAsString(size_t N, char *buf, size_t size); + + private: + const char *sentence_; + size_t size_; + double theta_; + double Z_; + int request_type_; + std::string what_; + std::vector end_nodes_; + std::vector begin_nodes_; + std::vector feature_constraint_; + std::vector boundary_constraint_; + const Writer *writer_; + scoped_ptr ostrs_; + scoped_ptr > allocator_; + + StringBuffer *stream() { + if (!ostrs_.get()) { + ostrs_.reset(new StringBuffer); + } + return ostrs_.get(); + } + + const char *toStringInternal(StringBuffer *os); + const char *toStringInternal(const Node *node, StringBuffer *os); + const char *enumNBestAsStringInternal(size_t N, StringBuffer *os); +}; + +ModelImpl::ModelImpl() + : viterbi_(new Viterbi), writer_(new Writer), + request_type_(MECAB_ONE_BEST), theta_(0.0) {} + +ModelImpl::~ModelImpl() { + delete viterbi_; + viterbi_ = 0; +} + +bool ModelImpl::open(int argc, char **argv) { + Param param; + if (!param.open(argc, argv, long_options) || + !load_dictionary_resource(¶m)) { + setGlobalError(param.what()); + return false; + } + return open(param); +} + +bool ModelImpl::open(const char *arg) { + Param param; + if (!param.open(arg, long_options) || + !load_dictionary_resource(¶m)) { + setGlobalError(param.what()); + return false; + } + return open(param); +} + +bool ModelImpl::open(const Param ¶m) { + if (!writer_->open(param) || !viterbi_->open(param)) { + std::string error = viterbi_->what(); + if (!error.empty()) { + error.append(" "); + } + error.append(writer_->what()); + setGlobalError(error.c_str()); + return false; + } + + request_type_ = load_request_type(param); + theta_ = param.get("theta"); + + return is_available(); +} + +bool ModelImpl::swap(Model *model) { + scoped_ptr model_data(model); + + if (!is_available()) { + setGlobalError("current model is not available"); + return false; + } +#ifndef HAVE_ATOMIC_OPS + setGlobalError("atomic model replacement is not supported"); + return false; +#else + ModelImpl *m = static_cast(model_data.get()); + if (!m) { + setGlobalError("Invalid model is passed"); + return false; + } + + if (!m->is_available()) { + setGlobalError("Passed model is not available"); + return false; + } + + Viterbi *current_viterbi = viterbi_; + { + scoped_writer_lock l(mutex()); + viterbi_ = m->take_viterbi(); + request_type_ = m->request_type(); + theta_ = m->theta(); + } + + delete current_viterbi; + + return true; +#endif +} + +Tagger *ModelImpl::createTagger() const { + if (!is_available()) { + setGlobalError("Model is not available"); + return 0; + } + TaggerImpl *tagger = new TaggerImpl; + if (!tagger->open(*this)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + tagger->set_theta(theta_); + tagger->set_request_type(request_type_); + return tagger; +} + +Lattice *ModelImpl::createLattice() const { + if (!is_available()) { + setGlobalError("Model is not available"); + return 0; + } + return new LatticeImpl(writer_.get()); +} + +TaggerImpl::TaggerImpl() + : current_model_(0), + request_type_(MECAB_ONE_BEST), theta_(kDefaultTheta) {} + +TaggerImpl::~TaggerImpl() {} + +const char *TaggerImpl::what() const { + return what_.c_str(); +} + +bool TaggerImpl::open(int argc, char **argv) { + model_.reset(new ModelImpl); + if (!model_->open(argc, argv)) { + model_.reset(0); + return false; + } + current_model_ = model_.get(); + request_type_ = model()->request_type(); + theta_ = model()->theta(); + return true; +} + +bool TaggerImpl::open(const char *arg) { + model_.reset(new ModelImpl); + if (!model_->open(arg)) { + model_.reset(0); + return false; + } + current_model_ = model_.get(); + request_type_ = model()->request_type(); + theta_ = model()->theta(); + return true; +} + +bool TaggerImpl::open(const ModelImpl &model) { + if (!model.is_available()) { + return false; + } + model_.reset(0); + current_model_ = &model; + request_type_ = current_model_->request_type(); + theta_ = current_model_->theta(); + return true; +} + +void TaggerImpl::set_request_type(int request_type) { + request_type_ = request_type; +} + +int TaggerImpl::request_type() const { + return request_type_; +} + +void TaggerImpl::set_partial(bool partial) { + if (partial) { + request_type_ |= MECAB_PARTIAL; + } else { + request_type_ &= ~MECAB_PARTIAL; + } +} + +bool TaggerImpl::partial() const { + return request_type_ & MECAB_PARTIAL; +} + +void TaggerImpl::set_theta(float theta) { + theta_ = theta; +} + +float TaggerImpl::theta() const { + return theta_; +} + +void TaggerImpl::set_lattice_level(int level) { + switch (level) { + case 0: request_type_ |= MECAB_ONE_BEST; + break; + case 1: request_type_ |= MECAB_NBEST; + break; + case 2: request_type_ |= MECAB_MARGINAL_PROB; + break; + default: + break; + } +} + +int TaggerImpl::lattice_level() const { + if (request_type_ & MECAB_MARGINAL_PROB) { + return 2; + } else if (request_type_ & MECAB_NBEST) { + return 1; + } else { + return 0; + } +} + +void TaggerImpl::set_all_morphs(bool all_morphs) { + if (all_morphs) { + request_type_ |= MECAB_ALL_MORPHS; + } else { + request_type_ &= ~MECAB_ALL_MORPHS; + } +} + +bool TaggerImpl::all_morphs() const { + return request_type_ & MECAB_ALL_MORPHS; +} + +bool TaggerImpl::parse(Lattice *lattice) const { +#ifdef HAVE_ATOMIC_OPS + scoped_reader_lock l(model()->mutex()); +#endif + + return model()->viterbi()->analyze(lattice); +} + +const char *TaggerImpl::parse(const char *str) { + return parse(str, std::strlen(str)); +} + +const char *TaggerImpl::parse(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + const char *result = lattice->toString(); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char *TaggerImpl::parse(const char *str, size_t len, + char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + const char *result = lattice->toString(out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const Node *TaggerImpl::parseToNode(const char *str) { + return parseToNode(str, std::strlen(str)); +} + +const Node *TaggerImpl::parseToNode(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + return lattice->bos_node(); +} + +bool TaggerImpl::parseNBestInit(const char *str) { + return parseNBestInit(str, std::strlen(str)); +} + +bool TaggerImpl::parseNBestInit(const char *str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + if (!parse(lattice)) { + set_what(lattice->what()); + return false; + } + return true; +} + +const Node* TaggerImpl::nextNode() { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + return lattice->bos_node(); +} + +const char* TaggerImpl::next() { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + const char *result = lattice->toString(); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::next(char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + if (!lattice->next()) { + lattice->set_what("no more results"); + return 0; + } + const char *result = lattice->toString(out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::parseNBest(size_t N, const char* str) { + return parseNBest(N, str, std::strlen(str)); +} + +const char* TaggerImpl::parseNBest(size_t N, + const char* str, size_t len) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + + const char *result = lattice->enumNBestAsString(N); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::parseNBest(size_t N, const char* str, size_t len, + char *out, size_t len2) { + Lattice *lattice = mutable_lattice(); + initRequestType(); + lattice->add_request_type(MECAB_NBEST); + lattice->set_sentence(str, len); + + if (!parse(lattice)) { + set_what(lattice->what()); + return 0; + } + + const char *result = lattice->enumNBestAsString(N, out, len2); + if (!result) { + set_what(lattice->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::formatNode(const Node* node) { + const char *result = mutable_lattice()->toString(node); + if (!result) { + set_what(mutable_lattice()->what()); + return 0; + } + return result; +} + +const char* TaggerImpl::formatNode(const Node* node, + char *out, size_t len) { + const char *result = mutable_lattice()->toString(node, out, len); + if (!result) { + set_what(mutable_lattice()->what()); + return 0; + } + return result; +} + +const DictionaryInfo *TaggerImpl::dictionary_info() const { + return model()->dictionary_info(); +} + +LatticeImpl::LatticeImpl(const Writer *writer) + : sentence_(0), size_(0), theta_(kDefaultTheta), Z_(0.0), + request_type_(MECAB_ONE_BEST), + writer_(writer), + ostrs_(0), + allocator_(new Allocator) { + begin_nodes_.reserve(MIN_INPUT_BUFFER_SIZE); + end_nodes_.reserve(MIN_INPUT_BUFFER_SIZE); +} + +LatticeImpl::~LatticeImpl() {} + +void LatticeImpl::clear() { + allocator_->free(); + if (ostrs_.get()) { + ostrs_->clear(); + } + begin_nodes_.clear(); + end_nodes_.clear(); + feature_constraint_.clear(); + boundary_constraint_.clear(); + size_ = 0; + theta_ = kDefaultTheta; + Z_ = 0.0; + sentence_ = 0; +} + +void LatticeImpl::set_sentence(const char *sentence) { + return set_sentence(sentence, strlen(sentence)); +} + +void LatticeImpl::set_sentence(const char *sentence, size_t len) { + clear(); + end_nodes_.resize(len + 4); + begin_nodes_.resize(len + 4); + + if (has_request_type(MECAB_ALLOCATE_SENTENCE) || + has_request_type(MECAB_PARTIAL)) { + char *new_sentence = allocator()->strdup(sentence, len); + sentence_ = new_sentence; + } else { + sentence_ = sentence; + } + + size_ = len; + std::memset(&end_nodes_[0], 0, + sizeof(end_nodes_[0]) * (len + 4)); + std::memset(&begin_nodes_[0], 0, + sizeof(begin_nodes_[0]) * (len + 4)); +} + +bool LatticeImpl::next() { + if (!has_request_type(MECAB_NBEST)) { + set_what("MECAB_NBEST request type is not set"); + return false; + } + + if (!allocator()->nbest_generator()->next()) { + return false; + } + + Viterbi::buildResultForNBest(this); + return true; +} + +void LatticeImpl::set_result(const char *result) { + char *str = allocator()->strdup(result, std::strlen(result)); + std::vector lines; + const size_t lsize = tokenize(str, "\n", + std::back_inserter(lines), + std::strlen(result)); + CHECK_DIE(lsize == lines.size()); + + std::string sentence; + std::vector surfaces, features; + for (size_t i = 0; i < lines.size(); ++i) { + if (::strcmp("EOS", lines[i]) == 0) { + break; + } + char *cols[2]; + if (tokenize(lines[i], "\t", cols, 2) != 2) { + break; + } + sentence += cols[0]; + surfaces.push_back(cols[0]); + features.push_back(cols[1]); + } + + CHECK_DIE(features.size() == surfaces.size()); + + set_sentence(allocator()->strdup(sentence.c_str(), sentence.size())); + + Node *bos_node = allocator()->newNode(); + bos_node->surface = const_cast(BOS_KEY); // dummy + bos_node->feature = "BOS/EOS"; + bos_node->isbest = 1; + bos_node->stat = MECAB_BOS_NODE; + + Node *eos_node = allocator()->newNode(); + eos_node->surface = const_cast(BOS_KEY); // dummy + eos_node->feature = "BOS/EOS"; + eos_node->isbest = 1; + eos_node->stat = MECAB_EOS_NODE; + + bos_node->surface = sentence_; + end_nodes_[0] = bos_node; + + size_t offset = 0; + Node *prev = bos_node; + for (size_t i = 0; i < surfaces.size(); ++i) { + Node *node = allocator()->newNode(); + node->prev = prev; + prev->next = node; + node->surface = sentence_ + offset; + node->length = surfaces[i].size(); + node->rlength = surfaces[i].size(); + node->isbest = 1; + node->stat = MECAB_NOR_NODE; + node->wcost = 0; + node->cost = 0; + node->feature = allocator()->strdup(features[i].c_str(), + features[i].size()); + begin_nodes_[offset] = node; + end_nodes_[offset + node->length] = node; + offset += node->length; + prev = node; + } + + prev->next = eos_node; + eos_node->prev = prev; +} + +// default implementation of Lattice formatter. +namespace { +void writeLattice(Lattice *lattice, StringBuffer *os) { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature; + *os << '\n'; + } + *os << "EOS\n"; +} +} // namespace + +const char *LatticeImpl::toString() { + return toStringInternal(stream()); +} + +const char *LatticeImpl::toString(char *buf, size_t size) { + StringBuffer os(buf, size); + return toStringInternal(&os); +} + +const char *LatticeImpl::toStringInternal(StringBuffer *os) { + os->clear(); + if (writer_) { + if (!writer_->write(this, os)) { + return 0; + } + } else { + writeLattice(this, os); + } + *os << '\0'; + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + return os->str(); +} + +const char *LatticeImpl::toString(const Node *node) { + return toStringInternal(node, stream()); +} + +const char *LatticeImpl::toString(const Node *node, + char *buf, size_t size) { + StringBuffer os(buf, size); + return toStringInternal(node, &os); +} + +const char *LatticeImpl::toStringInternal(const Node *node, + StringBuffer *os) { + os->clear(); + if (!node) { + set_what("node is NULL"); + return 0; + } + if (writer_) { + if (!writer_->writeNode(this, node, os)) { + return 0; + } + } else { + os->write(node->surface, node->length); + *os << '\t' << node->feature; + } + *os << '\0'; + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + return os->str(); +} + +const char *LatticeImpl::enumNBestAsString(size_t N) { + return enumNBestAsStringInternal(N, stream()); +} + +const char *LatticeImpl::enumNBestAsString(size_t N, char *buf, size_t size) { + StringBuffer os(buf, size); + return enumNBestAsStringInternal(N, &os); +} + +const char *LatticeImpl::enumNBestAsStringInternal(size_t N, + StringBuffer *os) { + os->clear(); + + if (N == 0 || N > NBEST_MAX) { + set_what("nbest size must be 1 <= nbest <= 512"); + return 0; + } + + for (size_t i = 0; i < N; ++i) { + if (!next()) { + break; + } + if (writer_) { + if (!writer_->write(this, os)) { + return 0; + } + } else { + writeLattice(this, os); + } + } + + // make a dummy node for EON + if (writer_) { + Node eon_node; + memset(&eon_node, 0, sizeof(eon_node)); + eon_node.stat = MECAB_EON_NODE; + eon_node.next = 0; + eon_node.surface = this->sentence() + this->size(); + if (!writer_->writeNode(this, &eon_node, os)) { + return 0; + } + } + *os << '\0'; + + if (!os->str()) { + set_what("output buffer overflow"); + return 0; + } + + return os->str(); +} + +bool LatticeImpl::has_constraint() const { + return !boundary_constraint_.empty(); +} + +int LatticeImpl::boundary_constraint(size_t pos) const { + if (!boundary_constraint_.empty()) { + return boundary_constraint_[pos]; + } + return MECAB_ANY_BOUNDARY; +} + +const char *LatticeImpl::feature_constraint(size_t begin_pos) const { + if (!feature_constraint_.empty()) { + return feature_constraint_[begin_pos]; + } + return 0; +} + +void LatticeImpl::set_boundary_constraint(size_t pos, + int boundary_constraint_type) { + if (boundary_constraint_.empty()) { + boundary_constraint_.resize(size() + 4, MECAB_ANY_BOUNDARY); + } + boundary_constraint_[pos] = boundary_constraint_type; +} + +void LatticeImpl::set_feature_constraint(size_t begin_pos, size_t end_pos, + const char *feature) { + if (begin_pos >= end_pos || !feature) { + return; + } + + if (feature_constraint_.empty()) { + feature_constraint_.resize(size() + 4, 0); + } + + end_pos = std::min(end_pos, size()); + + set_boundary_constraint(begin_pos, MECAB_TOKEN_BOUNDARY); + set_boundary_constraint(end_pos, MECAB_TOKEN_BOUNDARY); + for (size_t i = begin_pos + 1; i < end_pos; ++i) { + set_boundary_constraint(i, MECAB_INSIDE_TOKEN); + } + + feature_constraint_[begin_pos] = feature; +} +} // namespace + +Tagger *Tagger::create(int argc, char **argv) { + return createTagger(argc, argv); +} + +Tagger *Tagger::create(const char *arg) { + return createTagger(arg); +} + +const char *Tagger::version() { + return VERSION; +} + +Tagger *createTagger(int argc, char **argv) { + TaggerImpl *tagger = new TaggerImpl(); + if (!tagger->open(argc, argv)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + return tagger; +} + +Tagger *createTagger(const char *argv) { + TaggerImpl *tagger = new TaggerImpl(); + if (!tagger->open(argv)) { + setGlobalError(tagger->what()); + delete tagger; + return 0; + } + return tagger; +} + +void deleteTagger(Tagger *tagger) { + delete tagger; +} + +const char *getTaggerError() { + return getLastError(); +} + +const char *getLastError() { + return getGlobalError(); +} + +Model *createModel(int argc, char **argv) { + ModelImpl *model = new ModelImpl; + if (!model->open(argc, argv)) { + delete model; + return 0; + } + return model; +} + +Model *createModel(const char *arg) { + ModelImpl *model = new ModelImpl; + if (!model->open(arg)) { + delete model; + return 0; + } + return model; +} + +void deleteModel(Model *model) { + delete model; +} + +Model *Model::create(int argc, char **argv) { + return createModel(argc, argv); +} + +Model *Model::create(const char *arg) { + return createModel(arg); +} + +const char *Model::version() { + return VERSION; +} + +bool Tagger::parse(const Model &model, Lattice *lattice) { + scoped_ptr tagger(model.createTagger()); + return tagger->parse(lattice); +} + +Lattice *Lattice::create() { + return createLattice(); +} + +Lattice *createLattice() { + return new LatticeImpl; +} + +void deleteLattice(Lattice *lattice) { + delete lattice; +} +} // MeCab + +int mecab_do(int argc, char **argv) { +#define WHAT_ERROR(msg) do { \ + std::cout << msg << std::endl; \ + return EXIT_FAILURE; } \ + while (0); + + MeCab::Param param; + if (!param.open(argc, argv, MeCab::long_options)) { + std::cout << param.what() << std::endl; + return EXIT_FAILURE; + } + + if (param.get("help")) { + std::cout << param.help() << std::endl; + return EXIT_SUCCESS; + } + + if (param.get("version")) { + std::cout << param.version() << std::endl; + return EXIT_SUCCESS; + } + + if (!load_dictionary_resource(¶m)) { + std::cout << param.what() << std::endl; + return EXIT_SUCCESS; + } + + if (param.get("lattice-level") >= 1) { + std::cerr << "lattice-level is DEPERCATED. " + << "use --marginal or --nbest." << std::endl; + } + + MeCab::scoped_ptr model(new MeCab::ModelImpl); + if (!model->open(param)) { + std::cout << MeCab::getLastError() << std::endl; + return EXIT_FAILURE; + } + + std::string ofilename = param.get("output"); + if (ofilename.empty()) { + ofilename = "-"; + } + + const int nbest = param.get("nbest"); + if (nbest <= 0 || nbest > NBEST_MAX) { + WHAT_ERROR("invalid N value"); + } + + MeCab::ostream_wrapper ofs(ofilename.c_str()); + if (!*ofs) { + WHAT_ERROR("no such file or directory: " << ofilename); + } + + if (param.get("dump-config")) { + param.dump_config(&*ofs); + return EXIT_FAILURE; + } + + if (param.get("dictionary-info")) { + for (const MeCab::DictionaryInfo *d = model->dictionary_info(); + d; d = d->next) { + *ofs << "filename:\t" << d->filename << std::endl; + *ofs << "version:\t" << d->version << std::endl; + *ofs << "charset:\t" << d->charset << std::endl; + *ofs << "type:\t" << d->type << std::endl; + *ofs << "size:\t" << d->size << std::endl; + *ofs << "left size:\t" << d->lsize << std::endl; + *ofs << "right size:\t" << d->rsize << std::endl; + *ofs << std::endl; + } + return EXIT_FAILURE; + } + + const std::vector& rest_ = param.rest_args(); + std::vector rest = rest_; + + if (rest.empty()) { + rest.push_back("-"); + } + + size_t ibufsize = std::min(MAX_INPUT_BUFFER_SIZE, + std::max(param.get + ("input-buffer-size"), + MIN_INPUT_BUFFER_SIZE)); + + const bool partial = param.get("partial"); + if (partial) { + ibufsize *= 8; + } + + MeCab::scoped_array ibuf_data(new char[ibufsize]); + char *ibuf = ibuf_data.get(); + + MeCab::scoped_ptr tagger(model->createTagger()); + + if (!tagger.get()) { + WHAT_ERROR("cannot create tagger"); + } + + for (size_t i = 0; i < rest.size(); ++i) { + MeCab::istream_wrapper ifs(rest[i].c_str()); + if (!*ifs) { + WHAT_ERROR("no such file or directory: " << rest[i]); + } + + while (true) { + if (!partial) { + ifs->getline(ibuf, ibufsize); + } else { + std::string sentence; + MeCab::scoped_fixed_array line; + for (;;) { + if (!ifs->getline(line.get(), line.size())) { + ifs->clear(std::ios::eofbit|std::ios::badbit); + break; + } + sentence += line.get(); + sentence += '\n'; + if (std::strcmp(line.get(), "EOS") == 0 || line[0] == '\0') { + break; + } + } + std::strncpy(ibuf, sentence.c_str(), ibufsize); + } + if (ifs->eof() && !ibuf[0]) { + return false; + } + if (ifs->fail()) { + std::cerr << "input-buffer overflow. " + << "The line is split. use -b #SIZE option." << std::endl; + ifs->clear(); + } + const char *r = (nbest >= 2) ? tagger->parseNBest(nbest, ibuf) : + tagger->parse(ibuf); + if (!r) { + WHAT_ERROR(tagger->what()); + } + *ofs << r << std::flush; + } + } + + return EXIT_SUCCESS; + +#undef WHAT_ERROR +} diff --git a/fts/third_party/mecab/src/thread.h b/fts/third_party/mecab/src/thread.h new file mode 100644 index 00000000..35282a4f --- /dev/null +++ b/fts/third_party/mecab/src/thread.h @@ -0,0 +1,189 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_THREAD_H +#define MECAB_THREAD_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_PTHREAD_H +#include +#else +#ifdef _WIN32 +#include +#include +#endif +#endif + +#if defined HAVE_GCC_ATOMIC_OPS || defined HAVE_OSX_ATOMIC_OPS +#include +#endif + +#if defined HAVE_OSX_ATOMIC_OPS +#include +#endif + +#if defined HAVE_PTHREAD_H +#define MECAB_USE_THREAD 1 +#endif + +#if (defined(_WIN32) && !defined(__CYGWIN__)) +#define MECAB_USE_THREAD 1 +#define BEGINTHREAD(src, stack, func, arg, flag, id) \ + (HANDLE)_beginthreadex((void *)(src), (unsigned)(stack), \ + (unsigned(_stdcall *)(void *))(func), (void *)(arg), \ + (unsigned)(flag), (unsigned *)(id)) +#endif + +namespace MeCab { + +#if (defined(_WIN32) && !defined(__CYGWIN__)) +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) ::InterlockedExchangeAdd(a, b) +#define compare_and_swap(a, b, c) ::InterlockedCompareExchange(a, c, b) +#define yield_processor() YieldProcessor() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_GCC_ATOMIC_OPS +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) __sync_add_and_fetch(a, b) +#define compare_and_swap(a, b, c) __sync_val_compare_and_swap(a, b, c) +#define yield_processor() sched_yield() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_OSX_ATOMIC_OPS +#undef atomic_add +#undef compare_and_swap +#undef yield_processor +#define atomic_add(a, b) OSAtomicAdd32(b, a) +#define compare_and_swap(a, b, c) OSAtomicCompareAndSwapInt(b, c, a) +#define yield_processor() sched_yield() +#define HAVE_ATOMIC_OPS 1 +#endif + +#ifdef HAVE_ATOMIC_OPS +// This is a simple non-scalable writer-preference lock. +// Slightly modified the following paper. +// "Scalable Reader-Writer Synchronization for Shared-Memory Multiprocessors" +// PPoPP '91. John M. Mellor-Crummey and Michael L. Scott. T +class read_write_mutex { + public: + inline void write_lock() { + atomic_add(&write_pending_, 1); + while (compare_and_swap(&l_, 0, kWaFlag)) { + yield_processor(); + } + } + inline void read_lock() { + while (write_pending_ > 0) { + yield_processor(); + } + atomic_add(&l_, kRcIncr); + while ((l_ & kWaFlag) != 0) { + yield_processor(); + } + } + inline void write_unlock() { + atomic_add(&l_, -kWaFlag); + atomic_add(&write_pending_, -1); + } + inline void read_unlock() { + atomic_add(&l_, -kRcIncr); + } + + read_write_mutex(): l_(0), write_pending_(0) {} + + private: + static const int kWaFlag = 0x1; + static const int kRcIncr = 0x2; +#ifdef HAVE_OSX_ATOMIC_OPS + volatile int l_; + volatile int write_pending_; +#else + long l_; + long write_pending_; +#endif +}; + +class scoped_writer_lock { + public: + scoped_writer_lock(read_write_mutex *mutex) : mutex_(mutex) { + mutex_->write_lock(); + } + ~scoped_writer_lock() { + mutex_->write_unlock(); + } + private: + read_write_mutex *mutex_; +}; + +class scoped_reader_lock { + public: + scoped_reader_lock(read_write_mutex *mutex) : mutex_(mutex) { + mutex_->read_lock(); + } + ~scoped_reader_lock() { + mutex_->read_unlock(); + } + private: + read_write_mutex *mutex_; +}; +#endif // HAVE_ATOMIC_OPS + +class thread { + private: +#ifdef HAVE_PTHREAD_H + pthread_t hnd; +#else +#ifdef _WIN32 + HANDLE hnd; +#endif +#endif + + public: + static void* wrapper(void *ptr) { + thread *p = static_cast(ptr); + p->run(); + return 0; + } + + virtual void run() {} + + void start() { +#ifdef HAVE_PTHREAD_H + pthread_create(&hnd, 0, &thread::wrapper, + static_cast(this)); + +#else +#ifdef _WIN32 + DWORD id; + hnd = BEGINTHREAD(0, 0, &thread::wrapper, this, 0, &id); +#endif +#endif + } + + void join() { +#ifdef HAVE_PTHREAD_H + pthread_join(hnd, 0); +#else +#ifdef _WIN32 + WaitForSingleObject(hnd, INFINITE); + CloseHandle(hnd); +#endif +#endif + } + + virtual ~thread() {} +}; +} +#endif diff --git a/fts/third_party/mecab/src/tokenizer.cpp b/fts/third_party/mecab/src/tokenizer.cpp new file mode 100644 index 00000000..bb59ebc9 --- /dev/null +++ b/fts/third_party/mecab/src/tokenizer.cpp @@ -0,0 +1,393 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include "common.h" +#include "connector.h" +#include "darts.h" +#include "learner_node.h" +#include "param.h" +#include "scoped_ptr.h" +#include "tokenizer.h" +#include "utils.h" +#include "viterbi.h" + +namespace MeCab { +namespace { + +void inline read_node_info(const Dictionary &dic, + const Token &token, + LearnerNode **node) { + (*node)->lcAttr = token.lcAttr; + (*node)->rcAttr = token.rcAttr; + (*node)->posid = token.posid; + (*node)->wcost2 = token.wcost; + (*node)->feature = dic.feature(token); +} + +void inline read_node_info(const Dictionary &dic, + const Token &token, + Node **node) { + (*node)->lcAttr = token.lcAttr; + (*node)->rcAttr = token.rcAttr; + (*node)->posid = token.posid; + (*node)->wcost = token.wcost; + (*node)->feature = dic.feature(token); +} +} // namespace + +template class Tokenizer; +template class Tokenizer; +template Tokenizer::Tokenizer(); +template void Tokenizer::close(); +template const DictionaryInfo +*Tokenizer::dictionary_info() const; +template Node* Tokenizer::getBOSNode(Allocator *) const; +template Node* Tokenizer::getEOSNode(Allocator *) const; +template Node* Tokenizer::lookup( + const char *, + const char *, + Allocator *, + Lattice *) const; +template Node* Tokenizer::lookup( + const char *, + const char *, + Allocator *, + Lattice *) const; +template bool Tokenizer::open(const Param &); +template Tokenizer::Tokenizer(); +template void Tokenizer::close(); +template const DictionaryInfo +*Tokenizer::dictionary_info() const; +template LearnerNode * Tokenizer::getEOSNode( + Allocator *) const; +template LearnerNode * Tokenizer::getBOSNode( + Allocator *) const; +template LearnerNode *Tokenizer::lookup( + const char *, + const char *, + Allocator *, Lattice *) const; +template bool Tokenizer::open(const Param &); + +template +Tokenizer::Tokenizer() + : dictionary_info_freelist_(4), + dictionary_info_(0), + max_grouping_size_(0) {} + +template +N *Tokenizer::getBOSNode(Allocator *allocator) const { + N *bos_node = allocator->newNode(); + bos_node->surface = const_cast(BOS_KEY); // dummy + bos_node->feature = bos_feature_.get(); + bos_node->isbest = 1; + bos_node->stat = MECAB_BOS_NODE; + return bos_node; +} + +template +N *Tokenizer::getEOSNode(Allocator *allocator) const { + N *eos_node = getBOSNode(allocator); // same + eos_node->stat = MECAB_EOS_NODE; + return eos_node; +} + +template +bool Tokenizer::open(const Param ¶m) { + close(); + + const std::string prefix = param.template get("dicdir"); + + CHECK_FALSE(unkdic_.open(create_filename + (prefix, UNK_DIC_FILE).c_str())) + << unkdic_.what(); + CHECK_FALSE(property_.open(param)) << property_.what(); + + Dictionary *sysdic = new Dictionary; + + CHECK_FALSE(sysdic->open + (create_filename(prefix, SYS_DIC_FILE).c_str())) + << sysdic->what(); + + CHECK_FALSE(sysdic->type() == 0) + << "not a system dictionary: " << prefix; + + property_.set_charset(sysdic->charset()); + dic_.push_back(sysdic); + + const std::string userdic = param.template get("userdic"); + if (!userdic.empty()) { + scoped_fixed_array buf; + scoped_fixed_array dicfile; + std::strncpy(buf.get(), userdic.c_str(), buf.size()); + const size_t n = tokenizeCSV(buf.get(), dicfile.get(), dicfile.size()); + for (size_t i = 0; i < n; ++i) { + Dictionary *d = new Dictionary; + CHECK_FALSE(d->open(dicfile[i])) << d->what(); + CHECK_FALSE(d->type() == 1) + << "not a user dictionary: " << dicfile[i]; + CHECK_FALSE(sysdic->isCompatible(*d)) + << "incompatible dictionary: " << dicfile[i]; + dic_.push_back(d); + } + } + + dictionary_info_ = 0; + dictionary_info_freelist_.free(); + for (int i = static_cast(dic_.size() - 1); i >= 0; --i) { + DictionaryInfo *d = dictionary_info_freelist_.alloc(); + d->next = dictionary_info_; + d->filename = dic_[i]->filename(); + d->charset = dic_[i]->charset(); + d->size = dic_[i]->size(); + d->lsize = dic_[i]->lsize(); + d->rsize = dic_[i]->rsize(); + d->type = dic_[i]->type(); + d->version = dic_[i]->version(); + dictionary_info_ = d; + } + + unk_tokens_.clear(); + for (size_t i = 0; i < property_.size(); ++i) { + const char *key = property_.name(i); + const Dictionary::result_type n = unkdic_.exactMatchSearch(key); + CHECK_FALSE(n.value != -1) << "cannot find UNK category: " << key; + const Token *token = unkdic_.token(n); + size_t size = unkdic_.token_size(n); + unk_tokens_.push_back(std::make_pair(token, size)); + } + + space_ = property_.getCharInfo(0x20); // ad-hoc + + bos_feature_.reset_string(param.template get("bos-feature")); + + const std::string tmp = param.template get("unk-feature"); + unk_feature_.reset(0); + if (!tmp.empty()) { + unk_feature_.reset_string(tmp); + } + + CHECK_FALSE(*bos_feature_ != '\0') + << "bos-feature is undefined in dicrc"; + + max_grouping_size_ = param.template get("max-grouping-size"); + if (max_grouping_size_ == 0) { + max_grouping_size_ = DEFAULT_MAX_GROUPING_SIZE; + } + + return true; +} + +namespace { +inline bool partial_match(const char *f1, const char *f2) { + if (std::strcmp(f1, "*") == 0) { + return true; + } + + scoped_fixed_array buf1; + scoped_fixed_array buf2; + scoped_fixed_array c1; + scoped_fixed_array c2; + + std::strncpy(buf1.get(), f1, buf1.size()); + std::strncpy(buf2.get(), f2, buf2.size()); + + const size_t n1 = tokenizeCSV(buf1.get(), c1.get(), c1.size()); + const size_t n2 = tokenizeCSV(buf2.get(), c2.get(), c2.size()); + const size_t n = std::min(n1, n2); + + for (size_t i = 0; i < n; ++i) { + if (std::strcmp(c1[i], "*") != 0 && + std::strcmp(c1[i], c2[i]) != 0) { + return false; + } + } + + return true; +} + +template +bool is_valid_node(const Lattice *lattice, N *node) { + const size_t end_pos = node->surface - lattice->sentence() + node->length; + if (lattice->boundary_constraint(end_pos) == MECAB_INSIDE_TOKEN) { + return false; + } + const size_t begin_pos = + node->surface - lattice->sentence() + node->length - node->rlength; + const char *feature = lattice->feature_constraint(begin_pos); + if (!feature) { + return true; + } + if (lattice->boundary_constraint(begin_pos) == MECAB_TOKEN_BOUNDARY && + lattice->boundary_constraint(end_pos) == MECAB_TOKEN_BOUNDARY && + partial_match(feature, node->feature)) { + return true; + } + return false; +} +} // namespace + +#define ADDUNKNWON do { \ + const Token *token = unk_tokens_[cinfo.default_type].first; \ + size_t size = unk_tokens_[cinfo.default_type].second; \ + for (size_t k = 0; k < size; ++k) { \ + N *new_node = allocator->newNode(); \ + read_node_info(unkdic_, *(token + k), &new_node); \ + new_node->char_type = cinfo.default_type; \ + new_node->surface = begin2; \ + new_node->length = begin3 - begin2; \ + new_node->rlength = begin3 - begin; \ + new_node->stat = MECAB_UNK_NODE; \ + new_node->bnext = result_node; \ + if (unk_feature_.get()) new_node->feature = unk_feature_.get(); \ + if (isPartial && !is_valid_node(lattice, new_node)) { continue; } \ + result_node = new_node; } } while (0) + +template +template +N *Tokenizer::lookup(const char *begin, const char *end, + Allocator *allocator, Lattice *lattice) const { + CharInfo cinfo; + N *result_node = 0; + size_t mblen = 0; + size_t clen = 0; + + end = static_cast(end - begin) >= 65535 ? begin + 65535 : end; + + if (isPartial) { + const size_t begin_pos = begin - lattice->sentence(); + for (size_t n = begin_pos + 1; n < lattice->size(); ++n) { + if (lattice->boundary_constraint(n) == MECAB_TOKEN_BOUNDARY) { + end = lattice->sentence() + n; + break; + } + } + } + + const char *begin2 = property_.seekToOtherType(begin, end, space_, + &cinfo, &mblen, &clen); + + Dictionary::result_type *daresults = allocator->mutable_results(); + const size_t results_size = allocator->results_size(); + + for (std::vector::const_iterator it = dic_.begin(); + it != dic_.end(); ++it) { + const size_t n = (*it)->commonPrefixSearch( + begin2, + static_cast(end - begin2), + daresults, results_size); + for (size_t i = 0; i < n; ++i) { + size_t size = (*it)->token_size(daresults[i]); + const Token *token = (*it)->token(daresults[i]); + for (size_t j = 0; j < size; ++j) { + N *new_node = allocator->newNode(); + read_node_info(**it, *(token + j), &new_node); + new_node->length = daresults[i].length; + new_node->rlength = begin2 - begin + new_node->length; + new_node->surface = begin2; + new_node->stat = MECAB_NOR_NODE; + new_node->char_type = cinfo.default_type; + if (isPartial && !is_valid_node(lattice, new_node)) { + continue; + } + new_node->bnext = result_node; + result_node = new_node; + } + } + } + + if (result_node && !cinfo.invoke) { + return result_node; + } + + const char *begin3 = begin2 + mblen; + const char *group_begin3 = 0; + + if (begin3 > end) { + ADDUNKNWON; + if (result_node) { + return result_node; + } + } + + if (cinfo.group) { + const char *tmp = begin3; + CharInfo fail; + begin3 = property_.seekToOtherType(begin3, end, cinfo, + &fail, &mblen, &clen); + if (clen <= max_grouping_size_) { + ADDUNKNWON; + } + group_begin3 = begin3; + begin3 = tmp; + } + + for (size_t i = 1; i <= cinfo.length; ++i) { + if (begin3 > end) { + break; + } + if (begin3 == group_begin3) { + continue; + } + clen = i; + ADDUNKNWON; + if (!cinfo.isKindOf(property_.getCharInfo(begin3, end, &mblen))) { + break; + } + begin3 += mblen; + } + + if (!result_node) { + ADDUNKNWON; + } + + if (isPartial && !result_node) { + begin3 = begin2; + while (true) { + cinfo = property_.getCharInfo(begin3, end, &mblen); + begin3 += mblen; + if (begin3 > end || + lattice->boundary_constraint(begin3 - lattice->sentence()) + != MECAB_INSIDE_TOKEN) { + break; + } + } + ADDUNKNWON; + + if (!result_node) { + N *new_node = allocator->newNode(); + new_node->char_type = cinfo.default_type; + new_node->surface = begin2; + new_node->length = begin3 - begin2; + new_node->rlength = begin3 - begin; + new_node->stat = MECAB_UNK_NODE; + new_node->bnext = result_node; + new_node->feature = + lattice->feature_constraint(begin - lattice->sentence()); + CHECK_DIE(new_node->feature); + result_node = new_node; + } + } + + return result_node; +} + +#undef ADDUNKNWON + +template +const DictionaryInfo *Tokenizer::dictionary_info() const { + return const_cast(dictionary_info_); +} + +template +void Tokenizer::close() { + for (std::vector::iterator it = dic_.begin(); + it != dic_.end(); ++it) { + delete *it; + } + dic_.clear(); + unk_tokens_.clear(); + property_.close(); +} +} diff --git a/fts/third_party/mecab/src/tokenizer.h b/fts/third_party/mecab/src/tokenizer.h new file mode 100644 index 00000000..e1d6fb74 --- /dev/null +++ b/fts/third_party/mecab/src/tokenizer.h @@ -0,0 +1,134 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_TOKENIZER_H_ +#define MECAB_TOKENIZER_H_ + +#include "mecab.h" +#include "freelist.h" +#include "dictionary.h" +#include "char_property.h" +#include "nbest_generator.h" +#include "scoped_ptr.h" + +namespace MeCab { + +class Param; +class NBestGenerator; + +template +class Allocator { + public: + N *newNode() { + N *node = node_freelist_->alloc(); + std::memset(node, 0, sizeof(N)); + node->id = id_++; + return node; + } + + P *newPath() { + if (!path_freelist_.get()) { + path_freelist_.reset(new FreeList

(PATH_FREELIST_SIZE)); + } + return path_freelist_->alloc(); + } + + Dictionary::result_type *mutable_results() { + return results_.get(); + } + + char *alloc(size_t size) { + if (!char_freelist_.get()) { + char_freelist_.reset(new ChunkFreeList(BUF_SIZE)); + } + return char_freelist_->alloc(size + 1); + } + + char *strdup(const char *str, size_t size) { + char *n = alloc(size + 1); + std::strncpy(n, str, size + 1); + return n; + } + + NBestGenerator *nbest_generator() { + if (!nbest_generator_.get()) { + nbest_generator_.reset(new NBestGenerator); + } + return nbest_generator_.get(); + } + + char *partial_buffer(size_t size) { + partial_buffer_.resize(size); + return &partial_buffer_[0]; + } + + size_t results_size() const { + return kResultsSize; + } + + void free() { + id_ = 0; + node_freelist_->free(); + if (path_freelist_.get()) { + path_freelist_->free(); + } + if (char_freelist_.get()) { + char_freelist_->free(); + } + } + + Allocator() + : id_(0), + node_freelist_(new FreeList(NODE_FREELIST_SIZE)), + path_freelist_(0), + char_freelist_(0), + nbest_generator_(0), + results_(new Dictionary::result_type[kResultsSize]) {} + virtual ~Allocator() {} + + private: + static const size_t kResultsSize = 512; + size_t id_; + scoped_ptr > node_freelist_; + scoped_ptr > path_freelist_; + scoped_ptr > char_freelist_; + scoped_ptr nbest_generator_; + std::vector partial_buffer_; + scoped_array results_; +}; + +template +class Tokenizer { + private: + std::vector dic_; + Dictionary unkdic_; + scoped_string bos_feature_; + scoped_string unk_feature_; + FreeList dictionary_info_freelist_; + std::vector > unk_tokens_; + DictionaryInfo *dictionary_info_; + CharInfo space_; + CharProperty property_; + size_t max_grouping_size_; + whatlog what_; + + public: + N *getBOSNode(Allocator *allocator) const; + N *getEOSNode(Allocator *allocator) const; + template N *lookup(const char *begin, const char *end, + Allocator *allocator, + Lattice *lattice) const; + bool open(const Param ¶m); + void close(); + + const DictionaryInfo *dictionary_info() const; + + const char *what() { return what_.str(); } + + explicit Tokenizer(); + virtual ~Tokenizer() { this->close(); } +}; +} +#endif // MECAB_TOKENIZER_H_ diff --git a/fts/third_party/mecab/src/ucs.h b/fts/third_party/mecab/src/ucs.h new file mode 100644 index 00000000..8600161b --- /dev/null +++ b/fts/third_party/mecab/src/ucs.h @@ -0,0 +1,148 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_UCS_H +#define MECAB_UCS_H + +#ifndef MECAB_USE_UTF8_ONLY +#include "ucstable.h" +#endif + +namespace MeCab { + +// All internal codes are represented in UCS2, +// if you want to use specific local codes, e.g, big5/euc-kr, +// make a function which maps the local code to the UCS code. + +inline unsigned short utf8_to_ucs2(const char *begin, const char *end, + size_t* mblen) { + const size_t len = end - begin; + + if (static_cast(begin[0]) < 0x80) { + *mblen = 1; + return static_cast(begin[0]); + + } else if (len >= 2 && (begin[0] & 0xe0) == 0xc0) { + *mblen = 2; + return((begin[0] & 0x1f) << 6) |(begin[1] & 0x3f); + + } else if (len >= 3 && (begin[0] & 0xf0) == 0xe0) { + *mblen = 3; + return ((begin[0] & 0x0f) << 12) | + ((begin[1] & 0x3f) << 6) |(begin[2] & 0x3f); + + /* belows are out of UCS2 */ + } else if (len >= 4 && (begin[0] & 0xf8) == 0xf0) { + *mblen = 4; + return 0; + + } else if (len >= 5 && (begin[0] & 0xfc) == 0xf8) { + *mblen = 5; + return 0; + + } else if (len >= 6 && (begin[0] & 0xfe) == 0xfc) { + *mblen = 6; + return 0; + + } else { + *mblen = 1; + return 0; + } +} + +inline unsigned short ascii_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + *mblen = 1; + return static_cast(begin[0]); +} + +inline unsigned short utf16be_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + if (len <= 1) { + *mblen = 1; + return 0; + } + *mblen = 2; +#if defined WORDS_BIGENDIAN + return (begin[0] << 8 | begin[1]); +#else + return (begin[1] << 8 | begin[0]); +#endif + return 0; +} + +inline unsigned short utf16le_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + if (len <= 1) { + *mblen = 1; + return 0; + } + *mblen = 2; +#if defined WORDS_BIGENDIAN + return (begin[1] << 8 | begin[0]); +#else + return (begin[0] << 8 | begin[1]); +#endif +} + +inline unsigned short utf16_to_ucs2(const char *begin, const char *end, + size_t *mblen) { +#if defined WORDS_BIGENDIAN + return utf16be_to_ucs2(begin, end, mblen); +#else + return utf16le_to_ucs2(begin, end, mblen); +#endif +} + + +#ifndef MECAB_USE_UTF8_ONLY +inline unsigned short euc_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + + // JISX 0212, 0213 + if (static_cast(begin[0]) == 0x8f && len >= 3) { + unsigned short key = (static_cast(begin[1]) << 8) + + static_cast(begin[2]); + if (key < 0xA0A0) { // offset violation + *mblen = 1; + return static_cast(begin[0]); + } + *mblen = 3; + return euc_hojo_tbl[ key - 0xA0A0 ]; + // JISX 0208 + 0201 + } else if ((static_cast(begin[0]) & 0x80) && len >= 2) { + *mblen = 2; + return euc_tbl[(static_cast(begin[0]) << 8) + + static_cast(begin[1]) ]; + } else { + *mblen = 1; + return static_cast(begin[0]); + } +} + +inline unsigned short cp932_to_ucs2(const char *begin, const char *end, + size_t *mblen) { + const size_t len = end - begin; + + if ((static_cast(begin[0]) >= 0xA1 && + static_cast(begin[0]) <= 0xDF)) { + *mblen = 1; + return cp932_tbl[static_cast(begin[0]) ]; + } else if ((static_cast(begin[0]) & 0x80) && len >= 2) { + *mblen = 2; + return cp932_tbl[(static_cast(begin[0]) << 8) + + static_cast(begin[1]) ]; + } else { + *mblen = 1; + return static_cast(begin[0]); + } +} +#endif +} + +#endif diff --git a/fts/third_party/mecab/src/utils.cpp b/fts/third_party/mecab/src/utils.cpp new file mode 100644 index 00000000..80fd613a --- /dev/null +++ b/fts/third_party/mecab/src/utils.cpp @@ -0,0 +1,564 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#ifdef HAVE_DIRENT_H +#include +#endif + +#ifdef HAVE_WINDOWS_H +#define NOMINMAX +#include +#include +#endif + +#include + +#if defined(_WIN32) && !defined(__CYGWIN__) +extern HINSTANCE DllInstance; +#endif + +#include "common.h" +#include "mecab.h" +#include "param.h" +#include "utils.h" + +namespace MeCab { + +#if defined(_WIN32) && !defined(__CYGWIN__) +std::wstring Utf8ToWide(const std::string &input) { + int output_length = ::MultiByteToWideChar(CP_UTF8, 0, + input.c_str(), -1, NULL, 0); + output_length = output_length <= 0 ? 0 : output_length - 1; + if (output_length == 0) { + return L""; + } + scoped_array input_wide(new wchar_t[output_length + 1]); + const int result = ::MultiByteToWideChar(CP_UTF8, 0, input.c_str(), -1, + input_wide.get(), output_length + 1); + std::wstring output; + if (result > 0) { + output.assign(input_wide.get()); + } + return output; +} + +std::string WideToUtf8(const std::wstring &input) { + const int output_length = ::WideCharToMultiByte(CP_UTF8, 0, + input.c_str(), -1, NULL, 0, + NULL, NULL); + if (output_length == 0) { + return ""; + } + + scoped_array input_encoded(new char[output_length + 1]); + const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1, + input_encoded.get(), + output_length + 1, NULL, NULL); + std::string output; + if (result > 0) { + output.assign(input_encoded.get()); + } + return output; +} +#endif + +int decode_charset(const char *charset) { + std::string tmp = charset; + toLower(&tmp); + if (tmp == "sjis" || tmp == "shift-jis" || + tmp == "shift_jis" || tmp == "cp932") + return CP932; + else if (tmp == "euc" || tmp == "euc_jp" || + tmp == "euc-jp") + return EUC_JP; + else if (tmp == "utf8" || tmp == "utf_8" || + tmp == "utf-8") + return UTF8; + else if (tmp == "utf16" || tmp == "utf_16" || + tmp == "utf-16") + return UTF16; + else if (tmp == "utf16be" || tmp == "utf_16be" || + tmp == "utf-16be") + return UTF16BE; + else if (tmp == "utf16le" || tmp == "utf_16le" || + tmp == "utf-16le") + return UTF16LE; + else if (tmp == "ascii") + return ASCII; + + return UTF8; // default is UTF8 +} + +std::string create_filename(const std::string &path, + const std::string &file) { + std::string s = path; +#if defined(_WIN32) && !defined(__CYGWIN__) + if (s.size() && s[s.size()-1] != '\\') s += '\\'; +#else + if (s.size() && s[s.size()-1] != '/') s += '/'; +#endif + s += file; + return s; +} + +void remove_filename(std::string *s) { + int len = static_cast(s->size()) - 1; + bool ok = false; + for (; len >= 0; --len) { +#if defined(_WIN32) && !defined(__CYGWIN__) + if ((*s)[len] == '\\') { + ok = true; + break; + } +#else + if ((*s)[len] == '/') { + ok = true; + break; + } +#endif + } + if (ok) + *s = s->substr(0, len); + else + *s = "."; +} + +void remove_pathname(std::string *s) { + int len = static_cast(s->size()) - 1; + bool ok = false; + for (; len >= 0; --len) { +#if defined(_WIN32) && !defined(__CYGWIN__) + if ((*s)[len] == '\\') { + ok = true; + break; + } +#else + if ((*s)[len] == '/') { + ok = true; + break; + } +#endif + } + if (ok) + *s = s->substr(len + 1, s->size() - len); + else + *s = "."; +} + +void replace_string(std::string *s, + const std::string &src, + const std::string &dst) { + const std::string::size_type pos = s->find(src); + if (pos != std::string::npos) { + s->replace(pos, src.size(), dst); + } +} + +void enum_csv_dictionaries(const char *path, + std::vector *dics) { + dics->clear(); + +#if defined(_WIN32) && !defined(__CYGWIN__) + WIN32_FIND_DATAW wfd; + HANDLE hFind; + const std::wstring pat = Utf8ToWide(create_filename(path, "*.csv")); + hFind = ::FindFirstFileW(pat.c_str(), &wfd); + CHECK_DIE(hFind != INVALID_HANDLE_VALUE) + << "Invalid File Handle. Get Last Error reports"; + do { + std::string tmp = create_filename(path, WideToUtf8(wfd.cFileName)); + dics->push_back(tmp); + } while (::FindNextFileW(hFind, &wfd)); + ::FindClose(hFind); +#else + DIR *dir = opendir(path); + CHECK_DIE(dir) << "no such directory: " << path; + + for (struct dirent *dp = readdir(dir); + dp; + dp = readdir(dir)) { + const std::string tmp = dp->d_name; + if (tmp.size() >= 5) { + std::string ext = tmp.substr(tmp.size() - 4, 4); + toLower(&ext); + if (ext == ".csv") { + dics->push_back(create_filename(path, tmp)); + } + } + } + closedir(dir); +#endif +} + +bool toLower(std::string *s) { + for (size_t i = 0; i < s->size(); ++i) { + char c = (*s)[i]; + if ((c >= 'A') && (c <= 'Z')) { + c += 'a' - 'A'; + (*s)[i] = c; + } + } + return true; +} + +bool escape_csv_element(std::string *w) { + if (w->find(',') != std::string::npos || + w->find('"') != std::string::npos) { + std::string tmp = "\""; + for (size_t j = 0; j < w->size(); j++) { + if ((*w)[j] == '"') tmp += '"'; + tmp += (*w)[j]; + } + tmp += '"'; + *w = tmp; + } + return true; +} + +int progress_bar(const char* message, size_t current, size_t total) { + static char bar[] = "###########################################"; + static int scale = sizeof(bar) - 1; + static int prev = 0; + + int cur_percentage = static_cast(100.0 * current/total); + int bar_len = static_cast(1.0 * current*scale/total); + + if (prev != cur_percentage) { + printf("%s: %3d%% |%.*s%*s| ", message, cur_percentage, + bar_len, bar, scale - bar_len, ""); + if (cur_percentage == 100) + printf("\n"); + else + printf("\r"); + fflush(stdout); + } + + prev = cur_percentage; + + return 1; +} + +int load_request_type(const Param ¶m) { + int request_type = MECAB_ONE_BEST; + + if (param.get("allocate-sentence")) { + request_type |= MECAB_ALLOCATE_SENTENCE; + } + + if (param.get("partial")) { + request_type |= MECAB_PARTIAL; + } + + if (param.get("all-morphs")) { + request_type |= MECAB_ALL_MORPHS; + } + + if (param.get("marginal")) { + request_type |= MECAB_MARGINAL_PROB; + } + + const int nbest = param.get("nbest"); + if (nbest >= 2) { + request_type |= MECAB_NBEST; + } + + // DEPRECATED: + const int lattice_level = param.get("lattice-level"); + if (lattice_level >= 1) { + request_type |= MECAB_NBEST; + } + + if (lattice_level >= 2) { + request_type |= MECAB_MARGINAL_PROB; + } + + return request_type; +} + +bool load_dictionary_resource(Param *param) { + std::string rcfile = param->get("rcfile"); + +#ifdef HAVE_GETENV + if (rcfile.empty()) { + const char *homedir = getenv("HOME"); + if (homedir) { + const std::string s = MeCab::create_filename(std::string(homedir), + ".mecabrc"); + std::ifstream ifs(WPATH(s.c_str())); + if (ifs) { + rcfile = s; + } + } + } + + if (rcfile.empty()) { + const char *rcenv = getenv("MECABRC"); + if (rcenv) { + rcfile = rcenv; + } + } +#endif + +#if defined (HAVE_GETENV) && defined(_WIN32) && !defined(__CYGWIN__) + if (rcfile.empty()) { + scoped_fixed_array buf; + const DWORD len = ::GetEnvironmentVariableW(L"MECABRC", + buf.get(), + buf.size()); + if (len < buf.size() && len > 0) { + rcfile = WideToUtf8(buf.get()); + } + } +#endif + +#if defined(_WIN32) && !defined(__CYGWIN__) + HKEY hKey; + scoped_fixed_array v; + DWORD vt; + DWORD size = v.size() * sizeof(v[0]); + DWORD qvres; + + if (rcfile.empty()) { + ::RegOpenKeyExW(HKEY_LOCAL_MACHINE, L"software\\mecab", 0, KEY_READ, &hKey); + qvres = ::RegQueryValueExW(hKey, L"mecabrc", 0, &vt, + reinterpret_cast(v.get()), &size); + ::RegCloseKey(hKey); + if (qvres == ERROR_SUCCESS && vt == REG_SZ) { + rcfile = WideToUtf8(v.get()); + } + } + + if (rcfile.empty()) { + ::RegOpenKeyExW(HKEY_CURRENT_USER, L"software\\mecab", 0, KEY_READ, &hKey); + qvres = ::RegQueryValueExW(hKey, L"mecabrc", 0, &vt, + reinterpret_cast(v.get()), &size); + ::RegCloseKey(hKey); + if (qvres == ERROR_SUCCESS && vt == REG_SZ) { + rcfile = WideToUtf8(v.get()); + } + } + + if (rcfile.empty()) { + vt = ::GetModuleFileNameW(DllInstance, v.get(), size); + if (vt != 0) { + scoped_fixed_array drive; + scoped_fixed_array dir; + _wsplitpath(v.get(), drive.get(), dir.get(), NULL, NULL); + const std::wstring path = + std::wstring(drive.get()) + std::wstring(dir.get()) + L"mecabrc"; + if (::GetFileAttributesW(path.c_str()) != -1) { + rcfile = WideToUtf8(path); + } + } + } +#endif + + if (rcfile.empty()) { + rcfile = MECAB_DEFAULT_RC; + } + + if (!param->load(rcfile.c_str())) { + return false; + } + + std::string dicdir = param->get("dicdir"); + if (dicdir.empty()) { + dicdir = "."; // current + } + remove_filename(&rcfile); + replace_string(&dicdir, "$(rcpath)", rcfile); + param->set("dicdir", dicdir, true); + dicdir = create_filename(dicdir, DICRC); + + if (!param->load(dicdir.c_str())) { + return false; + } + + return true; +} + +namespace { +// Copied from MurmurHash3.cpp +// http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp +//----------------------------------------------------------------------------- +// Platform-specific functions and macros +// Microsoft Visual Studio +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#define ROTL32(x,y) _rotl(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32 ( uint32_t x, uint8_t r ) { + return (x << r) | (x >> (32 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint32_t getblock ( const uint32_t * p, int i ) { + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint32_t fmix (uint32_t h) { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +void MurmurHash3_x86_128(const void * key, const int len, + uint32_t seed, char *out) { + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + uint32_t c1 = 0x239b961b; + uint32_t c2 = 0xab0e9789; + uint32_t c3 = 0x38b34ae5; + uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock(blocks,i*4+0); + uint32_t k2 = getblock(blocks,i*4+1); + uint32_t k3 = getblock(blocks,i*4+2); + uint32_t k4 = getblock(blocks,i*4+3); + + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + + h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; + + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; + + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; + + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch(len & 15) + { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = fmix(h1); + h2 = fmix(h2); + h3 = fmix(h3); + h4 = fmix(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + std::memcpy(out, reinterpret_cast(&h1), 4); + std::memcpy(out + 4, reinterpret_cast(&h2), 4); + std::memcpy(out + 8, reinterpret_cast(&h3), 4); + std::memcpy(out+ 12, reinterpret_cast(&h4), 4); +} +} + +uint64_t fingerprint(const char *str, size_t size) { + uint64_t result[2] = { 0 }; + const uint32_t kFingerPrint32Seed = 0xfd14deff; + MurmurHash3_x86_128(str, size, kFingerPrint32Seed, + reinterpret_cast(result)); + return result[0]; +} + +uint64_t fingerprint(const std::string &str) { + return fingerprint(str.data(), str.size()); +} + +bool file_exists(const char *filename) { + std::ifstream ifs(WPATH(filename)); + if (!ifs) { + return false; + } + return true; +} +} // namespace MeCab diff --git a/fts/third_party/mecab/src/utils.h b/fts/third_party/mecab/src/utils.h new file mode 100644 index 00000000..28ce2afd --- /dev/null +++ b/fts/third_party/mecab/src/utils.h @@ -0,0 +1,258 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_UTILS_H +#define MECAB_UTILS_H + +#include +#include +#include +#include +#include +#include +#include "common.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_STDINT_H +#include +#else // HAVE_STDINT_H +#if defined(_WIN32) && !defined(__CYGWIN__) +#if defined(_MSC_VER) && (_MSC_VER <= 1500) +typedef unsigned char uint8_t; +typedef unsigned long uint32_t; +typedef unsigned long long uint64_t; +#else // _MSC_VER +#include +#endif // _MSC_VER +#else // _WIN32 +typedef unsigned char uint8_t; +typedef unsigned long uint32_t; +typedef unsigned __int64 uint64_t; +#endif // _WIN32 +#endif // HAVE_STDINT_H + +namespace MeCab { + +class Param; + +enum { EUC_JP, CP932, UTF8, UTF16, UTF16LE, UTF16BE, ASCII }; +int decode_charset(const char *charset); + +void inline dtoa(double val, char *s) { + std::sprintf(s, "%-16f", val); + char *p = s; + for (; *p != ' '; ++p) {} + *p = '\0'; + return; +} + +template +inline void itoa(T val, char *s) { + char *t; + T mod; + + if (val < 0) { + *s++ = '-'; + val = -val; + } + t = s; + + while (val) { + mod = val % 10; + *t++ = static_cast(mod) + '0'; + val /= 10; + } + + if (s == t) *t++ = '0'; + *t = '\0'; + std::reverse(s, t); + + return; +} + +template +inline void uitoa(T val, char *s) { + char *t; + T mod; + t = s; + while (val) { + mod = val % 10; + *t++ = static_cast(mod) + '0'; + val /= 10; + } + + if (s == t) *t++ = '0'; + *t = '\0'; + std::reverse(s, t); + return; +} + +inline const char *read_ptr(const char **ptr, size_t size) { + const char *r = *ptr; + *ptr += size; + return r; +} + +template +inline void read_static(const char **ptr, T& value) { + const char *r = read_ptr(ptr, sizeof(T)); + memcpy(&value, r, sizeof(T)); +} + +bool file_exists(const char *filename); + +int load_request_type(const Param ¶m); + +bool load_dictionary_resource(Param *); + +bool escape_csv_element(std::string *w); + +void enum_csv_dictionaries(const char *path, + std::vector *dics); + +int progress_bar(const char* message, size_t current, size_t total); + +bool toLower(std::string *); + +std::string create_filename(const std::string &path, + const std::string &file); +void remove_filename(std::string *s); +void remove_pathname(std::string *s); +void replace_string(std::string *s, + const std::string &src, + const std::string &dst); + +template +inline size_t tokenizeCSV(char *str, + Iterator out, size_t max) { + char *eos = str + std::strlen(str); + char *start = 0; + char *end = 0; + size_t n = 0; + + for (; str < eos; ++str) { + // skip white spaces + while (*str == ' ' || *str == '\t') ++str; + if (*str == '"') { + start = ++str; + end = start; + for (; str < eos; ++str) { + if (*str == '"') { + str++; + if (*str != '"') + break; + } + *end++ = *str; + } + str = std::find(str, eos, ','); + } else { + start = str; + str = std::find(str, eos, ','); + end = str; + } + if (max-- > 1) *end = '\0'; + *out++ = start; + ++n; + if (max == 0) break; + } + + return n; +} + +template +inline size_t tokenize(char *str, const char *del, + Iterator out, size_t max) { + char *stre = str + std::strlen(str); + const char *dele = del + std::strlen(del); + size_t size = 0; + + while (size < max) { + char *n = std::find_first_of(str, stre, del, dele); + *n = '\0'; + *out++ = str; + ++size; + if (n == stre) break; + str = n + 1; + } + + return size; +} + +// continus run of space is regarded as one space +template +inline size_t tokenize2(char *str, const char *del, + Iterator out, size_t max) { + char *stre = str + std::strlen(str); + const char *dele = del + std::strlen(del); + size_t size = 0; + + while (size < max) { + char *n = std::find_first_of(str, stre, del, dele); + *n = '\0'; + if (*str != '\0') { + *out++ = str; + ++size; + } + if (n == stre) break; + str = n + 1; + } + + return size; +} + +inline double logsumexp(double x, double y, bool flg) { +#define MINUS_LOG_EPSILON 50 + + if (flg) return y; // init mode + double vmin = std::min(x, y); + double vmax = std::max(x, y); + if (vmax > vmin + MINUS_LOG_EPSILON) { + return vmax; + } else { + return vmax + std::log(std::exp(vmin - vmax) + 1.0); + } +} + +inline short int tocost(double d, int n) { + static const short max = +32767; + static const short min = -32767; + return static_cast(std::max( + std::min( + -n * d, + static_cast(max)), + static_cast(min)) ); +} + +inline char getEscapedChar(const char p) { + switch (p) { + case '0': return '\0'; + case 'a': return '\a'; + case 'b': return '\b'; + case 't': return '\t'; + case 'n': return '\n'; + case 'v': return '\v'; + case 'f': return '\f'; + case 'r': return '\r'; + case 's': return ' '; + case '\\': return '\\'; + default: break; + } + + return '\0'; // never be here +} + +// return 64 bit hash +uint64_t fingerprint(const char *str, size_t size); +uint64_t fingerprint(const std::string &str); + +#if defined(_WIN32) && !defined(__CYGWIN__) +std::wstring Utf8ToWide(const std::string &input); +std::string WideToUtf8(const std::wstring &input); +#endif +} +#endif diff --git a/fts/third_party/mecab/src/viterbi.cpp b/fts/third_party/mecab/src/viterbi.cpp new file mode 100644 index 00000000..ca089a94 --- /dev/null +++ b/fts/third_party/mecab/src/viterbi.cpp @@ -0,0 +1,413 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "connector.h" +#include "mecab.h" +#include "nbest_generator.h" +#include "param.h" +#include "viterbi.h" +#include "scoped_ptr.h" +#include "string_buffer.h" +#include "tokenizer.h" + +namespace MeCab { + +namespace { +void calc_alpha(Node *n, double beta) { + n->alpha = 0.0; + for (Path *path = n->lpath; path; path = path->lnext) { + n->alpha = logsumexp(n->alpha, + -beta * path->cost + path->lnode->alpha, + path == n->lpath); + } +} + +void calc_beta(Node *n, double beta) { + n->beta = 0.0; + for (Path *path = n->rpath; path; path = path->rnext) { + n->beta = logsumexp(n->beta, + -beta * path->cost + path->rnode->beta, + path == n->rpath); + } +} +} // namespace + +Viterbi::Viterbi() + : tokenizer_(0), connector_(0), + cost_factor_(0) {} + +Viterbi::~Viterbi() {} + +bool Viterbi::open(const Param ¶m) { + tokenizer_.reset(new Tokenizer); + CHECK_FALSE(tokenizer_->open(param)) << tokenizer_->what(); + CHECK_FALSE(tokenizer_->dictionary_info()) << "Dictionary is empty"; + + connector_.reset(new Connector); + CHECK_FALSE(connector_->open(param)) << connector_->what(); + + CHECK_FALSE(tokenizer_->dictionary_info()->lsize == + connector_->left_size() && + tokenizer_->dictionary_info()->rsize == + connector_->right_size()) + << "Transition table and dictionary are not compatible"; + + cost_factor_ = param.get("cost-factor"); + if (cost_factor_ == 0) { + cost_factor_ = 800; + } + + return true; +} + +bool Viterbi::analyze(Lattice *lattice) const { + if (!lattice || !lattice->sentence()) { + return false; + } + + if (!initPartial(lattice)) { + return false; + } + + bool result = false; + if (lattice->has_request_type(MECAB_NBEST) || + lattice->has_request_type(MECAB_MARGINAL_PROB)) { + // IsAllPath=true + if (lattice->has_constraint()) { + result = viterbi(lattice); + } else { + result = viterbi(lattice); + } + } else { + // IsAllPath=false + if (lattice->has_constraint()) { + result = viterbi(lattice); + } else { + result = viterbi(lattice); + } + } + + if (!result) { + return false; + } + + if (!forwardbackward(lattice)) { + return false; + } + + if (!buildBestLattice(lattice)) { + return false; + } + + if (!buildAllLattice(lattice)) { + return false; + } + + if (!initNBest(lattice)) { + return false; + } + + return true; +} + +const Tokenizer *Viterbi::tokenizer() const { + return tokenizer_.get(); +} + +const Connector *Viterbi::connector() const { + return connector_.get(); +} + +// static +bool Viterbi::forwardbackward(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_MARGINAL_PROB)) { + return true; + } + + Node **end_node_list = lattice->end_nodes(); + Node **begin_node_list = lattice->begin_nodes(); + + const size_t len = lattice->size(); + const double theta = lattice->theta(); + + end_node_list[0]->alpha = 0.0; + for (int pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + calc_alpha(node, theta); + } + } + + begin_node_list[len]->beta = 0.0; + for (int pos = static_cast(len); pos >= 0; --pos) { + for (Node *node = end_node_list[pos]; node; node = node->enext) { + calc_beta(node, theta); + } + } + + const double Z = begin_node_list[len]->alpha; + lattice->set_Z(Z); // alpha of EOS + + for (int pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + node->prob = std::exp(node->alpha + node->beta - Z); + for (Path *path = node->lpath; path; path = path->lnext) { + path->prob = std::exp(path->lnode->alpha + - theta * path->cost + + path->rnode->beta - Z); + } + } + } + + return true; +} + +// static +bool Viterbi::buildResultForNBest(Lattice *lattice) { + return buildAllLattice(lattice); +} + +// static +bool Viterbi::buildAllLattice(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_ALL_MORPHS)) { + return true; + } + + Node *prev = lattice->bos_node(); + const size_t len = lattice->size(); + Node **begin_node_list = lattice->begin_nodes(); + + for (long pos = 0; pos <= static_cast(len); ++pos) { + for (Node *node = begin_node_list[pos]; node; node = node->bnext) { + prev->next = node; + node->prev = prev; + prev = node; + } + } + + return true; +} + +// static +bool Viterbi::buildAlternative(Lattice *lattice) { + Node **begin_node_list = lattice->begin_nodes(); + + const Node *bos_node = lattice->bos_node(); + for (const Node *node = bos_node; node; node = node->next) { + if (node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE) { + continue; + } + const size_t pos = node->surface - lattice->sentence() - + node->rlength + node->length; + std::cout.write(node->surface, node->length); + std::cout << "\t" << node->feature << std::endl; + for (const Node *anode = begin_node_list[pos]; + anode; anode = anode->bnext) { + if (anode->rlength == node->rlength && + anode->length == node->length) { + std::cout << "@ "; + std::cout.write(anode->surface, anode->length); + std::cout << "\t" << anode->feature << std::endl; + } + } + } + + std::cout << "EOS" << std::endl; + + return true; +} + +// static +bool Viterbi::buildBestLattice(Lattice *lattice) { + Node *node = lattice->eos_node(); + for (Node *prev_node; node->prev;) { + node->isbest = 1; + prev_node = node->prev; + prev_node->next = node; + node = prev_node; + } + + return true; +} + +// static +bool Viterbi::initNBest(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_NBEST)) { + return true; + } + lattice->allocator()->nbest_generator()->set(lattice); + return true; +} + +// static +bool Viterbi::initPartial(Lattice *lattice) { + if (!lattice->has_request_type(MECAB_PARTIAL)) { + if (lattice->has_constraint()) { + lattice->set_boundary_constraint(0, MECAB_TOKEN_BOUNDARY); + lattice->set_boundary_constraint(lattice->size(), + MECAB_TOKEN_BOUNDARY); + } + return true; + } + + Allocator *allocator = lattice->allocator(); + char *str = allocator->partial_buffer(lattice->size() + 1); + strncpy(str, lattice->sentence(), lattice->size() + 1); + + std::vector lines; + const size_t lsize = tokenize(str, "\n", + std::back_inserter(lines), + lattice->size() + 1); + char* column[2]; + scoped_array buf(new char[lattice->size() + 1]); + StringBuffer os(buf.get(), lattice->size() + 1); + + std::vector > tokens; + tokens.reserve(lsize); + + size_t pos = 0; + for (size_t i = 0; i < lsize; ++i) { + const size_t size = tokenize(lines[i], "\t", column, 2); + if (size == 1 && std::strcmp(column[0], "EOS") == 0) { + break; + } + const size_t len = std::strlen(column[0]); + if (size == 2) { + tokens.push_back(std::make_pair(column[0], column[1])); + } else { + tokens.push_back(std::make_pair(column[0], reinterpret_cast(0))); + } + os << column[0]; + pos += len; + } + + os << '\0'; + + lattice->set_sentence(os.str()); + + pos = 0; + for (size_t i = 0; i < tokens.size(); ++i) { + const char *surface = tokens[i].first; + const char *feature = tokens[i].second; + const size_t len = std::strlen(surface); + lattice->set_boundary_constraint(pos, MECAB_TOKEN_BOUNDARY); + lattice->set_boundary_constraint(pos + len, MECAB_TOKEN_BOUNDARY); + if (feature) { + lattice->set_feature_constraint(pos, pos + len, feature); + for (size_t n = 1; n < len; ++n) { + lattice->set_boundary_constraint(pos + n, + MECAB_INSIDE_TOKEN); + } + } + pos += len; + } + + return true; +} + +namespace { +template bool connect(size_t pos, Node *rnode, + Node **begin_node_list, + Node **end_node_list, + const Connector *connector, + Allocator *allocator) { + for (;rnode; rnode = rnode->bnext) { + long best_cost = 2147483647; + Node* best_node = 0; + for (Node *lnode = end_node_list[pos]; lnode; lnode = lnode->enext) { + int lcost = connector->cost(lnode, rnode); // local cost + long cost = lnode->cost + lcost; + + if (cost < best_cost) { + best_node = lnode; + best_cost = cost; + } + + if (IsAllPath) { + Path *path = allocator->newPath(); + path->cost = lcost; + path->rnode = rnode; + path->lnode = lnode; + path->lnext = rnode->lpath; + rnode->lpath = path; + path->rnext = lnode->rpath; + lnode->rpath = path; + } + } + + // overflow check 2003/03/09 + if (!best_node) { + return false; + } + + rnode->prev = best_node; + rnode->next = 0; + rnode->cost = best_cost; + const size_t x = rnode->rlength + pos; + rnode->enext = end_node_list[x]; + end_node_list[x] = rnode; + } + + return true; +} +} // namespace + +template +bool Viterbi::viterbi(Lattice *lattice) const { + Node **end_node_list = lattice->end_nodes(); + Node **begin_node_list = lattice->begin_nodes(); + Allocator *allocator = lattice->allocator(); + const size_t len = lattice->size(); + const char *begin = lattice->sentence(); + const char *end = begin + len; + + Node *bos_node = tokenizer_->getBOSNode(lattice->allocator()); + bos_node->surface = lattice->sentence(); + end_node_list[0] = bos_node; + + for (size_t pos = 0; pos < len; ++pos) { + if (end_node_list[pos]) { + Node *right_node = tokenizer_->lookup(begin + pos, end, + allocator, lattice); + begin_node_list[pos] = right_node; + if (!connect(pos, right_node, + begin_node_list, + end_node_list, + connector_.get(), + allocator)) { + lattice->set_what("too long sentence."); + return false; + } + } + } + + Node *eos_node = tokenizer_->getEOSNode(lattice->allocator()); + eos_node->surface = lattice->sentence() + lattice->size(); + begin_node_list[lattice->size()] = eos_node; + + for (long pos = len; static_cast(pos) >= 0; --pos) { + if (end_node_list[pos]) { + if (!connect(pos, eos_node, + begin_node_list, + end_node_list, + connector_.get(), + allocator)) { + lattice->set_what("too long sentence."); + return false; + } + break; + } + } + + end_node_list[0] = bos_node; + begin_node_list[lattice->size()] = eos_node; + + return true; +} +} // Mecab diff --git a/fts/third_party/mecab/src/viterbi.h b/fts/third_party/mecab/src/viterbi.h new file mode 100644 index 00000000..b40667f2 --- /dev/null +++ b/fts/third_party/mecab/src/viterbi.h @@ -0,0 +1,53 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_VITERBI_H_ +#define MECAB_VITERBI_H_ + +#include +#include "mecab.h" +#include "thread.h" + +namespace MeCab { + +class Lattice; +class Param; +class Connector; +template class Tokenizer; + +class Viterbi { + public: + bool open(const Param ¶m); + + bool analyze(Lattice *lattice) const; + + const Tokenizer *tokenizer() const; + + const Connector *connector() const; + + const char *what() { return what_.str(); } + + static bool buildResultForNBest(Lattice *lattice); + + Viterbi(); + virtual ~Viterbi(); + + private: + template bool viterbi(Lattice *lattice) const; + + static bool forwardbackward(Lattice *lattice); + static bool initPartial(Lattice *lattice); + static bool initNBest(Lattice *lattice); + static bool buildBestLattice(Lattice *lattice); + static bool buildAllLattice(Lattice *lattice); + static bool buildAlternative(Lattice *lattice); + + scoped_ptr > tokenizer_; + scoped_ptr connector_; + int cost_factor_; + whatlog what_; +}; +} +#endif // MECAB_VITERBI_H_ diff --git a/fts/third_party/mecab/src/winmain.h b/fts/third_party/mecab/src/winmain.h new file mode 100644 index 00000000..64ca02bc --- /dev/null +++ b/fts/third_party/mecab/src/winmain.h @@ -0,0 +1,69 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#if defined(_WIN32) || defined(__CYGWIN__) + +#include +#include + +namespace { +class CommandLine { + public: + CommandLine(int argc, wchar_t **argv) : argc_(argc), argv_(0) { + argv_ = new char * [argc_]; + for (int i = 0; i < argc_; ++i) { + const std::string arg = WideToUtf8(argv[i]); + argv_[i] = new char[arg.size() + 1]; + ::memcpy(argv_[i], arg.data(), arg.size()); + argv_[i][arg.size()] = '\0'; + } + } + ~CommandLine() { + for (int i = 0; i < argc_; ++i) { + delete [] argv_[i]; + } + delete [] argv_; + } + + int argc() const { return argc_; } + char **argv() const { return argv_; } + + private: + static std::string WideToUtf8(const std::wstring &input) { + const int output_length = ::WideCharToMultiByte(CP_UTF8, 0, + input.c_str(), -1, NULL, 0, + NULL, NULL); + if (output_length == 0) { + return ""; + } + + char *input_encoded = new char[output_length + 1]; + const int result = ::WideCharToMultiByte(CP_UTF8, 0, input.c_str(), -1, + input_encoded, + output_length + 1, NULL, NULL); + std::string output; + if (result > 0) { + output.assign(input_encoded); + } + delete [] input_encoded; + return output; + } + + int argc_; + char **argv_; +}; +} // namespace + +#define main(argc, argv) wmain_to_main_wrapper(argc, argv) + +int wmain_to_main_wrapper(int argc, char **argv); + +#if defined(__MINGW32__) +extern "C" +#endif +int wmain(int argc, wchar_t **argv) { + CommandLine cmd(argc, argv); + return wmain_to_main_wrapper(cmd.argc(), cmd.argv()); +} +#endif diff --git a/fts/third_party/mecab/src/writer.cpp b/fts/third_party/mecab/src/writer.cpp new file mode 100644 index 00000000..14ea0178 --- /dev/null +++ b/fts/third_party/mecab/src/writer.cpp @@ -0,0 +1,412 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2011 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#include +#include +#include +#include +#include "common.h" +#include "param.h" +#include "string_buffer.h" +#include "utils.h" +#include "writer.h" + +namespace MeCab { + +Writer::Writer() : write_(&Writer::writeLattice) {} +Writer::~Writer() {} + +void Writer::close() { + write_ = &Writer::writeLattice; +} + +bool Writer::open(const Param ¶m) { + const std::string ostyle = param.get("output-format-type"); + write_ = &Writer::writeLattice; + + if (ostyle == "wakati") { + write_ = &Writer::writeWakati; + } else if (ostyle == "none") { + write_ = &Writer::writeNone; + } else if (ostyle == "dump") { + write_ = &Writer::writeDump; + } else if (ostyle == "em") { + write_ = &Writer::writeEM; + } else { + // default values + std::string node_format = "%m\\t%H\\n"; + std::string unk_format = "%m\\t%H\\n"; + std::string bos_format = ""; + std::string eos_format = "EOS\\n"; + std::string eon_format = ""; + + std::string node_format_key = "node-format"; + std::string bos_format_key = "bos-format"; + std::string eos_format_key = "eos-format"; + std::string unk_format_key = "unk-format"; + std::string eon_format_key = "eon-format"; + + if (!ostyle.empty()) { + node_format_key += "-"; + node_format_key += ostyle; + bos_format_key += "-"; + bos_format_key += ostyle; + eos_format_key += "-"; + eos_format_key += ostyle; + unk_format_key += "-"; + unk_format_key += ostyle; + eon_format_key += "-"; + eon_format_key += ostyle; + const std::string tmp = param.get(node_format_key.c_str()); + CHECK_FALSE(!tmp.empty()) << "unknown format type [" << ostyle << "]"; + } + + const std::string node_format2 = + param.get(node_format_key.c_str()); + const std::string bos_format2 = + param.get(bos_format_key.c_str()); + const std::string eos_format2 = + param.get(eos_format_key.c_str()); + const std::string unk_format2 = + param.get(unk_format_key.c_str()); + const std::string eon_format2 = + param.get(eon_format_key.c_str()); + + if (node_format != node_format2 || bos_format != bos_format2 || + eos_format != eos_format2 || unk_format != unk_format2) { + write_ = &Writer::writeUser; + if (node_format != node_format2) { + node_format = node_format2; + } + if (bos_format != bos_format2) { + bos_format = bos_format2; + } + if (eos_format != eos_format2) { + eos_format = eos_format2; + } + if (unk_format != unk_format2) { + unk_format = unk_format2; + } else if (node_format != node_format2) { + unk_format = node_format2; + } else { + unk_format = node_format; + } + if (eon_format != eon_format2) { + eon_format = eon_format2; + } + node_format_.reset_string(node_format.c_str()); + bos_format_.reset_string(bos_format.c_str()); + eos_format_.reset_string(eos_format.c_str()); + unk_format_.reset_string(unk_format.c_str()); + eon_format_.reset_string(eon_format.c_str()); + } + } + + return true; +} + +bool Writer::write(Lattice *lattice, StringBuffer *os) const { + if (!lattice || !lattice->is_available()) { + return false; + } + return (this->*write_)(lattice, os); +} + +bool Writer::writeLattice(Lattice *lattice, StringBuffer *os) const { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << '\t' << node->feature; // << '\t'; + *os << '\n'; + } + *os << "EOS\n"; + return true; +} + +bool Writer::writeWakati(Lattice *lattice, StringBuffer *os) const { + for (const Node *node = lattice->bos_node()->next; + node->next; node = node->next) { + os->write(node->surface, node->length); + *os << ' '; + } + *os << '\n'; + return true; +} + +bool Writer::writeNone(Lattice *lattice, StringBuffer *os) const { + return true; // do nothing +} + +bool Writer::writeEM(Lattice *lattice, StringBuffer *os) const { + static const float min_prob = 0.0001; + for (const Node *node = lattice->bos_node(); node; node = node->next) { + if (node->prob >= min_prob) { + *os << "U\t"; + if (node->stat == MECAB_BOS_NODE) { + *os << "BOS"; + } else if (node->stat == MECAB_EOS_NODE) { + *os << "EOS"; + } else { + os->write(node->surface, node->length); + } + *os << '\t' << node->feature << '\t' << node->prob << '\n'; + } + for (const Path *path = node->lpath; path; path = path->lnext) { + if (path->prob >= min_prob) { + *os << "B\t" << path->lnode->feature << '\t' + << node->feature << '\t' << path->prob << '\n'; + } + } + } + *os << "EOS\n"; + return true; +} + +bool Writer::writeDump(Lattice *lattice, StringBuffer *os) const { + const char *str = lattice->sentence(); + for (const Node *node = lattice->bos_node(); node; node = node->next) { + *os << node->id << ' '; + if (node->stat == MECAB_BOS_NODE) { + *os << "BOS"; + } else if (node->stat == MECAB_EOS_NODE) { + *os << "EOS"; + } else { + os->write(node->surface, node->length); + } + + *os << ' ' << node->feature + << ' ' << static_cast(node->surface - str) + << ' ' << static_cast(node->surface - str + node->length) + << ' ' << node->rcAttr + << ' ' << node->lcAttr + << ' ' << node->posid + << ' ' << static_cast(node->char_type) + << ' ' << static_cast(node->stat) + << ' ' << static_cast(node->isbest) + << ' ' << node->alpha + << ' ' << node->beta + << ' ' << node->prob + << ' ' << node->cost; + + for (const Path *path = node->lpath; path; path = path->lnext) { + *os << ' ' << path->lnode->id << ':' << path->cost << ':' << path->prob; + } + *os << '\n'; + } + return true; +} + +bool Writer::writeUser(Lattice *lattice, StringBuffer *os) const { + if (!writeNode(lattice, bos_format_.get(), lattice->bos_node(), os)) { + return false; + } + const Node *node = 0; + for (node = lattice->bos_node()->next; node->next; node = node->next) { + const char *fmt = (node->stat == MECAB_UNK_NODE ? unk_format_.get() : + node_format_.get()); + if (!writeNode(lattice, fmt, node, os)) { + return false; + } + } + if (!writeNode(lattice, eos_format_.get(), node, os)) { + return false; + } + return true; +} + +bool Writer::writeNode(Lattice *lattice, const Node *node, + StringBuffer *os) const { + switch (node->stat) { + case MECAB_BOS_NODE: + return writeNode(lattice, bos_format_.get(), node, os); + case MECAB_EOS_NODE: + return writeNode(lattice, eos_format_.get(), node, os); + case MECAB_UNK_NODE: + return writeNode(lattice, unk_format_.get(), node, os); + case MECAB_NOR_NODE: + return writeNode(lattice, node_format_.get(), node, os); + case MECAB_EON_NODE: + return writeNode(lattice, eon_format_.get(), node, os); + } + return true; +} + +bool Writer::writeNode(Lattice *lattice, + const char *p, + const Node *node, + StringBuffer *os) const { + scoped_fixed_array buf; + scoped_fixed_array ptr; + size_t psize = 0; + + for (; *p; p++) { + switch (*p) { + default: *os << *p; break; + + case '\\': *os << getEscapedChar(*++p); break; + + case '%': { // macros + switch (*++p) { + default: { + const std::string error = "unknown meta char: " + *p; + lattice->set_what(error.c_str()); + return false; + } + // input sentence + case 'S': os->write(lattice->sentence(), lattice->size()); break; + // sentence length + case 'L': *os << lattice->size(); break; + // morph + case 'm': os->write(node->surface, node->length); break; + case 'M': os->write(reinterpret_cast + (node->surface - node->rlength + node->length), + node->rlength); + break; + case 'h': *os << node->posid; break; // Part-Of-Speech ID + case '%': *os << '%'; break; // % + case 'c': *os << static_cast(node->wcost); break; // word cost + case 'H': *os << node->feature; break; + case 't': *os << static_cast(node->char_type); break; + case 's': *os << static_cast(node->stat); break; + case 'P': *os << node->prob; break; + case 'p': { + switch (*++p) { + default: + lattice->set_what("[iseSCwcnblLh] is required after %p"); + return false; + case 'i': *os << node->id; break; // node id + case 'S': os->write(reinterpret_cast + (node->surface - + node->rlength + node->length), + node->rlength - node->length); + break; // space + // start position + case 's': *os << static_cast( + node->surface - lattice->sentence()); + break; + // end position + case 'e': *os << static_cast + (node->surface - lattice->sentence() + node->length); + break; + // connection cost + case 'C': *os << node->cost - + node->prev->cost - node->wcost; + break; + case 'w': *os << node->wcost; break; // word cost + case 'c': *os << node->cost; break; // best cost + case 'n': *os << (node->cost - node->prev->cost); break; + // node cost + // * if best path, otherwise ' ' + case 'b': *os << (node->isbest ? '*' : ' '); break; + case 'P': *os << node->prob; break; + case 'A': *os << node->alpha; break; + case 'B': *os << node->beta; break; + case 'l': *os << node->length; break; // length of morph + // length of morph including the spaces + case 'L': *os << node->rlength; break; + case 'h': { // Hidden Layer ID + switch (*++p) { + default: + lattice->set_what("lr is required after %ph"); + return false; + case 'l': *os << node->lcAttr; break; // current + case 'r': *os << node->rcAttr; break; // prev + } + } break; + + case 'p': { + char mode = *++p; + char sep = *++p; + if (sep == '\\') { + sep = getEscapedChar(*++p); + } + if (!node->lpath) { + lattice->set_what("no path information is available"); + return false; + } + for (Path *path = node->lpath; path; path = path->lnext) { + if (path != node->lpath) *os << sep; + switch (mode) { + case 'i': *os << path->lnode->id; break; + case 'c': *os << path->cost; break; + case 'P': *os << path->prob; break; + default: + lattice->set_what("[icP] is required after %pp"); + return false; + } + } + } break; + + } + } break; + + case 'F': + case 'f': { + if (node->feature[0] == '\0') { + lattice->set_what("no feature information available"); + return false; + } + if (!psize) { + std::strncpy(buf.get(), node->feature, buf.size()); + psize = tokenizeCSV(buf.get(), ptr.get(), ptr.size()); + } + + // separator + char separator = '\t'; // default separator + if (*p == 'F') { // change separator + if (*++p == '\\') { + separator = getEscapedChar(*++p); + } else { + separator = *p; + } + } + + if (*++p !='[') { + lattice->set_what("cannot find '['"); + return false; + } + size_t n = 0; + bool sep = false; + bool isfil = false; + p++; + + for (;; ++p) { + switch (*p) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + n = 10 * n +(*p - '0'); + break; + case ',': case ']': + if (n >= psize) { + lattice->set_what("given index is out of range"); + return false; + } + isfil = (ptr[n][0] != '*'); + if (isfil) { + if (sep) { + *os << separator; + } + *os << ptr[n]; + } + if (*p == ']') { + goto last; + } + sep = isfil; + n = 0; + break; + default: + lattice->set_what("cannot find ']'"); + return false; + } + } + } last: break; + } // end switch + } break; // end case '%' + } // end switch + } + + return true; +} +} diff --git a/fts/third_party/mecab/src/writer.h b/fts/third_party/mecab/src/writer.h new file mode 100644 index 00000000..defde406 --- /dev/null +++ b/fts/third_party/mecab/src/writer.h @@ -0,0 +1,57 @@ +// MeCab -- Yet Another Part-of-Speech and Morphological Analyzer +// +// +// Copyright(C) 2001-2006 Taku Kudo +// Copyright(C) 2004-2006 Nippon Telegraph and Telephone Corporation +#ifndef MECAB_WRITER_H_ +#define MECAB_WRITER_H_ + +#include +#include "common.h" +#include "mecab.h" +#include "utils.h" +#include "scoped_ptr.h" +#include "string_buffer.h" + +namespace MeCab { + +class Param; + +class Writer { + public: + Writer(); + virtual ~Writer(); + bool open(const Param ¶m); + void close(); + + bool writeNode(Lattice *lattice, + const char *format, + const Node *node, StringBuffer *s) const; + bool writeNode(Lattice *lattice, + const Node *node, + StringBuffer *s) const; + + bool write(Lattice *lattice, StringBuffer *node) const; + + const char *what() { return what_.str(); } + + private: + scoped_string node_format_; + scoped_string bos_format_; + scoped_string eos_format_; + scoped_string unk_format_; + scoped_string eon_format_; + whatlog what_; + + bool writeLattice(Lattice *lattice, StringBuffer *s) const; + bool writeWakati(Lattice *lattice, StringBuffer *s) const; + bool writeNone(Lattice *lattice, StringBuffer *s) const; + bool writeUser(Lattice *lattice, StringBuffer *s) const; + bool writeDump(Lattice *lattice, StringBuffer *s) const; + bool writeEM(Lattice *lattice, StringBuffer *s) const; + + bool (Writer::*write_)(Lattice *lattice, StringBuffer *s) const; +}; +} + +#endif // WRITER_H_