[WIP] Merge Chinese Word Segmentation work

.gitattributes

@@ -50,6 +50,7 @@ sconstruct    text diff=python
 *.c     text diff=c
 *.cpp   text diff=cpp
 *.h     text diff=c
+*.hpp   text diff=cpp
 *.idl   text diff=c
 *.acf   text diff=c
 

.github/workflows/testAndPublish.yml

@@ -77,7 +77,7 @@ jobs:
     - name: Checkout NVDA
       uses: actions/checkout@v5
       with:
-        submodules: true
+        submodules: recursive
     - name: Install Python
       uses: actions/setup-python@v5
       with:

.gitignore

@@ -14,6 +14,7 @@ source/lib
 source/lib64
 source/typelibs
 source/louis
+source/cppjieba
 *.obj
 *.exp
 *.lib

.gitmodules

@@ -42,3 +42,6 @@
 [submodule "include/nvda-mathcat"]
 	path = include/nvda-mathcat
 	url = https://github.com/nvaccess/nvda-mathcat.git
+[submodule "include/cppjieba"]
+	path = include/cppjieba
+	url = https://github.com/yanyiwu/cppjieba

copying.txt

@@ -1027,6 +1027,7 @@ In addition to these dependencies, the following are also included in NVDA:
 * Microsoft Detours: MIT
 * Python: PSF
 * NSIS: zlib/libpng
+* cppjieba: MIT
 
 Furthermore, NVDA also utilises some static/binary dependencies, details of which can be found at the following URL:
 

include/readme.md

@@ -61,3 +61,10 @@ Used in chrome system tests.
 <https://github.com/microsoft/wil/>
 
 Fetch latest from master.
+
+### cppjieba
+
+[cppjieba](https://github.com/yanyiwu/cppjieba).
+
+Fetch latest from master.
+Used for Chinese text segmentation.

nvdaHelper/archBuild_sconscript

@@ -229,6 +229,10 @@ Export("detoursLib")
 apiHookObj = env.Object("apiHook", "common/apiHook.cpp")
 Export("apiHookObj")
 
+cppjiebaLib = env.SConscript("cppjieba/sconscript")
+Export("cppjiebaLib")
+env.Install(libInstallDir, cppjiebaLib)
+
 if isNVDAHelperLocalArch:
 	localLib = env.SConscript("local/sconscript")
 	Export("localLib")

nvdaHelper/cppjieba/cppjieba.cpp

@@ -0,0 +1,124 @@
+/*
+A part of NonVisual Desktop Access (NVDA)
+Copyright (C) 2025 NV Access Limited, Wang Chong
+This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license.
+For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt
+*/
+
+#include "cppjieba.hpp"
+
+
+using namespace std;
+
+// static members for singleton bookkeeping
+JiebaSingleton* JiebaSingleton::instance = nullptr;
+std::once_flag JiebaSingleton::initFlag;
+
+JiebaSingleton& JiebaSingleton::getInstance(const char* dictDir) {
+    // convert incoming C-string+length to std::string (handles dictDir == nullptr)
+    std::string dir = dictDir;
+
+    // ensure singleton is constructed exactly once
+    std::call_once(JiebaSingleton::initFlag, [&]() {
+        // allocate on heap, so we avoid copy/move and control lifetime
+        JiebaSingleton::instance = new JiebaSingleton(dir.c_str());
+        // optional: register deleter at exit
+        std::atexit([]() {
+            delete JiebaSingleton::instance;
+            JiebaSingleton::instance = nullptr;
+        });
+    });
+
+    // after call_once, instance must be non-null
+    return *JiebaSingleton::instance;
+}
+
+JiebaSingleton& JiebaSingleton::getInstance() {
+    if (!JiebaSingleton::instance) {
+        throw std::runtime_error("JiebaSingleton::getInstance() called before initialization. Call getInstance(dictDir) or initJieba() first.");
+    }
+    return *JiebaSingleton::instance;
+}
+
+JiebaSingleton::JiebaSingleton(const char* dictDir)
+: cppjieba::JiebaSegmenter(
+    std::string(dictDir),
+    std::string(dictDir),
+    std::string(dictDir)
+  )
+{
+    // base class ctor will load dictionaries/models
+}
+
+void JiebaSingleton::getWordEndOffsets(const std::string& text, std::vector<int>& wordEndOffsets) {
+    std::lock_guard<std::mutex> lock(segMutex);
+    wordEndOffsets.clear();
+    std::vector<cppjieba::Word> words;
+    this->Cut(text, words, true);
+
+    for (const auto& word : words) {
+        wordEndOffsets.push_back(word.unicode_offset + word.unicode_length);
+    }
+}
+
+extern "C" {
+
+bool initJieba(const char* dictDir) {
+    try {
+        // simply force the singleton into existence
+        (void)JiebaSingleton::getInstance(dictDir);
+        return true;
+    } catch (...) {
+        return false;
+    }
+}
+
+bool calculateWordOffsets(const char* text, int** wordEndOffsets, int* outLen) {
+    if (!text || !wordEndOffsets || !outLen) return false;
+
+    try {
+        std::string textStr(text);
+        std::vector<int> offs;
+        JiebaSingleton::getInstance().getWordEndOffsets(textStr, offs);
+
+        int n = static_cast<int>(offs.size());
+        if (n == 0) {
+            *wordEndOffsets = nullptr;
+            *outLen = 0;
+            return true; // success, but no offsets
+        }
+
+        int* buf = static_cast<int*>(std::malloc(sizeof(int) * n));
+        if (!buf) {
+            *wordEndOffsets = nullptr;
+            *outLen = 0;
+            return false;
+        }
+        for (int i = 0; i < n; ++i) buf[i] = offs[i];
+        *wordEndOffsets = buf;
+        *outLen = n;
+        return true;
+    } catch (...) {
+        *wordEndOffsets = nullptr;
+        *outLen = 0;
+        return false;
+    }
+}
+
+bool insertUserWord(const char* word, int freq, const char* tag = cppjieba::UNKNOWN_TAG) {
+	return JiebaSingleton::getInstance().InsertUserWord(string(word), freq, string(tag));
+}
+
+bool deleteUserWord(const char* word, const char* tag = cppjieba::UNKNOWN_TAG) {
+	return JiebaSingleton::getInstance().DeleteUserWord(string(word), string(tag));
+}
+
+bool find(const char* word) {
+	return JiebaSingleton::getInstance().Find(string(word));
+}
+
+void freeOffsets(int* ptr) {
+    if (ptr) std::free(ptr);
+}
+
+} // extern "C"

nvdaHelper/cppjieba/cppjieba.def

@@ -0,0 +1,8 @@
+LIBRARY cppjieba
+EXPORTS
+	initJieba
+	calculateWordOffsets
+	insertUserWord
+	deleteUserWord
+	find
+	freeOffsets

nvdaHelper/cppjieba/cppjieba.hpp

@@ -0,0 +1,165 @@
+/*
+A part of NonVisual Desktop Access (NVDA)
+Copyright (C) 2025 NV Access Limited, Wang Chong
+This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license.
+For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt
+*/
+
+#ifndef CPPJIEBA_DLL_H
+#define CPPJIEBA_DLL_H
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstring>
+#include <mutex>
+#include <cstdlib>
+#include <set>
+#include <stdexcept>
+#include "QuerySegment.hpp"
+
+using namespace std;
+
+namespace cppjieba {  // copied from Jieba.hpp and modified to drop off its keyword extractor we don't use
+
+class JiebaSegmenter {
+ public:
+  JiebaSegmenter(const string& dict_path,
+        const string& model_path,
+        const string& user_dict_path)
+    : dict_trie_(pathJoin(dict_path, "jieba.dict.utf8"), pathJoin(user_dict_path, "user.dict.utf8")),
+      model_(pathJoin(model_path, "hmm_model.utf8")),
+      mix_seg_(&dict_trie_, &model_) {
+  }
+  ~JiebaSegmenter() {
+  }
+
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    mix_seg_.Cut(sentence, words, hmm);
+  }
+
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.InsertUserWord(word,freq, tag);
+  }
+
+  bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.DeleteUserWord(word, tag);
+  }
+
+  bool Find(const string& word)
+  {
+    return dict_trie_.Find(word);
+  }
+
+  void ResetSeparators(const string& s) {
+    mix_seg_.ResetSeparators(s);
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return &dict_trie_;
+  }
+
+  const HMMModel* GetHMMModel() const {
+    return &model_;
+  }
+
+  void LoadUserDict(const vector<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const set<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const string& path)  {
+    dict_trie_.LoadUserDict(path);
+  }
+
+ private:
+  static string pathJoin(const string& dir, const string& filename) {
+    if (dir.empty()) {
+        return filename;
+    }
+
+    char last_char = dir[dir.length() - 1];
+    if (last_char == '/' || last_char == '\\') {
+        return dir + filename;
+    } else {
+        #ifdef _WIN32
+        return dir + '\\' + filename;
+        #else
+        return dir + '/' + filename;
+        #endif
+    }
+  }
+
+  static string getCurrentDirectory() {
+    string path(__FILE__);
+    size_t pos = path.find_last_of("/\\");
+    return (pos == string::npos) ? "" : path.substr(0, pos);
+  }
+
+  DictTrie dict_trie_;
+  HMMModel model_;
+
+  MixSegment mix_seg_;
+}; // class JiebaSegmenter
+
+} // namespace cppjieba
+
+
+/// @brief Singleton wrapper around cppjieba::Jieba.
+class JiebaSingleton : public cppjieba::JiebaSegmenter {
+public:
+    /// @brief Returns the single instance, constructing on first call.
+    static JiebaSingleton& getInstance(const char* dictDir);
+
+    static JiebaSingleton& getInstance();
+
+    /// @brief Do thread-safe segmentation and compute word end offsets.
+    /// @param text The input text in UTF-8 encoding.
+    /// @param wordEndOffsets Output vector to hold byte offsets of word ends.
+    void getWordEndOffsets(const std::string& text, std::vector<int>& wordEndOffsets);
+
+    // singleton bookkeeping
+    static JiebaSingleton* instance;
+    static std::once_flag initFlag;
+
+private:
+    JiebaSingleton(const char* dictDir);         ///< private ctor initializes base Jieba
+
+    /// Disable copy and move
+    JiebaSingleton(const JiebaSingleton&) = delete;
+    JiebaSingleton& operator = (const JiebaSingleton&) = delete;
+    JiebaSingleton(JiebaSingleton&&) = delete;
+    JiebaSingleton& operator = (JiebaSingleton&&) = delete;
+
+    std::mutex segMutex;      ///< guards concurrent Cut() calls
+};
+
+#ifdef _WIN32
+#  define JIEBA_API __declspec(dllexport)
+#else
+#  define JIEBA_API
+#endif
+
+extern "C" {
+
+/// @brief Force singleton construction (load dicts, etc.) before any segmentation.
+JIEBA_API bool initJieba(const char* dictDir);
+
+/// @brief Segment UTF-8 text into character offsets.
+/// @return 0 on success, -1 on failure.
+JIEBA_API bool calculateWordOffsets(const char* text, int** wordEndOffsets, int* outLen);
+
+/// Wrapper for word management
+JIEBA_API bool insertUserWord(const char* word, int freq, const char* tag);
+JIEBA_API bool deleteUserWord(const char* word, const char* tag);
+JIEBA_API bool find(const char* word);
+
+/// @brief Free memory allocated by calculateWordOffsets.
+JIEBA_API void freeOffsets(int* ptr);
+
+} // extern "C"
+
+#endif // CPPJIEBA_DLL_H