// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved.

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

// https://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.

#define PCRE2_CODE_UNIT_WIDTH 8

#include "opencc/openccxx.h"

#include "pcre2.h"

#include "string_utils.h"

#include "rag_analyzer.h"

#include "re2/re2.h"

#include <cassert>

#include <sstream> // std::ostringstream / std::istringstream — explicit for libc++ (macOS)

#include <cstdint>

#include <filesystem>

#include <iostream>

#include <cmath>

#include <fstream>

// import :term;

// import :stemmer;

// import :analyzer;

// import :darts_trie;

// import :wordnet_lemmatizer;

// import :stemmer;

// import :term;

//

// import std.compat;

namespace fs = std::filesystem;

static const std::string DICT_PATH = "rag/huqie.txt";

static const std::string POS_DEF_PATH = "rag/pos-id.def";

static const std::string TRIE_PATH = "rag/huqie.trie";

static const std::string WORDNET_PATH = "wordnet";

static const std::string OPENCC_PATH = "opencc";

static const std::string REGEX_SPLIT_CHAR =

R"#(([ ,\.<>/?;'\[\]\`!@#$%^&*$$\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-zA-Z\.-]+|[0-9,\.-]+))#";

static const std::string NLTK_TOKENIZE_PATTERN =

R"((?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)|(?=[^\(\"\`{\[:;&\#\*@\)}\]\-,])\S+?(?=\s|$|(?:[)\";}\]\*:@\'\({\[\?!])|(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)|,(?=$|\s|(?:[)\";}\]\*:@\'\({\[\?!])|(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)))|\S)";

static constexpr std::size_t MAX_SENTENCE_LEN = 100;

static inline int32_t Encode(int32_t freq, int32_t idx) {

uint32_t encoded_value = 0;

if (freq < 0) {

encoded_value |= static_cast<uint32_t>(-freq);

encoded_value |= (1U << 23);

} else {

encoded_value = static_cast<uint32_t>(freq & 0x7FFFFF);

}

encoded_value |= static_cast<uint32_t>(idx) << 24;

return static_cast<int32_t>(encoded_value);

}

static inline int32_t DecodeFreq(int32_t value) {

uint32_t v1 = static_cast<uint32_t>(value) & 0xFFFFFF;

if (v1 & (1 << 23)) {

v1 &= 0x7FFFFF;

return -static_cast<int32_t>(v1);

} else {

v1 = static_cast<int32_t>(v1);

}

return v1;

}

static inline int32_t DecodePOSIndex(int32_t value) {

// POS index is stored in the high 8 bits (bits 24-31)

return static_cast<int32_t>(static_cast<uint32_t>(value) >> 24);

}

void Split(const std::string &input, const std::string &split_pattern, std::vector<std::string> &result, bool keep_delim = false) {

re2::RE2 pattern(split_pattern);

re2::StringPiece leftover(input.data());

re2::StringPiece last_end = leftover;

re2::StringPiece extracted_delim_token;

while (RE2::FindAndConsume(&leftover, pattern, &extracted_delim_token)) {

std::string_view token(last_end.data(), extracted_delim_token.data() - last_end.data());

if (!token.empty()) {

result.emplace_back(token.data(), token.size());

}

if (keep_delim)

result.emplace_back(extracted_delim_token.data(), extracted_delim_token.size());

last_end = leftover;

}

if (!leftover.empty()) {

result.emplace_back(leftover.data(), leftover.size());

}

}

void Split(const std::string &input, const re2::RE2 &pattern, std::vector<std::string> &result, bool keep_delim = false) {

re2::StringPiece leftover(input.data());

re2::StringPiece last_end = leftover;

re2::StringPiece extracted_delim_token;

while (RE2::FindAndConsume(&leftover, pattern, &extracted_delim_token)) {

std::string_view token(last_end.data(), extracted_delim_token.data() - last_end.data());

if (!token.empty()) {

result.emplace_back(token.data(), token.size());

}

if (keep_delim)

result.emplace_back(extracted_delim_token.data(), extracted_delim_token.size());

last_end = leftover;

}

if (!leftover.empty()) {

result.emplace_back(leftover.data(), leftover.size());

}

}

std::string Replace(const re2::RE2 &re, const std::string &replacement, const std::string &input) {

std::string output = input;

re2::RE2::GlobalReplace(&output, re, replacement);

return output;

}

template <typename T>

std::string Join(const std::vector<T> &tokens, int start, int end, const std::string &delim = " ") {

std::ostringstream oss;

for (int i = start; i < end; ++i) {

if (i > start)

oss << delim;

oss << tokens[i];

}

return std::move(oss).str();

}

template <typename T>

std::string Join(const std::vector<T> &tokens, int start, const std::string &delim = " ") {

// C++23 strict overload resolution refuses the implicit size_t → int

// narrowing conversion; the explicit cast makes the 4-arg overload above

// unambiguous on libc++ (macOS) without changing behaviour on libstdc++.

return Join(tokens, start, static_cast<int>(tokens.size()), delim);

}

std::string Join(const TermList &tokens, int start, int end, const std::string &delim = " ") {

std::ostringstream oss;

for (int i = start; i < end; ++i) {

if (i > start)

oss << delim;

oss << tokens[i].text_;

}

return std::move(oss).str();

}

bool IsChinese(const std::string &str) {

for (std::size_t i = 0; i < str.length(); ++i) {

unsigned char c = str[i];

if (c >= 0xE4 && c <= 0xE9) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 >= 0x80 && c2 <= 0xBF) && (c3 >= 0x80 && c3 <= 0xBF)) {

return true;

}

}

}

}

return false;

}

bool IsAlphabet(const std::string &str) {

for (std::size_t i = 0; i < str.length(); ++i) {

unsigned char c = str[i];

if (c > 0x7F) {

return false;

}

}

return true;

}

bool IsKorean(const std::string &str) {

for (std::size_t i = 0; i < str.length(); ++i) {

unsigned char c = str[i];

if (c == 0xE1) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 == 0x84 || c2 == 0x85 || c2 == 0x86 || c2 == 0x87) && (c3 >= 0x80 && c3 <= 0xBF)) {

return true;

}

}

}

}

return false;

}

bool IsJapanese(const std::string &str) {

for (std::size_t i = 0; i < str.length(); ++i) {

unsigned char c = str[i];

if (c == 0xE3) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 == 0x81 || c2 == 0x82 || c2 == 0x83) && (c3 >= 0x81 && c3 <= 0xBF)) {

return true;

}

}

}

}

return false;

}

bool IsCJK(const std::string &str) {

for (std::size_t i = 0; i < str.length(); ++i) {

unsigned char c = str[i];

// Check Chinese

if (c >= 0xE4 && c <= 0xE9) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 >= 0x80 && c2 <= 0xBF) && (c3 >= 0x80 && c3 <= 0xBF)) {

return true;

}

}

}

// Check Japanese

if (c == 0xE3) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 == 0x81 || c2 == 0x82 || c2 == 0x83) && (c3 >= 0x81 && c3 <= 0xBF)) {

return true;

}

}

}

// Check Korean

if (c == 0xE1) {

if (i + 2 < str.length()) {

unsigned char c2 = str[i + 1];

unsigned char c3 = str[i + 2];

if ((c2 == 0x84 || c2 == 0x85 || c2 == 0x86 || c2 == 0x87) && (c3 >= 0x80 && c3 <= 0xBF)) {

return true;

}

}

}

}

return false;

}

class RegexTokenizer {

public:

RegexTokenizer() {

int errorcode = 0;

PCRE2_SIZE erroffset = 0;

re_ = pcre2_compile((PCRE2_SPTR)(NLTK_TOKENIZE_PATTERN.c_str()),

PCRE2_ZERO_TERMINATED,

PCRE2_MULTILINE | PCRE2_UTF,

&errorcode,

&erroffset,

nullptr);

}

~RegexTokenizer() {

pcre2_code_free(re_);

}

void RegexTokenize(const std::string &input, TermList &tokens) {

PCRE2_SPTR subject = (PCRE2_SPTR)input.c_str();

PCRE2_SIZE subject_length = input.length();

pcre2_match_data_8 *match_data = pcre2_match_data_create_8(1024, nullptr);

PCRE2_SIZE start_offset = 0;

while (start_offset < subject_length) {

int res = pcre2_match(re_, subject, subject_length, start_offset, 0, match_data, nullptr);

if (res < 0) {

if (res == PCRE2_ERROR_NOMATCH) {

break; // No more matches

} else {

std::cerr << "Matching error code: " << res << std::endl;

break; // Other error

}

}

// Extract matched substring

PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);

for (int i = 0; i < res; ++i) {

PCRE2_SIZE start = ovector[2 * i];

PCRE2_SIZE end = ovector[2 * i + 1];

tokens.Add(input.c_str() + start, end - start, start, end);

}

// Update the start offset for the next search

start_offset = ovector[1]; // Move to the end of the last match

}

// Free memory

pcre2_match_data_free(match_data);

}

private:

pcre2_code_8 *re_{nullptr};

};

class MacIntyreContractions {

public:

// List of contractions adapted from Robert MacIntyre's tokenizer.

std::vector<std::string> CONTRACTIONS2 = {R"((?i)\b(can)(?#X)(not)\b)",

R"((?i)\b(d)(?#X)('ye)\b)",

R"((?i)\b(gim)(?#X)(me)\b)",

R"((?i)\b(gon)(?#X)(na)\b)",

R"((?i)\b(got)(?#X)(ta)\b)",

R"((?i)\b(lem)(?#X)(me)\b)",

R"((?i)\b(more)(?#X)('n)\b)",

R"((?i)\b(wan)(?#X)(na)(?=\s))"};

std::vector<std::string> CONTRACTIONS3 = {R"((?i) ('t)(?#X)(is)\b)", R"((?i) ('t)(?#X)(was)\b)"};

std::vector<std::string> CONTRACTIONS4 = {R"((?i)\b(whad)(dd)(ya)\b)", R"((?i)\b(wha)(t)(cha)\b)"};

};

// Structure to hold precompiled regex patterns

struct CompiledRegex {

pcre2_code *re{nullptr};

std::string substitution;

CompiledRegex(pcre2_code *r, std::string sub) : re(r), substitution(std::move(sub)) {

}

CompiledRegex(const CompiledRegex &) = delete;

CompiledRegex &operator=(const CompiledRegex &) = delete;

CompiledRegex(CompiledRegex &&other) noexcept : re(other.re), substitution(std::move(other.substitution)) { other.re = nullptr; }

CompiledRegex &operator=(CompiledRegex &&other) noexcept {

if (this != &other) {

if (re)

pcre2_code_free(re);

re = other.re;

substitution = std::move(other.substitution);

other.re = nullptr;

}

return *this;

}

~CompiledRegex() {

if (re) {

pcre2_code_free(re);

}

}

};

class NLTKWordTokenizer {

MacIntyreContractions contractions_;

// Static singleton instance

static std::unique_ptr<NLTKWordTokenizer> instance_;

static std::once_flag init_flag_;

public:

// Static method to get the singleton instance

static NLTKWordTokenizer &GetInstance() {

std::call_once(init_flag_, []() { instance_ = std::make_unique<NLTKWordTokenizer>(); });

return *instance_;

}

// Starting quotes.

std::vector<std::pair<std::string, std::string>> STARTING_QUOTES = {

{std::string(R"(([«“‘„]|[`]+))"), std::string(R"( $1 )")},

{std::string(R"(^\")"), std::string(R"(``)")},

{std::string(R"((``))"), std::string(R"( $1 )")},

{std::string(R"(([ \(\[{<])(\"|\'{2}))"), std::string(R"($1 `` )")},

{std::string(R"((?i)(\')(?!re|ve|ll|m|t|s|d|n)(\w)\b)"), std::string(R"($1 $2)")}};

// Ending quotes.

std::vector<std::pair<std::string, std::string>> ENDING_QUOTES = {

{std::string(R"(([»”’]))"), std::string(R"( $1 )")},

{std::string(R"('')"), std::string(R"( '' )")},

{std::string(R"(")"), std::string(R"( '' )")},

{std::string(R"(\s+)"), std::string(R"( )")},

{std::string(R"(([^' ])('[sS]|'[mM]|'[dD]|') )"), std::string(R"($1 $2 )")},

{std::string(R"(([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) )"), std::string(R"($1 $2 )")}};

// Punctuation.

std::vector<std::pair<std::string, std::string>> PUNCTUATION = {

{std::string(R"(([^\.])(\.)([\]\)}>"\'»”’ ]*)\s*$)"), std::string(R"($1 $2 $3 )")},

{std::string(R"(([:,])([^\d]))"), std::string(R"( $1 $2)")},

{std::string(R"(([:,])$)"), std::string(R"($1 )")},

{std::string(R"(\.{2,})"), std::string(R"($0 )")},

{std::string(R"([;@#$%&])"), std::string(R"($0 )")},

{std::string(R"(([^\.])(\.)([\]\)}>"\']*)\s*$)"), std::string(R"($1 $2 $3 )")},

{std::string(R"([?!])"), std::string(R"($0 )")},

{std::string(R"(([^'])' )"), std::string(R"($1 ' )")},

{std::string(R"([*])"), std::string(R"($0 )")}};

// Pads parentheses

std::pair<std::string, std::string> PARENS_BRACKETS = {std::string(R"([\]\[\(\)\{\}\<\>])"), std::string(R"( $0 )")};

std::vector<std::pair<std::string, std::string>> CONVERT_PARENTHESES = {{std::string(R"(\()"), std::string("-LRB-")},

{std::string(R"(\))"), std::string("-RRB-")},

{std::string(R"(\[)"), std::string("-LSB-")},

{std::string(R"(\])"), std::string("-RSB-")},

{std::string(R"(\{)"), std::string("-LCB-")},

{std::string(R"(\})"), std::string("-RCB-")}};

std::pair<std::string, std::string> DOUBLE_DASHES = {std::string(R"(--)"), std::string(R"( -- )")};

// Cache for compiled regex patterns

std::vector<CompiledRegex> compiled_starting_quotes_;

std::vector<CompiledRegex> compiled_ending_quotes_;

std::vector<CompiledRegex> compiled_punctuation_;

CompiledRegex compiled_parens_brackets_;

std::vector<CompiledRegex> compiled_convert_parentheses_;

CompiledRegex compiled_double_dashes_;

std::vector<CompiledRegex> compiled_contractions2_;

std::vector<CompiledRegex> compiled_contractions3_;

// Constructor that precompiles all regex patterns

NLTKWordTokenizer() : compiled_parens_brackets_(nullptr, ""), compiled_double_dashes_(nullptr, "") { CompileRegexPatterns(); }

void Tokenize(const std::string &text, std::vector<std::string> &tokens, bool convert_parentheses = false) {

std::string result = text;

for (const auto &compiled : compiled_starting_quotes_) {

result = ApplyRegex(result, compiled);

}

for (const auto &compiled : compiled_punctuation_) {

result = ApplyRegex(result, compiled);

}

// Handles parentheses.

result = ApplyRegex(result, compiled_parens_brackets_);

// Optionally convert parentheses

if (convert_parentheses) {

for (const auto &compiled : compiled_convert_parentheses_) {

result = ApplyRegex(result, compiled);

}

}

// Handles double dash.

result = ApplyRegex(result, compiled_double_dashes_);

// Add extra space to make things easier

result = " " + result + " ";

for (const auto &compiled : compiled_ending_quotes_) {

result = ApplyRegex(result, compiled);

}

for (const auto &compiled : compiled_contractions2_) {

result = ApplyRegex(result, compiled);

}

for (const auto &compiled : compiled_contractions3_) {

result = ApplyRegex(result, compiled);

}

// Split the result into tokens

size_t start = 0;

size_t end = result.find(' ');

while (end != std::string::npos) {

if (end != start) {

std::string token = result.substr(start, end - start);

// Handle underscore tokens properly

if (token == "_") {

// Single underscore token

tokens.push_back("_");

} else if (token.find('_') != std::string::npos) {

// Split tokens containing underscores and keep underscores as separate tokens

std::stringstream ss(token);

std::string sub_token;

bool first = true;

while (std::getline(ss, sub_token, '_')) {

if (!first) {

tokens.push_back("_");

}

if (!sub_token.empty()) {

tokens.push_back(sub_token);

}

first = false;

}

// Handle case where token ends with underscore

if (token.back() == '_') {

tokens.push_back("_");

}

} else {

tokens.push_back(token);

}

}

start = end + 1;

end = result.find(' ', start);

}

if (start != result.length()) {

std::string token = result.substr(start);

// Handle underscore tokens properly

if (token == "_") {

// Single underscore token

tokens.push_back("_");

} else if (token.find('_') != std::string::npos) {

// Split tokens containing underscores and keep underscores as separate tokens

std::stringstream ss(token);

std::string sub_token;

bool first = true;

while (std::getline(ss, sub_token, '_')) {

if (!first) {

tokens.push_back("_");

}

if (!sub_token.empty()) {

tokens.push_back(sub_token);

}

first = false;

}

// Handle case where token ends with underscore

if (token.back() == '_') {

tokens.push_back("_");

}

} else {

tokens.push_back(token);

}

}

}

private:

void CompileRegexPatterns() {

compiled_starting_quotes_.reserve(STARTING_QUOTES.size());

for (const auto &[pattern, substitution] : STARTING_QUOTES) {

compiled_starting_quotes_.emplace_back(CompilePattern(pattern), substitution);

}

compiled_ending_quotes_.reserve(ENDING_QUOTES.size());

for (const auto &[pattern, substitution] : ENDING_QUOTES) {

compiled_ending_quotes_.emplace_back(CompilePattern(pattern), substitution);

}

compiled_punctuation_.reserve(PUNCTUATION.size());

for (const auto &[pattern, substitution] : PUNCTUATION) {

compiled_punctuation_.emplace_back(CompilePattern(pattern), substitution);

}

compiled_parens_brackets_ = CompiledRegex(CompilePattern(PARENS_BRACKETS.first), PARENS_BRACKETS.second);

compiled_convert_parentheses_.reserve(CONVERT_PARENTHESES.size());

for (const auto &[pattern, substitution] : CONVERT_PARENTHESES) {

compiled_convert_parentheses_.emplace_back(CompilePattern(pattern), substitution);

}

compiled_double_dashes_ = CompiledRegex(CompilePattern(DOUBLE_DASHES.first), DOUBLE_DASHES.second);

compiled_contractions2_.reserve(contractions_.CONTRACTIONS2.size());

for (const auto &pattern : contractions_.CONTRACTIONS2) {

compiled_contractions2_.emplace_back(CompilePattern(pattern), R"( $1 $2 )");

}

compiled_contractions3_.reserve(contractions_.CONTRACTIONS3.size());

for (const auto &pattern : contractions_.CONTRACTIONS3) {

compiled_contractions3_.emplace_back(CompilePattern(pattern), R"( $1 $2 )");

}

}

pcre2_code *CompilePattern(const std::string &pattern) {

int errorcode = 0;

PCRE2_SIZE erroffset = 0;

pcre2_code *re = pcre2_compile(reinterpret_cast<PCRE2_SPTR>(pattern.c_str()),

PCRE2_ZERO_TERMINATED,

PCRE2_MULTILINE | PCRE2_UTF,

&errorcode,

&erroffset,

nullptr);

if (re == nullptr) {

PCRE2_UCHAR buffer[256];

pcre2_get_error_message(errorcode, buffer, sizeof(buffer));

std::cerr << "PCRE2 compilation failed at offset " << erroffset << ": " << buffer << std::endl;

return nullptr;

}

return re;

}

std::string ApplyRegex(const std::string &text, const CompiledRegex &compiled) {

if (compiled.re == nullptr) {

return text;

}

PCRE2_SPTR pcre2_subject = reinterpret_cast<PCRE2_SPTR>(text.c_str());

PCRE2_SPTR pcre2_replacement = reinterpret_cast<PCRE2_SPTR>(compiled.substitution.c_str());

size_t outlength = text.length() * 2 < 1024 ? 1024 : text.length() * 2;

auto buffer = std::make_unique<PCRE2_UCHAR[]>(outlength);

int rc = pcre2_substitute(compiled.re,

pcre2_subject,

text.length(),

0,

PCRE2_SUBSTITUTE_GLOBAL,

nullptr,

nullptr,

pcre2_replacement,

PCRE2_ZERO_TERMINATED,

buffer.get(),

&outlength);

if (rc < 0) {

return text;

}

return std::string(reinterpret_cast<char *>(buffer.get()), outlength);

}

};

// Static member definitions for NLTKWordTokenizer singleton

std::unique_ptr<NLTKWordTokenizer> NLTKWordTokenizer::instance_ = nullptr;

std::once_flag NLTKWordTokenizer::init_flag_;

void SentenceSplitter(const std::string &text, std::vector<std::string> &result) {

int error_code;

PCRE2_SIZE error_offset;

const char *pattern = R"( *[\.\?!]['"\)\]]* *)";

pcre2_code *re = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, PCRE2_MULTILINE | PCRE2_UTF, &error_code, &error_offset, nullptr);

if (re == nullptr) {

PCRE2_UCHAR buffer[256];

pcre2_get_error_message(error_code, buffer, sizeof(buffer));

std::cerr << "PCRE2 compilation failed at offset " << error_offset << ": " << buffer << std::endl;

return;

}

pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, nullptr);

PCRE2_SIZE start_offset = 0;

while (start_offset < text.size()) {

int rc = pcre2_match(re, (PCRE2_SPTR)text.c_str(), text.size(), start_offset, 0, match_data, nullptr);

if (rc < 0) {

result.push_back(text.substr(start_offset));

break;

}

PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);

PCRE2_SIZE match_start = ovector[0];

PCRE2_SIZE match_end = ovector[1];

if (match_start > start_offset) {

result.push_back(text.substr(start_offset, match_end - start_offset));

}

start_offset = match_end;

}

pcre2_match_data_free(match_data);

pcre2_code_free(re);

}

RAGAnalyzer::RAGAnalyzer(const std::string &path)

: dict_path_(path), stemmer_(std::make_unique<Stemmer>()), lowercase_string_buffer_(term_string_buffer_limit_) {

InitStemmer(STEM_LANG_ENGLISH);

}

RAGAnalyzer::RAGAnalyzer(const RAGAnalyzer &other)

: own_dict_(false), trie_(other.trie_), pos_table_(other.pos_table_), wordnet_lemma_(other.wordnet_lemma_), stemmer_(std::make_unique<Stemmer>()),

opencc_(other.opencc_), lowercase_string_buffer_(term_string_buffer_limit_), fine_grained_(other.fine_grained_) {

InitStemmer(STEM_LANG_ENGLISH);

}

RAGAnalyzer::~RAGAnalyzer() {

if (own_dict_) {

delete trie_;

delete pos_table_;

delete wordnet_lemma_;

delete opencc_;

}

}

int32_t RAGAnalyzer::Load() {

fs::path root(dict_path_);

fs::path dict_path(root / DICT_PATH);

if (!fs::exists(dict_path)) {

printf("Invalid analyzer file: %s", dict_path.string().c_str());

// return Status::InvalidAnalyzerFile(dict_path);

return -1;

}

fs::path pos_def_path(root / POS_DEF_PATH);

if (!fs::exists(pos_def_path)) {

printf("Invalid post file: %s", pos_def_path.string().c_str());

// return Status::InvalidAnalyzerFile(pos_def_path);

return -1;

}

own_dict_ = true;

trie_ = new DartsTrie();

pos_table_ = new POSTable(pos_def_path.string());

if (pos_table_->Load() != 0) {

printf("Fail to load post table: %s", pos_def_path.string().c_str());

return -1;

// return Status::InvalidAnalyzerFile("Failed to load RAGAnalyzer POS definition");

}

fs::path trie_path(root / TRIE_PATH);

if (fs::exists(trie_path)) {

trie_->Load(trie_path.string());

} else {

// Build trie

try {

std::ifstream from(dict_path.string());

std::string line;

re2::RE2 re_pattern(R"([\r\n]+)");

std::string split_pattern("([ \t])");

while (getline(from, line)) {

line = line.substr(0, line.find('\r'));

if (line.empty())

continue;

line = Replace(re_pattern, "", line);

std::vector<std::string> results;

Split(line, split_pattern, results);

if (results.size() != 3)

throw std::runtime_error("Invalid dictionary format");

int32_t freq = std::stoi(results[1]);

freq = int32_t(std::log(float(freq) / DENOMINATOR) + 0.5);

int32_t pos_idx = pos_table_->GetPOSIndex(results[2]);

int value = Encode(freq, pos_idx);

trie_->Add(results[0], value);

std::string rkey = RKey(results[0]);

trie_->Add(rkey, Encode(1, 0));

}

trie_->Build();

} catch (const std::exception &e) {

return -1;

// return Status::InvalidAnalyzerFile("Failed to load RAGAnalyzer analyzer");

}

trie_->Save(trie_path.string());

}

fs::path lemma_path(root / WORDNET_PATH);

if (!fs::exists(lemma_path)) {

printf("Fail to load wordnet: %s", lemma_path.string().c_str());

return -1;

// return Status::InvalidAnalyzerFile(lemma_path);

}

wordnet_lemma_ = new WordNetLemmatizer(lemma_path.string());

fs::path opencc_path(root / OPENCC_PATH);

if (!fs::exists(opencc_path)) {

printf("Fail to load opencc_path: %s", opencc_path.string().c_str());

return -1;

// return Status::InvalidAnalyzerFile(opencc_path);

}

try {

opencc_ = new ::OpenCC(opencc_path.string());

} catch (const std::exception &e) {

return -1;

// return Status::InvalidAnalyzerFile("Failed to load OpenCC");

}

// return Status::OK();

return 0;

}

void RAGAnalyzer::BuildPositionMapping(const std::string &original, const std::string &converted, std::vector<unsigned> &pos_mapping) {

pos_mapping.clear();

pos_mapping.resize(converted.size() + 1);

size_t orig_pos = 0;

size_t conv_pos = 0;

// Map each character position from converted string to original string

while (orig_pos < original.size() && conv_pos < converted.size()) {

// Get character lengths

size_t orig_char_len = UTF8_BYTE_LENGTH_TABLE[static_cast<uint8_t>(original[orig_pos])];

size_t conv_char_len = UTF8_BYTE_LENGTH_TABLE[static_cast<uint8_t>(converted[conv_pos])];

// Map all bytes of current converted character to current original position

for (size_t i = 0; i < conv_char_len && conv_pos + i < pos_mapping.size(); ++i) {

pos_mapping[conv_pos + i] = static_cast<unsigned>(orig_pos);

}

// Move to next character in both strings

orig_pos += orig_char_len;

conv_pos += conv_char_len;

}

// Fill any remaining positions

for (size_t i = conv_pos; i < pos_mapping.size(); ++i) {

pos_mapping[i] = static_cast<unsigned>(original.size());

}

}

std::string RAGAnalyzer::StrQ2B(const std::string &input) {

std::string output;

size_t i = 0;

while (i < input.size()) {

unsigned char c = input[i];

uint32_t codepoint = 0;

if (c < 0x80) {

codepoint = c;

i += 1;

} else if ((c & 0xE0) == 0xC0) {

codepoint = (c & 0x1F) << 6;

codepoint |= (input[i + 1] & 0x3F);

i += 2;

} else if ((c & 0xF0) == 0xE0) {

codepoint = (c & 0x0F) << 12;

codepoint |= (input[i + 1] & 0x3F) << 6;

codepoint |= (input[i + 2] & 0x3F);

i += 3;

} else {

output += c;

i += 1;

continue;

}

if (codepoint >= 0xFF01 && codepoint <= 0xFF5E) {

output += static_cast<char>(codepoint - 0xFEE0);

} else if (codepoint == 0x3000) {

output += ' ';

} else {

if (codepoint < 0x80) {

output += static_cast<char>(codepoint);

} else if (codepoint < 0x800) {

output += static_cast<char>(0xC0 | (codepoint >> 6));

output += static_cast<char>(0x80 | (codepoint & 0x3F));

} else if (codepoint < 0x10000) {

output += static_cast<char>(0xE0 | (codepoint >> 12));

output += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));

output += static_cast<char>(0x80 | (codepoint & 0x3F));

}

}

}

return output;

}

int32_t RAGAnalyzer::Freq(const std::string_view key) const {

int32_t v = trie_->Get(key);

v = DecodeFreq(v);

return static_cast<int32_t>(std::exp(v) * DENOMINATOR + 0.5);

}

std::string RAGAnalyzer::Tag(std::string_view key) const {

std::string lower_key = Key(std::string(key));

int32_t encoded_value = trie_->Get(lower_key);

if (encoded_value == -1) {

return "";

}

int32_t pos_idx = DecodePOSIndex(encoded_value);

if (pos_table_ == nullptr) {

return "";

}

const char* pos_tag = pos_table_->GetPOS(pos_idx);

return pos_tag ? std::string(pos_tag) : "";

}

std::string RAGAnalyzer::Key(const std::string_view line) { return ToLowerString(line); }

std::string RAGAnalyzer::RKey(const std::string_view line) {

std::string reversed;

reversed.reserve(line.size() + 2);

reversed += "DD";

for (size_t i = line.size(); i > 0;) {

size_t start = i - 1;

while (start > 0 && (line[start] & 0xC0) == 0x80) {

--start;

}

reversed += line.substr(start, i - start);

i = start;

}

ToLower(reversed.data() + 2, reversed.size() - 2);

return reversed;

}

std::pair<std::vector<std::string>, double> RAGAnalyzer::Score(const std::vector<std::pair<std::string, int>> &token_freqs) {

constexpr int64_t B = 30;

int64_t F = 0, L = 0;

std::vector<std::string> tokens;

tokens.reserve(token_freqs.size());

for (const auto &[token, freq_tag] : token_freqs) {

F += DecodeFreq(freq_tag);

L += (UTF8Length(token) < 2) ? 0 : 1;

tokens.push_back(token);

}

const auto score = B / static_cast<double>(tokens.size()) + L / static_cast<double>(tokens.size()) + F;

return {std::move(tokens), score};

}

void RAGAnalyzer::SortTokens(const std::vector<std::vector<std::pair<std::string, int>>> &token_list,

std::vector<std::pair<std::vector<std::string>, double>> &res) {

for (const auto &tfts : token_list) {

res.push_back(Score(tfts));

}

std::sort(res.begin(), res.end(), [](const auto &a, const auto &b) { return a.second > b.second; });

}

std::pair<std::vector<std::string>, double> RAGAnalyzer::MaxForward(const std::string &line) const {

std::vector<std::pair<std::string, int>> res;

std::size_t s = 0;

std::size_t len = UTF8Length(line);

while (s < len) {

std::size_t e = s + 1;

std::string t = UTF8Substr(line, s, e - s);

while (e < len && trie_->HasKeysWithPrefix(Key(t))) {

e += 1;

t = UTF8Substr(line, s, e - s);

}

while (e - 1 > s && trie_->Get(Key(t)) == -1) {

e -= 1;

t = UTF8Substr(line, s, e - s);

}

int v = trie_->Get(Key(t));

if (v != -1) {

res.emplace_back(std::move(t), v);

} else {

res.emplace_back(std::move(t), 0);

}

s = e;

}

return Score(res);

}

std::pair<std::vector<std::string>, double> RAGAnalyzer::MaxBackward(const std::string &line) const {

std::vector<std::pair<std::string, int>> res;

int s = UTF8Length(line) - 1;

while (s >= 0) {

const int e = s + 1;

std::string t = UTF8Substr(line, s, e - s);

while (s > 0 && trie_->HasKeysWithPrefix(RKey(t))) {

s -= 1;

t = UTF8Substr(line, s, e - s);

}

while (s + 1 < e && trie_->Get(Key(t)) == -1) {

s += 1;

t = UTF8Substr(line, s, e - s);

}

int v = trie_->Get(Key(t));

if (v != -1) {

res.emplace_back(std::move(t), v);

} else {

res.emplace_back(std::move(t), 0);

}

s -= 1;

}

std::reverse(res.begin(), res.end());

return Score(res);

}

static constexpr int MAX_DFS_DEPTH = 10;

int RAGAnalyzer::DFS(const std::string &chars,

const int s,

std::vector<std::pair<std::string, int>> &pre_tokens,

std::vector<std::vector<std::pair<std::string, int>>> &token_list,

std::vector<std::string> &best_tokens,

double &max_score,

const bool memo_all,

const int depth) const {

int res = s;

const int len = UTF8Length(chars);

// Check max recursion depth - graceful degradation like Python version

if (depth > MAX_DFS_DEPTH) {

if (s < len) {

auto pretks = pre_tokens;

std::string remaining = UTF8Substr(chars, s, len - s);

pretks.emplace_back(std::move(remaining), Encode(-12, 0));

if (memo_all) {

token_list.push_back(std::move(pretks));

} else if (auto [vec_str, current_score] = Score(pretks); current_score > max_score) {

best_tokens = std::move(vec_str);

max_score = current_score;

}

}

return len;

}

if (s >= len) {

if (memo_all) {

token_list.push_back(pre_tokens);