Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support phrases replacement #139

Merged
merged 2 commits into from
Mar 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions src/Engine/PhraseReplacementMap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,6 @@ void PhraseReplacementMap::close() {
}

bool PhraseReplacementMap::load(const char* data, size_t length) {
if (mmapedFile_.data() != nullptr) {
// Cannot load while mmapedFile_ is already open.
return false;
}

if (data == nullptr || length == 0) {
return false;
}
Expand Down
61 changes: 52 additions & 9 deletions src/LanguageModelLoader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ constexpr char kUserPhraseFilename[] = "data.txt"; // same as macOS version
constexpr char kExcludedPhraseFilename[] = "exclude-phrases.txt"; // ditto
constexpr char kAssociatedPhrasesV2Path[] =
"data/mcbopomofo-associated-phrases-v2.txt";
constexpr char kPhrasesReplacementFilename[] = "phrases-replacement.txt";

LanguageModelLoader::LanguageModelLoader(
std::unique_ptr<LocalizedStrings> localizedStrings)
Expand Down Expand Up @@ -103,6 +104,7 @@ LanguageModelLoader::LanguageModelLoader(
userDataPath_ = userDataPath;
userPhrasesPath_ = userDataPath + "/" + kUserPhraseFilename;
excludedPhrasesPath_ = userDataPath + "/" + kExcludedPhraseFilename;
phrasesReplacementPath_ = userDataPath + "/" + kPhrasesReplacementFilename;
populateUserDataFilesIfNeeded();
reloadUserModelsIfNeeded();
}
Expand Down Expand Up @@ -142,35 +144,76 @@ void LanguageModelLoader::addUserPhrase(const std::string_view& reading,
}

void LanguageModelLoader::reloadUserModelsIfNeeded() {
bool shouldReload = false;
const char* userPhrasesPathPtr = nullptr;
const char* excludedPhrasesPathPtr = nullptr;
bool shouldReloadUserPhrases = false;
bool shouldReloadPhrasesReplacement = false;

if (!userPhrasesPath_.empty() && std::filesystem::exists(userPhrasesPath_)) {
std::filesystem::file_time_type t =
std::filesystem::last_write_time(userPhrasesPath_);
if (t != userPhrasesTimestamp_) {
shouldReload = true;
shouldReloadUserPhrases = true;
userPhrasesTimestamp_ = t;
FCITX_MCBOPOMOFO_INFO() << "Will load: " << userPhrasesPath_;
}
userPhrasesPathPtr = userPhrasesPath_.c_str();
}

if (!excludedPhrasesPath_.empty() &&
std::filesystem::exists(excludedPhrasesPath_)) {
std::filesystem::file_time_type t =
std::filesystem::last_write_time(excludedPhrasesPath_);
if (t != excludedPhrasesTimestamp_) {
shouldReload = true;
shouldReloadUserPhrases = true;
excludedPhrasesTimestamp_ = t;
FCITX_MCBOPOMOFO_INFO() << "Will load: " << excludedPhrasesPath_;
}
excludedPhrasesPathPtr = excludedPhrasesPath_.c_str();
}

if (shouldReload) {
lm_->loadUserPhrases(userPhrasesPathPtr, excludedPhrasesPathPtr);
// Phrases replacement is considered an advanced feature. We only enable
// it and check for updates if the file exists. If the file disappears,
// disable it.
if (!phrasesReplacementPath_.empty()) {
bool isEnabled = lm_->phraseReplacementEnabled();
bool fileExists = std::filesystem::exists(phrasesReplacementPath_);

if (isEnabled && !fileExists) {
// Disable phrases replacement now that the file is gone.
lm_->setPhraseReplacementEnabled(false);
// Reset the timestamp.
phrasesReplacementTimestamp_ = {};
FCITX_MCBOPOMOFO_INFO() << "Phrases replacement disabled, file gone: "
<< phrasesReplacementPath_;
} else if (fileExists) {
std::filesystem::file_time_type t =
std::filesystem::last_write_time(phrasesReplacementPath_);

// phrasesReplacementTimestamp_ default-inits to 0, so the reload flag
// is guaranteed to be set to true for the first time, assuming the file
// does not have a POSIX timestamp of 0, of course.
if (t != phrasesReplacementTimestamp_) {
shouldReloadPhrasesReplacement = true;
phrasesReplacementTimestamp_ = t;
}

if (shouldReloadPhrasesReplacement) {
if (isEnabled) {
FCITX_MCBOPOMOFO_INFO() << "Will reload phrases replacement file: "
<< phrasesReplacementPath_;
} else {
lm_->setPhraseReplacementEnabled(true);
FCITX_MCBOPOMOFO_INFO() << "Phrases replacement enabled, file: "
<< phrasesReplacementPath_;
}
}
}
}

if (shouldReloadUserPhrases) {
lm_->loadUserPhrases(userPhrasesPath_.c_str(),
excludedPhrasesPath_.c_str());
}

if (shouldReloadPhrasesReplacement) {
lm_->loadPhraseReplacementMap(phrasesReplacementPath_.c_str());
}
}

Expand Down
2 changes: 2 additions & 0 deletions src/LanguageModelLoader.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ class LanguageModelLoader : public UserPhraseAdder {
std::filesystem::file_time_type userPhrasesTimestamp_;
std::string excludedPhrasesPath_;
std::filesystem::file_time_type excludedPhrasesTimestamp_;
std::string phrasesReplacementPath_;
std::filesystem::file_time_type phrasesReplacementTimestamp_;
McBopomofo::InputMacroController inputMacroController_;

public:
Expand Down