fix: don't compress the token during collecting dict entries (#762) (#…

…768)

follow the default split behavior of boost::split

src/rime/algo/encoder.cc

@@ -20,7 +20,8 @@ string RawCode::ToString() const {
 }
 
 void RawCode::FromString(const string& code_str) {
-  *dynamic_cast<vector<string>*>(this) = strings::split(code_str, " ");
+  *dynamic_cast<vector<string>*>(this) =
+      strings::split(code_str, " ", strings::SplitBehavior::SkipToken);
 }
 
 TableEncoder::TableEncoder(PhraseCollector* collector)

src/rime/algo/strings.cc

@@ -8,7 +8,7 @@ vector<string> split(const string& str,
                      SplitBehavior behavior) {
   vector<string> strings;
   size_t lastPos, pos;
-  if (behavior == SplitBehavior::SkipEmpty) {
+  if (behavior == SplitBehavior::SkipToken) {
     lastPos = str.find_first_not_of(delim, 0);
   } else {
     lastPos = 0;
@@ -17,7 +17,7 @@ vector<string> split(const string& str,
 
   while (std::string::npos != pos || std::string::npos != lastPos) {
     strings.emplace_back(str.substr(lastPos, pos - lastPos));
-    if (behavior == SplitBehavior::SkipEmpty) {
+    if (behavior == SplitBehavior::SkipToken) {
       lastPos = str.find_first_not_of(delim, pos);
     } else {
       if (pos == std::string::npos) {
@@ -31,7 +31,7 @@ vector<string> split(const string& str,
 };
 
 vector<string> split(const string& str, const string& delim) {
-  return split(str, delim, SplitBehavior::SkipEmpty);
+  return split(str, delim, SplitBehavior::KeepToken);
 };
 
 }  // namespace strings

src/rime/algo/strings.h

@@ -7,7 +7,7 @@
 namespace rime {
 namespace strings {
 
-enum class SplitBehavior { KeepEmpty, SkipEmpty };
+enum class SplitBehavior { KeepToken, SkipToken };
 
 vector<string> split(const string& str,
                      const string& delim,