diff options
Diffstat (limited to 'decoder')
-rw-r--r-- | decoder/dict.cc | 22 |
1 files changed, 19 insertions, 3 deletions
diff --git a/decoder/dict.cc b/decoder/dict.cc index 4a842f82..2d6986c8 100644 --- a/decoder/dict.cc +++ b/decoder/dict.cc @@ -2,10 +2,26 @@ #include <string> #include <vector> -#include <boost/regex.hpp> -#include <boost/algorithm/string/regex.hpp> + +void TokenizeStringSeparator( + const std::string& str, + const std::string& separator, + std::vector<std::string>* tokens) { + + size_t pos = 0; + std::string::size_type nextPos = str.find(separator, pos); + + while (nextPos != std::string::npos) { + tokens->push_back(str.substr(pos, nextPos - pos)); + pos = nextPos + separator.size(); + nextPos = str.find(separator, pos); + } + tokens->push_back(str.substr(pos, nextPos - pos)); +} + void Dict::AsVector(const WordID& id, std::vector<std::string>* results) const { - boost::algorithm::split_regex(*results, Convert(id), boost::regex("\\s\\|\\|\\|\\s")); + results->clear(); + TokenizeStringSeparator(Convert(id), " ||| ", results); } |