diff options
Diffstat (limited to 'decoder')
| -rw-r--r-- | decoder/dict.cc | 22 | 
1 files changed, 19 insertions, 3 deletions
| diff --git a/decoder/dict.cc b/decoder/dict.cc index 4a842f82..2d6986c8 100644 --- a/decoder/dict.cc +++ b/decoder/dict.cc @@ -2,10 +2,26 @@  #include <string>  #include <vector> -#include <boost/regex.hpp> -#include <boost/algorithm/string/regex.hpp> + +void TokenizeStringSeparator( +          const std::string& str, +          const std::string& separator, +          std::vector<std::string>* tokens) { + +  size_t pos = 0; +  std::string::size_type nextPos = str.find(separator, pos); + +  while (nextPos != std::string::npos) { +    tokens->push_back(str.substr(pos, nextPos - pos)); +    pos = nextPos + separator.size(); +    nextPos = str.find(separator, pos); +  } +  tokens->push_back(str.substr(pos, nextPos - pos)); +} +  void Dict::AsVector(const WordID& id, std::vector<std::string>* results) const { -  boost::algorithm::split_regex(*results, Convert(id), boost::regex("\\s\\|\\|\\|\\s")); +  results->clear(); +  TokenizeStringSeparator(Convert(id), " ||| ", results);  } | 
