diff options
| author | graehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f> | 2010-07-20 23:03:54 +0000 |
|---|---|---|
| committer | graehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f> | 2010-07-20 23:03:54 +0000 |
| commit | 82e104f38ef76858588490114d01a20a489a6062 (patch) | |
| tree | f30c0d24b267593c9c5e3754f66aac8ec007ff34 /decoder/stringlib.h | |
| parent | 77b1850d21d0a6a77009e5112badaa3698151a1b (diff) | |
tokenization works. oops.
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@346 ec762483-ff6d-05da-a07a-a48fb63a330f
Diffstat (limited to 'decoder/stringlib.h')
| -rw-r--r-- | decoder/stringlib.h | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/decoder/stringlib.h b/decoder/stringlib.h index a21ffd59..a7c6c3c4 100644 --- a/decoder/stringlib.h +++ b/decoder/stringlib.h @@ -137,6 +137,11 @@ void VisitTokens(char *p,char *const end,F f) { template <class F> void VisitTokens(std::string const& s,F f) { + std::vector<std::string> ss=SplitOnWhitespace(s); + for (int i=0;i<ss.size();++i) + f(ss[i]); + return; + //FIXME: if (s.empty()) return; mutable_c_str mp(s); VisitTokens(mp.p,mp.p+s.size(),f); |