From 42c1346c6bce064601beb81bb954ea5e30e9f43d Mon Sep 17 00:00:00 2001 From: graehl Date: Tue, 20 Jul 2010 23:03:54 +0000 Subject: tokenization works. oops. git-svn-id: https://ws10smt.googlecode.com/svn/trunk@346 ec762483-ff6d-05da-a07a-a48fb63a330f --- decoder/stringlib.h | 5 +++++ decoder/tdict.cc | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'decoder') diff --git a/decoder/stringlib.h b/decoder/stringlib.h index a21ffd59..a7c6c3c4 100644 --- a/decoder/stringlib.h +++ b/decoder/stringlib.h @@ -137,6 +137,11 @@ void VisitTokens(char *p,char *const end,F f) { template void VisitTokens(std::string const& s,F f) { + std::vector ss=SplitOnWhitespace(s); + for (int i=0;i Ws; Ws *ids; explicit add_wordids(Ws *i) : ids(i) { } + add_wordids(const add_wordids& o) : ids(o.ids) { } void operator()(char const* s) { ids->push_back(TD::Convert(s)); } + void operator()(std::string const& s) { + ids->push_back(TD::Convert(s)); + } }; } -- cgit v1.2.3