From 9b83a2e82aba73b5ff0e848182a8726481a10485 Mon Sep 17 00:00:00 2001
From: Chris Dyer <redpony@gmail.com>
Date: Sun, 9 Feb 2014 20:50:41 -0500
Subject: adaptive hope-fear learner
---
utils/stringlib.h | 15 +++++++++++++++
utils/tdict.cc | 4 ++--
utils/tdict.h | 2 +-
3 files changed, 18 insertions(+), 3 deletions(-)
(limited to 'utils')
diff --git a/utils/stringlib.h b/utils/stringlib.h
index f60b7867..2fdbfff8 100644
--- a/utils/stringlib.h
+++ b/utils/stringlib.h
@@ -242,6 +242,21 @@ void VisitTokens(std::string const& s,F f) {
VisitTokens(mp.p,mp.p+s.size(),f);
}
+template <class F>
+void VisitTokens(std::string const& s,F f, unsigned start) {
+ if (0) {
+ std::vector<std::string> ss=SplitOnWhitespace(s);
+ for (unsigned i=0;i<ss.size();++i)
+ f(ss[i]);
+ return;
+ }
+ //FIXME:
+ if (s.empty()) return;
+ mutable_c_str mp(s);
+ SLIBDBG("mp="<<mp.p);
+ VisitTokens(mp.p+start,mp.p+s.size(),f);
+}
+
inline void SplitCommandAndParam(const std::string& in, std::string* cmd, std::string* param) {
cmd->clear();
param->clear();
diff --git a/utils/tdict.cc b/utils/tdict.cc
index fd2b76cb..c99f1697 100644
--- a/utils/tdict.cc
+++ b/utils/tdict.cc
@@ -70,7 +70,7 @@ struct add_wordids {
}
-void TD::ConvertSentence(std::string const& s, std::vector<WordID>* ids) {
+void TD::ConvertSentence(std::string const& s, std::vector<WordID>* ids, unsigned start) {
ids->clear();
- VisitTokens(s,add_wordids(ids));
+ VisitTokens(s,add_wordids(ids),start);
}
diff --git a/utils/tdict.h b/utils/tdict.h
index 03afc2e6..bb19ecd5 100644
--- a/utils/tdict.h
+++ b/utils/tdict.h
@@ -9,7 +9,7 @@
struct TD {
static WordID end(); // next id to be assigned; [begin,end) give the non-reserved tokens seen so far
- static void ConvertSentence(std::string const& sent, std::vector<WordID>* ids);
+ static void ConvertSentence(std::string const& sent, std::vector<WordID>* ids, unsigned start=0);
static void GetWordIDs(const std::vector<std::string>& strings, std::vector<WordID>* ids);
static std::string GetString(const std::vector<WordID>& str);
static std::string GetString(WordID const* i,WordID const* e);
--
cgit v1.2.3