factor forest stats (show size post pruning, and portion kept) and cdec --scale_prune_srclen

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@96 ec762483-ff6d-05da-a07a-a48fb63a330f

1 files changed, 7 insertions, 0 deletions

diff --git a/decoder/stringlib.h b/decoder/stringlib.h
index 76efee8f..22863945 100644
--- a/decoder/stringlib.h
+++ b/decoder/stringlib.h
@@ -36,6 +36,13 @@ inline void Tokenize(const std::string& str, char delimiter, std::vector<std::st
     res->push_back(&s[last]);
 }
 
+inline unsigned NTokens(const std::string& str, char delimiter)
+{
+  std::vector<std::string> r;
+  Tokenize(str,delimiter,&r);
+  return r.size();
+}
+
 inline std::string LowercaseString(const std::string& in) {
   std::string res(in.size(),' ');
   for (int i = 0; i < in.size(); ++i)