major refactor, break bad circular deps

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@509 ec762483-ff6d-05da-a07a-a48fb63a330f

1 files changed, 27 insertions, 0 deletions

diff --git a/utils/dict.cc b/utils/dict.cc
new file mode 100644
index 00000000..2d6986c8
--- /dev/null
+++ b/utils/dict.cc
@@ -0,0 +1,27 @@
+#include "dict.h"
+
+#include <string>
+#include <vector>
+
+void TokenizeStringSeparator(
+          const std::string& str,
+          const std::string& separator,
+          std::vector<std::string>* tokens) {
+
+  size_t pos = 0;
+  std::string::size_type nextPos = str.find(separator, pos);
+
+  while (nextPos != std::string::npos) {
+    tokens->push_back(str.substr(pos, nextPos - pos));
+    pos = nextPos + separator.size();
+    nextPos = str.find(separator, pos);
+  }
+  tokens->push_back(str.substr(pos, nextPos - pos));
+}
+
+
+void Dict::AsVector(const WordID& id, std::vector<std::string>* results) const {
+  results->clear();
+  TokenizeStringSeparator(Convert(id), " ||| ", results);
+}
+