From 71df4aeddd3d0de93e96da0e77c5aca119fec37f Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@cs.cmu.edu>
Date: Wed, 29 Feb 2012 01:12:40 -0500
Subject: cleanup, mpi-ify lblmodel

---
 utils/corpus_tools.h | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 utils/corpus_tools.h

(limited to 'utils/corpus_tools.h')

diff --git a/utils/corpus_tools.h b/utils/corpus_tools.h
new file mode 100644
index 00000000..97bdaa94
--- /dev/null
+++ b/utils/corpus_tools.h
@@ -0,0 +1,19 @@
+#ifndef _CORPUS_TOOLS_H_
+#define _CORPUS_TOOLS_H_
+
+#include <string>
+#include <set>
+#include <vector>
+#include "wordid.h"
+
+struct CorpusTools {
+  static void ReadFromFile(const std::string& filename,
+                           std::vector<std::vector<WordID> >* src,
+                           std::set<WordID>* src_vocab = NULL,
+                           std::vector<std::vector<WordID> >* trg = NULL,
+                           std::set<WordID>* trg_vocab = NULL,
+                           int rank = 0,
+                           int size = 1);
+};
+
+#endif
-- 
cgit v1.2.3