summaryrefslogtreecommitdiff
path: root/utils/corpus_tools.h
diff options
context:
space:
mode:
Diffstat (limited to 'utils/corpus_tools.h')
-rw-r--r--utils/corpus_tools.h19
1 files changed, 19 insertions, 0 deletions
diff --git a/utils/corpus_tools.h b/utils/corpus_tools.h
new file mode 100644
index 00000000..97bdaa94
--- /dev/null
+++ b/utils/corpus_tools.h
@@ -0,0 +1,19 @@
+#ifndef _CORPUS_TOOLS_H_
+#define _CORPUS_TOOLS_H_
+
+#include <string>
+#include <set>
+#include <vector>
+#include "wordid.h"
+
+struct CorpusTools {
+ static void ReadFromFile(const std::string& filename,
+ std::vector<std::vector<WordID> >* src,
+ std::set<WordID>* src_vocab = NULL,
+ std::vector<std::vector<WordID> >* trg = NULL,
+ std::set<WordID>* trg_vocab = NULL,
+ int rank = 0,
+ int size = 1);
+};
+
+#endif