utils/corpus_tools.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

#ifndef CORPUS_TOOLS_H_
#define CORPUS_TOOLS_H_

#include <string>
#include <set>
#include <vector>
#include "wordid.h"

struct CorpusTools {
  static void ReadLine(const std::string& line,
                       std::vector<WordID>* src,
                       std::vector<WordID>* trg);

  static void ReadFromFile(const std::string& filename,
                           std::vector<std::vector<WordID> >* src,
                           std::set<WordID>* src_vocab = NULL,
                           std::vector<std::vector<WordID> >* trg = NULL,
                           std::set<WordID>* trg_vocab = NULL,
                           int rank = 0,
                           int size = 1);
};

#endif