1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
#include <iostream>
#include <string>
#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>
#include "alignment.h"
#include "data_array.h"
#include "grammar_extractor.h"
#include "precomputation.h"
#include "suffix_array.h"
#include "translation_table.h"
namespace po = boost::program_options;
using namespace std;
int main(int argc, char** argv) {
// TODO(pauldb): Also take arguments from config file.
po::options_description desc("Command line options");
desc.add_options()
("help,h", "Show available options")
("source,f", po::value<string>(), "Source language corpus")
("target,e", po::value<string>(), "Target language corpus")
("bitext,b", po::value<string>(), "Parallel text (source ||| target)")
("alignment,a", po::value<string>()->required(), "Bitext word alignment")
("frequent", po::value<int>()->default_value(100),
"Number of precomputed frequent patterns")
("super_frequent", po::value<int>()->default_value(10),
"Number of precomputed super frequent patterns")
("max_rule_span,s", po::value<int>()->default_value(15),
"Maximum rule span")
("max_rule_symbols,l", po::value<int>()->default_value(5),
"Maximum number of symbols (terminals + nontermals) in a rule")
("min_gap_size,g", po::value<int>()->default_value(1), "Minimum gap size")
("max_phrase_len,p", po::value<int>()->default_value(4),
"Maximum frequent phrase length")
("max_nonterminals", po::value<int>()->default_value(2),
"Maximum number of nonterminals in a rule")
("min_frequency", po::value<int>()->default_value(1000),
"Minimum number of occurences for a pharse to be considered frequent")
("baeza_yates", po::value<bool>()->default_value(true),
"Use double binary search");
po::variables_map vm;
po::store(po::parse_command_line(argc, argv, desc), vm);
// Check for help argument before notify, so we don't need to pass in the
// required parameters.
if (vm.count("help")) {
cout << desc << endl;
return 0;
}
po::notify(vm);
if (!((vm.count("source") && vm.count("target")) || vm.count("bitext"))) {
cerr << "A paralel corpus is required. "
<< "Use -f (source) with -e (target) or -b (bitext)."
<< endl;
return 1;
}
shared_ptr<DataArray> source_data_array, target_data_array;
if (vm.count("bitext")) {
source_data_array = make_shared<DataArray>(
vm["bitext"].as<string>(), SOURCE);
target_data_array = make_shared<DataArray>(
vm["bitext"].as<string>(), TARGET);
} else {
source_data_array = make_shared<DataArray>(vm["source"].as<string>());
target_data_array = make_shared<DataArray>(vm["target"].as<string>());
}
shared_ptr<SuffixArray> source_suffix_array =
make_shared<SuffixArray>(source_data_array);
Alignment alignment(vm["alignment"].as<string>());
Precomputation precomputation(
source_suffix_array,
vm["frequent"].as<int>(),
vm["super_frequent"].as<int>(),
vm["max_rule_span"].as<int>(),
vm["max_rule_symbols"].as<int>(),
vm["min_gap_size"].as<int>(),
vm["max_phrase_len"].as<int>(),
vm["min_frequency"].as<int>());
TranslationTable table(source_data_array, target_data_array, alignment);
// TODO(pauldb): Add parallelization.
GrammarExtractor extractor(
source_suffix_array,
target_data_array,
alignment,
precomputation,
vm["min_gap_size"].as<int>(),
vm["max_rule_span"].as<int>(),
vm["max_nonterminals"].as<int>(),
vm["max_rule_symbols"].as<int>(),
vm["baeza_yates"].as<bool>());
string sentence;
while (getline(cin, sentence)) {
extractor.GetGrammar(sentence);
}
return 0;
}
|