summaryrefslogtreecommitdiff
path: root/extractor/run_extractor.cc
diff options
context:
space:
mode:
authorPaul Baltescu <pauldb89@gmail.com>2013-02-19 21:23:48 +0000
committerPaul Baltescu <pauldb89@gmail.com>2013-02-19 21:23:48 +0000
commit54a1c0e2bde259e3acc9c0a8ec8da3c7704e80ca (patch)
tree6171a6a2e904d00b86ea108b2ef1d572c2d36d87 /extractor/run_extractor.cc
parent63b30ed9c8510da8c8e2f6a456576424fddacc0e (diff)
Timing every part of the extractor.
Diffstat (limited to 'extractor/run_extractor.cc')
-rw-r--r--extractor/run_extractor.cc44
1 files changed, 43 insertions, 1 deletions
diff --git a/extractor/run_extractor.cc b/extractor/run_extractor.cc
index ed30e6fe..38f10a5f 100644
--- a/extractor/run_extractor.cc
+++ b/extractor/run_extractor.cc
@@ -1,3 +1,4 @@
+#include <chrono>
#include <fstream>
#include <iostream>
#include <string>
@@ -23,6 +24,7 @@
#include "rule.h"
#include "scorer.h"
#include "suffix_array.h"
+#include "time_util.h"
#include "translation_table.h"
namespace fs = boost::filesystem;
@@ -56,6 +58,9 @@ int main(int argc, char** argv) {
"Minimum number of occurences for a pharse to be considered frequent")
("max_samples", po::value<int>()->default_value(300),
"Maximum number of samples")
+ ("fast_intersect", po::value<bool>()->default_value(false),
+ "Enable fast intersect")
+ // TODO(pauldb): Check if this works when set to false.
("tight_phrases", po::value<bool>()->default_value(true),
"False if phrases may be loose (better, but slower)")
("baeza_yates", po::value<bool>()->default_value(true),
@@ -80,6 +85,9 @@ int main(int argc, char** argv) {
return 1;
}
+ Clock::time_point preprocess_start_time = Clock::now();
+ cerr << "Reading source and target data..." << endl;
+ Clock::time_point start_time = Clock::now();
shared_ptr<DataArray> source_data_array, target_data_array;
if (vm.count("bitext")) {
source_data_array = make_shared<DataArray>(
@@ -90,13 +98,28 @@ int main(int argc, char** argv) {
source_data_array = make_shared<DataArray>(vm["source"].as<string>());
target_data_array = make_shared<DataArray>(vm["target"].as<string>());
}
+ Clock::time_point stop_time = Clock::now();
+ cerr << "Reading data took " << GetDuration(start_time, stop_time)
+ << " seconds" << endl;
+
+ cerr << "Creating source suffix array..." << endl;
+ start_time = Clock::now();
shared_ptr<SuffixArray> source_suffix_array =
make_shared<SuffixArray>(source_data_array);
+ stop_time = Clock::now();
+ cerr << "Creating suffix array took "
+ << GetDuration(start_time, stop_time) << " seconds" << endl;
-
+ cerr << "Reading alignment..." << endl;
+ start_time = Clock::now();
shared_ptr<Alignment> alignment =
make_shared<Alignment>(vm["alignment"].as<string>());
+ stop_time = Clock::now();
+ cerr << "Reading alignment took "
+ << GetDuration(start_time, stop_time) << " seconds" << endl;
+ cerr << "Precomputating collocations..." << endl;
+ start_time = Clock::now();
shared_ptr<Precomputation> precomputation = make_shared<Precomputation>(
source_suffix_array,
vm["frequent"].as<int>(),
@@ -106,10 +129,24 @@ int main(int argc, char** argv) {
vm["min_gap_size"].as<int>(),
vm["max_phrase_len"].as<int>(),
vm["min_frequency"].as<int>());
+ stop_time = Clock::now();
+ cerr << "Precomputing collocations took "
+ << GetDuration(start_time, stop_time) << " seconds" << endl;
+ cerr << "Precomputing conditional probabilities..." << endl;
+ start_time = Clock::now();
shared_ptr<TranslationTable> table = make_shared<TranslationTable>(
source_data_array, target_data_array, alignment);
+ stop_time = Clock::now();
+ cerr << "Precomputing conditional probabilities took "
+ << GetDuration(start_time, stop_time) << " seconds" << endl;
+
+ Clock::time_point preprocess_stop_time = Clock::now();
+ cerr << "Overall preprocessing step took "
+ << GetDuration(preprocess_start_time, preprocess_stop_time)
+ << " seconds" << endl;
+ Clock::time_point extraction_start_time = Clock::now();
vector<shared_ptr<Feature> > features = {
make_shared<TargetGivenSourceCoherent>(),
make_shared<SampleSourceCount>(),
@@ -133,6 +170,7 @@ int main(int argc, char** argv) {
vm["max_nonterminals"].as<int>(),
vm["max_rule_symbols"].as<int>(),
vm["max_samples"].as<int>(),
+ vm["fast_intersect"].as<bool>(),
vm["baeza_yates"].as<bool>(),
vm["tight_phrases"].as<bool>());
@@ -161,6 +199,10 @@ int main(int argc, char** argv) {
<< "\"> " << sentence << " </seg> " << suffix << endl;
++grammar_id;
}
+ Clock::time_point extraction_stop_time = Clock::now();
+ cerr << "Overall extraction step took "
+ << GetDuration(extraction_start_time, extraction_stop_time)
+ << " seconds" << endl;
return 0;
}