Timing every part of the extractor.

1 files changed, 43 insertions, 1 deletions

diff --git a/extractor/run_extractor.cc b/extractor/run_extractor.cc
index ed30e6fe..38f10a5f 100644
--- a/extractor/run_extractor.cc
+++ b/extractor/run_extractor.cc
@@ -1,3 +1,4 @@
+#include <chrono>
 #include <fstream>
 #include <iostream>
 #include <string>
@@ -23,6 +24,7 @@
 #include "rule.h"
 #include "scorer.h"
 #include "suffix_array.h"
+#include "time_util.h"
 #include "translation_table.h"
 
 namespace fs = boost::filesystem;
@@ -56,6 +58,9 @@ int main(int argc, char** argv) {
         "Minimum number of occurences for a pharse to be considered frequent")
     ("max_samples", po::value<int>()->default_value(300),
         "Maximum number of samples")
+    ("fast_intersect", po::value<bool>()->default_value(false),
+        "Enable fast intersect")
+    // TODO(pauldb): Check if this works when set to false.
     ("tight_phrases", po::value<bool>()->default_value(true),
         "False if phrases may be loose (better, but slower)")
     ("baeza_yates", po::value<bool>()->default_value(true),
@@ -80,6 +85,9 @@ int main(int argc, char** argv) {
     return 1;
   }
 
+  Clock::time_point preprocess_start_time = Clock::now();
+  cerr << "Reading source and target data..." << endl;
+  Clock::time_point start_time = Clock::now();
   shared_ptr<DataArray> source_data_array, target_data_array;
   if (vm.count("bitext")) {
     source_data_array = make_shared<DataArray>(
@@ -90,13 +98,28 @@ int main(int argc, char** argv) {
     source_data_array = make_shared<DataArray>(vm["source"].as<string>());
     target_data_array = make_shared<DataArray>(vm["target"].as<string>());
   }
+  Clock::time_point stop_time = Clock::now();
+  cerr << "Reading data took " << GetDuration(start_time, stop_time)
+       << " seconds" << endl;
+
+  cerr << "Creating source suffix array..." << endl;
+  start_time = Clock::now();
   shared_ptr<SuffixArray> source_suffix_array =
       make_shared<SuffixArray>(source_data_array);
+  stop_time = Clock::now();
+  cerr << "Creating suffix array took "
+       << GetDuration(start_time, stop_time) << " seconds" << endl;
 
-
+  cerr << "Reading alignment..." << endl;
+  start_time = Clock::now();
   shared_ptr<Alignment> alignment =
       make_shared<Alignment>(vm["alignment"].as<string>());
+  stop_time = Clock::now();
+  cerr << "Reading alignment took "
+       << GetDuration(start_time, stop_time) << " seconds" << endl;
 
+  cerr << "Precomputating collocations..." << endl;
+  start_time = Clock::now();
   shared_ptr<Precomputation> precomputation = make_shared<Precomputation>(
       source_suffix_array,
       vm["frequent"].as<int>(),
@@ -106,10 +129,24 @@ int main(int argc, char** argv) {
       vm["min_gap_size"].as<int>(),
       vm["max_phrase_len"].as<int>(),
       vm["min_frequency"].as<int>());
+  stop_time = Clock::now();
+  cerr << "Precomputing collocations took "
+       << GetDuration(start_time, stop_time) << " seconds" << endl;
 
+  cerr << "Precomputing conditional probabilities..." << endl;
+  start_time = Clock::now();
   shared_ptr<TranslationTable> table = make_shared<TranslationTable>(
       source_data_array, target_data_array, alignment);
+  stop_time = Clock::now();
+  cerr << "Precomputing conditional probabilities took "
+       << GetDuration(start_time, stop_time) << " seconds" << endl;
+
+  Clock::time_point preprocess_stop_time = Clock::now();
+  cerr << "Overall preprocessing step took "
+       << GetDuration(preprocess_start_time, preprocess_stop_time)
+       << " seconds" << endl;
 
+  Clock::time_point extraction_start_time = Clock::now();
   vector<shared_ptr<Feature> > features = {
       make_shared<TargetGivenSourceCoherent>(),
       make_shared<SampleSourceCount>(),
@@ -133,6 +170,7 @@ int main(int argc, char** argv) {
       vm["max_nonterminals"].as<int>(),
       vm["max_rule_symbols"].as<int>(),
       vm["max_samples"].as<int>(),
+      vm["fast_intersect"].as<bool>(),
       vm["baeza_yates"].as<bool>(),
       vm["tight_phrases"].as<bool>());
 
@@ -161,6 +199,10 @@ int main(int argc, char** argv) {
          << "\"> " << sentence << " </seg> " << suffix << endl;
     ++grammar_id;
   }
+  Clock::time_point extraction_stop_time = Clock::now();
+  cerr << "Overall extraction step took "
+       << GetDuration(extraction_start_time, extraction_stop_time)
+       << " seconds" << endl;
 
   return 0;
 }