From 44a54840b888a9cdd6444472da88f480ceafced2 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 20 Jan 2014 20:00:16 -0500 Subject: fix for build --- extractor/Makefile.am | 11 +-- extractor/compile.cc | 194 --------------------------------------------- extractor/extract.cc | 6 +- extractor/run_extractor.cc | 4 + extractor/sacompile.cc | 194 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+), 202 deletions(-) delete mode 100644 extractor/compile.cc create mode 100644 extractor/sacompile.cc (limited to 'extractor') diff --git a/extractor/Makefile.am b/extractor/Makefile.am index e5b439f9..40112e5e 100644 --- a/extractor/Makefile.am +++ b/extractor/Makefile.am @@ -1,7 +1,5 @@ -bin_PROGRAMS = compile run_extractor extract - -if HAVE_CXX11 +bin_PROGRAMS = sacompile run_extractor extract EXTRA_PROGRAMS = alignment_test \ data_array_test \ @@ -114,8 +112,8 @@ vocabulary_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libextractor.a noinst_LIBRARIES = libextractor.a -compile_SOURCES = compile.cc -compile_LDADD = libextractor.a +sacompile_SOURCES = sacompile.cc +sacompile_LDADD = libextractor.a run_extractor_SOURCES = run_extractor.cc run_extractor_LDADD = libextractor.a extract_SOURCES = extract.cc @@ -156,6 +154,5 @@ libextractor_a_SOURCES = \ translation_table.cc \ vocabulary.cc -AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(CXX11_SWITCH) -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) +AM_CPPFLAGS = -W -Wall -Wno-sign-compare -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) AM_LDFLAGS = -fopenmp -endif diff --git a/extractor/compile.cc b/extractor/compile.cc deleted file mode 100644 index 3ee668ce..00000000 --- a/extractor/compile.cc +++ /dev/null @@ -1,194 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - -#include "alignment.h" -#include "data_array.h" -#include "precomputation.h" -#include "suffix_array.h" -#include "time_util.h" -#include "translation_table.h" -#include "vocabulary.h" - -namespace ar = boost::archive; -namespace fs = boost::filesystem; -namespace po = boost::program_options; -using namespace std; -using namespace extractor; - -int main(int argc, char** argv) { - po::options_description desc("Command line options"); - desc.add_options() - ("help,h", "Show available options") - ("source,f", po::value(), "Source language corpus") - ("target,e", po::value(), "Target language corpus") - ("bitext,b", po::value(), "Parallel text (source ||| target)") - ("alignment,a", po::value()->required(), "Bitext word alignment") - ("output,o", po::value()->required(), "Output path") - ("config,c", po::value()->required(), - "Path where the config file will be generated") - ("frequent", po::value()->default_value(100), - "Number of precomputed frequent patterns") - ("super_frequent", po::value()->default_value(10), - "Number of precomputed super frequent patterns") - ("max_rule_span,s", po::value()->default_value(15), - "Maximum rule span") - ("max_rule_symbols,l", po::value()->default_value(5), - "Maximum number of symbols (terminals + nontermals) in a rule") - ("min_gap_size,g", po::value()->default_value(1), "Minimum gap size") - ("max_phrase_len,p", po::value()->default_value(4), - "Maximum frequent phrase length") - ("min_frequency", po::value()->default_value(1000), - "Minimum number of occurrences for a pharse to be considered frequent"); - - po::variables_map vm; - po::store(po::parse_command_line(argc, argv, desc), vm); - - // Check for help argument before notify, so we don't need to pass in the - // required parameters. - if (vm.count("help")) { - cout << desc << endl; - return 0; - } - - po::notify(vm); - - if (!((vm.count("source") && vm.count("target")) || vm.count("bitext"))) { - cerr << "A paralel corpus is required. " - << "Use -f (source) with -e (target) or -b (bitext)." - << endl; - return 1; - } - - fs::path output_dir(vm["output"].as()); - if (!fs::exists(output_dir)) { - fs::create_directory(output_dir); - } - - // Reading source and target data. - Clock::time_point start_time = Clock::now(); - cerr << "Reading source and target data..." << endl; - shared_ptr source_data_array, target_data_array; - if (vm.count("bitext")) { - source_data_array = make_shared( - vm["bitext"].as(), SOURCE); - target_data_array = make_shared( - vm["bitext"].as(), TARGET); - } else { - source_data_array = make_shared(vm["source"].as()); - target_data_array = make_shared(vm["target"].as()); - } - - ofstream config_stream(vm["config"].as()); - - Clock::time_point start_write = Clock::now(); - string target_path = (output_dir / fs::path("target.bin")).string(); - config_stream << "target = " << target_path << endl; - ofstream target_fstream(target_path); - ar::binary_oarchive target_stream(target_fstream); - target_stream << *target_data_array; - Clock::time_point stop_write = Clock::now(); - double write_duration = GetDuration(start_write, stop_write); - - Clock::time_point stop_time = Clock::now(); - cerr << "Reading data took " << GetDuration(start_time, stop_time) - << " seconds" << endl; - - // Constructing and compiling the suffix array. - start_time = Clock::now(); - cerr << "Constructing source suffix array..." << endl; - shared_ptr source_suffix_array = - make_shared(source_data_array); - - start_write = Clock::now(); - string source_path = (output_dir / fs::path("source.bin")).string(); - config_stream << "source = " << source_path << endl; - ofstream source_fstream(source_path); - ar::binary_oarchive output_stream(source_fstream); - output_stream << *source_suffix_array; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - cerr << "Constructing suffix array took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - // Reading alignment. - start_time = Clock::now(); - cerr << "Reading alignment..." << endl; - shared_ptr alignment = - make_shared(vm["alignment"].as()); - - start_write = Clock::now(); - string alignment_path = (output_dir / fs::path("alignment.bin")).string(); - config_stream << "alignment = " << alignment_path << endl; - ofstream alignment_fstream(alignment_path); - ar::binary_oarchive alignment_stream(alignment_fstream); - alignment_stream << *alignment; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Reading alignment took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - shared_ptr vocabulary = make_shared(); - - start_time = Clock::now(); - cerr << "Precomputing collocations..." << endl; - Precomputation precomputation( - vocabulary, - source_suffix_array, - vm["frequent"].as(), - vm["super_frequent"].as(), - vm["max_rule_span"].as(), - vm["max_rule_symbols"].as(), - vm["min_gap_size"].as(), - vm["max_phrase_len"].as(), - vm["min_frequency"].as()); - - start_write = Clock::now(); - string precomputation_path = (output_dir / fs::path("precomp.bin")).string(); - config_stream << "precomputation = " << precomputation_path << endl; - ofstream precomp_fstream(precomputation_path); - ar::binary_oarchive precomp_stream(precomp_fstream); - precomp_stream << precomputation; - - string vocabulary_path = (output_dir / fs::path("vocab.bin")).string(); - config_stream << "vocabulary = " << vocabulary_path << endl; - ofstream vocab_fstream(vocabulary_path); - ar::binary_oarchive vocab_stream(vocab_fstream); - vocab_stream << *vocabulary; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Precomputing collocations took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - start_time = Clock::now(); - cerr << "Precomputing conditional probabilities..." << endl; - TranslationTable table(source_data_array, target_data_array, alignment); - - start_write = Clock::now(); - string table_path = (output_dir / fs::path("bilex.bin")).string(); - config_stream << "ttable = " << table_path << endl; - ofstream table_fstream(table_path); - ar::binary_oarchive table_stream(table_fstream); - table_stream << table; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Precomputing conditional probabilities took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - cerr << "Total time spent writing: " << write_duration - << " seconds" << endl; - - return 0; -} diff --git a/extractor/extract.cc b/extractor/extract.cc index 387cbe9b..e5b6f6ff 100644 --- a/extractor/extract.cc +++ b/extractor/extract.cc @@ -8,7 +8,11 @@ #include #include #include -#include +#if HAVE_OPEN_MP + #include +#else + const unsigned omp_get_num_threads() { return 1; } +#endif #include "alignment.h" #include "data_array.h" diff --git a/extractor/run_extractor.cc b/extractor/run_extractor.cc index f1aa5e35..00564a36 100644 --- a/extractor/run_extractor.cc +++ b/extractor/run_extractor.cc @@ -8,7 +8,11 @@ #include #include #include +#if HAVE_OPEN_MP #include +#else + const unsigned omp_get_num_threads() { return 1; } +#endif #include "alignment.h" #include "data_array.h" diff --git a/extractor/sacompile.cc b/extractor/sacompile.cc new file mode 100644 index 00000000..3ee668ce --- /dev/null +++ b/extractor/sacompile.cc @@ -0,0 +1,194 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include "alignment.h" +#include "data_array.h" +#include "precomputation.h" +#include "suffix_array.h" +#include "time_util.h" +#include "translation_table.h" +#include "vocabulary.h" + +namespace ar = boost::archive; +namespace fs = boost::filesystem; +namespace po = boost::program_options; +using namespace std; +using namespace extractor; + +int main(int argc, char** argv) { + po::options_description desc("Command line options"); + desc.add_options() + ("help,h", "Show available options") + ("source,f", po::value(), "Source language corpus") + ("target,e", po::value(), "Target language corpus") + ("bitext,b", po::value(), "Parallel text (source ||| target)") + ("alignment,a", po::value()->required(), "Bitext word alignment") + ("output,o", po::value()->required(), "Output path") + ("config,c", po::value()->required(), + "Path where the config file will be generated") + ("frequent", po::value()->default_value(100), + "Number of precomputed frequent patterns") + ("super_frequent", po::value()->default_value(10), + "Number of precomputed super frequent patterns") + ("max_rule_span,s", po::value()->default_value(15), + "Maximum rule span") + ("max_rule_symbols,l", po::value()->default_value(5), + "Maximum number of symbols (terminals + nontermals) in a rule") + ("min_gap_size,g", po::value()->default_value(1), "Minimum gap size") + ("max_phrase_len,p", po::value()->default_value(4), + "Maximum frequent phrase length") + ("min_frequency", po::value()->default_value(1000), + "Minimum number of occurrences for a pharse to be considered frequent"); + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + + // Check for help argument before notify, so we don't need to pass in the + // required parameters. + if (vm.count("help")) { + cout << desc << endl; + return 0; + } + + po::notify(vm); + + if (!((vm.count("source") && vm.count("target")) || vm.count("bitext"))) { + cerr << "A paralel corpus is required. " + << "Use -f (source) with -e (target) or -b (bitext)." + << endl; + return 1; + } + + fs::path output_dir(vm["output"].as()); + if (!fs::exists(output_dir)) { + fs::create_directory(output_dir); + } + + // Reading source and target data. + Clock::time_point start_time = Clock::now(); + cerr << "Reading source and target data..." << endl; + shared_ptr source_data_array, target_data_array; + if (vm.count("bitext")) { + source_data_array = make_shared( + vm["bitext"].as(), SOURCE); + target_data_array = make_shared( + vm["bitext"].as(), TARGET); + } else { + source_data_array = make_shared(vm["source"].as()); + target_data_array = make_shared(vm["target"].as()); + } + + ofstream config_stream(vm["config"].as()); + + Clock::time_point start_write = Clock::now(); + string target_path = (output_dir / fs::path("target.bin")).string(); + config_stream << "target = " << target_path << endl; + ofstream target_fstream(target_path); + ar::binary_oarchive target_stream(target_fstream); + target_stream << *target_data_array; + Clock::time_point stop_write = Clock::now(); + double write_duration = GetDuration(start_write, stop_write); + + Clock::time_point stop_time = Clock::now(); + cerr << "Reading data took " << GetDuration(start_time, stop_time) + << " seconds" << endl; + + // Constructing and compiling the suffix array. + start_time = Clock::now(); + cerr << "Constructing source suffix array..." << endl; + shared_ptr source_suffix_array = + make_shared(source_data_array); + + start_write = Clock::now(); + string source_path = (output_dir / fs::path("source.bin")).string(); + config_stream << "source = " << source_path << endl; + ofstream source_fstream(source_path); + ar::binary_oarchive output_stream(source_fstream); + output_stream << *source_suffix_array; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + cerr << "Constructing suffix array took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + // Reading alignment. + start_time = Clock::now(); + cerr << "Reading alignment..." << endl; + shared_ptr alignment = + make_shared(vm["alignment"].as()); + + start_write = Clock::now(); + string alignment_path = (output_dir / fs::path("alignment.bin")).string(); + config_stream << "alignment = " << alignment_path << endl; + ofstream alignment_fstream(alignment_path); + ar::binary_oarchive alignment_stream(alignment_fstream); + alignment_stream << *alignment; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Reading alignment took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + shared_ptr vocabulary = make_shared(); + + start_time = Clock::now(); + cerr << "Precomputing collocations..." << endl; + Precomputation precomputation( + vocabulary, + source_suffix_array, + vm["frequent"].as(), + vm["super_frequent"].as(), + vm["max_rule_span"].as(), + vm["max_rule_symbols"].as(), + vm["min_gap_size"].as(), + vm["max_phrase_len"].as(), + vm["min_frequency"].as()); + + start_write = Clock::now(); + string precomputation_path = (output_dir / fs::path("precomp.bin")).string(); + config_stream << "precomputation = " << precomputation_path << endl; + ofstream precomp_fstream(precomputation_path); + ar::binary_oarchive precomp_stream(precomp_fstream); + precomp_stream << precomputation; + + string vocabulary_path = (output_dir / fs::path("vocab.bin")).string(); + config_stream << "vocabulary = " << vocabulary_path << endl; + ofstream vocab_fstream(vocabulary_path); + ar::binary_oarchive vocab_stream(vocab_fstream); + vocab_stream << *vocabulary; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Precomputing collocations took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + start_time = Clock::now(); + cerr << "Precomputing conditional probabilities..." << endl; + TranslationTable table(source_data_array, target_data_array, alignment); + + start_write = Clock::now(); + string table_path = (output_dir / fs::path("bilex.bin")).string(); + config_stream << "ttable = " << table_path << endl; + ofstream table_fstream(table_path); + ar::binary_oarchive table_stream(table_fstream); + table_stream << table; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Precomputing conditional probabilities took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + cerr << "Total time spent writing: " << write_duration + << " seconds" << endl; + + return 0; +} -- cgit v1.2.3