From 44a54840b888a9cdd6444472da88f480ceafced2 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 20 Jan 2014 20:00:16 -0500 Subject: fix for build --- extractor/Makefile.am | 11 +-- extractor/compile.cc | 194 --------------------------------------------- extractor/extract.cc | 6 +- extractor/run_extractor.cc | 4 + extractor/sacompile.cc | 194 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+), 202 deletions(-) delete mode 100644 extractor/compile.cc create mode 100644 extractor/sacompile.cc (limited to 'extractor') diff --git a/extractor/Makefile.am b/extractor/Makefile.am index e5b439f9..40112e5e 100644 --- a/extractor/Makefile.am +++ b/extractor/Makefile.am @@ -1,7 +1,5 @@ -bin_PROGRAMS = compile run_extractor extract - -if HAVE_CXX11 +bin_PROGRAMS = sacompile run_extractor extract EXTRA_PROGRAMS = alignment_test \ data_array_test \ @@ -114,8 +112,8 @@ vocabulary_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libextractor.a noinst_LIBRARIES = libextractor.a -compile_SOURCES = compile.cc -compile_LDADD = libextractor.a +sacompile_SOURCES = sacompile.cc +sacompile_LDADD = libextractor.a run_extractor_SOURCES = run_extractor.cc run_extractor_LDADD = libextractor.a extract_SOURCES = extract.cc @@ -156,6 +154,5 @@ libextractor_a_SOURCES = \ translation_table.cc \ vocabulary.cc -AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(CXX11_SWITCH) -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) +AM_CPPFLAGS = -W -Wall -Wno-sign-compare -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) AM_LDFLAGS = -fopenmp -endif diff --git a/extractor/compile.cc b/extractor/compile.cc deleted file mode 100644 index 3ee668ce..00000000 --- a/extractor/compile.cc +++ /dev/null @@ -1,194 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - -#include "alignment.h" -#include "data_array.h" -#include "precomputation.h" -#include "suffix_array.h" -#include "time_util.h" -#include "translation_table.h" -#include "vocabulary.h" - -namespace ar = boost::archive; -namespace fs = boost::filesystem; -namespace po = boost::program_options; -using namespace std; -using namespace extractor; - -int main(int argc, char** argv) { - po::options_description desc("Command line options"); - desc.add_options() - ("help,h", "Show available options") - ("source,f", po::value(), "Source language corpus") - ("target,e", po::value(), "Target language corpus") - ("bitext,b", po::value(), "Parallel text (source ||| target)") - ("alignment,a", po::value()->required(), "Bitext word alignment") - ("output,o", po::value()->required(), "Output path") - ("config,c", po::value()->required(), - "Path where the config file will be generated") - ("frequent", po::value()->default_value(100), - "Number of precomputed frequent patterns") - ("super_frequent", po::value()->default_value(10), - "Number of precomputed super frequent patterns") - ("max_rule_span,s", po::value()->default_value(15), - "Maximum rule span") - ("max_rule_symbols,l", po::value()->default_value(5), - "Maximum number of symbols (terminals + nontermals) in a rule") - ("min_gap_size,g", po::value()->default_value(1), "Minimum gap size") - ("max_phrase_len,p", po::value()->default_value(4), - "Maximum frequent phrase length") - ("min_frequency", po::value()->default_value(1000), - "Minimum number of occurrences for a pharse to be considered frequent"); - - po::variables_map vm; - po::store(po::parse_command_line(argc, argv, desc), vm); - - // Check for help argument before notify, so we don't need to pass in the - // required parameters. - if (vm.count("help")) { - cout << desc << endl; - return 0; - } - - po::notify(vm); - - if (!((vm.count("source") && vm.count("target")) || vm.count("bitext"))) { - cerr << "A paralel corpus is required. " - << "Use -f (source) with -e (target) or -b (bitext)." - << endl; - return 1; - } - - fs::path output_dir(vm["output"].as()); - if (!fs::exists(output_dir)) { - fs::create_directory(output_dir); - } - - // Reading source and target data. - Clock::time_point start_time = Clock::now(); - cerr << "Reading source and target data..." << endl; - shared_ptr source_data_array, target_data_array; - if (vm.count("bitext")) { - source_data_array = make_shared( - vm["bitext"].as(), SOURCE); - target_data_array = make_shared( - vm["bitext"].as(), TARGET); - } else { - source_data_array = make_shared(vm["source"].as()); - target_data_array = make_shared(vm["target"].as()); - } - - ofstream config_stream(vm["config"].as()); - - Clock::time_point start_write = Clock::now(); - string target_path = (output_dir / fs::path("target.bin")).string(); - config_stream << "target = " << target_path << endl; - ofstream target_fstream(target_path); - ar::binary_oarchive target_stream(target_fstream); - target_stream << *target_data_array; - Clock::time_point stop_write = Clock::now(); - double write_duration = GetDuration(start_write, stop_write); - - Clock::time_point stop_time = Clock::now(); - cerr << "Reading data took " << GetDuration(start_time, stop_time) - << " seconds" << endl; - - // Constructing and compiling the suffix array. - start_time = Clock::now(); - cerr << "Constructing source suffix array..." << endl; - shared_ptr source_suffix_array = - make_shared(source_data_array); - - start_write = Clock::now(); - string source_path = (output_dir / fs::path("source.bin")).string(); - config_stream << "source = " << source_path << endl; - ofstream source_fstream(source_path); - ar::binary_oarchive output_stream(source_fstream); - output_stream << *source_suffix_array; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - cerr << "Constructing suffix array took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - // Reading alignment. - start_time = Clock::now(); - cerr << "Reading alignment..." << endl; - shared_ptr alignment = - make_shared(vm["alignment"].as()); - - start_write = Clock::now(); - string alignment_path = (output_dir / fs::path("alignment.bin")).string(); - config_stream << "alignment = " << alignment_path << endl; - ofstream alignment_fstream(alignment_path); - ar::binary_oarchive alignment_stream(alignment_fstream); - alignment_stream << *alignment; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Reading alignment took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - shared_ptr vocabulary = make_shared(); - - start_time = Clock::now(); - cerr << "Precomputing collocations..." << endl; - Precomputation precomputation( - vocabulary, - source_suffix_array, - vm["frequent"].as(), - vm["super_frequent"].as(), - vm["max_rule_span"].as(), - vm["max_rule_symbols"].as(), - vm["min_gap_size"].as(), - vm["max_phrase_len"].as(), - vm["min_frequency"].as()); - - start_write = Clock::now(); - string precomputation_path = (output_dir / fs::path("precomp.bin")).string(); - config_stream << "precomputation = " << precomputation_path << endl; - ofstream precomp_fstream(precomputation_path); - ar::binary_oarchive precomp_stream(precomp_fstream); - precomp_stream << precomputation; - - string vocabulary_path = (output_dir / fs::path("vocab.bin")).string(); - config_stream << "vocabulary = " << vocabulary_path << endl; - ofstream vocab_fstream(vocabulary_path); - ar::binary_oarchive vocab_stream(vocab_fstream); - vocab_stream << *vocabulary; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Precomputing collocations took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - start_time = Clock::now(); - cerr << "Precomputing conditional probabilities..." << endl; - TranslationTable table(source_data_array, target_data_array, alignment); - - start_write = Clock::now(); - string table_path = (output_dir / fs::path("bilex.bin")).string(); - config_stream << "ttable = " << table_path << endl; - ofstream table_fstream(table_path); - ar::binary_oarchive table_stream(table_fstream); - table_stream << table; - stop_write = Clock::now(); - write_duration += GetDuration(start_write, stop_write); - - stop_time = Clock::now(); - cerr << "Precomputing conditional probabilities took " - << GetDuration(start_time, stop_time) << " seconds" << endl; - - cerr << "Total time spent writing: " << write_duration - << " seconds" << endl; - - return 0; -} diff --git a/extractor/extract.cc b/extractor/extract.cc index 387cbe9b..e5b6f6ff 100644 --- a/extractor/extract.cc +++ b/extractor/extract.cc @@ -8,7 +8,11 @@ #include #include #include -#include +#if HAVE_OPEN_MP + #include +#else + const unsigned omp_get_num_threads() { return 1; } +#endif #include "alignment.h" #include "data_array.h" diff --git a/extractor/run_extractor.cc b/extractor/run_extractor.cc index f1aa5e35..00564a36 100644 --- a/extractor/run_extractor.cc +++ b/extractor/run_extractor.cc @@ -8,7 +8,11 @@ #include #include #include +#if HAVE_OPEN_MP #include +#else + const unsigned omp_get_num_threads() { return 1; } +#endif #include "alignment.h" #include "data_array.h" diff --git a/extractor/sacompile.cc b/extractor/sacompile.cc new file mode 100644 index 00000000..3ee668ce --- /dev/null +++ b/extractor/sacompile.cc @@ -0,0 +1,194 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include "alignment.h" +#include "data_array.h" +#include "precomputation.h" +#include "suffix_array.h" +#include "time_util.h" +#include "translation_table.h" +#include "vocabulary.h" + +namespace ar = boost::archive; +namespace fs = boost::filesystem; +namespace po = boost::program_options; +using namespace std; +using namespace extractor; + +int main(int argc, char** argv) { + po::options_description desc("Command line options"); + desc.add_options() + ("help,h", "Show available options") + ("source,f", po::value(), "Source language corpus") + ("target,e", po::value(), "Target language corpus") + ("bitext,b", po::value(), "Parallel text (source ||| target)") + ("alignment,a", po::value()->required(), "Bitext word alignment") + ("output,o", po::value()->required(), "Output path") + ("config,c", po::value()->required(), + "Path where the config file will be generated") + ("frequent", po::value()->default_value(100), + "Number of precomputed frequent patterns") + ("super_frequent", po::value()->default_value(10), + "Number of precomputed super frequent patterns") + ("max_rule_span,s", po::value()->default_value(15), + "Maximum rule span") + ("max_rule_symbols,l", po::value()->default_value(5), + "Maximum number of symbols (terminals + nontermals) in a rule") + ("min_gap_size,g", po::value()->default_value(1), "Minimum gap size") + ("max_phrase_len,p", po::value()->default_value(4), + "Maximum frequent phrase length") + ("min_frequency", po::value()->default_value(1000), + "Minimum number of occurrences for a pharse to be considered frequent"); + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + + // Check for help argument before notify, so we don't need to pass in the + // required parameters. + if (vm.count("help")) { + cout << desc << endl; + return 0; + } + + po::notify(vm); + + if (!((vm.count("source") && vm.count("target")) || vm.count("bitext"))) { + cerr << "A paralel corpus is required. " + << "Use -f (source) with -e (target) or -b (bitext)." + << endl; + return 1; + } + + fs::path output_dir(vm["output"].as()); + if (!fs::exists(output_dir)) { + fs::create_directory(output_dir); + } + + // Reading source and target data. + Clock::time_point start_time = Clock::now(); + cerr << "Reading source and target data..." << endl; + shared_ptr source_data_array, target_data_array; + if (vm.count("bitext")) { + source_data_array = make_shared( + vm["bitext"].as(), SOURCE); + target_data_array = make_shared( + vm["bitext"].as(), TARGET); + } else { + source_data_array = make_shared(vm["source"].as()); + target_data_array = make_shared(vm["target"].as()); + } + + ofstream config_stream(vm["config"].as()); + + Clock::time_point start_write = Clock::now(); + string target_path = (output_dir / fs::path("target.bin")).string(); + config_stream << "target = " << target_path << endl; + ofstream target_fstream(target_path); + ar::binary_oarchive target_stream(target_fstream); + target_stream << *target_data_array; + Clock::time_point stop_write = Clock::now(); + double write_duration = GetDuration(start_write, stop_write); + + Clock::time_point stop_time = Clock::now(); + cerr << "Reading data took " << GetDuration(start_time, stop_time) + << " seconds" << endl; + + // Constructing and compiling the suffix array. + start_time = Clock::now(); + cerr << "Constructing source suffix array..." << endl; + shared_ptr source_suffix_array = + make_shared(source_data_array); + + start_write = Clock::now(); + string source_path = (output_dir / fs::path("source.bin")).string(); + config_stream << "source = " << source_path << endl; + ofstream source_fstream(source_path); + ar::binary_oarchive output_stream(source_fstream); + output_stream << *source_suffix_array; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + cerr << "Constructing suffix array took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + // Reading alignment. + start_time = Clock::now(); + cerr << "Reading alignment..." << endl; + shared_ptr alignment = + make_shared(vm["alignment"].as()); + + start_write = Clock::now(); + string alignment_path = (output_dir / fs::path("alignment.bin")).string(); + config_stream << "alignment = " << alignment_path << endl; + ofstream alignment_fstream(alignment_path); + ar::binary_oarchive alignment_stream(alignment_fstream); + alignment_stream << *alignment; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Reading alignment took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + shared_ptr vocabulary = make_shared(); + + start_time = Clock::now(); + cerr << "Precomputing collocations..." << endl; + Precomputation precomputation( + vocabulary, + source_suffix_array, + vm["frequent"].as(), + vm["super_frequent"].as(), + vm["max_rule_span"].as(), + vm["max_rule_symbols"].as(), + vm["min_gap_size"].as(), + vm["max_phrase_len"].as(), + vm["min_frequency"].as()); + + start_write = Clock::now(); + string precomputation_path = (output_dir / fs::path("precomp.bin")).string(); + config_stream << "precomputation = " << precomputation_path << endl; + ofstream precomp_fstream(precomputation_path); + ar::binary_oarchive precomp_stream(precomp_fstream); + precomp_stream << precomputation; + + string vocabulary_path = (output_dir / fs::path("vocab.bin")).string(); + config_stream << "vocabulary = " << vocabulary_path << endl; + ofstream vocab_fstream(vocabulary_path); + ar::binary_oarchive vocab_stream(vocab_fstream); + vocab_stream << *vocabulary; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Precomputing collocations took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + start_time = Clock::now(); + cerr << "Precomputing conditional probabilities..." << endl; + TranslationTable table(source_data_array, target_data_array, alignment); + + start_write = Clock::now(); + string table_path = (output_dir / fs::path("bilex.bin")).string(); + config_stream << "ttable = " << table_path << endl; + ofstream table_fstream(table_path); + ar::binary_oarchive table_stream(table_fstream); + table_stream << table; + stop_write = Clock::now(); + write_duration += GetDuration(start_write, stop_write); + + stop_time = Clock::now(); + cerr << "Precomputing conditional probabilities took " + << GetDuration(start_time, stop_time) << " seconds" << endl; + + cerr << "Total time spent writing: " << write_duration + << " seconds" << endl; + + return 0; +} -- cgit v1.2.3 From ff248364f8385ec2b60a7e7bfbab260c2695116d Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 20 Jan 2014 20:25:00 -0500 Subject: update date, copyright --- configure.ac | 2 +- decoder/decoder.cc | 2 +- extractor/Makefile.am | 36 +++++++++++++++++++++++++++++++++++- training/liblbfgs/Makefile.am | 7 +++---- 4 files changed, 40 insertions(+), 7 deletions(-) (limited to 'extractor') diff --git a/configure.ac b/configure.ac index d5261b8e..e5d2dadb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_CONFIG_MACRO_DIR([m4]) -AC_INIT([cdec],[2014-01-19]) +AC_INIT([cdec],[2014-01-20]) AC_CONFIG_SRCDIR([decoder/cdec.cc]) AM_INIT_AUTOMAKE AC_CONFIG_HEADERS(config.h) diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 5bb62710..e02c7730 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -86,7 +86,7 @@ struct ELengthWeightFunction { } }; inline void ShowBanner() { - cerr << "cdec (c) 2009--2013 by Chris Dyer\n"; + cerr << "cdec (c) 2009--2014 by Chris Dyer\n"; } inline string str(char const* name,po::variables_map const& conf) { diff --git a/extractor/Makefile.am b/extractor/Makefile.am index 40112e5e..a4ffac11 100644 --- a/extractor/Makefile.am +++ b/extractor/Makefile.am @@ -132,6 +132,14 @@ libextractor_a_SOURCES = \ features/max_lex_target_given_source.cc \ features/sample_source_count.cc \ features/target_given_source_coherent.cc \ + features/count_source_target.h \ + features/feature.h \ + features/is_source_singleton.h \ + features/is_source_target_singleton.h \ + features/max_lex_source_given_target.h \ + features/max_lex_target_given_source.h \ + features/sample_source_count.h \ + features/target_given_source_coherent.h \ grammar.cc \ grammar_extractor.cc \ matchings_finder.cc \ @@ -152,7 +160,33 @@ libextractor_a_SOURCES = \ target_phrase_extractor.cc \ time_util.cc \ translation_table.cc \ - vocabulary.cc + vocabulary.cc \ + alignment.h \ + backoff_sampler.h \ + data_array.h \ + fast_intersector.h \ + grammar.h \ + grammar_extractor.h \ + matchings_finder.h \ + matchings_sampler.h \ + matchings_trie.h \ + phrase.h \ + phrase_builder.h \ + phrase_location.h \ + phrase_location_sampler.h \ + precomputation.h \ + rule.h \ + rule_extractor.h \ + rule_extractor_helper.h \ + rule_factory.h \ + sampler.h \ + scorer.h \ + suffix_array.h \ + suffix_array_sampler.h \ + target_phrase_extractor.h \ + time_util.h \ + translation_table.h \ + vocabulary.h AM_CPPFLAGS = -W -Wall -Wno-sign-compare -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) AM_LDFLAGS = -fopenmp diff --git a/training/liblbfgs/Makefile.am b/training/liblbfgs/Makefile.am index 272d6f56..aa3af377 100644 --- a/training/liblbfgs/Makefile.am +++ b/training/liblbfgs/Makefile.am @@ -1,10 +1,9 @@ -TESTS = ll_test -noinst_PROGRAMS = ll_test +noinst_LIBRARIES = liblbfgs.a +noinst_PROGRAMS = ll_test ll_test_SOURCES = ll_test.cc ll_test_LDADD = liblbfgs.a -lz - -noinst_LIBRARIES = liblbfgs.a +TESTS = ll_test liblbfgs_a_SOURCES = \ lbfgs.c \ -- cgit v1.2.3 From 8d9cbc6605685401e14f9ed94c5be19c3c73a497 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 20 Jan 2014 20:35:10 -0500 Subject: fix openmp flag usage --- extractor/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'extractor') diff --git a/extractor/Makefile.am b/extractor/Makefile.am index a4ffac11..a406d9dc 100644 --- a/extractor/Makefile.am +++ b/extractor/Makefile.am @@ -188,5 +188,5 @@ libextractor_a_SOURCES = \ translation_table.h \ vocabulary.h -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -fopenmp $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) -AM_LDFLAGS = -fopenmp +AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(OPENMP_CXXFLAGS) $(GTEST_CPPFLAGS) $(GMOCK_CPPFLAGS) +AM_LDFLAGS = $(OPENMP_CXXFLAGS) -- cgit v1.2.3