From fc936db02d42cc3978a4cc2017efe7a15c78855d Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 27 May 2012 22:28:34 -0400 Subject: kbest repo --- training/kbest_repository.cc | 37 +++++++++++++++++++++++++++++++++++++ training/kbest_repository.h | 19 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 training/kbest_repository.cc create mode 100644 training/kbest_repository.h (limited to 'training') diff --git a/training/kbest_repository.cc b/training/kbest_repository.cc new file mode 100644 index 00000000..145b40a2 --- /dev/null +++ b/training/kbest_repository.cc @@ -0,0 +1,37 @@ +#include "kbest_repository.h" + +#include + +using namespace std; + +struct ApproxVectorHasher { + static const size_t MASK = 0xFFFFFFFFull; + union UType { + double f; // leave as double + size_t i; + }; + static inline double round(const double x) { + UType t; + t.f = x; + size_t r = t.i & MASK; + if ((r << 1) > MASK) + t.i += MASK - r + 1; + else + t.i &= (1ull - MASK); + return t.f; + } + size_t operator()(const SparseVector& x) const { + size_t h = 0x573915839; + for (SparseVector::const_iterator it = x.begin(); it != x.end(); ++it) { + UType t; + t.f = it->second; + if (t.f) { + size_t z = (t.i >> 32); + boost::hash_combine(h, it->first); + boost::hash_combine(h, z); + } + } + return h; + } +}; + diff --git a/training/kbest_repository.h b/training/kbest_repository.h new file mode 100644 index 00000000..0345394a --- /dev/null +++ b/training/kbest_repository.h @@ -0,0 +1,19 @@ +#ifndef _KBEST_REPOSITORY_H_ +#define _KBEST_REPOSITORY_H_ + +#include +#include "wordid.h" +#include "ns.h" +#include "sparse_vector.h" + +class KBestRepository { + struct HypInfo { + std::vector words; + SparseVector x; + SufficientStats score_stats; + }; + + std::vector candidates; +}; + +#endif -- cgit v1.2.3 From 104aad02a868c1fc6320276d9b3b9b0e1f41f457 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 27 May 2012 23:25:16 -0400 Subject: fix mapper to use common candidate set code --- pro-train/Makefile.am | 6 +- pro-train/mr_pro_map.cc | 174 +++---------------------------------------- training/Makefile.am | 30 +++++--- training/candidate_set.cc | 169 +++++++++++++++++++++++++++++++++++++++++ training/candidate_set.h | 53 +++++++++++++ training/kbest_repository.cc | 37 --------- training/kbest_repository.h | 19 ----- 7 files changed, 251 insertions(+), 237 deletions(-) create mode 100644 training/candidate_set.cc create mode 100644 training/candidate_set.h delete mode 100644 training/kbest_repository.cc delete mode 100644 training/kbest_repository.h (limited to 'training') diff --git a/pro-train/Makefile.am b/pro-train/Makefile.am index a98dd245..1e9d46b0 100644 --- a/pro-train/Makefile.am +++ b/pro-train/Makefile.am @@ -2,12 +2,10 @@ bin_PROGRAMS = \ mr_pro_map \ mr_pro_reduce -TESTS = lo_test - mr_pro_map_SOURCES = mr_pro_map.cc -mr_pro_map_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz +mr_pro_map_LDADD = $(top_srcdir)/training/libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz mr_pro_reduce_SOURCES = mr_pro_reduce.cc mr_pro_reduce_LDADD = $(top_srcdir)/training/liblbfgs/liblbfgs.a $(top_srcdir)/utils/libutils.a -lz -AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training +AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training diff --git a/pro-train/mr_pro_map.cc b/pro-train/mr_pro_map.cc index 52b67f32..2aa0dc6f 100644 --- a/pro-train/mr_pro_map.cc +++ b/pro-train/mr_pro_map.cc @@ -9,14 +9,13 @@ #include #include +#include "candidate_set.h" #include "sampler.h" #include "filelib.h" #include "stringlib.h" #include "weights.h" #include "inside_outside.h" #include "hg_io.h" -#include "kbest.h" -#include "viterbi.h" #include "ns.h" #include "ns_docscorer.h" @@ -25,52 +24,6 @@ using namespace std; namespace po = boost::program_options; -struct ApproxVectorHasher { - static const size_t MASK = 0xFFFFFFFFull; - union UType { - double f; // leave as double - size_t i; - }; - static inline double round(const double x) { - UType t; - t.f = x; - size_t r = t.i & MASK; - if ((r << 1) > MASK) - t.i += MASK - r + 1; - else - t.i &= (1ull - MASK); - return t.f; - } - size_t operator()(const SparseVector& x) const { - size_t h = 0x573915839; - for (SparseVector::const_iterator it = x.begin(); it != x.end(); ++it) { - UType t; - t.f = it->second; - if (t.f) { - size_t z = (t.i >> 32); - boost::hash_combine(h, it->first); - boost::hash_combine(h, z); - } - } - return h; - } -}; - -struct ApproxVectorEquals { - bool operator()(const SparseVector& a, const SparseVector& b) const { - SparseVector::const_iterator bit = b.begin(); - for (SparseVector::const_iterator ait = a.begin(); ait != a.end(); ++ait) { - if (bit == b.end() || - ait->first != bit->first || - ApproxVectorHasher::round(ait->second) != ApproxVectorHasher::round(bit->second)) - return false; - ++bit; - } - if (bit != b.end()) return false; - return true; - } -}; - boost::shared_ptr rng; void InitCommandLine(int argc, char** argv, po::variables_map* conf) { @@ -105,107 +58,6 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { } } -struct HypInfo { - HypInfo() : g_(-100.0f) {} - HypInfo(const vector& h, const SparseVector& feats) : hyp(h), g_(-100.0f), x(feats) {} - - // lazy evaluation - double g(const SegmentEvaluator& scorer, const EvaluationMetric* metric) const { - if (g_ == -100.0f) { - SufficientStats ss; - scorer.Evaluate(hyp, &ss); - g_ = metric->ComputeScore(ss); - } - return g_; - } - vector hyp; - mutable float g_; - SparseVector x; -}; - -struct HypInfoCompare { - bool operator()(const HypInfo& a, const HypInfo& b) const { - ApproxVectorEquals comp; - return (a.hyp == b.hyp && comp(a.x,b.x)); - } -}; - -struct HypInfoHasher { - size_t operator()(const HypInfo& x) const { - boost::hash > hhasher; - ApproxVectorHasher vhasher; - size_t ha = hhasher(x.hyp); - boost::hash_combine(ha, vhasher(x.x)); - return ha; - } -}; - -void WriteKBest(const string& file, const vector& kbest) { - WriteFile wf(file); - ostream& out = *wf.stream(); - out.precision(10); - for (int i = 0; i < kbest.size(); ++i) { - out << TD::GetString(kbest[i].hyp) << endl; - out << kbest[i].x << endl; - } -} - -void ParseSparseVector(string& line, size_t cur, SparseVector* out) { - SparseVector& x = *out; - size_t last_start = cur; - size_t last_comma = string::npos; - while(cur <= line.size()) { - if (line[cur] == ' ' || cur == line.size()) { - if (!(cur > last_start && last_comma != string::npos && cur > last_comma)) { - cerr << "[ERROR] " << line << endl << " position = " << cur << endl; - exit(1); - } - const int fid = FD::Convert(line.substr(last_start, last_comma - last_start)); - if (cur < line.size()) line[cur] = 0; - const double val = strtod(&line[last_comma + 1], NULL); - x.set_value(fid, val); - - last_comma = string::npos; - last_start = cur+1; - } else { - if (line[cur] == '=') - last_comma = cur; - } - ++cur; - } -} - -void ReadKBest(const string& file, vector* kbest) { - cerr << "Reading from " << file << endl; - ReadFile rf(file); - istream& in = *rf.stream(); - string cand; - string feats; - while(getline(in, cand)) { - getline(in, feats); - assert(in); - kbest->push_back(HypInfo()); - TD::ConvertSentence(cand, &kbest->back().hyp); - ParseSparseVector(feats, 0, &kbest->back().x); - } - cerr << " read " << kbest->size() << " hypotheses\n"; -} - -void Dedup(vector* h) { - cerr << "Dedup in=" << h->size(); - tr1::unordered_set u; - while(h->size() > 0) { - u.insert(h->back()); - h->pop_back(); - } - tr1::unordered_set::iterator it = u.begin(); - while (it != u.end()) { - h->push_back(*it); - it = u.erase(it); - } - cerr << " out=" << h->size() << endl; -} - struct ThresholdAlpha { explicit ThresholdAlpha(double t = 0.05) : threshold(t) {} double operator()(double mag) const { @@ -239,7 +91,7 @@ struct DiffOrder { void Sample(const unsigned gamma, const unsigned xi, - const vector& J_i, + const training::CandidateSet& J_i, const SegmentEvaluator& scorer, const EvaluationMetric* metric, vector* pv) { @@ -257,10 +109,10 @@ void Sample(const unsigned gamma, const float gdiff = fabs(ga - gb); if (!gdiff) continue; avg_diff += gdiff; - SparseVector xdiff = (J_i[a].x - J_i[b].x).erase_zeros(); + SparseVector xdiff = (J_i[a].fmap - J_i[b].fmap).erase_zeros(); if (xdiff.empty()) { - cerr << "Empty diff:\n " << TD::GetString(J_i[a].hyp) << endl << "x=" << J_i[a].x << endl; - cerr << " " << TD::GetString(J_i[b].hyp) << endl << "x=" << J_i[b].x << endl; + cerr << "Empty diff:\n " << TD::GetString(J_i[a].ewords) << endl << "x=" << J_i[a].fmap << endl; + cerr << " " << TD::GetString(J_i[b].ewords) << endl << "x=" << J_i[b].fmap << endl; continue; } v1.push_back(TrainingInstance(xdiff, positive, gdiff)); @@ -328,23 +180,15 @@ int main(int argc, char** argv) { is >> file >> sent_id; ReadFile rf(file); ostringstream os; - vector J_i; + training::CandidateSet J_i; os << kbest_repo << "/kbest." << sent_id << ".txt.gz"; const string kbest_file = os.str(); if (FileExists(kbest_file)) - ReadKBest(kbest_file, &J_i); + J_i.ReadFromFile(kbest_file); HypergraphIO::ReadFromJSON(rf.stream(), &hg); hg.Reweight(weights); - KBest::KBestDerivations, ESentenceTraversal> kbest(hg, kbest_size); - - for (int i = 0; i < kbest_size; ++i) { - const KBest::KBestDerivations, ESentenceTraversal>::Derivation* d = - kbest.LazyKthBest(hg.nodes_.size() - 1, i); - if (!d) break; - J_i.push_back(HypInfo(d->yield, d->feature_values)); - } - Dedup(&J_i); - WriteKBest(kbest_file, J_i); + J_i.AddKBestCandidates(hg, kbest_size); + J_i.WriteToFile(kbest_file); Sample(gamma, xi, J_i, *ds[sent_id], metric, &v); for (unsigned i = 0; i < v.size(); ++i) { diff --git a/training/Makefile.am b/training/Makefile.am index 991ac210..8124b107 100644 --- a/training/Makefile.am +++ b/training/Makefile.am @@ -23,11 +23,17 @@ noinst_PROGRAMS = \ TESTS = lbfgs_test optimize_test -mpi_online_optimize_SOURCES = mpi_online_optimize.cc online_optimizer.cc -mpi_online_optimize_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz +noinst_LIBRARIES = libtraining.a +libtraining_a_SOURCES = \ + candidate_set.cc \ + optimize.cc \ + online_optimizer.cc -mpi_flex_optimize_SOURCES = mpi_flex_optimize.cc online_optimizer.cc optimize.cc -mpi_flex_optimize_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz +mpi_online_optimize_SOURCES = mpi_online_optimize.cc +mpi_online_optimize_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz + +mpi_flex_optimize_SOURCES = mpi_flex_optimize.cc +mpi_flex_optimize_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz mpi_extract_reachable_SOURCES = mpi_extract_reachable.cc mpi_extract_reachable_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz @@ -35,8 +41,8 @@ mpi_extract_reachable_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mtev mpi_extract_features_SOURCES = mpi_extract_features.cc mpi_extract_features_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz -mpi_batch_optimize_SOURCES = mpi_batch_optimize.cc optimize.cc -mpi_batch_optimize_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz +mpi_batch_optimize_SOURCES = mpi_batch_optimize.cc +mpi_batch_optimize_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz mpi_compute_cllh_SOURCES = mpi_compute_cllh.cc mpi_compute_cllh_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz @@ -50,14 +56,14 @@ test_ngram_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteva model1_SOURCES = model1.cc ttables.cc model1_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz -lbl_model_SOURCES = lbl_model.cc optimize.cc -lbl_model_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz +lbl_model_SOURCES = lbl_model.cc +lbl_model_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz grammar_convert_SOURCES = grammar_convert.cc grammar_convert_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz -optimize_test_SOURCES = optimize_test.cc optimize.cc online_optimizer.cc -optimize_test_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz +optimize_test_SOURCES = optimize_test.cc +optimize_test_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz collapse_weights_SOURCES = collapse_weights.cc collapse_weights_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz @@ -65,8 +71,8 @@ collapse_weights_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/lib lbfgs_test_SOURCES = lbfgs_test.cc lbfgs_test_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz -mr_optimize_reduce_SOURCES = mr_optimize_reduce.cc optimize.cc -mr_optimize_reduce_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz +mr_optimize_reduce_SOURCES = mr_optimize_reduce.cc +mr_optimize_reduce_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz mr_em_map_adapter_SOURCES = mr_em_map_adapter.cc mr_em_map_adapter_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz diff --git a/training/candidate_set.cc b/training/candidate_set.cc new file mode 100644 index 00000000..5ab4558a --- /dev/null +++ b/training/candidate_set.cc @@ -0,0 +1,169 @@ +#include "candidate_set.h" + +#include + +#include + +#include "ns.h" +#include "filelib.h" +#include "wordid.h" +#include "tdict.h" +#include "hg.h" +#include "kbest.h" +#include "viterbi.h" + +using namespace std; + +namespace training { + +struct ApproxVectorHasher { + static const size_t MASK = 0xFFFFFFFFull; + union UType { + double f; // leave as double + size_t i; + }; + static inline double round(const double x) { + UType t; + t.f = x; + size_t r = t.i & MASK; + if ((r << 1) > MASK) + t.i += MASK - r + 1; + else + t.i &= (1ull - MASK); + return t.f; + } + size_t operator()(const SparseVector& x) const { + size_t h = 0x573915839; + for (SparseVector::const_iterator it = x.begin(); it != x.end(); ++it) { + UType t; + t.f = it->second; + if (t.f) { + size_t z = (t.i >> 32); + boost::hash_combine(h, it->first); + boost::hash_combine(h, z); + } + } + return h; + } +}; + +struct ApproxVectorEquals { + bool operator()(const SparseVector& a, const SparseVector& b) const { + SparseVector::const_iterator bit = b.begin(); + for (SparseVector::const_iterator ait = a.begin(); ait != a.end(); ++ait) { + if (bit == b.end() || + ait->first != bit->first || + ApproxVectorHasher::round(ait->second) != ApproxVectorHasher::round(bit->second)) + return false; + ++bit; + } + if (bit != b.end()) return false; + return true; + } +}; + +double Candidate::g(const SegmentEvaluator& scorer, const EvaluationMetric* metric) const { + if (g_ == -100.0f) { + SufficientStats ss; + scorer.Evaluate(ewords, &ss); + g_ = metric->ComputeScore(ss); + } + return g_; +} + +struct CandidateCompare { + bool operator()(const Candidate& a, const Candidate& b) const { + ApproxVectorEquals eq; + return (a.ewords == b.ewords && eq(a.fmap,b.fmap)); + } +}; + +struct CandidateHasher { + size_t operator()(const Candidate& x) const { + boost::hash > hhasher; + ApproxVectorHasher vhasher; + size_t ha = hhasher(x.ewords); + boost::hash_combine(ha, vhasher(x.fmap)); + return ha; + } +}; + +void CandidateSet::WriteToFile(const string& file) const { + WriteFile wf(file); + ostream& out = *wf.stream(); + out.precision(10); + for (unsigned i = 0; i < cs.size(); ++i) { + out << TD::GetString(cs[i].ewords) << endl; + out << cs[i].fmap << endl; + } +} + +static void ParseSparseVector(string& line, size_t cur, SparseVector* out) { + SparseVector& x = *out; + size_t last_start = cur; + size_t last_comma = string::npos; + while(cur <= line.size()) { + if (line[cur] == ' ' || cur == line.size()) { + if (!(cur > last_start && last_comma != string::npos && cur > last_comma)) { + cerr << "[ERROR] " << line << endl << " position = " << cur << endl; + exit(1); + } + const int fid = FD::Convert(line.substr(last_start, last_comma - last_start)); + if (cur < line.size()) line[cur] = 0; + const double val = strtod(&line[last_comma + 1], NULL); + x.set_value(fid, val); + + last_comma = string::npos; + last_start = cur+1; + } else { + if (line[cur] == '=') + last_comma = cur; + } + ++cur; + } +} + +void CandidateSet::ReadFromFile(const string& file) { + cerr << "Reading candidates from " << file << endl; + ReadFile rf(file); + istream& in = *rf.stream(); + string cand; + string feats; + while(getline(in, cand)) { + getline(in, feats); + assert(in); + cs.push_back(Candidate()); + TD::ConvertSentence(cand, &cs.back().ewords); + ParseSparseVector(feats, 0, &cs.back().fmap); + } + cerr << " read " << cs.size() << " candidates\n"; +} + +void CandidateSet::Dedup() { + cerr << "Dedup in=" << cs.size(); + tr1::unordered_set u; + while(cs.size() > 0) { + u.insert(cs.back()); + cs.pop_back(); + } + tr1::unordered_set::iterator it = u.begin(); + while (it != u.end()) { + cs.push_back(*it); + it = u.erase(it); + } + cerr << " out=" << cs.size() << endl; +} + +void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size) { + KBest::KBestDerivations, ESentenceTraversal> kbest(hg, kbest_size); + + for (unsigned i = 0; i < kbest_size; ++i) { + const KBest::KBestDerivations, ESentenceTraversal>::Derivation* d = + kbest.LazyKthBest(hg.nodes_.size() - 1, i); + if (!d) break; + cs.push_back(Candidate(d->yield, d->feature_values)); + } + Dedup(); +} + +} diff --git a/training/candidate_set.h b/training/candidate_set.h new file mode 100644 index 00000000..e2b0b1ba --- /dev/null +++ b/training/candidate_set.h @@ -0,0 +1,53 @@ +#ifndef _CANDIDATE_SET_H_ +#define _CANDIDATE_SET_H_ + +#include +#include + +#include "wordid.h" +#include "sparse_vector.h" + +class Hypergraph; +struct SegmentEvaluator; +struct EvaluationMetric; + +namespace training { + +struct Candidate { + Candidate() : g_(-100.0f) {} + Candidate(const std::vector& e, const SparseVector& fm) : ewords(e), fmap(fm), g_(-100.0f) {} + std::vector ewords; + SparseVector fmap; + double g(const SegmentEvaluator& scorer, const EvaluationMetric* metric) const; + void swap(Candidate& other) { + std::swap(g_, other.g_); + ewords.swap(other.ewords); + fmap.swap(other.fmap); + } + private: + mutable float g_; + //SufficientStats score_stats; +}; + +// represents some kind of collection of translation candidates, e.g. +// aggregated k-best lists, sample lists, etc. +class CandidateSet { + public: + CandidateSet() {} + inline size_t size() const { return cs.size(); } + const Candidate& operator[](size_t i) const { return cs[i]; } + + void ReadFromFile(const std::string& file); + void WriteToFile(const std::string& file) const; + void AddKBestCandidates(const Hypergraph& hg, size_t kbest_size); + // TODO add code to do unique k-best + // TODO add code to draw k samples + + private: + void Dedup(); + std::vector cs; +}; + +} + +#endif diff --git a/training/kbest_repository.cc b/training/kbest_repository.cc deleted file mode 100644 index 145b40a2..00000000 --- a/training/kbest_repository.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "kbest_repository.h" - -#include - -using namespace std; - -struct ApproxVectorHasher { - static const size_t MASK = 0xFFFFFFFFull; - union UType { - double f; // leave as double - size_t i; - }; - static inline double round(const double x) { - UType t; - t.f = x; - size_t r = t.i & MASK; - if ((r << 1) > MASK) - t.i += MASK - r + 1; - else - t.i &= (1ull - MASK); - return t.f; - } - size_t operator()(const SparseVector& x) const { - size_t h = 0x573915839; - for (SparseVector::const_iterator it = x.begin(); it != x.end(); ++it) { - UType t; - t.f = it->second; - if (t.f) { - size_t z = (t.i >> 32); - boost::hash_combine(h, it->first); - boost::hash_combine(h, z); - } - } - return h; - } -}; - diff --git a/training/kbest_repository.h b/training/kbest_repository.h deleted file mode 100644 index 0345394a..00000000 --- a/training/kbest_repository.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _KBEST_REPOSITORY_H_ -#define _KBEST_REPOSITORY_H_ - -#include -#include "wordid.h" -#include "ns.h" -#include "sparse_vector.h" - -class KBestRepository { - struct HypInfo { - std::vector words; - SparseVector x; - SufficientStats score_stats; - }; - - std::vector candidates; -}; - -#endif -- cgit v1.2.3 From ded34c668ca87b9e0a0ebca68944c6648602593a Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 28 May 2012 00:19:10 -0400 Subject: cache metric computation in pro --- pro-train/mr_pro_map.cc | 9 ++++----- training/candidate_set.cc | 39 +++++++++++++++++++-------------------- training/candidate_set.h | 18 ++++++++---------- 3 files changed, 31 insertions(+), 35 deletions(-) (limited to 'training') diff --git a/pro-train/mr_pro_map.cc b/pro-train/mr_pro_map.cc index 2aa0dc6f..bb13fdf4 100644 --- a/pro-train/mr_pro_map.cc +++ b/pro-train/mr_pro_map.cc @@ -92,7 +92,6 @@ struct DiffOrder { void Sample(const unsigned gamma, const unsigned xi, const training::CandidateSet& J_i, - const SegmentEvaluator& scorer, const EvaluationMetric* metric, vector* pv) { const bool invert_score = metric->IsErrorMetric(); @@ -102,8 +101,8 @@ void Sample(const unsigned gamma, const size_t a = rng->inclusive(0, J_i.size() - 1)(); const size_t b = rng->inclusive(0, J_i.size() - 1)(); if (a == b) continue; - float ga = J_i[a].g(scorer, metric); - float gb = J_i[b].g(scorer, metric); + float ga = metric->ComputeScore(J_i[a].score_stats); + float gb = metric->ComputeScore(J_i[b].score_stats); bool positive = gb < ga; if (invert_score) positive = !positive; const float gdiff = fabs(ga - gb); @@ -187,10 +186,10 @@ int main(int argc, char** argv) { J_i.ReadFromFile(kbest_file); HypergraphIO::ReadFromJSON(rf.stream(), &hg); hg.Reweight(weights); - J_i.AddKBestCandidates(hg, kbest_size); + J_i.AddKBestCandidates(hg, kbest_size, ds[sent_id]); J_i.WriteToFile(kbest_file); - Sample(gamma, xi, J_i, *ds[sent_id], metric, &v); + Sample(gamma, xi, J_i, metric, &v); for (unsigned i = 0; i < v.size(); ++i) { const TrainingInstance& vi = v[i]; cout << vi.y << "\t" << vi.x << endl; diff --git a/training/candidate_set.cc b/training/candidate_set.cc index 5ab4558a..e2ca9ad2 100644 --- a/training/candidate_set.cc +++ b/training/candidate_set.cc @@ -62,15 +62,6 @@ struct ApproxVectorEquals { } }; -double Candidate::g(const SegmentEvaluator& scorer, const EvaluationMetric* metric) const { - if (g_ == -100.0f) { - SufficientStats ss; - scorer.Evaluate(ewords, &ss); - g_ = metric->ComputeScore(ss); - } - return g_; -} - struct CandidateCompare { bool operator()(const Candidate& a, const Candidate& b) const { ApproxVectorEquals eq; @@ -88,16 +79,6 @@ struct CandidateHasher { } }; -void CandidateSet::WriteToFile(const string& file) const { - WriteFile wf(file); - ostream& out = *wf.stream(); - out.precision(10); - for (unsigned i = 0; i < cs.size(); ++i) { - out << TD::GetString(cs[i].ewords) << endl; - out << cs[i].fmap << endl; - } -} - static void ParseSparseVector(string& line, size_t cur, SparseVector* out) { SparseVector& x = *out; size_t last_start = cur; @@ -123,18 +104,34 @@ static void ParseSparseVector(string& line, size_t cur, SparseVector* ou } } +void CandidateSet::WriteToFile(const string& file) const { + WriteFile wf(file); + ostream& out = *wf.stream(); + out.precision(10); + string ss; + for (unsigned i = 0; i < cs.size(); ++i) { + out << TD::GetString(cs[i].ewords) << endl; + out << cs[i].fmap << endl; + cs[i].score_stats.Encode(&ss); + out << ss << endl; + } +} + void CandidateSet::ReadFromFile(const string& file) { cerr << "Reading candidates from " << file << endl; ReadFile rf(file); istream& in = *rf.stream(); string cand; string feats; + string ss; while(getline(in, cand)) { getline(in, feats); + getline(in, ss); assert(in); cs.push_back(Candidate()); TD::ConvertSentence(cand, &cs.back().ewords); ParseSparseVector(feats, 0, &cs.back().fmap); + cs.back().score_stats = SufficientStats(ss); } cerr << " read " << cs.size() << " candidates\n"; } @@ -154,7 +151,7 @@ void CandidateSet::Dedup() { cerr << " out=" << cs.size() << endl; } -void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size) { +void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size, const SegmentEvaluator* scorer) { KBest::KBestDerivations, ESentenceTraversal> kbest(hg, kbest_size); for (unsigned i = 0; i < kbest_size; ++i) { @@ -162,6 +159,8 @@ void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size) { kbest.LazyKthBest(hg.nodes_.size() - 1, i); if (!d) break; cs.push_back(Candidate(d->yield, d->feature_values)); + if (scorer) + scorer->Evaluate(d->yield, &cs.back().score_stats); } Dedup(); } diff --git a/training/candidate_set.h b/training/candidate_set.h index e2b0b1ba..824a4de2 100644 --- a/training/candidate_set.h +++ b/training/candidate_set.h @@ -4,29 +4,27 @@ #include #include +#include "ns.h" #include "wordid.h" #include "sparse_vector.h" class Hypergraph; -struct SegmentEvaluator; -struct EvaluationMetric; namespace training { struct Candidate { - Candidate() : g_(-100.0f) {} - Candidate(const std::vector& e, const SparseVector& fm) : ewords(e), fmap(fm), g_(-100.0f) {} + Candidate() {} + Candidate(const std::vector& e, const SparseVector& fm) : + ewords(e), + fmap(fm) {} std::vector ewords; SparseVector fmap; - double g(const SegmentEvaluator& scorer, const EvaluationMetric* metric) const; + SufficientStats score_stats; void swap(Candidate& other) { - std::swap(g_, other.g_); + score_stats.swap(other.score_stats); ewords.swap(other.ewords); fmap.swap(other.fmap); } - private: - mutable float g_; - //SufficientStats score_stats; }; // represents some kind of collection of translation candidates, e.g. @@ -39,7 +37,7 @@ class CandidateSet { void ReadFromFile(const std::string& file); void WriteToFile(const std::string& file) const; - void AddKBestCandidates(const Hypergraph& hg, size_t kbest_size); + void AddKBestCandidates(const Hypergraph& hg, size_t kbest_size, const SegmentEvaluator* scorer = NULL); // TODO add code to do unique k-best // TODO add code to draw k samples -- cgit v1.2.3 From 317d650f6cb1e24ac6f3be6f7bf9d4246a59e0e5 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 29 May 2012 21:39:22 -0400 Subject: add support to rampion for accumulating k-best lists --- pro-train/mr_pro_map.cc | 4 +-- rampion/Makefile.am | 4 +-- rampion/rampion.pl | 16 ++++++++-- rampion/rampion_cccp.cc | 69 ++++++++++++++++++++++++------------------- training/candidate_set.cc | 6 ++-- training/candidate_set.h | 21 +++++++++---- training/mpi_flex_optimize.cc | 10 +++---- 7 files changed, 78 insertions(+), 52 deletions(-) (limited to 'training') diff --git a/pro-train/mr_pro_map.cc b/pro-train/mr_pro_map.cc index bb13fdf4..eef40b8a 100644 --- a/pro-train/mr_pro_map.cc +++ b/pro-train/mr_pro_map.cc @@ -101,8 +101,8 @@ void Sample(const unsigned gamma, const size_t a = rng->inclusive(0, J_i.size() - 1)(); const size_t b = rng->inclusive(0, J_i.size() - 1)(); if (a == b) continue; - float ga = metric->ComputeScore(J_i[a].score_stats); - float gb = metric->ComputeScore(J_i[b].score_stats); + float ga = metric->ComputeScore(J_i[a].eval_feats); + float gb = metric->ComputeScore(J_i[b].eval_feats); bool positive = gb < ga; if (invert_score) positive = !positive; const float gdiff = fabs(ga - gb); diff --git a/rampion/Makefile.am b/rampion/Makefile.am index 12df39c2..f4dbb7cc 100644 --- a/rampion/Makefile.am +++ b/rampion/Makefile.am @@ -1,6 +1,6 @@ bin_PROGRAMS = rampion_cccp rampion_cccp_SOURCES = rampion_cccp.cc -rampion_cccp_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz +rampion_cccp_LDADD = $(top_srcdir)/training/libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz -AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training +AM_CPPFLAGS = -W -Wall $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training diff --git a/rampion/rampion.pl b/rampion/rampion.pl index 9884f453..55f7b3f1 100755 --- a/rampion/rampion.pl +++ b/rampion/rampion.pl @@ -65,12 +65,14 @@ my $cpbin=1; my $tune_regularizer = 0; my $reg = 500; my $reg_previous = 5000; +my $dont_accum = 0; # Process command-line options Getopt::Long::Configure("no_auto_abbrev"); if (GetOptions( "jobs=i" => \$jobs, "dont-clean" => \$disable_clean, + "dont-accumulate" => \$dont_accum, "pass-suffix=s" => \$pass_suffix, "qsub" => \$useqsub, "dry-run" => \$dryrun, @@ -163,8 +165,6 @@ my $decoderBase = check_output("basename $decoder"); chomp $decoderBase; my $newIniFile = "$dir/$decoderBase.ini"; my $inputFileName = "$dir/input"; my $user = $ENV{"USER"}; - - # process ini file -e $iniFile || die "Error: could not open $iniFile for reading\n"; open(INI, $iniFile); @@ -229,6 +229,13 @@ close F; unless($best_weights){ $best_weights = $weights; } unless($projected_score){ $projected_score = 0.0; } $seen_weights{$weights} = 1; +my $kbest = "$dir/kbest"; +if ($dont_accum) { + $kbest = ''; +} else { + check_call("mkdir -p $kbest"); + $kbest = "--kbest_repository $kbest"; +} my $random_seed = int(time / 1000); my $lastWeightsFile; @@ -305,7 +312,7 @@ while (1){ $cmd="$MAPINPUT $dir/hgs > $dir/agenda.$im1"; print STDERR "COMMAND:\n$cmd\n"; check_call($cmd); - $cmd="$MAPPER $refs_comma_sep -m $metric -i $dir/agenda.$im1 -w $inweights > $outweights"; + $cmd="$MAPPER $refs_comma_sep -m $metric -i $dir/agenda.$im1 $kbest -w $inweights > $outweights"; check_call($cmd); $lastWeightsFile = $outweights; $iteration++; @@ -445,6 +452,9 @@ General options: --help Print this message and exit. + --dont-accumulate + Don't accumulate k-best lists from multiple iterations. + --max-iterations Maximum number of iterations to run. If not specified, defaults to $default_max_iter. diff --git a/rampion/rampion_cccp.cc b/rampion/rampion_cccp.cc index 7a6f1f0c..1e36dc51 100644 --- a/rampion/rampion_cccp.cc +++ b/rampion/rampion_cccp.cc @@ -14,6 +14,7 @@ #include "viterbi.h" #include "ns.h" #include "ns_docscorer.h" +#include "candidate_set.h" using namespace std; namespace po = boost::program_options; @@ -25,6 +26,7 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { ("weights,w",po::value(), "[REQD] Weights files from current iterations") ("input,i",po::value()->default_value("-"), "Input file to map (- is STDIN)") ("evaluation_metric,m",po::value()->default_value("IBM_BLEU"), "Evaluation metric (ibm_bleu, koehn_bleu, nist_bleu, ter, meteor, etc.)") + ("kbest_repository,R",po::value(), "Accumulate k-best lists from previous iterations (parameter is path to repository)") ("kbest_size,k",po::value()->default_value(500u), "Top k-hypotheses to extract") ("cccp_iterations,I", po::value()->default_value(10u), "CCCP iterations (T')") ("ssd_iterations,J", po::value()->default_value(5u), "Stochastic subgradient iterations (T'')") @@ -50,38 +52,36 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { } } -struct HypInfo { - HypInfo() : g(-100.0f) {} - HypInfo(const vector& h, - const SparseVector& feats, - const SegmentEvaluator& scorer, const EvaluationMetric* metric) : hyp(h), x(feats) { - SufficientStats ss; - scorer.Evaluate(hyp, &ss); - g = metric->ComputeScore(ss); +struct GainFunction { + explicit GainFunction(const EvaluationMetric* m) : metric(m) {} + float operator()(const SufficientStats& eval_feats) const { + float g = metric->ComputeScore(eval_feats); if (!metric->IsErrorMetric()) g = 1 - g; + return g; } - - vector hyp; - float g; - SparseVector x; + const EvaluationMetric* metric; }; -void CostAugmentedSearch(const vector& kbest, +template +void CostAugmentedSearch(const GainFunc& gain, + const training::CandidateSet& cs, const SparseVector& w, double alpha, SparseVector* fmap) { unsigned best_i = 0; double best = -numeric_limits::infinity(); - for (unsigned i = 0; i < kbest.size(); ++i) { - double s = kbest[i].x.dot(w) + alpha * kbest[i].g; + for (unsigned i = 0; i < cs.size(); ++i) { + double s = cs[i].fmap.dot(w) + alpha * gain(cs[i].eval_feats); if (s > best) { best = s; best_i = i; } } - *fmap = kbest[best_i].x; + *fmap = cs[best_i].fmap; } + + // runs lines 4--15 of rampion algorithm int main(int argc, char** argv) { po::variables_map conf; @@ -97,6 +97,11 @@ int main(int argc, char** argv) { Hypergraph hg; string last_file; ReadFile in_read(conf["input"].as()); + string kbest_repo; + if (conf.count("kbest_repository")) { + kbest_repo = conf["kbest_repository"].as(); + MkDirP(kbest_repo); + } istream &in=*in_read.stream(); const unsigned kbest_size = conf["kbest_size"].as(); const unsigned tp = conf["cccp_iterations"].as(); @@ -112,40 +117,44 @@ int main(int argc, char** argv) { Weights::InitSparseVector(vweights, &weights); } string line, file; - vector > kis; + vector kis; cerr << "Loading hypergraphs...\n"; while(getline(in, line)) { istringstream is(line); int sent_id; kis.resize(kis.size() + 1); - vector& curkbest = kis.back(); + training::CandidateSet& curkbest = kis.back(); + string kbest_file; + if (kbest_repo.size()) { + ostringstream os; + os << kbest_repo << "/kbest." << sent_id << ".txt.gz"; + kbest_file = os.str(); + if (FileExists(kbest_file)) + curkbest.ReadFromFile(kbest_file); + } is >> file >> sent_id; ReadFile rf(file); if (kis.size() % 5 == 0) { cerr << '.'; } if (kis.size() % 200 == 0) { cerr << " [" << kis.size() << "]\n"; } HypergraphIO::ReadFromJSON(rf.stream(), &hg); hg.Reweight(weights); - KBest::KBestDerivations, ESentenceTraversal> kbest(hg, kbest_size); - - for (int i = 0; i < kbest_size; ++i) { - const KBest::KBestDerivations, ESentenceTraversal>::Derivation* d = - kbest.LazyKthBest(hg.nodes_.size() - 1, i); - if (!d) break; - curkbest.push_back(HypInfo(d->yield, d->feature_values, *ds[sent_id], metric)); - } + curkbest.AddKBestCandidates(hg, kbest_size, ds[sent_id]); + if (kbest_file.size()) + curkbest.WriteToFile(kbest_file); } cerr << "\nHypergraphs loaded.\n"; vector > goals(kis.size()); // f(x_i,y+,h+) SparseVector fear; // f(x,y-,h-) + const GainFunction gain(metric); for (unsigned iterp = 1; iterp <= tp; ++iterp) { cerr << "CCCP Iteration " << iterp << endl; - for (int i = 0; i < goals.size(); ++i) - CostAugmentedSearch(kis[i], weights, goodsign * alpha, &goals[i]); + for (unsigned i = 0; i < goals.size(); ++i) + CostAugmentedSearch(gain, kis[i], weights, goodsign * alpha, &goals[i]); for (unsigned iterpp = 1; iterpp <= tpp; ++iterpp) { cerr << " SSD Iteration " << iterpp << endl; - for (int i = 0; i < goals.size(); ++i) { - CostAugmentedSearch(kis[i], weights, badsign * alpha, &fear); + for (unsigned i = 0; i < goals.size(); ++i) { + CostAugmentedSearch(gain, kis[i], weights, badsign * alpha, &fear); weights -= weights * (eta * reg / goals.size()); weights += (goals[i] - fear) * eta; } diff --git a/training/candidate_set.cc b/training/candidate_set.cc index e2ca9ad2..8c086ece 100644 --- a/training/candidate_set.cc +++ b/training/candidate_set.cc @@ -112,7 +112,7 @@ void CandidateSet::WriteToFile(const string& file) const { for (unsigned i = 0; i < cs.size(); ++i) { out << TD::GetString(cs[i].ewords) << endl; out << cs[i].fmap << endl; - cs[i].score_stats.Encode(&ss); + cs[i].eval_feats.Encode(&ss); out << ss << endl; } } @@ -131,7 +131,7 @@ void CandidateSet::ReadFromFile(const string& file) { cs.push_back(Candidate()); TD::ConvertSentence(cand, &cs.back().ewords); ParseSparseVector(feats, 0, &cs.back().fmap); - cs.back().score_stats = SufficientStats(ss); + cs.back().eval_feats = SufficientStats(ss); } cerr << " read " << cs.size() << " candidates\n"; } @@ -160,7 +160,7 @@ void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size, c if (!d) break; cs.push_back(Candidate(d->yield, d->feature_values)); if (scorer) - scorer->Evaluate(d->yield, &cs.back().score_stats); + scorer->Evaluate(d->yield, &cs.back().eval_feats); } Dedup(); } diff --git a/training/candidate_set.h b/training/candidate_set.h index 824a4de2..9d326ed0 100644 --- a/training/candidate_set.h +++ b/training/candidate_set.h @@ -15,16 +15,25 @@ namespace training { struct Candidate { Candidate() {} Candidate(const std::vector& e, const SparseVector& fm) : - ewords(e), - fmap(fm) {} - std::vector ewords; - SparseVector fmap; - SufficientStats score_stats; + ewords(e), + fmap(fm) {} + Candidate(const std::vector& e, + const SparseVector& fm, + const SegmentEvaluator& se) : + ewords(e), + fmap(fm) { + se.Evaluate(ewords, &eval_feats); + } + void swap(Candidate& other) { - score_stats.swap(other.score_stats); + eval_feats.swap(other.eval_feats); ewords.swap(other.ewords); fmap.swap(other.fmap); } + + std::vector ewords; + SparseVector fmap; + SufficientStats eval_feats; }; // represents some kind of collection of translation candidates, e.g. diff --git a/training/mpi_flex_optimize.cc b/training/mpi_flex_optimize.cc index a9197208..a9ead018 100644 --- a/training/mpi_flex_optimize.cc +++ b/training/mpi_flex_optimize.cc @@ -179,18 +179,16 @@ double ApplyRegularizationTerms(const double C, const double T, const vector& weights, const vector& prev_weights, - vector* g) { - assert(weights.size() == g->size()); + double* g) { double reg = 0; for (size_t i = 0; i < weights.size(); ++i) { const double prev_w_i = (i < prev_weights.size() ? prev_weights[i] : 0.0); const double& w_i = weights[i]; - double& g_i = (*g)[i]; reg += C * w_i * w_i; - g_i += 2 * C * w_i; + g[i] += 2 * C * w_i; reg += T * (w_i - prev_w_i) * (w_i - prev_w_i); - g_i += 2 * T * (w_i - prev_w_i); + g[i] += 2 * T * (w_i - prev_w_i); } return reg; } @@ -365,7 +363,7 @@ int main(int argc, char** argv) { time_series_strength, // * (iter == 0 ? 0.0 : 1.0), cur_weights, prev_weights, - &gg); + &gg[0]); obj += r; if (mi == 0 || mi == (minibatch_iterations - 1)) { if (!mi) cerr << iter << ' '; else cerr << ' '; -- cgit v1.2.3