From 9b3306332a27f23a36e96a93b5ff97caee1b6e3c Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 25 Nov 2012 00:28:07 -0500 Subject: optionally optimize diagonal tension --- word-aligner/da.h | 10 +++++----- word-aligner/fast_align.cc | 47 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/word-aligner/da.h b/word-aligner/da.h index 5e6dfa92..c979b641 100644 --- a/word-aligner/da.h +++ b/word-aligner/da.h @@ -17,7 +17,7 @@ struct DiagonalAlignment { assert(m >= i); assert(n >= j); #endif - return exp(feat(i, j, m, n) * alpha); + return exp(Feature(i, j, m, n) * alpha); } static double ComputeZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) { @@ -51,21 +51,21 @@ struct DiagonalAlignment { double pct = 0; double pcb = 0; if (num_top) { - pct = arithmetico_geometric_series(feat(i, ceil, m, n), UnnormalizedProb(i, ceil, m, n, alpha), ratio, d, num_top); + pct = arithmetico_geometric_series(Feature(i, ceil, m, n), UnnormalizedProb(i, ceil, m, n, alpha), ratio, d, num_top); //cerr << "PCT = " << pct << endl; } if (floor) { - pcb = arithmetico_geometric_series(feat(i, floor, m, n), UnnormalizedProb(i, floor, m, n, alpha), ratio, d, floor); + pcb = arithmetico_geometric_series(Feature(i, floor, m, n), UnnormalizedProb(i, floor, m, n, alpha), ratio, d, floor); //cerr << "PCB = " << pcb << endl; } return (pct + pcb) / z; } - private: - inline static double feat(const unsigned i, const unsigned j, const unsigned m, const unsigned n) { + inline static double Feature(const unsigned i, const unsigned j, const unsigned m, const unsigned n) { return -fabs(double(j) / n - double(i) / m); } + private: inline static double arithmetico_geometric_series(const double a_1, const double g_1, const double r, const double d, const unsigned n) { const double g_np1 = g_1 * pow(r, n); const double a_n = d * (n - 1) + a_1; diff --git a/word-aligner/fast_align.cc b/word-aligner/fast_align.cc index 14f7cac8..9d698074 100644 --- a/word-aligner/fast_align.cc +++ b/word-aligner/fast_align.cc @@ -1,6 +1,9 @@ #include #include +#include +#include +#include #include #include @@ -14,6 +17,7 @@ namespace po = boost::program_options; using namespace std; +using namespace std::tr1; bool InitCommandLine(int argc, char** argv, po::variables_map* conf) { po::options_description opts("Configuration options"); @@ -25,6 +29,7 @@ bool InitCommandLine(int argc, char** argv, po::variables_map* conf) { ("favor_diagonal,d", "Use a static alignment distribution that assigns higher probabilities to alignments near the diagonal") ("prob_align_null", po::value()->default_value(0.08), "When --favor_diagonal is set, what's the probability of a null alignment?") ("diagonal_tension,T", po::value()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)") + ("optimize_tension,o", "Optimize diagonal tension during EM") ("variational_bayes,v","Infer VB estimate of parameters under a symmetric Dirichlet prior") ("alpha,a", po::value()->default_value(0.01), "Hyperparameter for optional Dirichlet prior") ("no_null_word,N","Do not generate from a null token") @@ -68,7 +73,8 @@ int main(int argc, char** argv) { const bool add_viterbi = (conf.count("no_add_viterbi") == 0); const bool variational_bayes = (conf.count("variational_bayes") > 0); const bool write_alignments = (conf.count("output_parameters") == 0); - const double diagonal_tension = conf["diagonal_tension"].as(); + double diagonal_tension = conf["diagonal_tension"].as(); + bool optimize_tension = conf.count("optimize_tension"); const bool hide_training_alignments = (conf.count("hide_training_alignments") > 0); string testset; if (conf.count("testset")) testset = conf["testset"].as(); @@ -83,8 +89,10 @@ int main(int argc, char** argv) { TTable s2t, t2s; TTable::Word2Word2Double s2t_viterbi; + unordered_map, unsigned, boost::hash > > size_counts; double tot_len_ratio = 0; double mean_srclen_multiplier = 0; + vector probs; for (int iter = 0; iter < ITERATIONS; ++iter) { const bool final_iteration = (iter == (ITERATIONS - 1)); cerr << "ITERATION " << (iter + 1) << (final_iteration ? " (FINAL)" : "") << endl; @@ -98,6 +106,7 @@ int main(int argc, char** argv) { string ssrc, strg; vector src, trg; double c0 = 0; + double emp_feat = 0; double toks = 0; while(true) { getline(in, line); @@ -115,7 +124,9 @@ int main(int argc, char** argv) { if (iter == 0) tot_len_ratio += static_cast(trg.size()) / static_cast(src.size()); denom += trg.size(); - vector probs(src.size() + 1); + probs.resize(src.size() + 1); + if (iter == 0) + ++size_counts[make_pair(trg.size(), src.size())]; bool first_al = true; // used for write_alignments toks += trg.size(); for (unsigned j = 0; j < trg.size(); ++j) { @@ -170,8 +181,11 @@ int main(int argc, char** argv) { c0 += count; s2t.Increment(kNULL, f_j, count); } - for (unsigned i = 1; i <= src.size(); ++i) - s2t.Increment(src[i-1], f_j, probs[i] / sum); + for (unsigned i = 1; i <= src.size(); ++i) { + const double p = probs[i] / sum; + s2t.Increment(src[i-1], f_j, p); + emp_feat += DiagonalAlignment::Feature(j, i, trg.size(), src.size()) * p; + } } likelihood += log(sum); } @@ -186,17 +200,38 @@ int main(int argc, char** argv) { mean_srclen_multiplier = tot_len_ratio / lc; cerr << "expected target length = source length * " << mean_srclen_multiplier << endl; } + emp_feat /= toks; cerr << " log_e likelihood: " << likelihood << endl; cerr << " log_2 likelihood: " << base2_likelihood << endl; cerr << " cross entropy: " << (-base2_likelihood / denom) << endl; cerr << " perplexity: " << pow(2.0, -base2_likelihood / denom) << endl; + cerr << " posterior p0: " << c0 / toks << endl; + cerr << " posterior al-feat: " << emp_feat << endl; + //cerr << " model tension: " << mod_feat / toks << endl; + cerr << " size counts: " << size_counts.size() << endl; if (!final_iteration) { + if (favor_diagonal && optimize_tension && iter > 0) { + for (int ii = 0; ii < 8; ++ii) { + double mod_feat = 0; + unordered_map,unsigned>::iterator it = size_counts.begin(); + for(; it != size_counts.end(); ++it) { + const pair& p = it->first; + for (short j = 1; j <= p.first; ++j) + mod_feat += it->second * DiagonalAlignment::ComputeDLogZ(j, p.first, p.second, diagonal_tension); + } + mod_feat /= toks; + cerr << " " << ii + 1 << " model al-feat: " << mod_feat << " (tension=" << diagonal_tension << ")\n"; + diagonal_tension += (emp_feat - mod_feat) * 20.0; + if (diagonal_tension <= 0.1) diagonal_tension = 0.1; + if (diagonal_tension > 14) diagonal_tension = 14; + } + cerr << " final tension: " << diagonal_tension << endl; + } if (variational_bayes) s2t.NormalizeVB(alpha); else s2t.Normalize(); - cerr << " p0: " << c0 / toks << endl; - //prob_align_null *= 0.8; + //prob_align_null *= 0.8; // XXX //prob_align_null += (c0 / toks) * 0.2; prob_align_not_null = 1.0 - prob_align_null; } -- cgit v1.2.3