From c0565f28cd45ba7d6f478c1dc36a6f1b8aa47669 Mon Sep 17 00:00:00 2001 From: redpony Date: Fri, 22 Oct 2010 23:29:11 +0000 Subject: handle translation from the null word git-svn-id: https://ws10smt.googlecode.com/svn/trunk@689 ec762483-ff6d-05da-a07a-a48fb63a330f --- decoder/aligner.cc | 12 +++++---- decoder/decoder.cc | 2 +- decoder/ff_wordalign.cc | 66 +++++++++++++++++++++++++++++++------------------ decoder/ff_wordalign.h | 6 ++++- 4 files changed, 55 insertions(+), 31 deletions(-) (limited to 'decoder') diff --git a/decoder/aligner.cc b/decoder/aligner.cc index 92431be4..3f0c7347 100644 --- a/decoder/aligner.cc +++ b/decoder/aligner.cc @@ -24,8 +24,10 @@ void SourceEdgeCoveragesUsingParseIndices(const Hypergraph& g, if (edge.rule_->EWords() == 0 || edge.rule_->FWords() == 0) continue; // aligned to NULL (crf ibm variant only) - if (edge.prev_i_ == -1 || edge.i_ == -1) + if (edge.prev_i_ == -1 || edge.i_ == -1) { + cov.insert(-1); continue; + } assert(edge.j_ >= 0); assert(edge.prev_j_ >= 0); if (edge.Arity() == 0) { @@ -211,7 +213,7 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice, // figure out the src and reference size; int src_size = src_sent.size(); int ref_size = trg_sent.size(); - Array2D align(src_size, ref_size, prob_t::Zero()); + Array2D align(src_size + 1, ref_size, prob_t::Zero()); for (int c = 0; c < g->edges_.size(); ++c) { const prob_t& p = edge_posteriors[c]; const set& srcs = src_cov[c]; @@ -220,7 +222,7 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice, si != srcs.end(); ++si) { for (set::const_iterator ti = trgs.begin(); ti != trgs.end(); ++ti) { - align(*si, *ti) += p; + align(*si + 1, *ti) += p; } } } @@ -234,12 +236,12 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice, for (int j = 0; j < ref_size; ++j) { if (use_soft_threshold) { threshold = prob_t::Zero(); - for (int i = 0; i < src_size; ++i) + for (int i = 0; i <= src_size; ++i) if (align(i, j) > threshold) threshold = align(i, j); //threshold *= prob_t(0.99); } for (int i = 0; i < src_size; ++i) - grid(i, j) = align(i, j) >= threshold; + grid(i, j) = align(i+1, j) >= threshold; } if (out == &cout) { // TODO need to do some sort of verbose flag diff --git a/decoder/decoder.cc b/decoder/decoder.cc index b975a5fc..eb983419 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -364,7 +364,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream ("beam_prune", po::value(), "Prune paths from +LM forest, keep paths within exp(alpha>=0)") ("scale_prune_srclen", "scale beams by the input length (in # of tokens; may not be what you want for lattices") ("promise_power",po::value()->default_value(0), "Give more beam budget to more promising previous-pass nodes when pruning - but allocate the same average beams. 0 means off, 1 means beam proportional to inside*outside prob, n means nth power (affects just --cubepruning_pop_limit). note: for the same pop_limit, this gives more search error unless very close to 0 (recommend disabled; even 0.01 is slightly worse than 0) which is a bad sign and suggests this isn't doing a good job; further it's slightly slower to LM cube rescore with 0.01 compared to 0, as well as giving (very insignificantly) lower BLEU. TODO: test under more conditions, or try idea with different formula, or prob. cube beams.") - ("lexalign_use_null", "Support source-side null words in lexical translation") + ("lextrans_use_null", "Support source-side null words in lexical translation") ("tagger_tagset,t", po::value(), "(Tagger) file containing tag set") ("csplit_output_plf", "(Compound splitter) Output lattice in PLF format") ("csplit_preserve_full_word", "(Compound splitter) Always include the unsegmented form in the output lattice") diff --git a/decoder/ff_wordalign.cc b/decoder/ff_wordalign.cc index f2f07033..5f42b438 100644 --- a/decoder/ff_wordalign.cc +++ b/decoder/ff_wordalign.cc @@ -16,6 +16,8 @@ static const int MAX_SENTENCE_SIZE = 100; +static const int kNULL_i = 255; // -1 as an unsigned char + using namespace std; Model2BinaryFeatures::Model2BinaryFeatures(const string& ) : @@ -149,7 +151,11 @@ void MarkovJumpFClass::FireFeature(const SentenceMetadata& smeta, int prev_src_pos, int cur_src_pos, SparseVector* features) const { + if (prev_src_pos == kNULL_i || cur_src_pos == kNULL_i) + return; + const int jumpsize = cur_src_pos - prev_src_pos; + assert(smeta.GetSentenceID() < pos_.size()); const WordID cur_fclass = pos_[smeta.GetSentenceID()][cur_src_pos]; const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second; @@ -189,10 +195,13 @@ void MarkovJumpFClass::TraversalFeaturesImpl(const SentenceMetadata& smeta, } } -// std::vector > flen2jump2fid_; MarkovJump::MarkovJump(const string& param) : FeatureFunction(1), fid_(FD::Convert("MarkovJump")), + fid_lex_null_(FD::Convert("JumpLexNull")), + fid_null_lex_(FD::Convert("JumpNullLex")), + fid_null_null_(FD::Convert("JumpNullNull")), + fid_lex_lex_(FD::Convert("JumpLexLex")), binary_params_(false) { cerr << " MarkovJump"; vector argv; @@ -218,7 +227,7 @@ MarkovJump::MarkovJump(const string& param) : cerr << endl; } -// TODO handle NULLs according to Och 2000 +// TODO handle NULLs according to Och 2000? void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta, const Hypergraph::Edge& edge, const vector& ant_states, @@ -229,19 +238,20 @@ void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta, const int flen = smeta.GetSourceLength(); if (edge.Arity() == 0) { dpstate = static_cast(edge.i_); - if (edge.prev_i_ == 0) { - if (binary_params_) { - // NULL will be tricky - // TODO initial state distribution, not normal jumps + if (edge.prev_i_ == 0) { // first word in sentence + if (edge.i_ >= 0 && binary_params_) { const int fid = flen2jump2fid_[flen].find(edge.i_ + 1)->second; features->set_value(fid, 1.0); + } else if (edge.i_ < 0 && binary_params_) { + // handled by bigram features } } else if (edge.prev_i_ == smeta.GetTargetLength() - 1) { - // NULL will be tricky - if (binary_params_) { + if (edge.i_ >= 0 && binary_params_) { int jumpsize = flen - edge.i_; const int fid = flen2jump2fid_[flen].find(jumpsize)->second; features->set_value(fid, 1.0); + } else if (edge.i_ < 0 && binary_params_) { + // handled by bigram features } } } else if (edge.Arity() == 1) { @@ -253,13 +263,24 @@ void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta, dpstate = static_cast(left_index); else dpstate = static_cast(right_index); - const int jumpsize = right_index - left_index; + if (left_index == kNULL_i || right_index == kNULL_i) { + if (left_index == kNULL_i && right_index == kNULL_i) + features->set_value(fid_null_null_, 1.0); + else if (left_index == kNULL_i) + features->set_value(fid_null_lex_, 1.0); + else + features->set_value(fid_lex_null_, 1.0); - if (binary_params_) { - const int fid = flen2jump2fid_[flen].find(jumpsize)->second; - features->set_value(fid, 1.0); } else { - features->set_value(fid_, fabs(jumpsize - 1)); // Blunsom and Cohn def + features->set_value(fid_lex_lex_, 1.0); // TODO should only use if NULL is enabled + const int jumpsize = right_index - left_index; + + if (binary_params_) { + const int fid = flen2jump2fid_[flen].find(jumpsize)->second; + features->set_value(fid, 1.0); + } else { + features->set_value(fid_, fabs(jumpsize - 1)); // Blunsom and Cohn def + } } } else { assert(!"something really unexpected is happening"); @@ -294,15 +315,6 @@ void SourceBigram::FireFeature(WordID left, if (fid == 0) fid = -1; } if (fid > 0) features->set_value(fid, 1.0); - int& ufid = ufmap_[left]; - if (!ufid) { - ostringstream os; - os << "SU:"; - if (left < 0) { os << "BOS"; } else { os << TD::Convert(left); } - ufid = FD::Convert(os.str()); - if (ufid == 0) fid = -1; - } - if (ufid > 0) features->set_value(ufid, 1.0); } void SourceBigram::TraversalFeaturesImpl(const SentenceMetadata& smeta, @@ -386,8 +398,14 @@ void SourcePOSBigram::TraversalFeaturesImpl(const SentenceMetadata& smeta, if (arity == 0) { assert(smeta.GetSentenceID() < pos_.size()); const vector& pos_sent = pos_[smeta.GetSentenceID()]; - assert(edge.i_ < pos_sent.size()); - out_context = pos_sent[edge.i_]; + if (edge.i_ >= 0) { // non-NULL source + assert(edge.i_ < pos_sent.size()); + out_context = pos_sent[edge.i_]; + } else { // NULL source + // should assert that source is kNULL? + static const WordID kNULL = TD::Convert(""); + out_context = kNULL; + } out_word_count = edge.rule_->EWords(); assert(out_word_count == 1); // this is only defined for lex translation! // revisit this if you want to translate into null words diff --git a/decoder/ff_wordalign.h b/decoder/ff_wordalign.h index 30ddf7a1..0714229c 100644 --- a/decoder/ff_wordalign.h +++ b/decoder/ff_wordalign.h @@ -49,6 +49,11 @@ class MarkovJump : public FeatureFunction { void* out_context) const; private: const int fid_; + const int fid_lex_null_; + const int fid_null_lex_; + const int fid_null_null_; + const int fid_lex_lex_; + bool binary_params_; std::vector > flen2jump2fid_; }; @@ -96,7 +101,6 @@ class SourceBigram : public FeatureFunction { WordID trg, SparseVector* features) const; mutable Class2Class2FID fmap_; - mutable Class2FID ufmap_; }; class SourcePOSBigram : public FeatureFunction { -- cgit v1.2.3