From 703fd668503e147ea2596c3715eb3a29f8ff89e1 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Thu, 3 Mar 2011 22:44:42 -0500 Subject: new span features --- decoder/ff_spans.cc | 45 +++++++++++++++++++++++++++++++++------------ decoder/ff_spans.h | 7 +++++-- 2 files changed, 38 insertions(+), 14 deletions(-) (limited to 'decoder') diff --git a/decoder/ff_spans.cc b/decoder/ff_spans.cc index 1cf72be9..17e1d650 100644 --- a/decoder/ff_spans.cc +++ b/decoder/ff_spans.cc @@ -13,6 +13,13 @@ using namespace std; +// log transform to make long spans cluster together +// but preserve differences +int SpanSizeTransform(unsigned span_size) { + if (!span_size) return 0; + return static_cast(log(span_size+1) / log(1.39)) - 1; +} + SpanFeatures::SpanFeatures(const string& param) : kS(TD::Convert("S") * -1), kX(TD::Convert("X") * -1), @@ -41,8 +48,8 @@ SpanFeatures::SpanFeatures(const string& param) : } word2class_[v[0]] = v[1]; } - word2class_[TD::Convert("")] = TD::Convert("BOS"); - word2class_[TD::Convert("")] = TD::Convert("EOS"); + word2class_[TD::Convert("BOS")] = TD::Convert("BOS"); + word2class_[TD::Convert("EOS")] = TD::Convert("EOS"); oov_ = TD::Convert("OOV"); } @@ -88,10 +95,15 @@ void SpanFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta, } else { // non-collapsed features: features->set_value(end_span_ids_[edge.j_], 1); features->set_value(beg_span_ids_[edge.i_], 1); - if (edge.rule_->lhs_ == kS) + features->set_value(end_bigram_ids_[edge.j_], 1); + features->set_value(beg_bigram_ids_[edge.i_], 1); + if (edge.rule_->lhs_ == kS) { features->set_value(span_feats_(edge.i_,edge.j_).second, 1); - else + features->set_value(len_span_feats_(edge.i_,edge.j_).second, 1); + } else { features->set_value(span_feats_(edge.i_,edge.j_).first, 1); + features->set_value(len_span_feats_(edge.i_,edge.j_).first, 1); + } } } @@ -104,11 +116,14 @@ WordID SpanFeatures::MapIfNecessary(const WordID& w) const { void SpanFeatures::PrepareForInput(const SentenceMetadata& smeta) { const Lattice& lattice = smeta.GetSourceLattice(); - const WordID eos = TD::Convert(""); - const WordID bos = TD::Convert(""); + const WordID eos = TD::Convert("EOS"); // right of the last source word + const WordID bos = TD::Convert("BOS"); // left of the first source word beg_span_ids_.resize(lattice.size() + 1); end_span_ids_.resize(lattice.size() + 1); span_feats_.resize(lattice.size() + 1, lattice.size() + 1); + beg_bigram_ids_.resize(lattice.size() + 1); + end_bigram_ids_.resize(lattice.size() + 1); + len_span_feats_.resize(lattice.size() + 1, lattice.size() + 1); if (use_collapsed_features_) { beg_span_vals_.resize(lattice.size() + 1); end_span_vals_.resize(lattice.size() + 1); @@ -126,6 +141,12 @@ void SpanFeatures::PrepareForInput(const SentenceMetadata& smeta) { ostringstream sfid; sfid << "ES:" << TD::Convert(word); end_span_ids_[i] = FD::Convert(sfid.str()); + ostringstream esbiid; + esbiid << "EBI:" << TD::Convert(bword) << "_" << TD::Convert(word); + end_bigram_ids_[i] = FD::Convert(esbiid.str()); + ostringstream bsbiid; + bsbiid << "BBI:" << TD::Convert(bword) << "_" << TD::Convert(word); + beg_bigram_ids_[i] = FD::Convert(bsbiid.str()); ostringstream bfid; bfid << "BS:" << TD::Convert(bword); beg_span_ids_[i] = FD::Convert(bfid.str()); @@ -145,9 +166,14 @@ void SpanFeatures::PrepareForInput(const SentenceMetadata& smeta) { word = lattice[j][0].label; word = MapIfNecessary(word); ostringstream pf; - pf << "SS:" << TD::Convert(bword) << "_" << TD::Convert(word); + pf << "S:" << TD::Convert(bword) << "_" << TD::Convert(word); span_feats_(i,j).first = FD::Convert(pf.str()); span_feats_(i,j).second = FD::Convert("S_" + pf.str()); + ostringstream lf; + const unsigned span_size = (i < j ? j - i : i - j); + lf << "LS:" << SpanSizeTransform(span_size) << "_" << TD::Convert(bword) << "_" << TD::Convert(word); + len_span_feats_(i,j).first = FD::Convert(lf.str()); + len_span_feats_(i,j).second = FD::Convert("S_" + lf.str()); if (use_collapsed_features_) { span_vals_(i,j).first = feat2val_[pf.str()]; span_vals_(i,j).second = feat2val_["S_" + pf.str()]; @@ -187,11 +213,6 @@ CMR2008ReorderingFeatures::CMR2008ReorderingFeatures(const std::string& param) : } } -int CMR2008ReorderingFeatures::SpanSizeTransform(unsigned span_size) { - if (!span_size) return 0; - return static_cast(log(span_size+1) / log(1.39)) - 1; -} - void CMR2008ReorderingFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta, const Hypergraph::Edge& edge, const vector& ant_contexts, diff --git a/decoder/ff_spans.h b/decoder/ff_spans.h index 9928d70f..67269d26 100644 --- a/decoder/ff_spans.h +++ b/decoder/ff_spans.h @@ -23,8 +23,11 @@ class SpanFeatures : public FeatureFunction { const int kS; const int kX; Array2D > span_feats_; // first for X, second for S + Array2D > len_span_feats_; // first for X, second for S, including length std::vector end_span_ids_; + std::vector end_bigram_ids_; std::vector beg_span_ids_; + std::vector beg_bigram_ids_; std::map word2class_; // optional projection to coarser class // collapsed feature values @@ -35,7 +38,9 @@ class SpanFeatures : public FeatureFunction { int fid_span_; std::map feat2val_; std::vector end_span_vals_; + std::vector end_bigram_vals_; std::vector beg_span_vals_; + std::vector beg_bigram_vals_; Array2D > span_vals_; WordID oov_; @@ -52,8 +57,6 @@ class CMR2008ReorderingFeatures : public FeatureFunction { SparseVector* estimated_features, void* context) const; private: - static int SpanSizeTransform(unsigned span_size); - const int kS; std::pair unconditioned_fids_; // first = monotone // second = inverse -- cgit v1.2.3