summaryrefslogtreecommitdiff
path: root/decoder
diff options
context:
space:
mode:
Diffstat (limited to 'decoder')
-rw-r--r--decoder/aligner.cc12
-rw-r--r--decoder/decoder.cc2
-rw-r--r--decoder/ff_wordalign.cc66
-rw-r--r--decoder/ff_wordalign.h6
4 files changed, 55 insertions, 31 deletions
diff --git a/decoder/aligner.cc b/decoder/aligner.cc
index 92431be4..3f0c7347 100644
--- a/decoder/aligner.cc
+++ b/decoder/aligner.cc
@@ -24,8 +24,10 @@ void SourceEdgeCoveragesUsingParseIndices(const Hypergraph& g,
if (edge.rule_->EWords() == 0 || edge.rule_->FWords() == 0)
continue;
// aligned to NULL (crf ibm variant only)
- if (edge.prev_i_ == -1 || edge.i_ == -1)
+ if (edge.prev_i_ == -1 || edge.i_ == -1) {
+ cov.insert(-1);
continue;
+ }
assert(edge.j_ >= 0);
assert(edge.prev_j_ >= 0);
if (edge.Arity() == 0) {
@@ -211,7 +213,7 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice,
// figure out the src and reference size;
int src_size = src_sent.size();
int ref_size = trg_sent.size();
- Array2D<prob_t> align(src_size, ref_size, prob_t::Zero());
+ Array2D<prob_t> align(src_size + 1, ref_size, prob_t::Zero());
for (int c = 0; c < g->edges_.size(); ++c) {
const prob_t& p = edge_posteriors[c];
const set<int>& srcs = src_cov[c];
@@ -220,7 +222,7 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice,
si != srcs.end(); ++si) {
for (set<int>::const_iterator ti = trgs.begin();
ti != trgs.end(); ++ti) {
- align(*si, *ti) += p;
+ align(*si + 1, *ti) += p;
}
}
}
@@ -234,12 +236,12 @@ void AlignerTools::WriteAlignment(const Lattice& src_lattice,
for (int j = 0; j < ref_size; ++j) {
if (use_soft_threshold) {
threshold = prob_t::Zero();
- for (int i = 0; i < src_size; ++i)
+ for (int i = 0; i <= src_size; ++i)
if (align(i, j) > threshold) threshold = align(i, j);
//threshold *= prob_t(0.99);
}
for (int i = 0; i < src_size; ++i)
- grid(i, j) = align(i, j) >= threshold;
+ grid(i, j) = align(i+1, j) >= threshold;
}
if (out == &cout) {
// TODO need to do some sort of verbose flag
diff --git a/decoder/decoder.cc b/decoder/decoder.cc
index b975a5fc..eb983419 100644
--- a/decoder/decoder.cc
+++ b/decoder/decoder.cc
@@ -364,7 +364,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream
("beam_prune", po::value<double>(), "Prune paths from +LM forest, keep paths within exp(alpha>=0)")
("scale_prune_srclen", "scale beams by the input length (in # of tokens; may not be what you want for lattices")
("promise_power",po::value<double>()->default_value(0), "Give more beam budget to more promising previous-pass nodes when pruning - but allocate the same average beams. 0 means off, 1 means beam proportional to inside*outside prob, n means nth power (affects just --cubepruning_pop_limit). note: for the same pop_limit, this gives more search error unless very close to 0 (recommend disabled; even 0.01 is slightly worse than 0) which is a bad sign and suggests this isn't doing a good job; further it's slightly slower to LM cube rescore with 0.01 compared to 0, as well as giving (very insignificantly) lower BLEU. TODO: test under more conditions, or try idea with different formula, or prob. cube beams.")
- ("lexalign_use_null", "Support source-side null words in lexical translation")
+ ("lextrans_use_null", "Support source-side null words in lexical translation")
("tagger_tagset,t", po::value<string>(), "(Tagger) file containing tag set")
("csplit_output_plf", "(Compound splitter) Output lattice in PLF format")
("csplit_preserve_full_word", "(Compound splitter) Always include the unsegmented form in the output lattice")
diff --git a/decoder/ff_wordalign.cc b/decoder/ff_wordalign.cc
index f2f07033..5f42b438 100644
--- a/decoder/ff_wordalign.cc
+++ b/decoder/ff_wordalign.cc
@@ -16,6 +16,8 @@
static const int MAX_SENTENCE_SIZE = 100;
+static const int kNULL_i = 255; // -1 as an unsigned char
+
using namespace std;
Model2BinaryFeatures::Model2BinaryFeatures(const string& ) :
@@ -149,7 +151,11 @@ void MarkovJumpFClass::FireFeature(const SentenceMetadata& smeta,
int prev_src_pos,
int cur_src_pos,
SparseVector<double>* features) const {
+ if (prev_src_pos == kNULL_i || cur_src_pos == kNULL_i)
+ return;
+
const int jumpsize = cur_src_pos - prev_src_pos;
+
assert(smeta.GetSentenceID() < pos_.size());
const WordID cur_fclass = pos_[smeta.GetSentenceID()][cur_src_pos];
const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second;
@@ -189,10 +195,13 @@ void MarkovJumpFClass::TraversalFeaturesImpl(const SentenceMetadata& smeta,
}
}
-// std::vector<std::map<int, int> > flen2jump2fid_;
MarkovJump::MarkovJump(const string& param) :
FeatureFunction(1),
fid_(FD::Convert("MarkovJump")),
+ fid_lex_null_(FD::Convert("JumpLexNull")),
+ fid_null_lex_(FD::Convert("JumpNullLex")),
+ fid_null_null_(FD::Convert("JumpNullNull")),
+ fid_lex_lex_(FD::Convert("JumpLexLex")),
binary_params_(false) {
cerr << " MarkovJump";
vector<string> argv;
@@ -218,7 +227,7 @@ MarkovJump::MarkovJump(const string& param) :
cerr << endl;
}
-// TODO handle NULLs according to Och 2000
+// TODO handle NULLs according to Och 2000?
void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta,
const Hypergraph::Edge& edge,
const vector<const void*>& ant_states,
@@ -229,19 +238,20 @@ void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta,
const int flen = smeta.GetSourceLength();
if (edge.Arity() == 0) {
dpstate = static_cast<unsigned int>(edge.i_);
- if (edge.prev_i_ == 0) {
- if (binary_params_) {
- // NULL will be tricky
- // TODO initial state distribution, not normal jumps
+ if (edge.prev_i_ == 0) { // first word in sentence
+ if (edge.i_ >= 0 && binary_params_) {
const int fid = flen2jump2fid_[flen].find(edge.i_ + 1)->second;
features->set_value(fid, 1.0);
+ } else if (edge.i_ < 0 && binary_params_) {
+ // handled by bigram features
}
} else if (edge.prev_i_ == smeta.GetTargetLength() - 1) {
- // NULL will be tricky
- if (binary_params_) {
+ if (edge.i_ >= 0 && binary_params_) {
int jumpsize = flen - edge.i_;
const int fid = flen2jump2fid_[flen].find(jumpsize)->second;
features->set_value(fid, 1.0);
+ } else if (edge.i_ < 0 && binary_params_) {
+ // handled by bigram features
}
}
} else if (edge.Arity() == 1) {
@@ -253,13 +263,24 @@ void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta,
dpstate = static_cast<unsigned int>(left_index);
else
dpstate = static_cast<unsigned int>(right_index);
- const int jumpsize = right_index - left_index;
+ if (left_index == kNULL_i || right_index == kNULL_i) {
+ if (left_index == kNULL_i && right_index == kNULL_i)
+ features->set_value(fid_null_null_, 1.0);
+ else if (left_index == kNULL_i)
+ features->set_value(fid_null_lex_, 1.0);
+ else
+ features->set_value(fid_lex_null_, 1.0);
- if (binary_params_) {
- const int fid = flen2jump2fid_[flen].find(jumpsize)->second;
- features->set_value(fid, 1.0);
} else {
- features->set_value(fid_, fabs(jumpsize - 1)); // Blunsom and Cohn def
+ features->set_value(fid_lex_lex_, 1.0); // TODO should only use if NULL is enabled
+ const int jumpsize = right_index - left_index;
+
+ if (binary_params_) {
+ const int fid = flen2jump2fid_[flen].find(jumpsize)->second;
+ features->set_value(fid, 1.0);
+ } else {
+ features->set_value(fid_, fabs(jumpsize - 1)); // Blunsom and Cohn def
+ }
}
} else {
assert(!"something really unexpected is happening");
@@ -294,15 +315,6 @@ void SourceBigram::FireFeature(WordID left,
if (fid == 0) fid = -1;
}
if (fid > 0) features->set_value(fid, 1.0);
- int& ufid = ufmap_[left];
- if (!ufid) {
- ostringstream os;
- os << "SU:";
- if (left < 0) { os << "BOS"; } else { os << TD::Convert(left); }
- ufid = FD::Convert(os.str());
- if (ufid == 0) fid = -1;
- }
- if (ufid > 0) features->set_value(ufid, 1.0);
}
void SourceBigram::TraversalFeaturesImpl(const SentenceMetadata& smeta,
@@ -386,8 +398,14 @@ void SourcePOSBigram::TraversalFeaturesImpl(const SentenceMetadata& smeta,
if (arity == 0) {
assert(smeta.GetSentenceID() < pos_.size());
const vector<WordID>& pos_sent = pos_[smeta.GetSentenceID()];
- assert(edge.i_ < pos_sent.size());
- out_context = pos_sent[edge.i_];
+ if (edge.i_ >= 0) { // non-NULL source
+ assert(edge.i_ < pos_sent.size());
+ out_context = pos_sent[edge.i_];
+ } else { // NULL source
+ // should assert that source is kNULL?
+ static const WordID kNULL = TD::Convert("<eps>");
+ out_context = kNULL;
+ }
out_word_count = edge.rule_->EWords();
assert(out_word_count == 1); // this is only defined for lex translation!
// revisit this if you want to translate into null words
diff --git a/decoder/ff_wordalign.h b/decoder/ff_wordalign.h
index 30ddf7a1..0714229c 100644
--- a/decoder/ff_wordalign.h
+++ b/decoder/ff_wordalign.h
@@ -49,6 +49,11 @@ class MarkovJump : public FeatureFunction {
void* out_context) const;
private:
const int fid_;
+ const int fid_lex_null_;
+ const int fid_null_lex_;
+ const int fid_null_null_;
+ const int fid_lex_lex_;
+
bool binary_params_;
std::vector<std::map<int, int> > flen2jump2fid_;
};
@@ -96,7 +101,6 @@ class SourceBigram : public FeatureFunction {
WordID trg,
SparseVector<double>* features) const;
mutable Class2Class2FID fmap_;
- mutable Class2FID ufmap_;
};
class SourcePOSBigram : public FeatureFunction {
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-25 20:54:44 +0000