diff options
| author | Chris Dyer <cdyer@cs.cmu.edu> | 2011-03-02 18:57:29 -0500 |
|---|---|---|
| committer | Chris Dyer <cdyer@cs.cmu.edu> | 2011-03-02 18:57:29 -0500 |
| commit | 4e232d33e56d8f1980f72105c48eb47392544c0c (patch) | |
| tree | 0817e66e73c9073c3247f7d8a581d08e661a4956 | |
| parent | d5d86d6513e34f0b26030814c9eb516271ce2aec (diff) | |
better sentence marker handling, part 2
| -rw-r--r-- | decoder/ff_klm.cc | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/decoder/ff_klm.cc b/decoder/ff_klm.cc index a55a0b41..a12d4a2d 100644 --- a/decoder/ff_klm.cc +++ b/decoder/ff_klm.cc @@ -100,9 +100,13 @@ class KLanguageModelImpl { const lm::WordIndex cur_word = IthUnscoredWord(k, astate); double p = 0; if (cur_word == kSOS_) { - if (has_some_history) { p = -100; } state = ngram_->BeginSentenceState(); - if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2; + if (has_some_history) { // this is immediately fully scored, and bad + p = -100; + context_complete = true; + } else { // this might be a real <s> + num_scored = max(0, order_ - 2); + } } else { const lm::ngram::State scopy(state); p = ngram_->Score(scopy, cur_word, state); @@ -132,9 +136,13 @@ class KLanguageModelImpl { const lm::WordIndex cur_word = MapWord(e[j]); double p = 0; if (cur_word == kSOS_) { - if (has_some_history) p = -100; state = ngram_->BeginSentenceState(); - if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2; + if (has_some_history) { // this is immediately fully scored, and bad + p = -100; + context_complete = true; + } else { // this might be a real <s> + num_scored = max(0, order_ - 2); + } } else { const lm::ngram::State scopy(state); p = ngram_->Score(scopy, cur_word, state); |