diff options
| author | Chris Dyer <cdyer@cs.cmu.edu> | 2011-03-02 18:57:29 -0500 |
|---|---|---|
| committer | Chris Dyer <cdyer@cs.cmu.edu> | 2011-03-02 18:57:29 -0500 |
| commit | cef04414dec9ee9e7bec53b94ab355c08b7cc38e (patch) | |
| tree | 164894688bf4b709a00647412d8acb0cff593759 /decoder | |
| parent | 9f8f30c5a8506272f9a3b74bc65c08e6cc62a4b2 (diff) | |
better sentence marker handling, part 2
Diffstat (limited to 'decoder')
| -rw-r--r-- | decoder/ff_klm.cc | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/decoder/ff_klm.cc b/decoder/ff_klm.cc index a55a0b41..a12d4a2d 100644 --- a/decoder/ff_klm.cc +++ b/decoder/ff_klm.cc @@ -100,9 +100,13 @@ class KLanguageModelImpl { const lm::WordIndex cur_word = IthUnscoredWord(k, astate); double p = 0; if (cur_word == kSOS_) { - if (has_some_history) { p = -100; } state = ngram_->BeginSentenceState(); - if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2; + if (has_some_history) { // this is immediately fully scored, and bad + p = -100; + context_complete = true; + } else { // this might be a real <s> + num_scored = max(0, order_ - 2); + } } else { const lm::ngram::State scopy(state); p = ngram_->Score(scopy, cur_word, state); @@ -132,9 +136,13 @@ class KLanguageModelImpl { const lm::WordIndex cur_word = MapWord(e[j]); double p = 0; if (cur_word == kSOS_) { - if (has_some_history) p = -100; state = ngram_->BeginSentenceState(); - if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2; + if (has_some_history) { // this is immediately fully scored, and bad + p = -100; + context_complete = true; + } else { // this might be a real <s> + num_scored = max(0, order_ - 2); + } } else { const lm::ngram::State scopy(state); p = ngram_->Score(scopy, cur_word, state); |