summaryrefslogtreecommitdiff
path: root/decoder
diff options
context:
space:
mode:
authorChris Dyer <cdyer@cs.cmu.edu>2011-03-02 18:57:29 -0500
committerChris Dyer <cdyer@cs.cmu.edu>2011-03-02 18:57:29 -0500
commitcef04414dec9ee9e7bec53b94ab355c08b7cc38e (patch)
tree164894688bf4b709a00647412d8acb0cff593759 /decoder
parent9f8f30c5a8506272f9a3b74bc65c08e6cc62a4b2 (diff)
better sentence marker handling, part 2
Diffstat (limited to 'decoder')
-rw-r--r--decoder/ff_klm.cc16
1 files changed, 12 insertions, 4 deletions
diff --git a/decoder/ff_klm.cc b/decoder/ff_klm.cc
index a55a0b41..a12d4a2d 100644
--- a/decoder/ff_klm.cc
+++ b/decoder/ff_klm.cc
@@ -100,9 +100,13 @@ class KLanguageModelImpl {
const lm::WordIndex cur_word = IthUnscoredWord(k, astate);
double p = 0;
if (cur_word == kSOS_) {
- if (has_some_history) { p = -100; }
state = ngram_->BeginSentenceState();
- if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+ if (has_some_history) { // this is immediately fully scored, and bad
+ p = -100;
+ context_complete = true;
+ } else { // this might be a real <s>
+ num_scored = max(0, order_ - 2);
+ }
} else {
const lm::ngram::State scopy(state);
p = ngram_->Score(scopy, cur_word, state);
@@ -132,9 +136,13 @@ class KLanguageModelImpl {
const lm::WordIndex cur_word = MapWord(e[j]);
double p = 0;
if (cur_word == kSOS_) {
- if (has_some_history) p = -100;
state = ngram_->BeginSentenceState();
- if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+ if (has_some_history) { // this is immediately fully scored, and bad
+ p = -100;
+ context_complete = true;
+ } else { // this might be a real <s>
+ num_scored = max(0, order_ - 2);
+ }
} else {
const lm::ngram::State scopy(state);
p = ngram_->Score(scopy, cur_word, state);