summaryrefslogtreecommitdiff
path: root/decoder/ff_klm.cc
diff options
context:
space:
mode:
authorChris Dyer <cdyer@cs.cmu.edu>2011-03-02 18:57:29 -0500
committerChris Dyer <cdyer@cs.cmu.edu>2011-03-02 18:57:29 -0500
commit4e232d33e56d8f1980f72105c48eb47392544c0c (patch)
tree0817e66e73c9073c3247f7d8a581d08e661a4956 /decoder/ff_klm.cc
parentd5d86d6513e34f0b26030814c9eb516271ce2aec (diff)
better sentence marker handling, part 2
Diffstat (limited to 'decoder/ff_klm.cc')
-rw-r--r--decoder/ff_klm.cc16
1 files changed, 12 insertions, 4 deletions
diff --git a/decoder/ff_klm.cc b/decoder/ff_klm.cc
index a55a0b41..a12d4a2d 100644
--- a/decoder/ff_klm.cc
+++ b/decoder/ff_klm.cc
@@ -100,9 +100,13 @@ class KLanguageModelImpl {
const lm::WordIndex cur_word = IthUnscoredWord(k, astate);
double p = 0;
if (cur_word == kSOS_) {
- if (has_some_history) { p = -100; }
state = ngram_->BeginSentenceState();
- if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+ if (has_some_history) { // this is immediately fully scored, and bad
+ p = -100;
+ context_complete = true;
+ } else { // this might be a real <s>
+ num_scored = max(0, order_ - 2);
+ }
} else {
const lm::ngram::State scopy(state);
p = ngram_->Score(scopy, cur_word, state);
@@ -132,9 +136,13 @@ class KLanguageModelImpl {
const lm::WordIndex cur_word = MapWord(e[j]);
double p = 0;
if (cur_word == kSOS_) {
- if (has_some_history) p = -100;
state = ngram_->BeginSentenceState();
- if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+ if (has_some_history) { // this is immediately fully scored, and bad
+ p = -100;
+ context_complete = true;
+ } else { // this might be a real <s>
+ num_scored = max(0, order_ - 2);
+ }
} else {
const lm::ngram::State scopy(state);
p = ngram_->Score(scopy, cur_word, state);