From 4e232d33e56d8f1980f72105c48eb47392544c0c Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@cs.cmu.edu>
Date: Wed, 2 Mar 2011 18:57:29 -0500
Subject: better sentence marker handling, part 2

---
 decoder/ff_klm.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'decoder')

diff --git a/decoder/ff_klm.cc b/decoder/ff_klm.cc
index a55a0b41..a12d4a2d 100644
--- a/decoder/ff_klm.cc
+++ b/decoder/ff_klm.cc
@@ -100,9 +100,13 @@ class KLanguageModelImpl {
           const lm::WordIndex cur_word = IthUnscoredWord(k, astate);
           double p = 0;
           if (cur_word == kSOS_) {
-            if (has_some_history) { p = -100; }
             state = ngram_->BeginSentenceState();
-            if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+            if (has_some_history) {  // this is immediately fully scored, and bad
+              p = -100;
+              context_complete = true;
+            } else {  // this might be a real <s>
+              num_scored = max(0, order_ - 2);
+            }
           } else {
             const lm::ngram::State scopy(state);
             p = ngram_->Score(scopy, cur_word, state);
@@ -132,9 +136,13 @@ class KLanguageModelImpl {
         const lm::WordIndex cur_word = MapWord(e[j]);
         double p = 0;
         if (cur_word == kSOS_) {
-          if (has_some_history) p = -100;
           state = ngram_->BeginSentenceState();
-          if (!context_complete && num_scored < (order_ - 2)) num_scored = order_ - 2;
+          if (has_some_history) {  // this is immediately fully scored, and bad
+            p = -100;
+            context_complete = true;
+          } else {  // this might be a real <s>
+            num_scored = max(0, order_ - 2);
+          }
         } else {
           const lm::ngram::State scopy(state);
           p = ngram_->Score(scopy, cur_word, state);
-- 
cgit v1.2.3