summaryrefslogtreecommitdiff
path: root/decoder
diff options
context:
space:
mode:
Diffstat (limited to 'decoder')
-rw-r--r--decoder/ff_lm.cc18
1 files changed, 11 insertions, 7 deletions
diff --git a/decoder/ff_lm.cc b/decoder/ff_lm.cc
index 7e3f6e2b..cdfbb96a 100644
--- a/decoder/ff_lm.cc
+++ b/decoder/ff_lm.cc
@@ -2,7 +2,7 @@
//TODO: allow features to reorder by heuristic*weight the rules' terminal phrases (or of hyperedges'). if first pass has pruning, then compute over whole ruleset as part of heuristic
-//TODO: verify that this is true: if ngram order is bigger than lm state's, then the longest possible ngram scores are still used. if you really want a lower order, a truncated copy of the LM should be small enough. otherwise, an option to null out words outside of the order's window would need to be implemented.
+//NOTE: if ngram order is bigger than lm state's, then the longest possible ngram scores are still used. if you really want a lower order, a truncated copy of the LM should be small enough. otherwise, an option to null out words outside of the order's window would need to be implemented.
#include "ff_lm.h"
@@ -160,13 +160,14 @@ struct LMClient {
class LanguageModelImpl {
public:
explicit LanguageModelImpl(int order) :
- ngram_(*TD::dict_, order), buffer_(), order_(order), state_size_(OrderToStateSize(order) - 1),
+ ngram_(*TD::dict_, order), buffer_(), order_(order), state_size_(OrderToStateSize(order) - 1),
floor_(-100.0),
kSTART(TD::Convert("<s>")),
kSTOP(TD::Convert("</s>")),
kUNKNOWN(TD::Convert("<unk>")),
kNONE(-1),
- kSTAR(TD::Convert("<{STAR}>")) {}
+ kSTAR(TD::Convert("<{STAR}>"))
+ , unigram(order<=1) {}
LanguageModelImpl(int order, const string& f) :
ngram_(*TD::dict_, order), buffer_(), order_(order), state_size_(OrderToStateSize(order) - 1),
@@ -175,7 +176,9 @@ class LanguageModelImpl {
kSTOP(TD::Convert("</s>")),
kUNKNOWN(TD::Convert("<unk>")),
kNONE(-1),
- kSTAR(TD::Convert("<{STAR}>")) {
+ kSTAR(TD::Convert("<{STAR}>"))
+ , unigram(order<=1)
+ {
File file(f.c_str(), "r", 0);
assert(file);
cerr << "Reading " << order_ << "-gram LM from " << f << endl;
@@ -264,7 +267,7 @@ class LanguageModelImpl {
//TODO: make sure that Vocab_None is set to kNONE in srilm (-1), or that SRILM otherwise interprets -1 as a terminator and not a word
double EstimateProb(const void* state) {
- if (!order_) return 0.;
+ if (unigram) return 0.;
int len = StateSize(state);
// cerr << "residual len: " << len << endl;
buffer_.resize(len + 1);
@@ -278,7 +281,7 @@ class LanguageModelImpl {
// for <s> (n-1 left words) and (n-1 right words) </s>
double FinalTraversalCost(const void* state) {
- if (!order_) return 0.;
+ if (unigram) return 0.;
int slen = StateSize(state);
int len = slen + 2;
// cerr << "residual len: " << len << endl;
@@ -328,7 +331,7 @@ class LanguageModelImpl {
//NOTE: this is where the scoring of words happens (heuristic happens in EstimateProb)
double LookupWords(const TRule& rule, const vector<const void*>& ant_states, void* vstate) {
- if (order_==0)
+ if (unigram)
return stateless_cost(rule);
int len = rule.ELength() - rule.Arity();
for (int i = 0; i < ant_states.size(); ++i)
@@ -399,6 +402,7 @@ public:
const WordID kUNKNOWN;
const WordID kNONE;
const WordID kSTAR;
+ const bool unigram;
};
struct ClientLMI : public LanguageModelImpl