summaryrefslogtreecommitdiff
path: root/decoder/ff_ngrams.cc
diff options
context:
space:
mode:
authorPatrick Simianer <p@simianer.de>2012-03-13 09:24:47 +0100
committerPatrick Simianer <p@simianer.de>2012-03-13 09:24:47 +0100
commitef6085e558e26c8819f1735425761103021b6470 (patch)
tree5cf70e4c48c64d838e1326b5a505c8c4061bff4a /decoder/ff_ngrams.cc
parent10a232656a0c882b3b955d2bcfac138ce11e8a2e (diff)
parentdfbc278c1057555fda9312291c8024049e00b7d8 (diff)
merge with upstream
Diffstat (limited to 'decoder/ff_ngrams.cc')
-rw-r--r--decoder/ff_ngrams.cc43
1 files changed, 39 insertions, 4 deletions
diff --git a/decoder/ff_ngrams.cc b/decoder/ff_ngrams.cc
index 04dd1906..d6d79f5e 100644
--- a/decoder/ff_ngrams.cc
+++ b/decoder/ff_ngrams.cc
@@ -57,6 +57,39 @@ namespace {
}
}
+static bool ParseArgs(string const& in, bool* explicit_markers, unsigned* order) {
+ vector<string> const& argv=SplitOnWhitespace(in);
+ *explicit_markers = false;
+ *order = 3;
+#define LMSPEC_NEXTARG if (i==argv.end()) { \
+ cerr << "Missing argument for "<<*last<<". "; goto usage; \
+ } else { ++i; }
+
+ for (vector<string>::const_iterator last,i=argv.begin(),e=argv.end();i!=e;++i) {
+ string const& s=*i;
+ if (s[0]=='-') {
+ if (s.size()>2) goto fail;
+ switch (s[1]) {
+ case 'x':
+ *explicit_markers = true;
+ break;
+ case 'o':
+ LMSPEC_NEXTARG; *order=atoi((*i).c_str());
+ break;
+#undef LMSPEC_NEXTARG
+ default:
+ fail:
+ cerr<<"Unknown option on NgramFeatures "<<s<<" ; ";
+ goto usage;
+ }
+ }
+ }
+ return true;
+usage:
+ cerr << "NgramFeatures is incorrect!\n";
+ return false;
+}
+
class NgramDetectorImpl {
// returns the number of unscored words at the left edge of a span
@@ -264,10 +297,10 @@ class NgramDetectorImpl {
}
public:
- explicit NgramDetectorImpl(bool explicit_markers) :
+ explicit NgramDetectorImpl(bool explicit_markers, unsigned order) :
kCDEC_UNK(TD::Convert("<unk>")) ,
add_sos_eos_(!explicit_markers) {
- order_ = 3;
+ order_ = order;
state_size_ = (order_ - 1) * sizeof(WordID) + 2 + (order_ - 1) * sizeof(WordID);
unscored_size_offset_ = (order_ - 1) * sizeof(WordID);
is_complete_offset_ = unscored_size_offset_ + 1;
@@ -316,8 +349,10 @@ class NgramDetectorImpl {
NgramDetector::NgramDetector(const string& param) {
string filename, mapfile, featname;
- bool explicit_markers = (param == "-x");
- pimpl_ = new NgramDetectorImpl(explicit_markers);
+ bool explicit_markers = false;
+ unsigned order = 3;
+ ParseArgs(param, &explicit_markers, &order);
+ pimpl_ = new NgramDetectorImpl(explicit_markers, order);
SetStateSize(pimpl_->ReserveStateSize());
}