summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Dyer <cdyer@cs.cmu.edu>2012-03-07 09:46:54 -0500
committerChris Dyer <cdyer@cs.cmu.edu>2012-03-07 09:46:54 -0500
commit4f19cd0c9a729cfd59d186492b3035c168f5e58f (patch)
treea3fffdc6a02f8a48b1cddfadf903a8734055d24e
parent27e0de58bf49a4fc74bbf58718d1b89525a154a6 (diff)
configure order of n-gram features
-rw-r--r--decoder/ff_ngrams.cc43
1 files changed, 39 insertions, 4 deletions
diff --git a/decoder/ff_ngrams.cc b/decoder/ff_ngrams.cc
index 04dd1906..d6d79f5e 100644
--- a/decoder/ff_ngrams.cc
+++ b/decoder/ff_ngrams.cc
@@ -57,6 +57,39 @@ namespace {
}
}
+static bool ParseArgs(string const& in, bool* explicit_markers, unsigned* order) {
+ vector<string> const& argv=SplitOnWhitespace(in);
+ *explicit_markers = false;
+ *order = 3;
+#define LMSPEC_NEXTARG if (i==argv.end()) { \
+ cerr << "Missing argument for "<<*last<<". "; goto usage; \
+ } else { ++i; }
+
+ for (vector<string>::const_iterator last,i=argv.begin(),e=argv.end();i!=e;++i) {
+ string const& s=*i;
+ if (s[0]=='-') {
+ if (s.size()>2) goto fail;
+ switch (s[1]) {
+ case 'x':
+ *explicit_markers = true;
+ break;
+ case 'o':
+ LMSPEC_NEXTARG; *order=atoi((*i).c_str());
+ break;
+#undef LMSPEC_NEXTARG
+ default:
+ fail:
+ cerr<<"Unknown option on NgramFeatures "<<s<<" ; ";
+ goto usage;
+ }
+ }
+ }
+ return true;
+usage:
+ cerr << "NgramFeatures is incorrect!\n";
+ return false;
+}
+
class NgramDetectorImpl {
// returns the number of unscored words at the left edge of a span
@@ -264,10 +297,10 @@ class NgramDetectorImpl {
}
public:
- explicit NgramDetectorImpl(bool explicit_markers) :
+ explicit NgramDetectorImpl(bool explicit_markers, unsigned order) :
kCDEC_UNK(TD::Convert("<unk>")) ,
add_sos_eos_(!explicit_markers) {
- order_ = 3;
+ order_ = order;
state_size_ = (order_ - 1) * sizeof(WordID) + 2 + (order_ - 1) * sizeof(WordID);
unscored_size_offset_ = (order_ - 1) * sizeof(WordID);
is_complete_offset_ = unscored_size_offset_ + 1;
@@ -316,8 +349,10 @@ class NgramDetectorImpl {
NgramDetector::NgramDetector(const string& param) {
string filename, mapfile, featname;
- bool explicit_markers = (param == "-x");
- pimpl_ = new NgramDetectorImpl(explicit_markers);
+ bool explicit_markers = false;
+ unsigned order = 3;
+ ParseArgs(param, &explicit_markers, &order);
+ pimpl_ = new NgramDetectorImpl(explicit_markers, order);
SetStateSize(pimpl_->ReserveStateSize());
}