diff options
| author | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2014-02-23 02:13:40 -0500 |
|---|---|---|
| committer | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2014-02-23 02:13:40 -0500 |
| commit | dd555f71427c72753bb0a2451de05d8d7125717c (patch) | |
| tree | eabc2d2fda8e4cd5f9ad34cd9bf18e1ff9bf93ef /decoder/ff_ngrams.cc | |
| parent | 3ec30b72f47e063d94648a9823653e6ec3e17401 (diff) | |
| parent | fbdc905f6f201e2cc0dbee89ef81e36a53bb3c42 (diff) | |
Merge branch 'master' of github.com:redpony/cdec
Diffstat (limited to 'decoder/ff_ngrams.cc')
| -rw-r--r-- | decoder/ff_ngrams.cc | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/decoder/ff_ngrams.cc b/decoder/ff_ngrams.cc index d337b28b..0bc14e5a 100644 --- a/decoder/ff_ngrams.cc +++ b/decoder/ff_ngrams.cc @@ -60,8 +60,9 @@ namespace { } } -static bool ParseArgs(string const& in, bool* explicit_markers, unsigned* order, vector<string>& prefixes, string& target_separator, string* cluster_file) { +static bool ParseArgs(string const& in, bool* explicit_markers, unsigned* order, vector<string>& prefixes, string& target_separator, string* cluster_file, string* featname) { vector<string> const& argv=SplitOnWhitespace(in); + *featname = ""; *explicit_markers = false; *order = 3; prefixes.push_back("NOT-USED"); @@ -83,6 +84,9 @@ static bool ParseArgs(string const& in, bool* explicit_markers, unsigned* order, case 'x': *explicit_markers = true; break; + case 'n': + LMSPEC_NEXTARG; *featname=*i; + break; case 'U': LMSPEC_NEXTARG; prefixes[1] = *i; @@ -226,6 +230,7 @@ class NgramDetectorImpl { ++n; if (!fid) { ostringstream os; + os << featname_; os << prefixes_[n]; for (int i = n-1; i >= 0; --i) { os << (i != n-1 ? target_separator_ : ""); @@ -404,7 +409,8 @@ class NgramDetectorImpl { public: explicit NgramDetectorImpl(bool explicit_markers, unsigned order, - vector<string>& prefixes, string& target_separator, const string& clusters) : + vector<string>& prefixes, string& target_separator, const string& clusters, + const string& featname) : kCDEC_UNK(TD::Convert("<unk>")) , add_sos_eos_(!explicit_markers) { order_ = order; @@ -414,6 +420,7 @@ class NgramDetectorImpl { unscored_words_offset_ = is_complete_offset_ + 1; prefixes_ = prefixes; target_separator_ = target_separator; + featname_ = featname; // special handling of beginning / ending sentence markers dummy_state_ = new char[state_size_]; @@ -454,6 +461,7 @@ class NgramDetectorImpl { TRulePtr dummy_rule_; vector<string> prefixes_; string target_separator_; + string featname_; struct FidTree { map<WordID, int> fids; map<WordID, FidTree> levels; @@ -467,9 +475,9 @@ NgramDetector::NgramDetector(const string& param) { bool explicit_markers = false; unsigned order = 3; string clusters; - ParseArgs(param, &explicit_markers, &order, prefixes, target_separator, &clusters); + ParseArgs(param, &explicit_markers, &order, prefixes, target_separator, &clusters, &featname); pimpl_ = new NgramDetectorImpl(explicit_markers, order, prefixes, - target_separator, clusters); + target_separator, clusters, featname); SetStateSize(pimpl_->ReserveStateSize()); } |