summaryrefslogtreecommitdiff
path: root/decoder/ff_lexical.h
diff options
context:
space:
mode:
authorPatrick Simianer <p@simianer.de>2014-01-13 17:15:24 +0100
committerPatrick Simianer <p@simianer.de>2014-01-13 17:15:24 +0100
commit6b6a2d966a0d341fe5abee8b332a9d89f6c95bc4 (patch)
tree4bcb7831741648c5817f6da6d12724f6b5a724d4 /decoder/ff_lexical.h
parentb60df3ce6c1a509f52fb19703963caefcfc9859b (diff)
Felix' https://github.com/felleh lexical word alignment features
Diffstat (limited to 'decoder/ff_lexical.h')
-rw-r--r--decoder/ff_lexical.h128
1 files changed, 128 insertions, 0 deletions
diff --git a/decoder/ff_lexical.h b/decoder/ff_lexical.h
new file mode 100644
index 00000000..21c85b27
--- /dev/null
+++ b/decoder/ff_lexical.h
@@ -0,0 +1,128 @@
+#ifndef FF_LEXICAL_H_
+#define FF_LEXICAL_H_
+
+#include <vector>
+#include <map>
+#include "trule.h"
+#include "ff.h"
+#include "hg.h"
+#include "array2d.h"
+#include "wordid.h"
+#include <sstream>
+#include <cassert>
+#include <cmath>
+
+#include "filelib.h"
+#include "stringlib.h"
+#include "sentence_metadata.h"
+#include "lattice.h"
+#include "fdict.h"
+#include "verbose.h"
+#include "tdict.h"
+#include "hg.h"
+
+using namespace std;
+
+namespace {
+ string Escape(const string& x) {
+ string y = x;
+ for (int i = 0; i < y.size(); ++i) {
+ if (y[i] == '=') y[i]='_';
+ if (y[i] == ';') y[i]='_';
+ }
+ return y;
+ }
+}
+
+class LexicalFeatures : public FeatureFunction {
+public:
+ LexicalFeatures(const std::string& param) {
+ if (param.empty()) {
+ cerr << "LexicalFeatures: using T,D,I\n";
+ T_ = true; I_ = true; D_ = true;
+ } else {
+ const vector<string> argv = SplitOnWhitespace(param);
+ assert(argv.size() == 3);
+ T_ = (bool) atoi(argv[0].c_str());
+ I_ = (bool) atoi(argv[1].c_str());
+ D_ = (bool) atoi(argv[2].c_str());
+ cerr << "T=" << T_ << " I=" << I_ << " D=" << D_ << endl;
+ }
+ };
+ static std::string usage(bool p,bool d) {
+ return usage_helper("LexicalFeatures","[0/1 0/1 0/1]","Sparse lexical word translation indicator features. If arguments are supplied, specify like this: translations insertions deletions",p,d);
+ }
+protected:
+ virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+ const HG::Edge& edge,
+ const std::vector<const void*>& ant_contexts,
+ SparseVector<double>* features,
+ SparseVector<double>* estimated_features,
+ void* context) const;
+ virtual void PrepareForInput(const SentenceMetadata& smeta);
+private:
+ mutable std::map<const TRule*, SparseVector<double> > rule2feats_;
+ bool T_;
+ bool I_;
+ bool D_;
+};
+
+void LexicalFeatures::PrepareForInput(const SentenceMetadata& smeta) {
+ rule2feats_.clear(); // std::map<const TRule*, SparseVector<double> >
+}
+
+void LexicalFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+ const HG::Edge& edge,
+ const std::vector<const void*>& ant_contexts,
+ SparseVector<double>* features,
+ SparseVector<double>* estimated_features,
+ void* context) const {
+
+ map<const TRule*, SparseVector<double> >::iterator it = rule2feats_.find(edge.rule_.get());
+ if (it == rule2feats_.end()) {
+ const TRule& rule = *edge.rule_;
+ it = rule2feats_.insert(make_pair(&rule, SparseVector<double>())).first;
+ SparseVector<double>& f = it->second;
+ std::vector<bool> sf(edge.rule_->FLength(),false); // stores if source tokens are visited by alignment points
+ std::vector<bool> se(edge.rule_->ELength(),false); // stores if target tokens are visited by alignment points
+ int fid = 0;
+ // translations
+ for (unsigned i=0;i<rule.a_.size();++i) {
+ const AlignmentPoint& ap = rule.a_[i];
+ sf[ap.s_] = true; // mark index as seen
+ se[ap.t_] = true; // mark index as seen
+ ostringstream os;
+ os << "LT:" << Escape(TD::Convert(rule.f_[ap.s_])) << ":" << Escape(TD::Convert(rule.e_[ap.t_]));
+ fid = FD::Convert(os.str());
+ if (fid <= 0) continue;
+ if (T_)
+ f.add_value(fid, 1.0);
+ }
+ // word deletions
+ for (unsigned i=0;i<sf.size();++i) {
+ if (!sf[i] && rule.f_[i] > 0) {// if not visited and is terminal
+ ostringstream os;
+ os << "LD:" << Escape(TD::Convert(rule.f_[i]));
+ fid = FD::Convert(os.str());
+ if (fid <= 0) continue;
+ if (D_)
+ f.add_value(fid, 1.0);
+ }
+ }
+ // word insertions
+ for (unsigned i=0;i<se.size();++i) {
+ if (!se[i] && rule.e_[i] >= 1) {// if not visited and is terminal
+ ostringstream os;
+ os << "LI:" << Escape(TD::Convert(rule.e_[i]));
+ fid = FD::Convert(os.str());
+ if (fid <= 0) continue;
+ if (I_)
+ f.add_value(fid, 1.0);
+ }
+ }
+ }
+ (*features) += it->second;
+}
+
+
+#endif