1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
//TODO: 0 size state != rule-local feature, i.e. still may depend on source span loc/context. identify truly rule-local features so if we want they can be added to grammar rules (minor speedup)
#include "ff.h"
#include "tdict.h"
#include "hg.h"
using namespace std;
FeatureFunction::~FeatureFunction() {}
void FeatureFunction::FinalTraversalFeatures(const void* ant_state,
SparseVector<double>* features) const {
(void) ant_state;
(void) features;
}
// Hiero and Joshua use log_10(e) as the value, so I do to
WordPenalty::WordPenalty(const string& param) :
fid_(FD::Convert("WordPenalty")),
value_(-1.0 / log(10)) {
if (!param.empty()) {
cerr << "Warning WordPenalty ignoring parameter: " << param << endl;
}
}
void WordPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
const Hypergraph::Edge& edge,
const std::vector<const void*>& ant_states,
SparseVector<double>* features,
SparseVector<double>* estimated_features,
void* state) const {
(void) smeta;
(void) ant_states;
(void) state;
(void) estimated_features;
features->set_value(fid_, edge.rule_->EWords() * value_);
}
SourceWordPenalty::SourceWordPenalty(const string& param) :
fid_(FD::Convert("SourceWordPenalty")),
value_(-1.0 / log(10)) {
if (!param.empty()) {
cerr << "Warning SourceWordPenalty ignoring parameter: " << param << endl;
}
}
void SourceWordPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
const Hypergraph::Edge& edge,
const std::vector<const void*>& ant_states,
SparseVector<double>* features,
SparseVector<double>* estimated_features,
void* state) const {
(void) smeta;
(void) ant_states;
(void) state;
(void) estimated_features;
features->set_value(fid_, edge.rule_->FWords() * value_);
}
ArityPenalty::ArityPenalty(const std::string& param) :
value_(-1.0 / log(10)) {
string fname = "Arity_X";
for (int i = 0; i < 10; ++i) {
fname[6]=i + '0';
fids_[i] = FD::Convert(fname);
}
}
void ArityPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
const Hypergraph::Edge& edge,
const std::vector<const void*>& ant_states,
SparseVector<double>* features,
SparseVector<double>* estimated_features,
void* state) const {
(void) smeta;
(void) ant_states;
(void) state;
(void) estimated_features;
features->set_value(fids_[edge.Arity()], value_);
}
ModelSet::ModelSet(const vector<double>& w, const vector<const FeatureFunction*>& models) :
models_(models),
weights_(w),
state_size_(0),
model_state_pos_(models.size()) {
for (int i = 0; i < models_.size(); ++i) {
model_state_pos_[i] = state_size_;
state_size_ += models_[i]->NumBytesContext();
}
}
void ModelSet::AddFeaturesToEdge(const SentenceMetadata& smeta,
const Hypergraph& hg,
const vector<string>& node_states,
Hypergraph::Edge* edge,
string* context,
prob_t* combination_cost_estimate) const {
context->resize(state_size_);
memset(&(*context)[0], 0, state_size_);
SparseVector<double> est_vals; // only computed if combination_cost_estimate is non-NULL
if (combination_cost_estimate) *combination_cost_estimate = prob_t::One();
for (int i = 0; i < models_.size(); ++i) {
const FeatureFunction& ff = *models_[i];
void* cur_ff_context = NULL;
vector<const void*> ants(edge->tail_nodes_.size());
bool has_context = ff.NumBytesContext() > 0;
if (has_context) {
int spos = model_state_pos_[i];
cur_ff_context = &(*context)[spos];
for (int i = 0; i < ants.size(); ++i) {
ants[i] = &node_states[edge->tail_nodes_[i]][spos];
}
}
ff.TraversalFeatures(smeta, *edge, ants, &edge->feature_values_, &est_vals, cur_ff_context);
}
if (combination_cost_estimate)
combination_cost_estimate->logeq(est_vals.dot(weights_));
edge->edge_prob_.logeq(edge->feature_values_.dot(weights_));
}
void ModelSet::AddFinalFeatures(const std::string& state, Hypergraph::Edge* edge) const {
assert(1 == edge->rule_->Arity());
for (int i = 0; i < models_.size(); ++i) {
const FeatureFunction& ff = *models_[i];
const void* ant_state = NULL;
bool has_context = ff.NumBytesContext() > 0;
if (has_context) {
int spos = model_state_pos_[i];
ant_state = &state[spos];
}
ff.FinalTraversalFeatures(ant_state, &edge->feature_values_);
}
edge->edge_prob_.logeq(edge->feature_values_.dot(weights_));
}
|
