1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
#ifndef _DTRAIN_H_
#define _DTRAIN_H_
#define DTRAIN_DOTS 10 // after how many inputs to display a '.'
#define DTRAIN_SCALE 100000
#include <iomanip>
#include <climits>
#include <string.h>
#include <boost/algorithm/string.hpp>
#include <boost/regex.hpp>
#include <boost/algorithm/string/regex.hpp>
#include <boost/program_options.hpp>
#include "decoder.h"
#include "ff_register.h"
#include "sentence_metadata.h"
#include "verbose.h"
#include "viterbi.h"
using namespace std;
namespace po = boost::program_options;
namespace dtrain
{
inline void register_and_convert(const vector<string>& strs, vector<WordID>& ids)
{
vector<string>::const_iterator it;
for (it = strs.begin(); it < strs.end(); it++)
ids.push_back(TD::Convert(*it));
}
inline string gettmpf(const string path, const string infix)
{
char fn[path.size() + infix.size() + 8];
strcpy(fn, path.c_str());
strcat(fn, "/");
strcat(fn, infix.c_str());
strcat(fn, "-XXXXXX");
if (!mkstemp(fn)) {
cerr << "Cannot make temp file in" << path << " , exiting." << endl;
exit(1);
}
return string(fn);
}
typedef double score_t;
struct ScoredHyp
{
vector<WordID> w;
SparseVector<double> f;
score_t model;
score_t score;
unsigned rank;
};
struct LocalScorer
{
unsigned N_;
vector<score_t> w_;
virtual score_t
Score(const vector<WordID>& hyp, const vector<WordID>& ref, const unsigned rank, const unsigned src_len)=0;
virtual void Reset() {} // only for ApproxBleuScorer, LinearBleuScorer
inline void
Init(unsigned N, vector<score_t> weights)
{
assert(N > 0);
N_ = N;
if (weights.empty()) for (unsigned i = 0; i < N_; i++) w_.push_back(1./N_);
else w_ = weights;
}
inline score_t
brevity_penalty(const unsigned hyp_len, const unsigned ref_len)
{
if (hyp_len > ref_len) return 1;
return exp(1 - (score_t)ref_len/hyp_len);
}
};
struct HypSampler : public DecoderObserver
{
LocalScorer* scorer_;
vector<WordID>* ref_;
unsigned f_count_, sz_;
virtual vector<ScoredHyp>* GetSamples()=0;
inline void SetScorer(LocalScorer* scorer) { scorer_ = scorer; }
inline void SetRef(vector<WordID>& ref) { ref_ = &ref; }
inline unsigned get_f_count() { return f_count_; }
inline unsigned get_sz() { return sz_; }
};
struct HSReporter
{
string task_id_;
HSReporter(string task_id) : task_id_(task_id) {}
inline void update_counter(string name, unsigned amount) {
cerr << "reporter:counter:" << task_id_ << "," << name << "," << amount << endl;
}
inline void update_gcounter(string name, unsigned amount) {
cerr << "reporter:counter:Global," << name << "," << amount << endl;
}
};
inline ostream& _np(ostream& out) { return out << resetiosflags(ios::showpos); }
inline ostream& _p(ostream& out) { return out << setiosflags(ios::showpos); }
inline ostream& _p2(ostream& out) { return out << setprecision(2); }
inline ostream& _p5(ostream& out) { return out << setprecision(5); }
inline void printWordIDVec(vector<WordID>& v, ostream& os=cerr)
{
for (unsigned i = 0; i < v.size(); i++) {
os << TD::Convert(v[i]);
if (i < v.size()-1) os << " ";
}
}
template<typename T>
inline T sign(T z)
{
if (z == 0) return 0;
return z < 0 ? -1 : +1;
}
} // namespace
#endif
|
