summaryrefslogtreecommitdiff
path: root/dtrain/dtrain.cc
diff options
context:
space:
mode:
Diffstat (limited to 'dtrain/dtrain.cc')
-rw-r--r--dtrain/dtrain.cc218
1 files changed, 148 insertions, 70 deletions
diff --git a/dtrain/dtrain.cc b/dtrain/dtrain.cc
index 25249c7f..8464a429 100644
--- a/dtrain/dtrain.cc
+++ b/dtrain/dtrain.cc
@@ -40,14 +40,15 @@ init(int argc, char** argv, boostpo::variables_map* conf)
boostpo::options_description opts( "Options" );
opts.add_options()
( "decoder-config,c", boostpo::value<string>(), "configuration file for cdec" )
- ( "kbest,k", boostpo::value<int>(), "k for kbest" )
+ ( "kbest,k", boostpo::value<size_t>(), "k for kbest" )
( "ngrams,n", boostpo::value<int>(), "n for Ngrams" )
- ( "filter,f", boostpo::value<string>(), "filter kbest list" );
+ ( "filter,f", boostpo::value<string>(), "filter kbest list" )
+ ( "test", "run tests and exit");
boostpo::options_description cmdline_options;
cmdline_options.add(opts);
boostpo::store( parse_command_line(argc, argv, cmdline_options), *conf );
boostpo::notify( *conf );
- if ( ! conf->count("decoder-config") ) {
+ if ( ! (conf->count("decoder-config") || conf->count("test")) ) {
cerr << cmdline_options << endl;
return false;
}
@@ -67,7 +68,7 @@ struct KBestList {
struct KBestGetter : public DecoderObserver
{
KBestGetter( const size_t k ) : k_(k) {}
- size_t k_;
+ const size_t k_;
KBestList kb;
virtual void
@@ -164,7 +165,7 @@ struct NgramCounts
map<size_t, size_t> clipped;
map<size_t, size_t> sum;
- NgramCounts&
+ void
operator+=( const NgramCounts& rhs )
{
assert( N_ == rhs.N_ );
@@ -247,6 +248,7 @@ brevity_penaly( const size_t hyp_len, const size_t ref_len )
/*
* bleu
* as in "BLEU: a Method for Automatic Evaluation of Machine Translation" (Papineni et al. '02)
+ * page TODO
* 0 if for N one of the counts = 0
*/
double
@@ -272,6 +274,7 @@ bleu( NgramCounts& counts, const size_t hyp_len, const size_t ref_len,
/*
* stupid_bleu
* as in "ORANGE: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation (Lin & Och '04)
+ * page TODO
* 0 iff no 1gram match
*/
double
@@ -298,6 +301,7 @@ stupid_bleu( NgramCounts& counts, const size_t hyp_len, const size_t ref_len,
/*
* smooth_bleu
* as in "An End-to-End Discriminative Approach to Machine Translation" (Liang et al. '06)
+ * page TODO
* max. 0.9375
*/
double
@@ -324,6 +328,7 @@ smooth_bleu( NgramCounts& counts, const size_t hyp_len, const size_t ref_len,
/*
* approx_bleu
* as in "Online Large-Margin Training for Statistical Machine Translation" (Watanabe et al. '07)
+ * page TODO
*
*/
double
@@ -348,11 +353,16 @@ register_and_convert(const vector<string>& strs, vector<WordID>& ids)
}
+/*
+ *
+ *
+ */
void
test_ngrams()
{
cout << "Testing ngrams..." << endl << endl;
size_t N = 5;
+ cout << "N = " << N << endl;
vector<int> a; // hyp
vector<int> b; // ref
cout << "a ";
@@ -373,18 +383,28 @@ test_ngrams()
c += c;
cout << endl;
c.print();
+ cout << endl;
}
+
+/*
+ *
+ *
+ */
double
approx_equal( double x, double y )
{
const double EPSILON = 1E-5;
- if ( x == 0 ) return fabs(y) <= EPSILON;
- if ( y == 0 ) return fabs(x) <= EPSILON;
+ if ( x == 0 ) return fabs( y ) <= EPSILON;
+ if ( y == 0 ) return fabs( x ) <= EPSILON;
return fabs( x - y ) / max( fabs(x), fabs(y) ) <= EPSILON;
}
+/*
+ *
+ *
+ */
#include <boost/assign/std/vector.hpp>
#include <iomanip>
void
@@ -423,104 +443,162 @@ test_metrics()
cout << setw(14) << "smooth bleu = " << smooth << endl;
cout << setw(14) << "stupid bleu = " << stupid << endl << endl;
}
+ cout << endl;
}
-
/*
- * main
+ *
*
*/
-int
-main(int argc, char** argv)
+void
+test_SetWeights()
{
- /*vector<string> v;
- for (int i = 0; i <= 10; i++) {
- v.push_back("asdf");
- }
- vector<vector<string> > ng = ngrams(v, 5);
- for (int i = 0; i < ng.size(); i++) {
- for (int j = 0; j < ng[i].size(); j++) {
- cout << " " << ng[i][j];
- }
- cout << endl;
- }*/
-
- test_metrics();
-
-
- //NgramCounts counts2 = make_ngram_counts( ref_ids, ref_ids, 4);
- //counts += counts2;
- //cout << counts.cNipped[1] << endl;
-
- //size_t c, r; // c length of candidates, r of references
- //c += cand.size();
- //r += ref.size();
- /*NgramMatches ngm; // for approx bleu
- ngm.sum = 1;
- ngm.clipped = 1;
+ cout << "Testing Weights::SetWeight..." << endl << endl;
+ Weights weights;
+ SparseVector<double> lambdas;
+ weights.InitSparseVector( &lambdas );
+ weights.SetWeight( &lambdas, "test", 0 );
+ weights.SetWeight( &lambdas, "test1", 1 );
+ WordID fid = FD::Convert( "test2" );
+ weights.SetWeight( &lambdas, fid, 2 );
+ string fn = "weights-test";
+ cout << "FD::NumFeats() " << FD::NumFeats() << endl;
+ assert( FD::NumFeats() == 4 );
+ weights.WriteToFile( fn, true );
+ cout << endl;
+}
- NgramMatches x;
- x.clipped = 1;
- x.sum = 1;
- x += ngm;
- x += x;
- x+= ngm;
+/*
+ *
+ *
+ */
+void
+run_tests()
+{
+ cout << endl;
+ test_ngrams();
+ cout << endl;
+ test_metrics();
+ cout << endl;
+ test_SetWeights();
+ exit(0);
+}
- cout << x.clipped << " " << x.sum << endl;*/
+void
+print_FD()
+{
+ for ( size_t i = 0; i < FD::NumFeats(); i++ ) cout << FD::Convert(i)<< endl;
+}
- /*register_feature_functions();
- SetSilent(true);
- boost::program_options::variables_map conf;
+/*
+ * main
+ *
+ */
+int
+main(int argc, char** argv)
+{
+ //SetSilent(true);
+ boostpo::variables_map conf;
if (!init(argc, argv, &conf)) return 1;
+ if ( conf.count("test") ) run_tests();
+ register_feature_functions();
+ size_t k = conf["kbest"].as<size_t>();
ReadFile ini_rf(conf["decoder-config"].as<string>());
Decoder decoder(ini_rf.stream());
+ KBestGetter observer(k);
+
+ // for approx. bleu
+ //NgramCounts global_counts;
+ //size_t global_hyp_len;
+ //size_t global_ref_len;
+
Weights weights;
SparseVector<double> lambdas;
weights.InitSparseVector(&lambdas);
+ vector<double> dense_weights;
- int k = conf["kbest"].as<int>();
+ lambdas.set_value(FD::Convert("logp"), 0);
- KBestGetter observer(k);
- string in, psg;
+
vector<string> strs;
- int i = 0;
- while(getline(cin, in)) {
- if (!SILENT) cerr << "getting kbest for sentence #" << i << endl;
+ string in, psg;
+ size_t i = 0;
+ while( getline(cin, in) ) {
+ if ( !SILENT ) cerr << endl << endl << "Getting kbest for sentence #" << i << endl;
+ // why? why!?
+ dense_weights.clear();
+ weights.InitFromVector( lambdas );
+ weights.InitVector( &dense_weights );
+ decoder.SetWeights( dense_weights );
+ //cout << "use_shell " << dense_weights[FD::Convert("use_shell")] << endl;
strs.clear();
- boost::split(strs, in, boost::is_any_of("\t"));
- psg = boost::replace_all_copy(strs[2], " __NEXT_RULE__ ", "\n"); psg += "\n";
+ boost::split( strs, in, boost::is_any_of("\t") );
+ psg = boost::replace_all_copy( strs[2], " __NEXT_RULE__ ", "\n" ); psg += "\n";
+ //decoder.SetId(i);
decoder.SetSentenceGrammar( psg );
decoder.Decode( strs[0], &observer );
KBestList* kb = observer.getkb();
- // FIXME not pretty iterating twice over k
- for (int i = 0; i < k; i++) {
- for (int j = 0; j < kb->sents[i].size(); ++j) {
- cout << TD::Convert(kb->sents[i][j]) << endl;
+ for ( size_t i = 0; i < k; i++ ) {
+ cout << i << " ";
+ for (size_t j = 0; j < kb->sents[i].size(); ++j ) {
+ cout << TD::Convert( kb->sents[i][j] ) << " ";
}
+ cout << kb->scores[i];
+ cout << endl;
}
+ lambdas.set_value( FD::Convert("use_shell"), 1 );
+ lambdas.set_value( FD::Convert("use_a"), 1 );
+ //print_FD();
}
+
+ weights.WriteToFile( "weights-final", true );
- return 0;*/
+ return 0;
}
+ // next: FMap, ->sofia, ->FMap, -> Weights
+ // learner gets all used features (binary! and dense (logprob is sum of logprobs!))
+ // only for those feats with weight > 0 after learning
+ // see decoder line 548
+
/*
* TODO
- * for t =1..T
- * mapper, reducer (average, handle ngram statistics for approx bleu)
- * 1st streaming
- * batch, non-batch in the mapper (what sofia gets)
- * filter yes/no
+ * iterate over training set, for t=1..T
+ * mapred impl
+ * mapper: main
+ * reducer: average weights, global NgramCounts for approx. bleu
+ * 1st cut: hadoop streaming?
+ * batch, non-batch in the mapper (what sofia gets, regenerated Kbest lists)
+ * filter kbest yes/no
* sofia: --eta_type explicit
- * psg preparation
- * set ref?
- * shared LM?
+ * psg preparation source\tref\tpsg
+ * set reference for cdec?
+ * LM
+ * shared?
+ * startup?
* X reference(s) for *bleu!?
- * kbest nicer!? shared_ptr
- * multipartite
+ * kbest nicer (do not iterate twice)!? -> shared_ptr
+ * multipartite ranking
* weights! global, per sentence from global, featuremap
- * todo const
+ * const decl...
+ * sketch: batch/iter options
+ * weights.cc: why wv_?
+ * --weights cmd line (for iterations): script to call again/hadoop streaming?
+ * I do not need to remember features, cdec does
+ * resocre hg?
+ * do not use Decoder::Decode!?
+ * what happens if feature not in FD? 0???
*/
+
+/*
+ * PROBLEMS
+ * cdec kbest vs 1best (no -k param)
+ * FD, Weights::wv_ grow too large, see utils/weights.cc; decoder/hg.h; decoder/scfg_translator.cc; utils/fdict.cc!?
+ * sparse vector instead of vector<double> for weights in Decoder?
+ * PhraseModel_* features for psg!? (seem to be generated)
+ */
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-11 08:11:33 +0000