diff options
46 files changed, 933 insertions, 1577 deletions
diff --git a/training/dtrain/Makefile.am b/training/dtrain/Makefile.am index 7717ec86..aadd376d 100644 --- a/training/dtrain/Makefile.am +++ b/training/dtrain/Makefile.am @@ -1,6 +1,6 @@ bin_PROGRAMS = dtrain -dtrain_SOURCES = dtrain.cc dtrain.h sample.h pairs.h score.h +dtrain_SOURCES = dtrain.cc dtrain.h sample.h update.h score.h dtrain_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval diff --git a/training/dtrain/dtrain.cc b/training/dtrain/dtrain.cc index 69630206..1b7047b0 100644 --- a/training/dtrain/dtrain.cc +++ b/training/dtrain/dtrain.cc @@ -1,541 +1,207 @@ #include "dtrain.h" #include "score.h" #include "sample.h" -#include "pairs.h" +#include "update.h" using namespace dtrain; - -bool -dtrain_init(int argc, char** argv, po::variables_map* conf) -{ - po::options_description ini("Configuration File Options"); - ini.add_options() - ("bitext,b", po::value<string>(), "bitext: 'src ||| tgt ||| tgt ||| ...'") - ("output,o", po::value<string>()->default_value("-"), "output weights file, '-' for STDOUT") - ("input_weights,w", po::value<string>(), "input weights file (e.g. from previous iteration)") - ("decoder_config,d", po::value<string>(), "configuration file for cdec") - ("print_weights", po::value<string>(), "weights to print on each iteration") - ("stop_after", po::value<unsigned>()->default_value(0), "stop after X input sentences") - ("keep", po::value<bool>()->zero_tokens(), "keep weights files for each iteration") - ("epochs", po::value<unsigned>()->default_value(10), "# of iterations T (per shard)") - ("k", po::value<unsigned>()->default_value(100), "how many translations to sample") - ("filter", po::value<string>()->default_value("uniq"), "filter kbest list: 'not', 'uniq'") - ("hi_lo", po::value<float>()->default_value(0.1), "hi and lo (X) for XYX (default 0.1), <= 0.5") - ("N", po::value<unsigned>()->default_value(4), "N for Ngrams (BLEU)") - ("scorer", po::value<string>()->default_value("stupid_bleu"), "scoring: bleu, stupid_, smooth_, approx_, lc_") - ("learning_rate", po::value<weight_t>()->default_value(1.0), "learning rate") - ("gamma", po::value<weight_t>()->default_value(0.), "gamma for SVM (0 for perceptron)") - ("select_weights", po::value<string>()->default_value("last"), "output best, last, avg weights ('VOID' to throw away)") - ("rescale", po::value<bool>()->zero_tokens(), "(re)scale data and weight vector to unit length") - ("l1_reg", po::value<string>()->default_value("none"), "apply l1 regularization with clipping as in 'Tsuroka et al' (2010)") - ("l1_reg_strength", po::value<weight_t>(), "l1 regularization strength") - ("fselect", po::value<weight_t>()->default_value(-1), "select top x percent (or by threshold) of features after each epoch NOT IMPLEMENTED") // TODO - ("loss_margin", po::value<weight_t>()->default_value(0.), "update if no error in pref pair but model scores this near") - ("pclr", po::value<string>()->default_value("no"), "use a (simple|adagrad) per-coordinate learning rate") - ("batch", po::value<bool>()->zero_tokens(), "do batch optimization") - ("repeat", po::value<unsigned>()->default_value(1), "repeat optimization over kbest list this number of times") - ("output_ranking", po::value<string>()->default_value(""), "output scored kbests to dir") - ("noup", po::value<bool>()->zero_tokens(), "dont't optimize"); - po::options_description cl("Command Line Options"); - cl.add_options() - ("config,c", po::value<string>(), "dtrain config file") - ("quiet,q", po::value<bool>()->zero_tokens(), "be quiet") - ("verbose,v", po::value<bool>()->zero_tokens(), "be verbose"); - cl.add(ini); - po::store(parse_command_line(argc, argv, cl), *conf); - if (conf->count("config")) { - ifstream ini_f((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(ini_f, ini), *conf); - } - po::notify(*conf); - if (!conf->count("decoder_config")) { - cerr << cl << endl; - return false; - } - if ((*conf)["hi_lo"].as<float>() > 0.5 || (*conf)["hi_lo"].as<float>() < 0.01) { - cerr << "hi_lo must lie in [0.01, 0.5]" << endl; - return false; - } - if (!conf->count("bitext")) { - cerr << "No training data given." << endl; - return false; - } - if ((*conf)["select_weights"].as<string>() != "last" && (*conf)["select_weights"].as<string>() != "best" && - (*conf)["select_weights"].as<string>() != "avg" && (*conf)["select_weights"].as<string>() != "VOID") { - cerr << "Wrong 'select_weights' param: '" << (*conf)["select_weights"].as<string>() << "', use 'last' or 'best'." << endl; - return false; - } - return true; -} - int main(int argc, char** argv) { - // handle most parameters + // get configuration po::variables_map conf; - if (!dtrain_init(argc, argv, &conf)) exit(1); // something is wrong - - bool quiet = false; - if (conf.count("quiet")) quiet = true; - bool verbose = false; - if (conf.count("verbose")) verbose = true; - bool noup = false; - if (conf.count("noup")) noup = true; - bool keep = false; - if (conf.count("keep")) keep = true; - bool rescale = false; - if (conf.count("rescale")) rescale = true; - - const unsigned k = conf["k"].as<unsigned>(); - const unsigned N = conf["N"].as<unsigned>(); - const unsigned T = conf["epochs"].as<unsigned>(); - const unsigned stop_after = conf["stop_after"].as<unsigned>(); - const string select_weights = conf["select_weights"].as<string>(); - const string output_ranking = conf["output_ranking"].as<string>(); - const float hi_lo = conf["hi_lo"].as<float>(); - int repeat = conf["repeat"].as<unsigned>(); - weight_t loss_margin = conf["loss_margin"].as<weight_t>(); - bool batch = false; - if (conf.count("batch")) batch = true; - if (loss_margin > 9998.) loss_margin = std::numeric_limits<float>::max(); - const string pclr = conf["pclr"].as<string>(); - bool average = false; - if (select_weights == "avg") - average = true; + if (!dtrain_init(argc, argv, &conf)) + exit(1); // something is wrong + const size_t k = conf["k"].as<size_t>(); + const size_t N = conf["N"].as<size_t>(); + const size_t T = conf["iterations"].as<size_t>(); + const weight_t eta = conf["learning_rate"].as<weight_t>(); + const weight_t error_margin = conf["error_margin"].as<weight_t>(); + const bool average = conf["average"].as<bool>(); + const bool keep = conf["keep"].as<bool>(); + const weight_t l1_reg = conf["l1_reg"].as<weight_t>(); + const string output_fn = conf["output"].as<string>(); vector<string> print_weights; - if (conf.count("print_weights")) - boost::split(print_weights, conf["print_weights"].as<string>(), boost::is_any_of(" ")); + boost::split(print_weights, conf["print_weights"].as<string>(), boost::is_any_of(" ")); // setup decoder register_feature_functions(); SetSilent(true); - ReadFile ini_rf(conf["decoder_config"].as<string>()); - if (!quiet) - cerr << setw(25) << "cdec conf " << "'" << conf["decoder_config"].as<string>() << "'" << endl; - Decoder decoder(ini_rf.stream()); + ReadFile f(conf["decoder_config"].as<string>()); + Decoder decoder(f.stream()); // setup decoder observer ScoredKbest* observer = new ScoredKbest(k, new PerSentenceBleuScorer(N)); - // init weights + // weights vector<weight_t>& decoder_weights = decoder.CurrentWeightVector(); - - SparseVector<weight_t> lambdas, cumulative_penalties, w_average, fixed; - if (conf.count("input_weights")) + SparseVector<weight_t> lambdas, w_average; + if (conf.count("input_weights")) { Weights::InitFromFile(conf["input_weights"].as<string>(), &decoder_weights); - Weights::InitSparseVector(decoder_weights, &lambdas); - - // meta params for perceptron, SVM - weight_t eta = conf["learning_rate"].as<weight_t>(); - weight_t gamma = conf["gamma"].as<weight_t>(); - - // faster perceptron: consider only misranked pairs, see - bool faster_perceptron = false; - if (gamma==0 && loss_margin==0) faster_perceptron = true; - - // l1 regularization - bool l1naive = false; - bool l1clip = false; - bool l1cumul = false; - weight_t l1_reg = 0; - if (conf["l1_reg"].as<string>() != "none") { - string s = conf["l1_reg"].as<string>(); - if (s == "naive") l1naive = true; - else if (s == "clip") l1clip = true; - else if (s == "cumul") l1cumul = true; - l1_reg = conf["l1_reg_strength"].as<weight_t>(); + Weights::InitSparseVector(decoder_weights, &lambdas); } - // output - string output_fn = conf["output"].as<string>(); // input - string input_fn; - ReadFile input(conf["bitext"].as<string>()); - // buffer input for t > 0 - vector<string> src_str_buf; // source strings (decoder takes only strings) - vector<vector<vector<WordID> > > refs_as_ids_buf; // references as WordID vecs - - unsigned in_sz = std::numeric_limits<unsigned>::max(); // input index, input size - vector<pair<score_t, score_t> > all_scores; - score_t max_score = 0.; - unsigned best_it = 0; - float overall_time = 0.; - - // output conf - if (!quiet) { - cerr << _p5; - cerr << endl << "dtrain" << endl << "Parameters:" << endl; - cerr << setw(25) << "k " << k << endl; - cerr << setw(25) << "N " << N << endl; - cerr << setw(25) << "T " << T << endl; - cerr << setw(25) << "batch " << batch << endl; - cerr << setw(25) << "learning rate " << eta << endl; - cerr << setw(25) << "gamma " << gamma << endl; - cerr << setw(25) << "loss margin " << loss_margin << endl; - cerr << setw(25) << "faster perceptron " << faster_perceptron << endl; - cerr << setw(25) << "hi lo " << hi_lo << endl; - cerr << setw(25) << "select weights " << "'" << select_weights << "'" << endl; - if (conf.count("l1_reg")) - cerr << setw(25) << "l1 reg " << l1_reg << " '" << conf["l1_reg"].as<string>() << "'" << endl; - if (rescale) - cerr << setw(25) << "rescale " << rescale << endl; - cerr << setw(25) << "pclr " << pclr << endl; - cerr << setw(25) << "repeat " << repeat << endl; - cerr << setw(25) << "cdec conf " << "'" << conf["decoder_config"].as<string>() << "'" << endl; - cerr << setw(25) << "input " << "'" << input_fn << "'" << endl; - cerr << setw(25) << "output " << "'" << output_fn << "'" << endl; - if (conf.count("input_weights")) - cerr << setw(25) << "weights in " << "'" << conf["input_weights"].as<string>() << "'" << endl; - if (stop_after > 0) - cerr << setw(25) << "stop_after " << stop_after << endl; - if (!verbose) cerr << "(a dot represents " << DTRAIN_DOTS << " inputs)" << endl; - } + string input_fn = conf["bitext"].as<string>(); + ReadFile input(input_fn); + vector<string> buf; // source strings (decoder takes only strings) + vector<vector<Ngrams> > buf_ngs; // compute ngrams and lengths of references + vector<vector<size_t> > buf_ls; // just once + size_t input_sz = 0; + + // output configuration + cerr << _p5 << "dtrain" << endl << "Parameters:" << endl; + cerr << setw(25) << "k " << k << endl; + cerr << setw(25) << "N " << N << endl; + cerr << setw(25) << "T " << T << endl; + cerr << setw(25) << "learning rate " << eta << endl; + cerr << setw(25) << "error margin " << error_margin << endl; + cerr << setw(25) << "l1 reg " << l1_reg << endl; + cerr << setw(25) << "decoder conf " << "'" << conf["decoder_config"].as<string>() << "'" << endl; + cerr << setw(25) << "input " << "'" << input_fn << "'" << endl; + cerr << setw(25) << "output " << "'" << output_fn << "'" << endl; + if (conf.count("input_weights")) + cerr << setw(25) << "weights in " << "'" << conf["input_weights"].as<string>() << "'" << endl; + cerr << "(a dot per input)" << endl; - // pclr - SparseVector<weight_t> learning_rates; - // batch - SparseVector<weight_t> batch_updates; - score_t batch_loss; + // meta + weight_t best=0., gold_prev=0.; + size_t best_iteration = 0; + time_t total_time = 0.; - for (unsigned t = 0; t < T; t++) // T epochs + for (size_t t = 0; t < T; t++) // T iterations { time_t start, end; time(&start); - score_t score_sum = 0.; - score_t model_sum(0); - unsigned ii = 0, rank_errors = 0, margin_violations = 0, npairs = 0, f_count = 0, list_sz = 0, kbest_loss_improve = 0; - batch_loss = 0.; - if (!quiet) cerr << "Iteration #" << t+1 << " of " << T << "." << endl; + weight_t gold_sum=0., model_sum=0.; + size_t i = 0, num_pairs = 0, feature_count = 0, list_sz = 0; + + cerr << "Iteration #" << t+1 << " of " << T << "." << endl; while(true) { + bool next = true; - string in; - vector<string> refs; - bool next = false, stop = false; // next iteration or premature stop + // getting input if (t == 0) { - if(!getline(*input, in)) next = true; - boost::algorithm::split_regex(refs, in, boost::regex(" \\|\\|\\| ")); - in = refs[0]; - refs.erase(refs.begin()); - } else { - if (ii == in_sz) next = true; // stop if we reach the end of our input - } - // stop after X sentences (but still go on for those) - if (stop_after > 0 && stop_after == ii && !next) stop = true; - - // produce some pretty output - if (!quiet && !verbose) { - if (ii == 0) cerr << " "; - if ((ii+1) % (DTRAIN_DOTS) == 0) { - cerr << "."; - cerr.flush(); - } - if ((ii+1) % (20*DTRAIN_DOTS) == 0) { - cerr << " " << ii+1 << endl; - if (!next && !stop) cerr << " "; - } - if (stop) { - if (ii % (20*DTRAIN_DOTS) != 0) cerr << " " << ii << endl; - cerr << "Stopping after " << stop_after << " input sentences." << endl; + string in; + if(!getline(*input, in)) { + next = false; } else { - if (next) { - if (ii % (20*DTRAIN_DOTS) != 0) cerr << " " << ii << endl; + vector<string> parts; + boost::algorithm::split_regex(parts, in, boost::regex(" \\|\\|\\| ")); + buf.push_back(parts[0]); + parts.erase(parts.begin()); + buf_ngs.push_back({}); + buf_ls.push_back({}); + for (auto s: parts) { + vector<WordID> r; + vector<string> tok; + boost::split(tok, s, boost::is_any_of(" ")); + RegisterAndConvert(tok, r); + buf_ngs.back().emplace_back(MakeNgrams(r, N)); + buf_ls.back().push_back(r.size()); } } + } else { + next = i<input_sz; } - // next iteration - if (next || stop) break; - - // weights - lambdas.init_vector(&decoder_weights); - - // getting input - if (t == 0) { - vector<vector<WordID> > cur_refs; - for (auto r: refs) { - vector<WordID> cur_ref; - vector<string> tok; - boost::split(tok, r, boost::is_any_of(" ")); - RegisterAndConvert(tok, cur_ref); - cur_refs.push_back(cur_ref); - } - refs_as_ids_buf.push_back(cur_refs); - src_str_buf.push_back(in); - } - observer->SetReference(refs_as_ids_buf[ii]); - if (t == 0) - decoder.Decode(in, observer); - else - decoder.Decode(src_str_buf[ii], observer); - - // get (scored) samples + // produce some pretty output + if (i == 0 || (i+1)%20==0) + cerr << " "; + cerr << "."; + cerr.flush(); + if (!next) + if (i%20 != 0) cerr << " " << i << endl; + + // stop iterating + if (!next) break; + + // decode + if (t > 0 || i > 0) + lambdas.init_vector(&decoder_weights); + observer->SetReference(buf_ngs[i], buf_ls[i]); + decoder.Decode(buf[i], observer); vector<ScoredHyp>* samples = observer->GetSamples(); - if (output_ranking != "") { - WriteFile of(output_ranking+"/"+to_string(t)+"."+to_string(ii)+".list"); // works with '-' - stringstream ss; - for (auto s: *samples) { - ss << ii << " ||| "; - PrintWordIDVec(s.w, ss); - ss << " ||| " << s.model << " ||| " << s.score << endl; - } - of.get() << ss.str(); - } - - if (verbose) { - cerr << "--- refs for " << ii << ": "; - for (auto r: refs_as_ids_buf[ii]) { - PrintWordIDVec(r); - cerr << endl; - } - for (unsigned u = 0; u < samples->size(); u++) { - cerr << _p2 << _np << "[" << u << ". '"; - PrintWordIDVec((*samples)[u].w); - cerr << "'" << endl; - cerr << "SCORE=" << (*samples)[u].score << ",model="<< (*samples)[u].model << endl; - cerr << "F{" << (*samples)[u].f << "} ]" << endl << endl; - } - } - - if (repeat == 1) { - score_sum += (*samples)[0].score; // stats for 1best - model_sum += (*samples)[0].model; - } - - f_count += observer->GetFeatureCount(); + // stats for 1best + gold_sum += samples->front().gold; + model_sum += samples->front().model; + feature_count += observer->GetFeatureCount(); list_sz += observer->GetSize(); - // weight updates - if (!noup) { - // get pairs - vector<pair<ScoredHyp,ScoredHyp> > pairs; - MakePairs(samples, pairs, faster_perceptron, hi_lo); - int cur_npairs = pairs.size(); - npairs += cur_npairs; - - score_t kbest_loss_first = 0.0, kbest_loss_last = 0.0; - - for (vector<pair<ScoredHyp,ScoredHyp> >::iterator it = pairs.begin(); - it != pairs.end(); it++) { - if (rescale) { - it->first.f /= it->first.f.l2norm(); - it->second.f /= it->second.f.l2norm(); - } - score_t model_diff = it->first.model - it->second.model; - score_t loss = max(0.0, -1.0 * model_diff); - kbest_loss_first += loss; - } - - score_t kbest_loss = 0.0; - for (int ki=0; ki < repeat; ki++) { - - SparseVector<weight_t> lambdas_copy; // for l1 regularization - SparseVector<weight_t> sum_up; // for pclr - if (l1naive||l1clip||l1cumul) lambdas_copy = lambdas; - - for (vector<pair<ScoredHyp,ScoredHyp> >::iterator it = pairs.begin(); - it != pairs.end(); it++) { - score_t model_diff = it->first.model - it->second.model; - score_t loss = max(0.0, -1.0 * model_diff); - - if (repeat > 1) { - model_diff = lambdas.dot(it->first.f) - lambdas.dot(it->second.f); - kbest_loss += loss; - } - bool rank_error = false; - score_t margin; - if (faster_perceptron) { // we only have considering misranked pairs - rank_error = true; // pair sampling already did this for us - margin = std::numeric_limits<float>::max(); - } else { - rank_error = model_diff<=0.0; - margin = fabs(model_diff); - if (!rank_error && margin < loss_margin) margin_violations++; - } - if (rank_error && ki==0) rank_errors++; - if (rank_error || margin < loss_margin) { - SparseVector<weight_t> diff_vec = it->first.f - it->second.f; - if (batch) { - batch_loss += max(0., -1.0 * model_diff); - batch_updates += diff_vec; - continue; - } - if (pclr != "no") { - sum_up += diff_vec; + // get pairs and update + vector<pair<ScoredHyp,ScoredHyp> > pairs; + SparseVector<weight_t> updates; + num_pairs += CollectUpdates(samples, updates, error_margin); + SparseVector<weight_t> lambdas_copy; + if (l1_reg) + lambdas_copy = lambdas; + lambdas.plus_eq_v_times_s(updates, eta); + + // l1 regularization + // NB: regularization is done after each sentence, + // not after every single pair! + if (l1_reg) { + SparseVector<weight_t>::iterator it = lambdas.begin(); + for (; it != lambdas.end(); ++it) { + if (it->second == 0) continue; + if (!lambdas_copy.get(it->first) // new or.. + || lambdas_copy.get(it->first)!=it->second) // updated feature + { + weight_t v = it->second; + if (v > 0) { + it->second = max(0., v - l1_reg); } else { - lambdas.plus_eq_v_times_s(diff_vec, eta); - if (gamma) lambdas.plus_eq_v_times_s(lambdas, -2*gamma*eta*(1./cur_npairs)); - } - } - } - - // per-coordinate learning rate - if (pclr != "no") { - SparseVector<weight_t>::iterator it = sum_up.begin(); - for (; it != sum_up.end(); ++it) { - if (pclr == "simple") { - lambdas[it->first] += it->second / max(1.0, learning_rates[it->first]); - learning_rates[it->first]++; - } else if (pclr == "adagrad") { - if (learning_rates[it->first] == 0) { - lambdas[it->first] += it->second * eta; - } else { - lambdas[it->first] += it->second * eta * learning_rates[it->first]; - } - learning_rates[it->first] += pow(it->second, 2.0); + it->second = min(0., v + l1_reg); } } } + } - // l1 regularization - // please note that this regularizations happen - // after a _sentence_ -- not after each example/pair! - if (l1naive) { - SparseVector<weight_t>::iterator it = lambdas.begin(); - for (; it != lambdas.end(); ++it) { - if (!lambdas_copy.get(it->first) || lambdas_copy.get(it->first)!=it->second) { - it->second *= max(0.0000001, eta/(eta+learning_rates[it->first])); // FIXME - learning_rates[it->first]++; - it->second -= sign(it->second) * l1_reg; - } - } - } else if (l1clip) { - SparseVector<weight_t>::iterator it = lambdas.begin(); - for (; it != lambdas.end(); ++it) { - if (!lambdas_copy.get(it->first) || lambdas_copy.get(it->first)!=it->second) { - if (it->second != 0) { - weight_t v = it->second; - if (v > 0) { - it->second = max(0., v - l1_reg); - } else { - it->second = min(0., v + l1_reg); - } - } - } - } - } else if (l1cumul) { - weight_t acc_penalty = (ii+1) * l1_reg; // ii is the index of the current input - SparseVector<weight_t>::iterator it = lambdas.begin(); - for (; it != lambdas.end(); ++it) { - if (!lambdas_copy.get(it->first) || lambdas_copy.get(it->first)!=it->second) { - if (it->second != 0) { - weight_t v = it->second; - weight_t penalized = 0.; - if (v > 0) { - penalized = max(0., v-(acc_penalty + cumulative_penalties.get(it->first))); - } else { - penalized = min(0., v+(acc_penalty - cumulative_penalties.get(it->first))); - } - it->second = penalized; - cumulative_penalties.set_value(it->first, cumulative_penalties.get(it->first)+penalized); - } - } - } - } - - if (ki==repeat-1) { // done - kbest_loss_last = kbest_loss; - if (repeat > 1) { - score_t best_model = -std::numeric_limits<score_t>::max(); - unsigned best_idx = 0; - for (unsigned i=0; i < samples->size(); i++) { - score_t s = lambdas.dot((*samples)[i].f); - if (s > best_model) { - best_idx = i; - best_model = s; - } - } - score_sum += (*samples)[best_idx].score; - model_sum += best_model; - } - } - } // repeat - - if ((kbest_loss_first - kbest_loss_last) >= 0) kbest_loss_improve++; - - } // noup - - if (rescale) lambdas /= lambdas.l2norm(); - - ++ii; + i++; } // input loop - if (t == 0) in_sz = ii; // remember size of input (# lines) - - if (batch) { - lambdas.plus_eq_v_times_s(batch_updates, eta); - if (gamma) lambdas.plus_eq_v_times_s(lambdas, -2*gamma*eta*(1./npairs)); - batch_updates.clear(); - } - - if (average) w_average += lambdas; - - // print some stats - score_t score_avg = score_sum/(score_t)in_sz; - score_t model_avg = model_sum/(score_t)in_sz; - score_t score_diff, model_diff; - if (t > 0) { - score_diff = score_avg - all_scores[t-1].first; - model_diff = model_avg - all_scores[t-1].second; - } else { - score_diff = score_avg; - model_diff = model_avg; + if (t == 0) + input_sz = i; // remember size of input (# lines) + + // update average + if (average) + w_average += lambdas; + + // stats + weight_t gold_avg = gold_sum/(weight_t)input_sz; + size_t non_zero = (size_t)lambdas.num_nonzero(); + cerr << _p5 << _p << "WEIGHTS" << endl; + for (auto name: print_weights) + cerr << setw(18) << name << " = " << lambdas.get(FD::Convert(name)) << endl; + cerr << " ---" << endl; + cerr << _np << " 1best avg score: " << gold_avg; + cerr << _p << " (" << gold_avg-gold_prev << ")" << endl; + cerr << _np << " 1best avg model score: " << model_sum/(weight_t)input_sz << endl; + cerr << " avg # pairs: "; + cerr << _np << num_pairs/(float)input_sz << endl; + cerr << " non-0 feature count: " << non_zero << endl; + cerr << " avg list sz: " << list_sz/(float)input_sz << endl; + cerr << " avg f count: " << feature_count/(float)list_sz << endl; + + if (gold_avg > best) { + best = gold_avg; + best_iteration = t; } + gold_prev = gold_avg; - unsigned nonz = 0; - if (!quiet) nonz = (unsigned)lambdas.num_nonzero(); - - if (!quiet) { - cerr << _p5 << _p << "WEIGHTS" << endl; - for (vector<string>::iterator it = print_weights.begin(); it != print_weights.end(); it++) { - cerr << setw(18) << *it << " = " << lambdas.get(FD::Convert(*it)) << endl; - } - cerr << " ---" << endl; - cerr << _np << " 1best avg score: " << score_avg; - cerr << _p << " (" << score_diff << ")" << endl; - cerr << _np << " 1best avg model score: " << model_avg; - cerr << _p << " (" << model_diff << ")" << endl; - cerr << " avg # pairs: "; - cerr << _np << npairs/(float)in_sz << endl; - cerr << " avg # rank err: "; - cerr << rank_errors/(float)in_sz; - if (faster_perceptron) cerr << " (meaningless)"; - cerr << endl; - cerr << " avg # margin viol: "; - cerr << margin_violations/(float)in_sz << endl; - if (batch) cerr << " batch loss: " << batch_loss << endl; - cerr << " k-best loss imp: " << ((float)kbest_loss_improve/in_sz)*100 << "%" << endl; - cerr << " non0 feature count: " << nonz << endl; - cerr << " avg list sz: " << list_sz/(float)in_sz << endl; - cerr << " avg f count: " << f_count/(float)list_sz << endl; - } - - pair<score_t,score_t> remember; - remember.first = score_avg; - remember.second = model_avg; - all_scores.push_back(remember); - if (score_avg > max_score) { - max_score = score_avg; - best_it = t; - } time (&end); - float time_diff = difftime(end, start); - overall_time += time_diff; - if (!quiet) { - cerr << _p2 << _np << "(time " << time_diff/60. << " min, "; - cerr << time_diff/in_sz << " s/S)" << endl; - } - if (t+1 != T && !quiet) cerr << endl; - - if (noup) break; + time_t time_diff = difftime(end, start); + total_time += time_diff; + cerr << _p2 << _np << "(time " << time_diff/60. << " min, "; + cerr << time_diff/input_sz << " s/S)" << endl; + if (t+1 != T) cerr << endl; - // write weights to file - if (select_weights == "best" || keep) { + if (keep) { // keep intermediate weights lambdas.init_vector(&decoder_weights); string w_fn = "weights." + boost::lexical_cast<string>(t) + ".gz"; Weights::WriteToFile(w_fn, decoder_weights, true); @@ -543,51 +209,19 @@ main(int argc, char** argv) } // outer loop - if (average) w_average /= (weight_t)T; - - if (!noup) { - if (!quiet) cerr << endl << "Writing weights file to '" << output_fn << "' ..." << endl; - if (select_weights == "last" || average) { // last, average - WriteFile of(output_fn); - ostream& o = *of.stream(); - o.precision(17); - o << _np; - if (average) { - for (SparseVector<weight_t>::iterator it = w_average.begin(); it != w_average.end(); ++it) { - if (it->second == 0) continue; - o << FD::Convert(it->first) << '\t' << it->second << endl; - } - } else { - for (SparseVector<weight_t>::iterator it = lambdas.begin(); it != lambdas.end(); ++it) { - if (it->second == 0) continue; - o << FD::Convert(it->first) << '\t' << it->second << endl; - } - } - } else if (select_weights == "VOID") { // do nothing with the weights - } else { // best - if (output_fn != "-") { - CopyFile("weights."+boost::lexical_cast<string>(best_it)+".gz", output_fn); - } else { - ReadFile bestw("weights."+boost::lexical_cast<string>(best_it)+".gz"); - string o; - cout.precision(17); - cout << _np; - while(getline(*bestw, o)) cout << o << endl; - } - if (!keep) { - for (unsigned i = 0; i < T; i++) { - string s = "weights." + boost::lexical_cast<string>(i) + ".gz"; - unlink(s.c_str()); - } - } - } - if (!quiet) cerr << "done" << endl; + // final weights + if (average) { + w_average /= (weight_t)T; + w_average.init_vector(decoder_weights); + } else if (!keep) { + lambdas.init_vector(decoder_weights); } + Weights::WriteToFile(output_fn, decoder_weights, true); - if (!quiet) { - cerr << _p5 << _np << endl << "---" << endl << "Best iteration: "; - cerr << best_it+1 << " [SCORE = " << max_score << "]." << endl; - cerr << "This took " << overall_time/60. << " min." << endl; - } + cerr << _p5 << _np << endl << "---" << endl << "Best iteration: "; + cerr << best_iteration+1 << " [GOLD = " << best << "]." << endl; + cerr << "This took " << total_time/60. << " min." << endl; + + return 0; } diff --git a/training/dtrain/dtrain.h b/training/dtrain/dtrain.h index 2b466930..728b0698 100644 --- a/training/dtrain/dtrain.h +++ b/training/dtrain/dtrain.h @@ -1,9 +1,6 @@ #ifndef _DTRAIN_H_ #define _DTRAIN_H_ -#define DTRAIN_DOTS 10 // after how many inputs to display a '.' -#define DTRAIN_SCALE 100000 - #include <iomanip> #include <climits> #include <string.h> @@ -25,20 +22,17 @@ namespace po = boost::program_options; namespace dtrain { -typedef double score_t; - struct ScoredHyp { - vector<WordID> w; + vector<WordID> w; SparseVector<weight_t> f; - score_t model, score; - unsigned rank; + weight_t model, gold; + size_t rank; }; inline void RegisterAndConvert(const vector<string>& strs, vector<WordID>& ids) { - vector<string>::const_iterator it; for (auto s: strs) ids.push_back(TD::Convert(s)); } @@ -46,7 +40,7 @@ RegisterAndConvert(const vector<string>& strs, vector<WordID>& ids) inline void PrintWordIDVec(vector<WordID>& v, ostream& os=cerr) { - for (unsigned i = 0; i < v.size(); i++) { + for (size_t i = 0; i < v.size(); i++) { os << TD::Convert(v[i]); if (i < v.size()-1) os << " "; } @@ -57,12 +51,45 @@ inline ostream& _p(ostream& out) { return out << setiosflags(ios::showpos); } inline ostream& _p2(ostream& out) { return out << setprecision(2); } inline ostream& _p5(ostream& out) { return out << setprecision(5); } -template<typename T> -inline T -sign(T z) +bool +dtrain_init(int argc, char** argv, po::variables_map* conf) { - if (z == 0) return 0; - return z < 0 ? -1 : +1; + po::options_description ini("Configuration File Options"); + ini.add_options() + ("bitext,b", po::value<string>(), "bitext") + ("decoder_config,C", po::value<string>(), "configuration file for decoder") + ("iterations,T", po::value<size_t>()->default_value(10), "number of iterations T (per shard)") + ("k", po::value<size_t>()->default_value(100), "size of kbest list") + ("learning_rate,l", po::value<weight_t>()->default_value(1.0), "learning rate") + ("l1_reg,r", po::value<weight_t>()->default_value(0.), "l1 regularization strength") + ("error_margin,m", po::value<weight_t>()->default_value(0.), "margin for margin perceptron") + ("N", po::value<size_t>()->default_value(4), "N for BLEU approximation") + ("input_weights,w", po::value<string>(), "input weights file") + ("average,a", po::value<bool>()->default_value(false), "output average weights") + ("keep,K", po::value<bool>()->default_value(false), "output a weight file per iteration") + ("output,o", po::value<string>()->default_value("-"), "output weights file, '-' for STDOUT") + ("print_weights,P", po::value<string>()->default_value("EgivenFCoherent SampleCountF CountEF MaxLexFgivenE MaxLexEgivenF IsSingletonF IsSingletonFE Glue WordPenalty PassThrough LanguageModel LanguageModel_OOV"), + "list of weights to print after each iteration"); + po::options_description cl("Command Line Options"); + cl.add_options() + ("config,c", po::value<string>(), "dtrain config file"); + cl.add(ini); + po::store(parse_command_line(argc, argv, cl), *conf); + if (conf->count("config")) { + ifstream f((*conf)["config"].as<string>().c_str()); + po::store(po::parse_config_file(f, ini), *conf); + } + po::notify(*conf); + if (!conf->count("decoder_config")) { + cerr << "Missing decoder configuration." << endl; + return false; + } + if (!conf->count("bitext")) { + cerr << "No training data given." << endl; + return false; + } + + return true; } } // namespace diff --git a/training/dtrain/examples/parallelized/README b/training/dtrain/examples/parallelized/README index 2fb3b54e..c4addd81 100644 --- a/training/dtrain/examples/parallelized/README +++ b/training/dtrain/examples/parallelized/README @@ -1,5 +1,5 @@ run for example - ../../parallelize.rb -c dtrain.ini -s 4 -e 3 -z -d ../../dtrain -p 2 -i in + ../../parallelize.rb -c dtrain.ini -s 4 -e 3 -d ../../dtrain -p 2 -i in final weights will be in the file work/weights.2 diff --git a/training/dtrain/examples/parallelized/dtrain.ini b/training/dtrain/examples/parallelized/dtrain.ini index 0b0932d6..9fc205a3 100644 --- a/training/dtrain/examples/parallelized/dtrain.ini +++ b/training/dtrain/examples/parallelized/dtrain.ini @@ -1,14 +1,7 @@ k=100 N=4 learning_rate=0.0001 -gamma=0 -loss_margin=1.0 -epochs=1 -scorer=stupid_bleu -sample_from=kbest -filter=uniq -pair_sampling=XYX -hi_lo=0.1 -select_weights=last -print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough +error_margin=1.0 +iterations=1 decoder_config=cdec.ini +print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough diff --git a/training/dtrain/examples/parallelized/work/out.0.0 b/training/dtrain/examples/parallelized/work/out.0.0 index 9154c906..77749404 100644 --- a/training/dtrain/examples/parallelized/work/out.0.0 +++ b/training/dtrain/examples/parallelized/work/out.0.0 @@ -1,65 +1,43 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 4087834873 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.0.0.in' output 'work/weights.0.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = +0.257 - WordPenalty = +0.026926 - LanguageModel = +0.67342 - LanguageModel_OOV = -0.046 - PhraseModel_0 = +0.25329 - PhraseModel_1 = +0.20036 - PhraseModel_2 = +0.00060731 - PhraseModel_3 = +0.65578 - PhraseModel_4 = +0.47916 - PhraseModel_5 = +0.004 - PhraseModel_6 = +0.1829 - PassThrough = -0.082 + Glue = +0.3404 + WordPenalty = -0.017632 + LanguageModel = +0.72958 + LanguageModel_OOV = -0.235 + PhraseModel_0 = -0.43721 + PhraseModel_1 = +1.01 + PhraseModel_2 = +1.3525 + PhraseModel_3 = -0.25541 + PhraseModel_4 = -0.78115 + PhraseModel_5 = +0 + PhraseModel_6 = -0.3681 + PassThrough = -0.3304 --- - 1best avg score: 0.04518 (+0.04518) - 1best avg model score: 32.803 (+32.803) - avg # pairs: 1266.3 - avg # rank err: 857 - avg # margin viol: 386.67 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.19474 (+0.19474) + 1best avg model score: 0.52232 + avg # pairs: 2513 + non-0 feature count: 11 avg list sz: 100 - avg f count: 10.853 -(time 0.47 min, 9.3 s/S) - -Writing weights file to 'work/weights.0.0' ... -done + avg f count: 11.42 +(time 0.32 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.04518]. -This took 0.46667 min. +Best iteration: 1 [GOLD = 0.19474]. +This took 0.31667 min. diff --git a/training/dtrain/examples/parallelized/work/out.0.1 b/training/dtrain/examples/parallelized/work/out.0.1 index 0dbc7bd3..d0dee623 100644 --- a/training/dtrain/examples/parallelized/work/out.0.1 +++ b/training/dtrain/examples/parallelized/work/out.0.1 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 2283043509 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.0.0.in' output 'work/weights.0.1' weights in 'work/weights.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.17905 - WordPenalty = +0.062126 - LanguageModel = +0.66825 - LanguageModel_OOV = -0.15248 - PhraseModel_0 = -0.55811 - PhraseModel_1 = +0.12741 - PhraseModel_2 = +0.60388 - PhraseModel_3 = -0.44464 - PhraseModel_4 = -0.63137 - PhraseModel_5 = -0.0084 - PhraseModel_6 = -0.20165 - PassThrough = -0.23468 + Glue = -0.40908 + WordPenalty = +0.12967 + LanguageModel = +0.39892 + LanguageModel_OOV = -0.6314 + PhraseModel_0 = -0.63992 + PhraseModel_1 = +0.74198 + PhraseModel_2 = +1.3096 + PhraseModel_3 = -0.1216 + PhraseModel_4 = -1.2274 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.21093 + PassThrough = -0.66155 --- - 1best avg score: 0.14066 (+0.14066) - 1best avg model score: -37.614 (-37.614) - avg # pairs: 1244.7 - avg # rank err: 728 - avg # margin viol: 516.67 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.15735 (+0.15735) + 1best avg model score: 46.831 + avg # pairs: 2132.3 + non-0 feature count: 12 avg list sz: 100 - avg f count: 11.507 -(time 0.45 min, 9 s/S) - -Writing weights file to 'work/weights.0.1' ... -done + avg f count: 10.64 +(time 0.38 min, 7 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.14066]. -This took 0.45 min. +Best iteration: 1 [GOLD = 0.15735]. +This took 0.38333 min. diff --git a/training/dtrain/examples/parallelized/work/out.0.2 b/training/dtrain/examples/parallelized/work/out.0.2 index fcecc7e1..9c4b110b 100644 --- a/training/dtrain/examples/parallelized/work/out.0.2 +++ b/training/dtrain/examples/parallelized/work/out.0.2 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 3693132895 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.0.0.in' output 'work/weights.0.2' weights in 'work/weights.1' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.019275 - WordPenalty = +0.022192 - LanguageModel = +0.40688 - LanguageModel_OOV = -0.36397 - PhraseModel_0 = -0.36273 - PhraseModel_1 = +0.56432 - PhraseModel_2 = +0.85638 - PhraseModel_3 = -0.20222 - PhraseModel_4 = -0.48295 - PhraseModel_5 = +0.03145 - PhraseModel_6 = -0.26092 - PassThrough = -0.38122 + Glue = -0.44422 + WordPenalty = +0.1032 + LanguageModel = +0.66474 + LanguageModel_OOV = -0.62252 + PhraseModel_0 = -0.59993 + PhraseModel_1 = +0.78992 + PhraseModel_2 = +1.3149 + PhraseModel_3 = +0.21434 + PhraseModel_4 = -1.0174 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.18452 + PassThrough = -0.65268 --- - 1best avg score: 0.18982 (+0.18982) - 1best avg model score: 1.7096 (+1.7096) - avg # pairs: 1524.3 - avg # rank err: 813.33 - avg # margin viol: 702.67 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.24722 (+0.24722) + 1best avg model score: 61.971 + avg # pairs: 2017.7 + non-0 feature count: 12 avg list sz: 100 - avg f count: 11.32 -(time 0.53 min, 11 s/S) - -Writing weights file to 'work/weights.0.2' ... -done + avg f count: 10.42 +(time 0.3 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.18982]. -This took 0.53333 min. +Best iteration: 1 [GOLD = 0.24722]. +This took 0.3 min. diff --git a/training/dtrain/examples/parallelized/work/out.1.0 b/training/dtrain/examples/parallelized/work/out.1.0 index 595dfc94..3dc4dca6 100644 --- a/training/dtrain/examples/parallelized/work/out.1.0 +++ b/training/dtrain/examples/parallelized/work/out.1.0 @@ -1,65 +1,43 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 859043351 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.1.0.in' output 'work/weights.1.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.3229 - WordPenalty = +0.27969 - LanguageModel = +1.3645 - LanguageModel_OOV = -0.0443 - PhraseModel_0 = -0.19049 - PhraseModel_1 = -0.077698 - PhraseModel_2 = +0.058898 - PhraseModel_3 = +0.017251 - PhraseModel_4 = -1.5474 - PhraseModel_5 = +0 - PhraseModel_6 = -0.1818 - PassThrough = -0.193 + Glue = -0.2722 + WordPenalty = +0.05433 + LanguageModel = +0.69948 + LanguageModel_OOV = -0.2641 + PhraseModel_0 = -1.4208 + PhraseModel_1 = -1.563 + PhraseModel_2 = -0.21051 + PhraseModel_3 = -0.17764 + PhraseModel_4 = -1.6583 + PhraseModel_5 = +0.0794 + PhraseModel_6 = +0.1528 + PassThrough = -0.2367 --- - 1best avg score: 0.070229 (+0.070229) - 1best avg model score: -44.01 (-44.01) - avg # pairs: 1294 - avg # rank err: 878.67 - avg # margin viol: 350.67 - k-best loss imp: 100% - non0 feature count: 11 + 1best avg score: 0.071329 (+0.071329) + 1best avg model score: -41.362 + avg # pairs: 1862.3 + non-0 feature count: 12 avg list sz: 100 - avg f count: 11.487 -(time 0.28 min, 5.7 s/S) - -Writing weights file to 'work/weights.1.0' ... -done + avg f count: 11.847 +(time 0.28 min, 5 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.070229]. +Best iteration: 1 [GOLD = 0.071329]. This took 0.28333 min. diff --git a/training/dtrain/examples/parallelized/work/out.1.1 b/training/dtrain/examples/parallelized/work/out.1.1 index 9346fc82..79ac35dc 100644 --- a/training/dtrain/examples/parallelized/work/out.1.1 +++ b/training/dtrain/examples/parallelized/work/out.1.1 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 3557309480 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.1.0.in' output 'work/weights.1.1' weights in 'work/weights.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.26425 - WordPenalty = +0.047881 - LanguageModel = +0.78496 - LanguageModel_OOV = -0.49307 - PhraseModel_0 = -0.58703 - PhraseModel_1 = -0.33425 - PhraseModel_2 = +0.20834 - PhraseModel_3 = -0.043346 - PhraseModel_4 = -0.60761 - PhraseModel_5 = +0.123 - PhraseModel_6 = -0.05415 - PassThrough = -0.42167 + Glue = -0.20488 + WordPenalty = -0.0091745 + LanguageModel = +0.79433 + LanguageModel_OOV = -0.4309 + PhraseModel_0 = -0.56242 + PhraseModel_1 = +0.85363 + PhraseModel_2 = +1.3458 + PhraseModel_3 = -0.13095 + PhraseModel_4 = -0.94762 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.16003 + PassThrough = -0.46105 --- - 1best avg score: 0.085952 (+0.085952) - 1best avg model score: -45.175 (-45.175) - avg # pairs: 1180.7 - avg # rank err: 668.33 - avg # margin viol: 512.33 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.13017 (+0.13017) + 1best avg model score: 14.53 + avg # pairs: 1968 + non-0 feature count: 12 avg list sz: 100 - avg f count: 12 -(time 0.27 min, 5.3 s/S) - -Writing weights file to 'work/weights.1.1' ... -done + avg f count: 11 +(time 0.33 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.085952]. -This took 0.26667 min. +Best iteration: 1 [GOLD = 0.13017]. +This took 0.33333 min. diff --git a/training/dtrain/examples/parallelized/work/out.1.2 b/training/dtrain/examples/parallelized/work/out.1.2 index 08f07a75..8c4f8b03 100644 --- a/training/dtrain/examples/parallelized/work/out.1.2 +++ b/training/dtrain/examples/parallelized/work/out.1.2 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 56743915 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.1.0.in' output 'work/weights.1.2' weights in 'work/weights.1' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.23608 - WordPenalty = +0.10931 - LanguageModel = +0.81339 - LanguageModel_OOV = -0.33238 - PhraseModel_0 = -0.53685 - PhraseModel_1 = -0.049658 - PhraseModel_2 = +0.40277 - PhraseModel_3 = +0.14601 - PhraseModel_4 = -0.72851 - PhraseModel_5 = +0.03475 - PhraseModel_6 = -0.27192 - PassThrough = -0.34763 + Glue = -0.49853 + WordPenalty = +0.07636 + LanguageModel = +1.3183 + LanguageModel_OOV = -0.60902 + PhraseModel_0 = -0.22481 + PhraseModel_1 = +0.86369 + PhraseModel_2 = +1.0747 + PhraseModel_3 = +0.18002 + PhraseModel_4 = -0.84661 + PhraseModel_5 = +0.02435 + PhraseModel_6 = +0.11247 + PassThrough = -0.63918 --- - 1best avg score: 0.10073 (+0.10073) - 1best avg model score: -38.422 (-38.422) - avg # pairs: 1505.3 - avg # rank err: 777 - avg # margin viol: 691.67 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.15478 (+0.15478) + 1best avg model score: -7.2154 + avg # pairs: 1776 + non-0 feature count: 12 avg list sz: 100 - avg f count: 12 -(time 0.35 min, 7 s/S) - -Writing weights file to 'work/weights.1.2' ... -done + avg f count: 11.327 +(time 0.27 min, 5 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.10073]. -This took 0.35 min. +Best iteration: 1 [GOLD = 0.15478]. +This took 0.26667 min. diff --git a/training/dtrain/examples/parallelized/work/out.2.0 b/training/dtrain/examples/parallelized/work/out.2.0 index 25ef6d4e..07c85963 100644 --- a/training/dtrain/examples/parallelized/work/out.2.0 +++ b/training/dtrain/examples/parallelized/work/out.2.0 @@ -1,65 +1,43 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 2662215673 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.2.0.in' output 'work/weights.2.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.1259 - WordPenalty = +0.048294 - LanguageModel = +0.36254 - LanguageModel_OOV = -0.1228 - PhraseModel_0 = +0.26357 - PhraseModel_1 = +0.24793 - PhraseModel_2 = +0.0063763 - PhraseModel_3 = -0.18966 - PhraseModel_4 = -0.226 + Glue = -0.2109 + WordPenalty = +0.14922 + LanguageModel = +0.79686 + LanguageModel_OOV = -0.6627 + PhraseModel_0 = +0.37999 + PhraseModel_1 = +0.69213 + PhraseModel_2 = +0.3422 + PhraseModel_3 = +1.1426 + PhraseModel_4 = -0.55413 PhraseModel_5 = +0 - PhraseModel_6 = +0.0743 - PassThrough = -0.1335 + PhraseModel_6 = +0.0676 + PassThrough = -0.6343 --- - 1best avg score: 0.072836 (+0.072836) - 1best avg model score: -0.56296 (-0.56296) - avg # pairs: 1094.7 - avg # rank err: 658 - avg # margin viol: 436.67 - k-best loss imp: 100% - non0 feature count: 11 + 1best avg score: 0.072374 (+0.072374) + 1best avg model score: -27.384 + avg # pairs: 2582 + non-0 feature count: 11 avg list sz: 100 - avg f count: 10.813 -(time 0.13 min, 2.7 s/S) - -Writing weights file to 'work/weights.2.0' ... -done + avg f count: 11.54 +(time 0.32 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.072836]. -This took 0.13333 min. +Best iteration: 1 [GOLD = 0.072374]. +This took 0.31667 min. diff --git a/training/dtrain/examples/parallelized/work/out.2.1 b/training/dtrain/examples/parallelized/work/out.2.1 index 8e4efde9..c54bb1b1 100644 --- a/training/dtrain/examples/parallelized/work/out.2.1 +++ b/training/dtrain/examples/parallelized/work/out.2.1 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 3092904479 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.2.0.in' output 'work/weights.2.1' weights in 'work/weights.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.10385 - WordPenalty = +0.038717 - LanguageModel = +0.49413 - LanguageModel_OOV = -0.24887 - PhraseModel_0 = -0.32102 - PhraseModel_1 = +0.34413 - PhraseModel_2 = +0.62366 - PhraseModel_3 = -0.49337 - PhraseModel_4 = -0.77005 - PhraseModel_5 = +0.007 - PhraseModel_6 = -0.05055 - PassThrough = -0.23928 + Glue = -0.76608 + WordPenalty = +0.15938 + LanguageModel = +1.5897 + LanguageModel_OOV = -0.521 + PhraseModel_0 = -0.58348 + PhraseModel_1 = +0.29828 + PhraseModel_2 = +0.78493 + PhraseModel_3 = +0.083222 + PhraseModel_4 = -0.93843 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.27382 + PassThrough = -0.55115 --- - 1best avg score: 0.10245 (+0.10245) - 1best avg model score: -20.384 (-20.384) - avg # pairs: 1741.7 - avg # rank err: 953.67 - avg # margin viol: 585.33 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.12881 (+0.12881) + 1best avg model score: -9.6731 + avg # pairs: 2020.3 + non-0 feature count: 12 avg list sz: 100 - avg f count: 11.977 -(time 0.12 min, 2.3 s/S) - -Writing weights file to 'work/weights.2.1' ... -done + avg f count: 12 +(time 0.32 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.10245]. -This took 0.11667 min. +Best iteration: 1 [GOLD = 0.12881]. +This took 0.31667 min. diff --git a/training/dtrain/examples/parallelized/work/out.2.2 b/training/dtrain/examples/parallelized/work/out.2.2 index e0ca2110..f5d6229f 100644 --- a/training/dtrain/examples/parallelized/work/out.2.2 +++ b/training/dtrain/examples/parallelized/work/out.2.2 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 2803362953 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.2.0.in' output 'work/weights.2.2' weights in 'work/weights.1' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 3 + .... 3 WEIGHTS - Glue = -0.32907 - WordPenalty = +0.049596 - LanguageModel = +0.33496 - LanguageModel_OOV = -0.44357 - PhraseModel_0 = -0.3068 - PhraseModel_1 = +0.59376 - PhraseModel_2 = +0.86416 - PhraseModel_3 = -0.21072 - PhraseModel_4 = -0.65734 - PhraseModel_5 = +0.03475 - PhraseModel_6 = -0.10653 - PassThrough = -0.46082 + Glue = -0.90863 + WordPenalty = +0.10819 + LanguageModel = +0.5239 + LanguageModel_OOV = -0.41623 + PhraseModel_0 = -0.86868 + PhraseModel_1 = +0.40784 + PhraseModel_2 = +1.1793 + PhraseModel_3 = -0.24698 + PhraseModel_4 = -1.2353 + PhraseModel_5 = +0.03375 + PhraseModel_6 = -0.17883 + PassThrough = -0.44638 --- - 1best avg score: 0.25055 (+0.25055) - 1best avg model score: -1.4459 (-1.4459) - avg # pairs: 1689 - avg # rank err: 755.67 - avg # margin viol: 829.33 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.12788 (+0.12788) + 1best avg model score: 41.302 + avg # pairs: 2246.3 + non-0 feature count: 12 avg list sz: 100 - avg f count: 10.53 -(time 0.13 min, 2.7 s/S) - -Writing weights file to 'work/weights.2.2' ... -done + avg f count: 10.98 +(time 0.35 min, 7 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.25055]. -This took 0.13333 min. +Best iteration: 1 [GOLD = 0.12788]. +This took 0.35 min. diff --git a/training/dtrain/examples/parallelized/work/out.3.0 b/training/dtrain/examples/parallelized/work/out.3.0 index 3c074f04..fa499523 100644 --- a/training/dtrain/examples/parallelized/work/out.3.0 +++ b/training/dtrain/examples/parallelized/work/out.3.0 @@ -1,65 +1,43 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 316107185 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.3.0.in' output 'work/weights.3.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 1 + .. 1 WEIGHTS - Glue = +0.046 - WordPenalty = +0.17328 - LanguageModel = +1.1667 - LanguageModel_OOV = +0.066 - PhraseModel_0 = -1.1694 - PhraseModel_1 = -0.9883 - PhraseModel_2 = +0.036205 - PhraseModel_3 = -0.77387 - PhraseModel_4 = -1.5019 - PhraseModel_5 = +0.024 - PhraseModel_6 = -0.514 - PassThrough = +0.031 + Glue = -0.09 + WordPenalty = +0.32442 + LanguageModel = +2.5769 + LanguageModel_OOV = -0.009 + PhraseModel_0 = -0.58972 + PhraseModel_1 = +0.063691 + PhraseModel_2 = +0.5366 + PhraseModel_3 = +0.12867 + PhraseModel_4 = -1.9801 + PhraseModel_5 = +0.018 + PhraseModel_6 = -0.486 + PassThrough = -0.09 --- - 1best avg score: 0.032916 (+0.032916) - 1best avg model score: 0 (+0) - avg # pairs: 900 - avg # rank err: 900 - avg # margin viol: 0 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.034204 (+0.034204) + 1best avg model score: 0 + avg # pairs: 1700 + non-0 feature count: 12 avg list sz: 100 - avg f count: 11.72 -(time 0.23 min, 14 s/S) - -Writing weights file to 'work/weights.3.0' ... -done + avg f count: 10.8 +(time 0.1 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.032916]. -This took 0.23333 min. +Best iteration: 1 [GOLD = 0.034204]. +This took 0.1 min. diff --git a/training/dtrain/examples/parallelized/work/out.3.1 b/training/dtrain/examples/parallelized/work/out.3.1 index 241d3455..c4b3aa3c 100644 --- a/training/dtrain/examples/parallelized/work/out.3.1 +++ b/training/dtrain/examples/parallelized/work/out.3.1 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 353677750 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.3.0.in' output 'work/weights.3.1' weights in 'work/weights.0' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 1 + .. 1 WEIGHTS - Glue = -0.08475 - WordPenalty = +0.11151 - LanguageModel = +1.0635 - LanguageModel_OOV = -0.11468 - PhraseModel_0 = -0.062922 - PhraseModel_1 = +0.0035552 - PhraseModel_2 = +0.039692 - PhraseModel_3 = +0.080265 - PhraseModel_4 = -0.57787 - PhraseModel_5 = +0.0174 - PhraseModel_6 = -0.17095 - PassThrough = -0.18248 + Glue = +0.31832 + WordPenalty = +0.11139 + LanguageModel = +0.95438 + LanguageModel_OOV = -0.0608 + PhraseModel_0 = -0.98113 + PhraseModel_1 = -0.090531 + PhraseModel_2 = +0.79088 + PhraseModel_3 = -0.57623 + PhraseModel_4 = -1.4382 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.10812 + PassThrough = -0.09095 --- - 1best avg score: 0.16117 (+0.16117) - 1best avg model score: -67.89 (-67.89) - avg # pairs: 1411 - avg # rank err: 460 - avg # margin viol: 951 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.084989 (+0.084989) + 1best avg model score: -52.323 + avg # pairs: 2487 + non-0 feature count: 12 avg list sz: 100 avg f count: 12 -(time 0.22 min, 13 s/S) - -Writing weights file to 'work/weights.3.1' ... -done +(time 0.1 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.16117]. -This took 0.21667 min. +Best iteration: 1 [GOLD = 0.084989]. +This took 0.1 min. diff --git a/training/dtrain/examples/parallelized/work/out.3.2 b/training/dtrain/examples/parallelized/work/out.3.2 index b995daf5..eb27dac2 100644 --- a/training/dtrain/examples/parallelized/work/out.3.2 +++ b/training/dtrain/examples/parallelized/work/out.3.2 @@ -1,66 +1,44 @@ - cdec cfg 'cdec.ini' Loading the LM will be faster if you build a binary file. Reading ../standard/nc-wmt11.en.srilm.gz ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 **************************************************************************************************** -Seeding random number sequence to 3001145976 - dtrain Parameters: k 100 N 4 T 1 - batch 0 - scorer 'stupid_bleu' - sample from 'kbest' - filter 'uniq' learning rate 0.0001 - gamma 0 - loss margin 1 - faster perceptron 0 - pairs 'XYX' - hi lo 0.1 - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' + error margin 1 + l1 reg 0 + decoder conf 'cdec.ini' + input 'work/shard.3.0.in' output 'work/weights.3.2' weights in 'work/weights.1' -(a dot represents 10 inputs) +(a dot per input) Iteration #1 of 1. - 1 + .. 1 WEIGHTS - Glue = -0.13247 - WordPenalty = +0.053592 - LanguageModel = +0.72105 - LanguageModel_OOV = -0.30827 - PhraseModel_0 = -0.37053 - PhraseModel_1 = +0.17551 - PhraseModel_2 = +0.5 - PhraseModel_3 = -0.1459 - PhraseModel_4 = -0.59563 - PhraseModel_5 = +0.03475 - PhraseModel_6 = -0.11143 - PassThrough = -0.32553 + Glue = -0.12993 + WordPenalty = +0.13651 + LanguageModel = +0.58946 + LanguageModel_OOV = -0.48362 + PhraseModel_0 = -0.81262 + PhraseModel_1 = +0.44273 + PhraseModel_2 = +1.1733 + PhraseModel_3 = -0.1826 + PhraseModel_4 = -1.2213 + PhraseModel_5 = +0.02435 + PhraseModel_6 = -0.18823 + PassThrough = -0.51378 --- - 1best avg score: 0.12501 (+0.12501) - 1best avg model score: -62.128 (-62.128) - avg # pairs: 979 - avg # rank err: 539 - avg # margin viol: 440 - k-best loss imp: 100% - non0 feature count: 12 + 1best avg score: 0.12674 (+0.12674) + 1best avg model score: -7.2878 + avg # pairs: 1769 + non-0 feature count: 12 avg list sz: 100 avg f count: 12 -(time 0.22 min, 13 s/S) - -Writing weights file to 'work/weights.3.2' ... -done +(time 0.1 min, 6 s/S) --- -Best iteration: 1 [SCORE 'stupid_bleu'=0.12501]. -This took 0.21667 min. +Best iteration: 1 [GOLD = 0.12674]. +This took 0.1 min. diff --git a/training/dtrain/examples/parallelized/work/shard.0.0.in b/training/dtrain/examples/parallelized/work/shard.0.0.in index d1b48321..a0ef6f54 100644 --- a/training/dtrain/examples/parallelized/work/shard.0.0.in +++ b/training/dtrain/examples/parallelized/work/shard.0.0.in @@ -1,3 +1,3 @@ -<seg grammar="grammar/grammar.out.8.gz" id="8">der erste schritt , um mit der rassenfrage umzugehen ist , ursache und folgen rassistischer feindseligkeiten zu verstehen , auch dann , wenn das bedeutet , unangenehme tatsachen aufzudecken .</seg> ||| the first step to address racial politics is to understand the origin and consequences of racial animosity , even if it means uncovering unpleasant truths . -<seg grammar="grammar/grammar.out.5.gz" id="5">die großen parteien der rechten und der linken mitte haben sich dem problem gestellt , in dem sie den kopf in den sand gesteckt und allen aussichten zuwider gehofft haben , es möge bald verschwinden .</seg> ||| mainstream parties of the center left and center right have confronted this prospect by hiding their heads in the ground , hoping against hope that the problem will disappear . +<seg grammar="grammar/grammar.out.0.gz" id="0">europas nach rassen geteiltes haus</seg> ||| europe 's divided racial house +<seg grammar="grammar/grammar.out.1.gz" id="1">ein gemeinsames merkmal aller extremen rechten in europa ist ihr rassismus und die tatsache , daß sie das einwanderungsproblem als politischen hebel benutzen .</seg> ||| a common feature of europe 's extreme right is its racism and use of the immigration issue as a political wedge . <seg grammar="grammar/grammar.out.2.gz" id="2">der lega nord in italien , der vlaams block in den niederlanden , die anhänger von le pens nationaler front in frankreich , sind beispiele für parteien oder bewegungen , die sich um das gemeinsame thema : ablehnung der zuwanderung gebildet haben und um forderung nach einer vereinfachten politik , um sie zu regeln .</seg> ||| the lega nord in italy , the vlaams blok in the netherlands , the supporters of le pen 's national front in france , are all examples of parties or movements formed on the common theme of aversion to immigrants and promotion of simplistic policies to control them . diff --git a/training/dtrain/examples/parallelized/work/shard.1.0.in b/training/dtrain/examples/parallelized/work/shard.1.0.in index a63f05bd..05f0273b 100644 --- a/training/dtrain/examples/parallelized/work/shard.1.0.in +++ b/training/dtrain/examples/parallelized/work/shard.1.0.in @@ -1,3 +1,3 @@ -<seg grammar="grammar/grammar.out.4.gz" id="4">eine alternde einheimische bevölkerung und immer offenere grenzen vermehren die rassistische zersplitterung in den europäischen ländern .</seg> ||| an aging population at home and ever more open borders imply increasing racial fragmentation in european countries . -<seg grammar="grammar/grammar.out.9.gz" id="9">genau das haben in den usa eine große anzahl an forschungsvorhaben in wirtschaft , soziologie , psychologie und politikwissenschaft geleistet . diese forschungen zeigten , daß menschen unterschiedlicher rasse einander deutlich weniger vertrauen .</seg> ||| this is precisely what a large amount of research in economics , sociology , psychology and political science has done for the us . <seg grammar="grammar/grammar.out.3.gz" id="3">während individuen wie jörg haidar und jean @-@ marie le pen kommen und ( leider nicht zu bald ) wieder gehen mögen , wird die rassenfrage aus der europäischer politik nicht so bald verschwinden .</seg> ||| while individuals like jorg haidar and jean @-@ marie le pen may come and ( never to soon ) go , the race question will not disappear from european politics anytime soon . +<seg grammar="grammar/grammar.out.4.gz" id="4">eine alternde einheimische bevölkerung und immer offenere grenzen vermehren die rassistische zersplitterung in den europäischen ländern .</seg> ||| an aging population at home and ever more open borders imply increasing racial fragmentation in european countries . +<seg grammar="grammar/grammar.out.5.gz" id="5">die großen parteien der rechten und der linken mitte haben sich dem problem gestellt , in dem sie den kopf in den sand gesteckt und allen aussichten zuwider gehofft haben , es möge bald verschwinden .</seg> ||| mainstream parties of the center left and center right have confronted this prospect by hiding their heads in the ground , hoping against hope that the problem will disappear . diff --git a/training/dtrain/examples/parallelized/work/shard.2.0.in b/training/dtrain/examples/parallelized/work/shard.2.0.in index fe542b40..0528d357 100644 --- a/training/dtrain/examples/parallelized/work/shard.2.0.in +++ b/training/dtrain/examples/parallelized/work/shard.2.0.in @@ -1,3 +1,3 @@ -<seg grammar="grammar/grammar.out.1.gz" id="1">ein gemeinsames merkmal aller extremen rechten in europa ist ihr rassismus und die tatsache , daß sie das einwanderungsproblem als politischen hebel benutzen .</seg> ||| a common feature of europe 's extreme right is its racism and use of the immigration issue as a political wedge . -<seg grammar="grammar/grammar.out.0.gz" id="0">europas nach rassen geteiltes haus</seg> ||| europe 's divided racial house <seg grammar="grammar/grammar.out.6.gz" id="6">das aber wird es nicht , wie die geschichte des rassismus in amerika deutlich zeigt .</seg> ||| it will not , as america 's racial history clearly shows . +<seg grammar="grammar/grammar.out.7.gz" id="7">die beziehungen zwischen den rassen standen in den usa über jahrzehnte - und tun das noch heute - im zentrum der politischen debatte . das ging so weit , daß rassentrennung genauso wichtig wie das einkommen wurde , - wenn nicht sogar noch wichtiger - um politische zuneigungen und einstellungen zu bestimmen .</seg> ||| race relations in the us have been for decades - and remain - at the center of political debate , to the point that racial cleavages are as important as income , if not more , as determinants of political preferences and attitudes . +<seg grammar="grammar/grammar.out.8.gz" id="8">der erste schritt , um mit der rassenfrage umzugehen ist , ursache und folgen rassistischer feindseligkeiten zu verstehen , auch dann , wenn das bedeutet , unangenehme tatsachen aufzudecken .</seg> ||| the first step to address racial politics is to understand the origin and consequences of racial animosity , even if it means uncovering unpleasant truths . diff --git a/training/dtrain/examples/parallelized/work/shard.3.0.in b/training/dtrain/examples/parallelized/work/shard.3.0.in index 4a8fa5b1..f7cbb3e3 100644 --- a/training/dtrain/examples/parallelized/work/shard.3.0.in +++ b/training/dtrain/examples/parallelized/work/shard.3.0.in @@ -1 +1 @@ -<seg grammar="grammar/grammar.out.7.gz" id="7">die beziehungen zwischen den rassen standen in den usa über jahrzehnte - und tun das noch heute - im zentrum der politischen debatte . das ging so weit , daß rassentrennung genauso wichtig wie das einkommen wurde , - wenn nicht sogar noch wichtiger - um politische zuneigungen und einstellungen zu bestimmen .</seg> ||| race relations in the us have been for decades - and remain - at the center of political debate , to the point that racial cleavages are as important as income , if not more , as determinants of political preferences and attitudes . +<seg grammar="grammar/grammar.out.9.gz" id="9">genau das haben in den usa eine große anzahl an forschungsvorhaben in wirtschaft , soziologie , psychologie und politikwissenschaft geleistet . diese forschungen zeigten , daß menschen unterschiedlicher rasse einander deutlich weniger vertrauen .</seg> ||| this is precisely what a large amount of research in economics , sociology , psychology and political science has done for the us . diff --git a/training/dtrain/examples/parallelized/work/weights.0 b/training/dtrain/examples/parallelized/work/weights.0 index c560fdbd..816269cd 100644 --- a/training/dtrain/examples/parallelized/work/weights.0 +++ b/training/dtrain/examples/parallelized/work/weights.0 @@ -1,12 +1,12 @@ -PhraseModel_4 -0.6990170657294328 -LanguageModel 0.891784887346263 -PhraseModel_0 -0.2107507586515428 -PhraseModel_1 -0.15442709655871997 -PhraseModel_3 -0.07262514338204715 -PhraseModel_6 -0.10965000000000148 -Glue -0.03644999999999783 -WordPenalty 0.13204723722268177 -PassThrough -0.09437500000000089 -LanguageModel_OOV -0.036775000000000564 -PhraseModel_2 0.025521702385571707 -PhraseModel_5 0.006999999999999977 +LanguageModel 1.200704259340465 +PhraseModel_4 -1.2434381298299035 +PhraseModel_1 0.050697726409824076 +PhraseModel_0 -0.516923312932941 +PhraseModel_2 0.5051987092783867 +PhraseModel_3 0.20955092377784057 +PassThrough -0.32285 +LanguageModel_OOV -0.29269999999999996 +PhraseModel_6 -0.158425 +Glue -0.05817500000000002 +WordPenalty 0.12758486142112804 +PhraseModel_5 0.02435 diff --git a/training/dtrain/examples/parallelized/work/weights.0.0 b/training/dtrain/examples/parallelized/work/weights.0.0 index 91eedc7b..be386c62 100644 --- a/training/dtrain/examples/parallelized/work/weights.0.0 +++ b/training/dtrain/examples/parallelized/work/weights.0.0 @@ -1,12 +1,11 @@ -PassThrough -0.082000000000001058 -Glue 0.25700000000000267 -LanguageModel_OOV -0.046000000000000034 -LanguageModel 0.67341721152744249 -PhraseModel_6 0.18290000000000028 -PhraseModel_5 0.0039999999999999975 -PhraseModel_4 0.47916377173928498 -PhraseModel_3 0.65577926367715722 -PhraseModel_2 0.00060731048591637909 -PhraseModel_0 0.25329462707903372 -WordPenalty 0.026926257878001431 -PhraseModel_1 0.20035945197369062 +WordPenalty -0.017632355965271129 +LanguageModel 0.72957628464102753 +LanguageModel_OOV -0.23499999999999999 +PhraseModel_0 -0.43720953659541578 +PhraseModel_1 1.0100170838129212 +PhraseModel_2 1.3524984123857073 +PhraseModel_3 -0.25541132249775761 +PhraseModel_4 -0.78115161368856911 +PhraseModel_6 -0.36810000000000004 +Glue 0.34040000000000004 +PassThrough -0.33040000000000003 diff --git a/training/dtrain/examples/parallelized/work/weights.0.1 b/training/dtrain/examples/parallelized/work/weights.0.1 index 6fcc9999..d4c77d07 100644 --- a/training/dtrain/examples/parallelized/work/weights.0.1 +++ b/training/dtrain/examples/parallelized/work/weights.0.1 @@ -1,12 +1,12 @@ -PassThrough -0.2346750000000028 -Glue -0.17904999999999763 -WordPenalty 0.062125825636256168 -LanguageModel 0.66824625053667575 -LanguageModel_OOV -0.15247500000000355 -PhraseModel_0 -0.5581144363944085 -PhraseModel_1 0.12740874153205478 -PhraseModel_2 0.6038779278708799 -PhraseModel_3 -0.44463820299544454 -PhraseModel_4 -0.63136538282212662 -PhraseModel_5 -0.0084000000000000324 -PhraseModel_6 -0.20164999999999911 +WordPenalty 0.12966947493426365 +LanguageModel 0.3989224621154368 +LanguageModel_OOV -0.63139999999999996 +PhraseModel_0 -0.63991953012355962 +PhraseModel_1 0.74197897612368646 +PhraseModel_2 1.3096163833051435 +PhraseModel_3 -0.12160001974680773 +PhraseModel_4 -1.2274031286515816 +PhraseModel_5 0.02435 +PhraseModel_6 -0.210925 +Glue -0.40907500000000002 +PassThrough -0.66155000000000008 diff --git a/training/dtrain/examples/parallelized/work/weights.0.2 b/training/dtrain/examples/parallelized/work/weights.0.2 index 5668915d..8ce1449b 100644 --- a/training/dtrain/examples/parallelized/work/weights.0.2 +++ b/training/dtrain/examples/parallelized/work/weights.0.2 @@ -1,12 +1,12 @@ -PassThrough -0.38122499999999337 -Glue -0.019274999999998679 -WordPenalty 0.022192448025253487 -LanguageModel 0.4068780855136106 -LanguageModel_OOV -0.363974999999992 -PhraseModel_0 -0.36273429313029715 -PhraseModel_1 0.56431752511029298 -PhraseModel_2 0.85638010019687694 -PhraseModel_3 -0.20222345248738063 -PhraseModel_4 -0.48295466434310252 -PhraseModel_5 0.031450000000000339 -PhraseModel_6 -0.26092499999998625 +WordPenalty 0.10319922626226019 +LanguageModel 0.6647396869692952 +LanguageModel_OOV -0.622525 +PhraseModel_0 -0.59993441316076157 +PhraseModel_1 0.78991513935858193 +PhraseModel_2 1.3148638774685031 +PhraseModel_3 0.2143393571820455 +PhraseModel_4 -1.0173894637028262 +PhraseModel_5 0.02435 +PhraseModel_6 -0.18452499999999999 +Glue -0.44422499999999998 +PassThrough -0.65267500000000012 diff --git a/training/dtrain/examples/parallelized/work/weights.1 b/training/dtrain/examples/parallelized/work/weights.1 index f52e07b8..2a00be2e 100644 --- a/training/dtrain/examples/parallelized/work/weights.1 +++ b/training/dtrain/examples/parallelized/work/weights.1 @@ -1,12 +1,12 @@ -LanguageModel 0.7527067666152598 -PhraseModel_4 -0.6467221787583058 -PhraseModel_2 0.36889175522051015 -PhraseModel_0 -0.38227173053779245 -PhraseModel_3 -0.2252732111174934 -LanguageModel_OOV -0.25227499999999975 -PassThrough -0.2695250000000011 -PhraseModel_1 0.03521067244127414 -Glue -0.1579749999999981 -PhraseModel_6 -0.11932500000000047 -WordPenalty 0.0650573133891042 -PhraseModel_5 0.03475000000000043 +PhraseModel_4 -1.1379250444170055 +PhraseModel_2 1.0578050661336098 +LanguageModel 0.9343385461706668 +PhraseModel_0 -0.6917392152965985 +PhraseModel_1 0.4508371141128957 +PassThrough -0.4411750000000001 +Glue -0.265425 +LanguageModel_OOV -0.411025 +PhraseModel_3 -0.186390082624459 +PhraseModel_6 -0.188225 +WordPenalty 0.09781397468665984 +PhraseModel_5 0.02435 diff --git a/training/dtrain/examples/parallelized/work/weights.1.0 b/training/dtrain/examples/parallelized/work/weights.1.0 index 31e08d81..cdcf959e 100644 --- a/training/dtrain/examples/parallelized/work/weights.1.0 +++ b/training/dtrain/examples/parallelized/work/weights.1.0 @@ -1,11 +1,12 @@ -LanguageModel_OOV -0.044300000000000235 -PassThrough -0.19300000000000087 -PhraseModel_6 -0.18180000000000701 -LanguageModel 1.3644969337716422 -PhraseModel_3 0.017250706134911725 -PhraseModel_4 -1.5473728273858063 -Glue -0.32289999999999447 -PhraseModel_1 -0.077697953502182365 -WordPenalty 0.27968564634568688 -PhraseModel_0 -0.19048660891012237 -PhraseModel_2 0.05889844333199834 +WordPenalty 0.05433023968609621 +LanguageModel 0.69947965605855011 +LanguageModel_OOV -0.2641 +PhraseModel_0 -1.4207505705360111 +PhraseModel_1 -1.563047680441811 +PhraseModel_2 -0.21050528366541305 +PhraseModel_3 -0.17764037275860439 +PhraseModel_4 -1.6583462458159566 +PhraseModel_5 0.079399999999999998 +PhraseModel_6 0.15280000000000002 +Glue -0.27220000000000005 +PassThrough -0.23670000000000002 diff --git a/training/dtrain/examples/parallelized/work/weights.1.1 b/training/dtrain/examples/parallelized/work/weights.1.1 index 544ff462..c1bb2cf0 100644 --- a/training/dtrain/examples/parallelized/work/weights.1.1 +++ b/training/dtrain/examples/parallelized/work/weights.1.1 @@ -1,12 +1,12 @@ -PassThrough -0.42167499999999858 -Glue -0.26424999999999721 -WordPenalty 0.04788096662983269 -LanguageModel 0.78495517342352483 -LanguageModel_OOV -0.49307499999999477 -PhraseModel_0 -0.58703462849498356 -PhraseModel_1 -0.33425278954714266 -PhraseModel_2 0.20834221229630179 -PhraseModel_3 -0.043345645640208569 -PhraseModel_4 -0.60760531115816907 -PhraseModel_5 0.12300000000000186 -PhraseModel_6 -0.054150000000001031 +WordPenalty -0.0091744709302067785 +LanguageModel 0.79433413663506514 +LanguageModel_OOV -0.43090000000000001 +PhraseModel_0 -0.56242499947237046 +PhraseModel_1 0.85362516703032698 +PhraseModel_2 1.3457900890481096 +PhraseModel_3 -0.13095079554478939 +PhraseModel_4 -0.94761908497413061 +PhraseModel_5 0.02435 +PhraseModel_6 -0.160025 +Glue -0.20487500000000003 +PassThrough -0.46105000000000007 diff --git a/training/dtrain/examples/parallelized/work/weights.1.2 b/training/dtrain/examples/parallelized/work/weights.1.2 index ac3284b9..c9598a04 100644 --- a/training/dtrain/examples/parallelized/work/weights.1.2 +++ b/training/dtrain/examples/parallelized/work/weights.1.2 @@ -1,12 +1,12 @@ -PassThrough -0.34762500000000224 -Glue -0.23607500000000026 -WordPenalty 0.10931192109504413 -LanguageModel 0.81339027211983694 -LanguageModel_OOV -0.33237500000000098 -PhraseModel_0 -0.53685104648974269 -PhraseModel_1 -0.049657790506137042 -PhraseModel_2 0.40277066454544108 -PhraseModel_3 0.14600791389785803 -PhraseModel_4 -0.72850673041349101 -PhraseModel_5 0.034750000000000433 -PhraseModel_6 -0.27192499999999448 +WordPenalty 0.076359827280638559 +LanguageModel 1.3183380272921175 +LanguageModel_OOV -0.60902499999999993 +PhraseModel_0 -0.2248075206657828 +PhraseModel_1 0.86368802571834491 +PhraseModel_2 1.0746702462261808 +PhraseModel_3 0.18002263643876637 +PhraseModel_4 -0.84660750337519441 +PhraseModel_5 0.02435 +PhraseModel_6 0.11247499999999999 +Glue -0.49852500000000005 +PassThrough -0.63917500000000005 diff --git a/training/dtrain/examples/parallelized/work/weights.2 b/training/dtrain/examples/parallelized/work/weights.2 index dedaf165..310973ec 100644 --- a/training/dtrain/examples/parallelized/work/weights.2 +++ b/training/dtrain/examples/parallelized/work/weights.2 @@ -1,12 +1,12 @@ -PhraseModel_2 0.6558266927225778 -PhraseModel_4 -0.6161090299356294 -LanguageModel 0.5690697644415413 -PhraseModel_1 0.32098232482479416 -PhraseModel_0 -0.39422813904895143 -PassThrough -0.37879999999999764 -LanguageModel_OOV -0.3620499999999963 -Glue -0.1792249999999967 -PhraseModel_6 -0.18769999999999526 -PhraseModel_3 -0.10321074877850786 -WordPenalty 0.05867318450512617 -PhraseModel_5 0.03392500000000041 +PhraseModel_2 1.185520780812669 +PhraseModel_4 -1.0801541070647134 +LanguageModel 0.7741099486587568 +PhraseModel_0 -0.6265095873268189 +PhraseModel_1 0.6260421233840029 +PassThrough -0.5630000000000002 +Glue -0.495325 +LanguageModel_OOV -0.53285 +PhraseModel_3 -0.008805626854390465 +PhraseModel_6 -0.10977500000000001 +WordPenalty 0.1060655698428214 +PhraseModel_5 0.026699999999999998 diff --git a/training/dtrain/examples/parallelized/work/weights.2.0 b/training/dtrain/examples/parallelized/work/weights.2.0 index f7ece54d..3e87fed4 100644 --- a/training/dtrain/examples/parallelized/work/weights.2.0 +++ b/training/dtrain/examples/parallelized/work/weights.2.0 @@ -1,11 +1,11 @@ -LanguageModel_OOV -0.12280000000000209 -PassThrough -0.13350000000000165 -Glue -0.1259000000000001 -PhraseModel_1 0.24792740418949952 -WordPenalty 0.048293546387642321 -PhraseModel_0 0.26356693580129958 -PhraseModel_2 0.0063762787517740458 -PhraseModel_3 -0.18966358382769741 -PhraseModel_4 -0.22599681869670471 -PhraseModel_6 0.074299999999999047 -LanguageModel 0.3625416478537038 +WordPenalty 0.14922358398195767 +LanguageModel 0.79685677298009394 +LanguageModel_OOV -0.66270000000000007 +PhraseModel_0 0.37998874905310187 +PhraseModel_1 0.69213063228111271 +PhraseModel_2 0.34219807728516061 +PhraseModel_3 1.1425846772648622 +PhraseModel_4 -0.55412548521619742 +PhraseModel_6 0.067599999999999993 +Glue -0.21090000000000003 +PassThrough -0.63429999999999997 diff --git a/training/dtrain/examples/parallelized/work/weights.2.1 b/training/dtrain/examples/parallelized/work/weights.2.1 index 0946609d..d129dc49 100644 --- a/training/dtrain/examples/parallelized/work/weights.2.1 +++ b/training/dtrain/examples/parallelized/work/weights.2.1 @@ -1,12 +1,12 @@ -PassThrough -0.23927500000000015 -Glue -0.10384999999999919 -WordPenalty 0.038717353061671053 -LanguageModel 0.49412782572695274 -LanguageModel_OOV -0.24887499999999915 -PhraseModel_0 -0.32101572713801541 -PhraseModel_1 0.34413149733472631 -PhraseModel_2 0.62365535622061474 -PhraseModel_3 -0.49337445280658987 -PhraseModel_4 -0.77004673375347765 -PhraseModel_5 0.0069999999999999767 -PhraseModel_6 -0.05055000000000108 +WordPenalty 0.1593752174964457 +LanguageModel 1.5897162231676281 +LanguageModel_OOV -0.52100000000000002 +PhraseModel_0 -0.5834836741748588 +PhraseModel_1 0.29827543837280185 +PhraseModel_2 0.78493316593562568 +PhraseModel_3 0.083221832554333464 +PhraseModel_4 -0.93843312963279457 +PhraseModel_5 0.02435 +PhraseModel_6 -0.27382499999999999 +Glue -0.76607500000000006 +PassThrough -0.55115000000000003 diff --git a/training/dtrain/examples/parallelized/work/weights.2.2 b/training/dtrain/examples/parallelized/work/weights.2.2 index b766fc75..bcc83b44 100644 --- a/training/dtrain/examples/parallelized/work/weights.2.2 +++ b/training/dtrain/examples/parallelized/work/weights.2.2 @@ -1,12 +1,12 @@ -PassThrough -0.46082499999999499 -Glue -0.32907499999998979 -WordPenalty 0.049596429833348527 -LanguageModel 0.33496341201347335 -LanguageModel_OOV -0.44357499999999361 -PhraseModel_0 -0.30679883980783829 -PhraseModel_1 0.5937585900939707 -PhraseModel_2 0.86415970329021152 -PhraseModel_3 -0.21072279838022553 -PhraseModel_4 -0.65734339854224544 -PhraseModel_5 0.034750000000000433 -PhraseModel_6 -0.10652500000000011 +WordPenalty 0.10819361280414735 +LanguageModel 0.52389743342585859 +LanguageModel_OOV -0.41622500000000001 +PhraseModel_0 -0.86867995703334211 +PhraseModel_1 0.40783818771767943 +PhraseModel_2 1.1792706530114188 +PhraseModel_3 -0.2469805689928464 +PhraseModel_4 -1.2352895858909159 +PhraseModel_5 0.033750000000000002 +PhraseModel_6 -0.17882500000000001 +Glue -0.90862500000000002 +PassThrough -0.44637500000000013 diff --git a/training/dtrain/examples/parallelized/work/weights.3.0 b/training/dtrain/examples/parallelized/work/weights.3.0 index 403ffbb3..e3586048 100644 --- a/training/dtrain/examples/parallelized/work/weights.3.0 +++ b/training/dtrain/examples/parallelized/work/weights.3.0 @@ -1,12 +1,12 @@ -PhraseModel_4 -1.501862388574505 -PhraseModel_3 -0.77386695951256013 -PhraseModel_6 -0.51399999999999824 -PhraseModel_5 0.02399999999999991 -LanguageModel 1.1666837562322641 -PhraseModel_2 0.036204776972598059 -PassThrough 0.030999999999999975 -Glue 0.046000000000000582 -PhraseModel_1 -0.98829728889588764 -WordPenalty 0.1732834982793964 -PhraseModel_0 -1.1693779885763822 -LanguageModel_OOV 0.066000000000000086 +WordPenalty 0.32441797798172944 +LanguageModel 2.5769043236821889 +LanguageModel_OOV -0.0090000000000000011 +PhraseModel_0 -0.58972189365343919 +PhraseModel_1 0.063690869987073351 +PhraseModel_2 0.53660363110809217 +PhraseModel_3 0.12867071310286207 +PhraseModel_4 -1.9801291745988916 +PhraseModel_5 0.018000000000000002 +PhraseModel_6 -0.48600000000000004 +Glue -0.090000000000000011 +PassThrough -0.090000000000000011 diff --git a/training/dtrain/examples/parallelized/work/weights.3.1 b/training/dtrain/examples/parallelized/work/weights.3.1 index c171d586..b27687d3 100644 --- a/training/dtrain/examples/parallelized/work/weights.3.1 +++ b/training/dtrain/examples/parallelized/work/weights.3.1 @@ -1,12 +1,12 @@ -PassThrough -0.18247500000000313 -Glue -0.084749999999998368 -WordPenalty 0.11150510822865688 -LanguageModel 1.063497816773886 -LanguageModel_OOV -0.1146750000000015 -PhraseModel_0 -0.062922130123762257 -PhraseModel_1 0.0035552404454581212 -PhraseModel_2 0.039691524494244249 -PhraseModel_3 0.080265456972269417 -PhraseModel_4 -0.57787128729945014 -PhraseModel_5 0.017399999999999922 -PhraseModel_6 -0.17095000000000066 +WordPenalty 0.11138567724613679 +LanguageModel 0.95438136276453733 +LanguageModel_OOV -0.060799999999999937 +PhraseModel_0 -0.98112865741560529 +PhraseModel_1 -0.090531125075232435 +PhraseModel_2 0.79088062624556033 +PhraseModel_3 -0.57623134776057228 +PhraseModel_4 -1.4382448344095151 +PhraseModel_5 0.02435 +PhraseModel_6 -0.108125 +Glue 0.31832499999999997 +PassThrough -0.090950000000000003 diff --git a/training/dtrain/examples/parallelized/work/weights.3.2 b/training/dtrain/examples/parallelized/work/weights.3.2 index 3ff0411d..ccb591a2 100644 --- a/training/dtrain/examples/parallelized/work/weights.3.2 +++ b/training/dtrain/examples/parallelized/work/weights.3.2 @@ -1,12 +1,12 @@ -PassThrough -0.32552500000000006 -Glue -0.13247499999999815 -WordPenalty 0.053591939066858545 -LanguageModel 0.72104728811924446 -LanguageModel_OOV -0.30827499999999869 -PhraseModel_0 -0.37052837676792744 -PhraseModel_1 0.17551097460105014 -PhraseModel_2 0.49999630285778179 -PhraseModel_3 -0.14590465814428336 -PhraseModel_4 -0.59563132644367889 -PhraseModel_5 0.034750000000000433 -PhraseModel_6 -0.11142500000000025 +WordPenalty 0.13650961302423945 +LanguageModel 0.58946464694775647 +LanguageModel_OOV -0.48362499999999997 +PhraseModel_0 -0.81261645844738917 +PhraseModel_1 0.44272714074140529 +PhraseModel_2 1.1732783465445731 +PhraseModel_3 -0.18260393204552733 +PhraseModel_4 -1.2213298752899167 +PhraseModel_5 0.02435 +PhraseModel_6 -0.188225 +Glue -0.12992500000000001 +PassThrough -0.51377500000000009 diff --git a/training/dtrain/examples/standard/dtrain.ini b/training/dtrain/examples/standard/dtrain.ini index a515db02..f2698007 100644 --- a/training/dtrain/examples/standard/dtrain.ini +++ b/training/dtrain/examples/standard/dtrain.ini @@ -1,27 +1,10 @@ -#input=./nc-wmt11.de.gz -#refs=./nc-wmt11.en.gz -bitext=./nc-wmt11.gz +bitext=./nc-wmt11.gz # input bitext output=- # a weights file (add .gz for gzip compression) or STDOUT '-' -select_weights=avg # output average (over epochs) weight vector decoder_config=./cdec.ini # config for cdec -# weights for these features will be printed on each iteration +iterations=3 # run over input 3 times +k=100 # use 100best lists +N=4 # optimize (approx.) BLEU4 +learning_rate=0.1 # learning rate +error_margin=1.0 # margin for margin perceptron print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough -# newer version of the grammar extractor use different feature names: -#print_weights= EgivenFCoherent SampleCountF CountEF MaxLexFgivenE MaxLexEgivenF IsSingletonF IsSingletonFE Glue WordPenalty PassThrough LanguageModel LanguageModel_OOV -stop_after=10 # stop epoch after 10 inputs -# interesting stuff -epochs=3 # run over input 3 times -k=100 # use 100best lists -N=4 # optimize (approx) BLEU4 -scorer=fixed_stupid_bleu # use 'stupid' BLEU+1 -learning_rate=0.1 # learning rate, don't care if gamma=0 (perceptron) and loss_margin=0 (not margin perceptron) -gamma=0 # use SVM reg -sample_from=kbest # use kbest lists (as opposed to forest) -filter=uniq # only unique entries in kbest (surface form) -pair_sampling=XYX # -hi_lo=0.1 # 10 vs 80 vs 10 and 80 vs 10 here -pair_threshold=0 # minimum distance in BLEU (here: > 0) -loss_margin=0 # update if correctly ranked, but within this margin -repeat=1 # repeat training on a kbest list 1 times -#batch=true # batch tuning, update after accumulating over all sentences and all kbest lists diff --git a/training/dtrain/examples/toy/dtrain.ini b/training/dtrain/examples/toy/dtrain.ini index 70c7331c..378224b8 100644 --- a/training/dtrain/examples/toy/dtrain.ini +++ b/training/dtrain/examples/toy/dtrain.ini @@ -1,12 +1,8 @@ decoder_config=cdec.ini bitext=in -output=- -print_weights=logp shell_rule house_rule small_rule little_rule PassThrough PassThrough_1 PassThrough_2 PassThrough_3 PassThrough_4 PassThrough_5 PassThrough_6 +output=weights k=4 N=4 -epochs=2 -scorer=bleu -sample_from=kbest -filter=uniq -pair_sampling=all +iterations=2 learning_rate=1 +print_weights=logp shell_rule house_rule small_rule little_rule PassThrough PassThrough_1 PassThrough_2 PassThrough_3 PassThrough_4 PassThrough_5 PassThrough_6 diff --git a/training/dtrain/examples/toy/expected-output b/training/dtrain/examples/toy/expected-output index 3c3a5a18..8c758d00 100644 --- a/training/dtrain/examples/toy/expected-output +++ b/training/dtrain/examples/toy/expected-output @@ -1,40 +1,24 @@ -Warning: hi_lo only works with pair_sampling XYX. - cdec cfg 'cdec.ini' -Seeding random number sequence to 3644621239 - dtrain Parameters: k 4 N 4 T 2 - batch 0 - scorer 'bleu' - sample from 'kbest' - filter 'uniq' learning rate 1 - gamma 0 - loss margin 0 - faster perceptron 1 - pairs 'all' - pair threshold 0 - select weights 'last' - l1 reg 0 'none' - pclr no - max pairs 4294967295 - repeat 1 - cdec cfg 'cdec.ini' - input '' - output '-' -(a dot represents 10 inputs) + error margin 0 + l1 reg 0 + decoder conf 'cdec.ini' + input 'in' + output 'weights' +(a dot per input) Iteration #1 of 2. - 2 + ... 2 WEIGHTS logp = +0 - shell_rule = -1 - house_rule = +2 - small_rule = -2 + shell_rule = +0 + house_rule = +3 + small_rule = +0 little_rule = +3 - PassThrough = -5 + PassThrough = -15 PassThrough_1 = +0 PassThrough_2 = +0 PassThrough_3 = +0 @@ -42,26 +26,23 @@ WEIGHTS PassThrough_5 = +0 PassThrough_6 = +0 --- - 1best avg score: 0.5 (+0.5) - 1best avg model score: 2.5 (+2.5) - avg # pairs: 1.5 - avg # rank err: 1.5 (meaningless) - avg # margin viol: 0 - k-best loss imp: 100% - non0 feature count: 6 + 1best avg score: 0.40937 (+0.40937) + 1best avg model score: 3 + avg # pairs: 2.5 + non-0 feature count: 4 avg list sz: 4 avg f count: 2.875 (time 0 min, 0 s/S) Iteration #2 of 2. - 2 + ... 2 WEIGHTS logp = +0 - shell_rule = -1 - house_rule = +2 - small_rule = -2 + shell_rule = +0 + house_rule = +3 + small_rule = +0 little_rule = +3 - PassThrough = -5 + PassThrough = -15 PassThrough_1 = +0 PassThrough_2 = +0 PassThrough_3 = +0 @@ -69,26 +50,14 @@ WEIGHTS PassThrough_5 = +0 PassThrough_6 = +0 --- - 1best avg score: 1 (+0.5) - 1best avg model score: 5 (+2.5) + 1best avg score: 0.81873 (+0.40937) + 1best avg model score: 6 avg # pairs: 0 - avg # rank err: 0 (meaningless) - avg # margin viol: 0 - k-best loss imp: 100% - non0 feature count: 6 + non-0 feature count: 4 avg list sz: 4 avg f count: 3 (time 0 min, 0 s/S) -Writing weights file to '-' ... -house_rule 2 -little_rule 3 -Glue -4 -PassThrough -5 -small_rule -2 -shell_rule -1 -done - --- -Best iteration: 2 [SCORE 'bleu'=1]. +Best iteration: 2 [GOLD = 0.81873]. This took 0 min. diff --git a/training/dtrain/examples/toy/weights b/training/dtrain/examples/toy/weights new file mode 100644 index 00000000..f6f32772 --- /dev/null +++ b/training/dtrain/examples/toy/weights @@ -0,0 +1,4 @@ +house_rule 3 +little_rule 3 +Glue -12 +PassThrough -15 diff --git a/training/dtrain/lplp.rb b/training/dtrain/lplp.rb index 86e835e8..a1fcd1a3 100755 --- a/training/dtrain/lplp.rb +++ b/training/dtrain/lplp.rb @@ -19,7 +19,8 @@ end # stats def median(feature_column, n) - return feature_column.concat(0.step(n-feature_column.size-1).map{|i|0}).sort[feature_column.size/2] + return feature_column.concat(0.step(n-feature_column.size-1).map{|i|0})\ + .sort[feature_column.size/2] end def mean(feature_column, n) @@ -28,7 +29,7 @@ end # selection def select_k(weights, norm_fun, n, k=10000) - weights.sort{|a,b| norm_fun.call(b[1], n) <=> norm_fun.call(a[1], n)}.each { |p| + weights.sort{|a,b| norm_fun.call(b[1], n)<=>norm_fun.call(a[1], n)}.each { |p| puts "#{p[0]}\t#{mean(p[1], n)}" k -= 1 if k == 0 then break end diff --git a/training/dtrain/pairs.h b/training/dtrain/pairs.h deleted file mode 100644 index dea0dabc..00000000 --- a/training/dtrain/pairs.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef _DTRAIN_PAIRS_H_ -#define _DTRAIN_PAIRS_H_ - -namespace dtrain -{ - -bool -CmpHypsByScore(ScoredHyp a, ScoredHyp b) -{ - return a.score > b.score; -} - -/* - * multipartite ranking - * sort (descending) by bleu - * compare top X (hi) to middle Y (med) and low X (lo) - * cmp middle Y to low X - */ -inline void -MakePairs(vector<ScoredHyp>* s, - vector<pair<ScoredHyp,ScoredHyp> >& training, - bool misranked_only, - float hi_lo) -{ - unsigned sz = s->size(); - if (sz < 2) return; - sort(s->begin(), s->end(), CmpHypsByScore); - unsigned sep = round(sz*hi_lo); - // hi vs. med vs. low - unsigned sep_hi = sep; - if (sz > 4) while (sep_hi < sz && (*s)[sep_hi-1].score == (*s)[sep_hi].score) ++sep_hi; - else sep_hi = 1; - for (unsigned i = 0; i < sep_hi; i++) { - for (unsigned j = sep_hi; j < sz; j++) { - if (misranked_only && !((*s)[i].model <= (*s)[j].model)) continue; - if ((*s)[i].score != (*s)[j].score) - training.push_back(make_pair((*s)[i], (*s)[j])); - } - } - // med vs. low - unsigned sep_lo = sz-sep; - while (sep_lo > 0 && (*s)[sep_lo-1].score == (*s)[sep_lo].score) --sep_lo; - for (unsigned i = sep_hi; i < sep_lo; i++) { - for (unsigned j = sep_lo; j < sz; j++) { - if (misranked_only && !((*s)[i].model <= (*s)[j].model)) continue; - if ((*s)[i].score != (*s)[j].score) - training.push_back(make_pair((*s)[i], (*s)[j])); - } - } -} - -} // namespace - -#endif - diff --git a/training/dtrain/parallelize.rb b/training/dtrain/parallelize.rb index 9315b7f7..29f3e609 100755 --- a/training/dtrain/parallelize.rb +++ b/training/dtrain/parallelize.rb @@ -3,61 +3,52 @@ require 'trollop' require 'zipf' -def usage - STDERR.write "Usage: " - STDERR.write "ruby parallelize.rb -c <dtrain.ini> [-e <epochs=10>] [--randomize/-z] [--reshard/-y] -s <#shards|0> [-p <at once=9999>] -i <input> [--qsub/-q] [--dtrain_binary <path to dtrain binary>] [-l \"l2 select_k 100000\"] [--extra_qsub \"-l mem_free=24G\"]\n" - exit 1 +conf = Trollop::options do + opt :config, "dtrain configuration", :type => :string + opt :input, "input as bitext (f ||| e)", :type => :string + opt :epochs, "number of epochs", :type => :int, :default => 10 + opt :lplp_args, "arguments for lplp.rb", :type => :string, :default => "l2 select_k 100000" + opt :randomize, "randomize shards once", :type => :bool, :default => false, :short => '-z' + opt :reshard, "randomize after each epoch", :type => :bool, :default => false, :short => '-y' + opt :shards, "number of shards", :type => :int + opt :weights, "input weights for first epoch", :type => :string, :default => '' + opt :per_shard_decoder_configs, "give custom decoder config per shard", :type => :string, :short => '-o' + opt :processes_at_once, "jobs to run at oce", :type => :int, :default => 9999 + opt :qsub, "use qsub", :type => :bool, :default => false + opt :qsub_args, "extra args for qsub", :type => :string, :default => "-l h_vmem=5G" + opt :dtrain_binary, "path to dtrain binary", :type => :string end -opts = Trollop::options do - opt :config, "dtrain config file", :type => :string - opt :epochs, "number of epochs", :type => :int, :default => 10 - opt :lplp_args, "arguments for lplp.rb", :type => :string, :default => "l2 select_k 100000" - opt :randomize, "randomize shards before each epoch", :type => :bool, :short => '-z', :default => false - opt :reshard, "reshard after each epoch", :type => :bool, :short => '-y', :default => false - opt :shards, "number of shards", :type => :int - opt :processes_at_once, "have this number (max) running at the same time", :type => :int, :default => 9999 - opt :input, "input (bitext f ||| e ||| ...)", :type => :string - opt :dtrain_binary, "path to dtrain binary", :type => :string - opt :qsub, "use qsub", :type => :bool, :default => false - opt :qsub_args, "extra args for qsub", :type => :string, :default => "-l h_vmem=5G" - opt :first_input_weights, "input weights for first iter", :type => :string, :default => '', :short => '-w' - opt :per_shard_decoder_configs, "give special decoder config per shard", :type => :string, :short => '-o' -end -usage if not opts[:config]&&opts[:shards]&&opts[:input] - dtrain_dir = File.expand_path File.dirname(__FILE__) -if not opts[:dtrain_binary] +if not conf[:dtrain_binary] dtrain_bin = "#{dtrain_dir}/dtrain" else - dtrain_bin = opts[:dtrain_binary] + dtrain_bin = conf[:dtrain_binary] end -ruby = '/usr/bin/ruby' lplp_rb = "#{dtrain_dir}/lplp.rb" -lplp_args = opts[:lplp_args] -cat = '/bin/cat' +lplp_args = conf[:lplp_args] -ini = opts[:config] -epochs = opts[:epochs] -rand = opts[:randomize] -reshard = opts[:reshard] -predefined_shards = false +dtrain_conf = conf[:config] +epochs = conf[:epochs] +rand = conf[:randomize] +reshard = conf[:reshard] +predefined_shards = false per_shard_decoder_configs = false -if opts[:shards] == 0 +if conf[:shards] == 0 predefined_shards = true num_shards = 0 - per_shard_decoder_configs = true if opts[:per_shard_decoder_configs] + per_shard_decoder_configs = true if conf[:per_shard_decoder_configs] else - num_shards = opts[:shards] + num_shards = conf[:shards] end -input = opts[:input] -use_qsub = opts[:qsub] -shards_at_once = opts[:processes_at_once] -first_input_weights = opts[:first_input_weights] +input = conf[:input] +use_qsub = conf[:qsub] +shards_at_once = conf[:processes_at_once] +first_input_weights = conf[:weights] `mkdir work` -def make_shards(input, num_shards, epoch, rand) +def make_shards input, num_shards, epoch, rand lc = `wc -l #{input}`.split.first.to_i index = (0..lc-1).to_a index.reverse! @@ -97,7 +88,8 @@ input_files = [] if predefined_shards input_files = File.new(input).readlines.map {|i| i.strip } if per_shard_decoder_configs - decoder_configs = File.new(opts[:per_shard_decoder_configs]).readlines.map {|i| i.strip} + decoder_configs = ReadFile.readlines_strip(conf[:per_shard_decoder_configs] + ).map { |i| i.strip } end num_shards = input_files.size else @@ -118,22 +110,29 @@ end qsub_str_start = qsub_str_end = '' local_end = '' if use_qsub - qsub_str_start = "qsub #{opts[:qsub_args]} -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \"" + qsub_str_start = "qsub #{conf[:qsub_args]} -cwd -sync y -b y -j y\ + -o work/out.#{shard}.#{epoch}\ + -N dtrain.#{shard}.#{epoch} \"" qsub_str_end = "\"" local_end = '' else local_end = "2>work/out.#{shard}.#{epoch}" end if per_shard_decoder_configs - cdec_cfg = "--decoder_config #{decoder_configs[shard]}" + cdec_conf = "--decoder_config #{decoder_configs[shard]}" else - cdec_cfg = "" + cdec_conf = "" end if first_input_weights!='' && epoch == 0 input_weights = "--input_weights #{first_input_weights}" end pids << Kernel.fork { - `#{qsub_str_start}#{dtrain_bin} -c #{ini} #{cdec_cfg} #{input_weights}\ + puts "#{qsub_str_start}#{dtrain_bin} -c #{dtrain_conf} #{cdec_conf}\ + #{input_weights}\ + --bitext #{input_files[shard]}\ + --output work/weights.#{shard}.#{epoch}#{qsub_str_end} #{local_end}" + `#{qsub_str_start}#{dtrain_bin} -c #{dtrain_conf} #{cdec_conf}\ + #{input_weights}\ --bitext #{input_files[shard]}\ --output work/weights.#{shard}.#{epoch}#{qsub_str_end} #{local_end}` } @@ -144,8 +143,9 @@ end pids.each { |pid| Process.wait(pid) } pids.clear end - `#{cat} work/weights.*.#{epoch} > work/weights_cat` - `#{ruby} #{lplp_rb} #{lplp_args} #{num_shards} < work/weights_cat > work/weights.#{epoch}` + `cat work/weights.*.#{epoch} > work/weights_cat` + `ruby #{lplp_rb} #{lplp_args} #{num_shards} < work/weights_cat\ + > work/weights.#{epoch}` if rand and reshard and epoch+1!=epochs input_files, num_shards = make_shards input, num_shards, epoch+1, rand end diff --git a/training/dtrain/sample.h b/training/dtrain/sample.h index 64d93cb0..c3586c58 100644 --- a/training/dtrain/sample.h +++ b/training/dtrain/sample.h @@ -9,14 +9,16 @@ namespace dtrain struct ScoredKbest : public DecoderObserver { - const unsigned k_; + const size_t k_; vector<ScoredHyp> s_; - unsigned src_len_; + size_t src_len_; PerSentenceBleuScorer* scorer_; vector<vector<WordID> >* refs_; - unsigned f_count_, sz_; + vector<Ngrams>* ref_ngs_; + vector<size_t>* ref_ls_; + size_t f_count_, sz_; - ScoredKbest(const unsigned k, PerSentenceBleuScorer* scorer) : + ScoredKbest(const size_t k, PerSentenceBleuScorer* scorer) : k_(k), scorer_(scorer) {} virtual void @@ -26,7 +28,7 @@ struct ScoredKbest : public DecoderObserver s_.clear(); sz_ = f_count_ = 0; KBest::KBestDerivations<vector<WordID>, ESentenceTraversal, KBest::FilterUnique, prob_t, EdgeProb> kbest(*hg, k_); - for (unsigned i = 0; i < k_; ++i) { + for (size_t i = 0; i < k_; ++i) { const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal, KBest::FilterUnique, prob_t, EdgeProb>::Derivation* d = kbest.LazyKthBest(hg->nodes_.size() - 1, i); @@ -36,7 +38,7 @@ struct ScoredKbest : public DecoderObserver h.f = d->feature_values; h.model = log(d->score); h.rank = i; - h.score = scorer_->Score(h.w, *refs_); + h.gold = scorer_->Score(h.w, *ref_ngs_, *ref_ls_); s_.push_back(h); sz_++; f_count_ += h.f.size(); @@ -44,9 +46,13 @@ struct ScoredKbest : public DecoderObserver } vector<ScoredHyp>* GetSamples() { return &s_; } - inline void SetReference(vector<vector<WordID> >& refs) { refs_ = &refs; } - inline unsigned GetFeatureCount() { return f_count_; } - inline unsigned GetSize() { return sz_; } + inline void SetReference(vector<Ngrams>& ngs, vector<size_t>& ls) + { + ref_ngs_ = &ngs; + ref_ls_ = &ls; + } + inline size_t GetFeatureCount() { return f_count_; } + inline size_t GetSize() { return sz_; } }; diff --git a/training/dtrain/score.h b/training/dtrain/score.h index c727dd30..d51aef82 100644 --- a/training/dtrain/score.h +++ b/training/dtrain/score.h @@ -8,17 +8,17 @@ namespace dtrain struct NgramCounts { - unsigned N_; - map<unsigned, score_t> clipped_; - map<unsigned, score_t> sum_; + size_t N_; + map<size_t, weight_t> clipped_; + map<size_t, weight_t> sum_; - NgramCounts(const unsigned N) : N_(N) { Zero(); } + NgramCounts(const size_t N) : N_(N) { Zero(); } inline void operator+=(const NgramCounts& rhs) { if (rhs.N_ > N_) Resize(rhs.N_); - for (unsigned i = 0; i < N_; i++) { + for (size_t i = 0; i < N_; i++) { this->clipped_[i] += rhs.clipped_.find(i)->second; this->sum_[i] += rhs.sum_.find(i)->second; } @@ -34,16 +34,16 @@ struct NgramCounts } inline void - operator*=(const score_t rhs) + operator*=(const weight_t rhs) { - for (unsigned i = 0; i < N_; i++) { + for (size_t i = 0; i < N_; i++) { this->clipped_[i] *= rhs; this->sum_[i] *= rhs; } } inline void - Add(const unsigned count, const unsigned ref_count, const unsigned i) + Add(const size_t count, const size_t ref_count, const size_t i) { assert(i < N_); if (count > ref_count) { @@ -57,40 +57,31 @@ struct NgramCounts inline void Zero() { - for (unsigned i = 0; i < N_; i++) { + for (size_t i = 0; i < N_; i++) { clipped_[i] = 0.; sum_[i] = 0.; } } inline void - One() + Print(ostream& os=cerr) { - for (unsigned i = 0; i < N_; i++) { - clipped_[i] = 1.; - sum_[i] = 1.; + for (size_t i = 0; i < N_; i++) { + os << i+1 << "grams (clipped):\t" << clipped_[i] << endl; + os << i+1 << "grams:\t\t\t" << sum_[i] << endl; } } - inline void - Print() - { - for (unsigned i = 0; i < N_; i++) { - cout << i+1 << "grams (clipped):\t" << clipped_[i] << endl; - cout << i+1 << "grams:\t\t\t" << sum_[i] << endl; - } - } - - inline void Resize(unsigned N) + inline void Resize(size_t N) { if (N == N_) return; else if (N > N_) { - for (unsigned i = N_; i < N; i++) { + for (size_t i = N_; i < N; i++) { clipped_[i] = 0.; sum_[i] = 0.; } } else { // N < N_ - for (unsigned i = N_-1; i > N-1; i--) { + for (size_t i = N_-1; i > N-1; i--) { clipped_.erase(i); sum_.erase(i); } @@ -99,16 +90,16 @@ struct NgramCounts } }; -typedef map<vector<WordID>, unsigned> Ngrams; +typedef map<vector<WordID>, size_t> Ngrams; inline Ngrams -MakeNgrams(const vector<WordID>& s, const unsigned N) +MakeNgrams(const vector<WordID>& s, const size_t N) { Ngrams ngrams; vector<WordID> ng; for (size_t i = 0; i < s.size(); i++) { ng.clear(); - for (unsigned j = i; j < min(i+N, s.size()); j++) { + for (size_t j = i; j < min(i+N, s.size()); j++) { ng.push_back(s[j]); ngrams[ng]++; } @@ -118,24 +109,21 @@ MakeNgrams(const vector<WordID>& s, const unsigned N) } inline NgramCounts -MakeNgramCounts(const vector<WordID>& hyp, const vector<vector<WordID> >& refs, const unsigned N) +MakeNgramCounts(const vector<WordID>& hyp, + const vector<Ngrams>& ref, + const size_t N) { Ngrams hyp_ngrams = MakeNgrams(hyp, N); - vector<Ngrams> refs_ngrams; - for (auto r: refs) { - Ngrams r_ng = MakeNgrams(r, N); - refs_ngrams.push_back(r_ng); - } NgramCounts counts(N); Ngrams::iterator it, ti; for (it = hyp_ngrams.begin(); it != hyp_ngrams.end(); it++) { - unsigned max_ref_count = 0; - for (auto ref_ngrams: refs_ngrams) { - ti = ref_ngrams.find(it->first); - if (ti != ref_ngrams.end()) + size_t max_ref_count = 0; + for (auto r: ref) { + ti = r.find(it->first); + if (ti != r.end()) max_ref_count = max(max_ref_count, ti->second); } - counts.Add(it->second, min(it->second, max_ref_count), it->first.size() - 1); + counts.Add(it->second, min(it->second, max_ref_count), it->first.size()-1); } return counts; @@ -150,56 +138,65 @@ MakeNgramCounts(const vector<WordID>& hyp, const vector<vector<WordID> >& refs, * [simply add 1 to reference length for calculation of BP] * */ - struct PerSentenceBleuScorer { - const unsigned N_; - vector<score_t> w_; + const size_t N_; + vector<weight_t> w_; - PerSentenceBleuScorer(unsigned n) : N_(n) + PerSentenceBleuScorer(size_t n) : N_(n) { - for (auto i = 1; i <= N_; i++) + for (size_t i = 1; i <= N_; i++) w_.push_back(1.0/N_); } - inline score_t - BrevityPenalty(const unsigned hyp_len, const unsigned ref_len) + inline weight_t + BrevityPenalty(const size_t hl, const size_t rl) { - if (hyp_len > ref_len) return 1; - return exp(1 - (score_t)ref_len/hyp_len); + if (hl > rl) + return 1; + + return exp(1 - (weight_t)rl/hl); } - score_t - Score(const vector<WordID>& hyp, const vector<vector<WordID> >& refs) + weight_t + Score(const vector<WordID>& hyp, + const vector<Ngrams>& ref_ngs, + const vector<size_t>& ref_ls) { - unsigned hyp_len = hyp.size(), ref_len = 0; + size_t hl = hyp.size(), rl = 0; + if (hl == 0) return 0.; // best match reference length - if (refs.size() == 1) { - ref_len = refs[0].size(); + if (ref_ls.size() == 1) { + rl = ref_ls.front(); } else { - unsigned i = 0, best_idx = 0; - unsigned best = std::numeric_limits<unsigned>::max(); - for (auto r: refs) { - unsigned d = abs(hyp_len-r.size()); - if (best > d) best_idx = i; + size_t i = 0, best_idx = 0; + size_t best = numeric_limits<size_t>::max(); + for (auto l: ref_ls) { + size_t d = abs(hl-l); + if (d < best) { + best_idx = i; + best = d; + } + i += 1; } - ref_len = refs[best_idx].size(); + rl = ref_ls[best_idx]; } - if (hyp_len == 0 || ref_len == 0) return 0.; - NgramCounts counts = MakeNgramCounts(hyp, refs, N_); - unsigned M = N_; - vector<score_t> v = w_; - if (ref_len < N_) { - M = ref_len; - for (unsigned i = 0; i < M; i++) v[i] = 1/((score_t)M); + if (rl == 0) return 0.; + NgramCounts counts = MakeNgramCounts(hyp, ref_ngs, N_); + size_t M = N_; + vector<weight_t> v = w_; + if (rl < N_) { + M = rl; + for (size_t i = 0; i < M; i++) v[i] = 1/((weight_t)M); } - score_t sum = 0, add = 0; - for (unsigned i = 0; i < M; i++) { + weight_t sum = 0, add = 0; + for (size_t i = 0; i < M; i++) { if (i == 0 && (counts.sum_[i] == 0 || counts.clipped_[i] == 0)) return 0.; if (i == 1) add = 1; - sum += v[i] * log(((score_t)counts.clipped_[i] + add)/((counts.sum_[i] + add))); + sum += v[i] * log(((weight_t)counts.clipped_[i] + add)/((counts.sum_[i] + add))); } - return BrevityPenalty(hyp_len, ref_len+1) * exp(sum); + + return BrevityPenalty(hl, rl+1) * exp(sum); } }; diff --git a/training/dtrain/update.h b/training/dtrain/update.h new file mode 100644 index 00000000..57671ce1 --- /dev/null +++ b/training/dtrain/update.h @@ -0,0 +1,65 @@ +#ifndef _DTRAIN_UPDATE_H_ +#define _DTRAIN_UPDATE_H_ + +namespace dtrain +{ + +bool +CmpHypsByGold(ScoredHyp a, ScoredHyp b) +{ + return a.gold > b.gold; +} + +/* + * multipartite ranking + * sort (descending) by bleu + * compare top X (hi) to middle Y (med) and low X (lo) + * cmp middle Y to low X + */ +inline size_t +CollectUpdates(vector<ScoredHyp>* s, + SparseVector<weight_t>& updates, + float margin=1.0) +{ + size_t num_pairs = 0; + size_t sz = s->size(); + if (sz < 2) return 0; + sort(s->begin(), s->end(), CmpHypsByGold); + size_t sep = round(sz*0.1); + size_t sep_hi = sep; + if (sz > 4) { + while + (sep_hi < sz && (*s)[sep_hi-1].gold == (*s)[sep_hi].gold) ++sep_hi; + } + else sep_hi = 1; + for (size_t i = 0; i < sep_hi; i++) { + for (size_t j = sep_hi; j < sz; j++) { + if (((*s)[i].model-(*s)[j].model) > margin) + continue; + if ((*s)[i].gold != (*s)[j].gold) { + updates += (*s)[i].f-(*s)[j].f; + num_pairs++; + } + } + } + size_t sep_lo = sz-sep; + while (sep_lo > 0 && (*s)[sep_lo-1].gold == (*s)[sep_lo].gold) + --sep_lo; + for (size_t i = sep_hi; i < sep_lo; i++) { + for (size_t j = sep_lo; j < sz; j++) { + if (((*s)[i].model-(*s)[j].model) > margin) + continue; + if ((*s)[i].gold != (*s)[j].gold) { + updates += (*s)[i].f-(*s)[j].f; + num_pairs++; + } + } + } + + return num_pairs; +} + +} // namespace + +#endif + |