From 95e9ea690b87f4648215782e820e177cbe17f18b Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Thu, 20 Oct 2011 15:21:54 +0100 Subject: bidir model1 base measure --- gi/pf/base_measures.cc | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ gi/pf/base_measures.h | 31 ++++++++++++++++++++++++++++ gi/pf/dpnaive.cc | 17 ++++++++++----- gi/pf/monotonic_pseg.h | 5 +++-- gi/pf/pfnaive.cc | 12 ++++++++++- 5 files changed, 113 insertions(+), 8 deletions(-) diff --git a/gi/pf/base_measures.cc b/gi/pf/base_measures.cc index f8ddfd32..8adb37d7 100644 --- a/gi/pf/base_measures.cc +++ b/gi/pf/base_measures.cc @@ -89,6 +89,62 @@ prob_t PhraseJointBase::p0(const vector& vsrc, return p; } +prob_t PhraseJointBase_BiDir::p0(const vector& vsrc, + const vector& vtrg, + int start_src, int start_trg) const { + const int flen = vsrc.size() - start_src; + const int elen = vtrg.size() - start_trg; + prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); + prob_t uniform_trg_alignment; uniform_trg_alignment.logeq(-log(elen + 1)); + + prob_t p1; + p1.logeq(log_poisson(flen, 1.0)); // flen ~Pois(1) + // elen | flen ~Pois(flen + 0.01) + prob_t ptrglen; ptrglen.logeq(log_poisson(elen, flen + 0.01)); + p1 *= ptrglen; + p1 *= kUNIFORM_SOURCE.pow(flen); // each f in F ~Uniform + for (int i = 0; i < elen; ++i) { // for each position i in E + const WordID trg = vtrg[i + start_trg]; + prob_t tp = prob_t::Zero(); + for (int j = -1; j < flen; ++j) { + const WordID src = j < 0 ? 0 : vsrc[j + start_src]; + tp += kM1MIXTURE * model1(src, trg); + tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; + } + tp *= uniform_src_alignment; // draw a_i ~uniform + p1 *= tp; // draw e_i ~Model1(f_a_i) / uniform + } + if (p1.is_0()) { + cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; + abort(); + } + + prob_t p2; + p2.logeq(log_poisson(elen, 1.0)); // elen ~Pois(1) + // flen | elen ~Pois(flen + 0.01) + prob_t psrclen; psrclen.logeq(log_poisson(flen, elen + 0.01)); + p2 *= psrclen; + p2 *= kUNIFORM_TARGET.pow(elen); // each f in F ~Uniform + for (int i = 0; i < flen; ++i) { // for each position i in E + const WordID src = vsrc[i + start_src]; + prob_t tp = prob_t::Zero(); + for (int j = -1; j < elen; ++j) { + const WordID trg = j < 0 ? 0 : vtrg[j + start_trg]; + tp += kM1MIXTURE * invmodel1(trg, src); + tp += kUNIFORM_MIXTURE * kUNIFORM_SOURCE; + } + tp *= uniform_trg_alignment; // draw a_i ~uniform + p2 *= tp; // draw e_i ~Model1(f_a_i) / uniform + } + if (p2.is_0()) { + cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; + abort(); + } + + static const prob_t kHALF(0.5); + return (p1 + p2) * kHALF; +} + JumpBase::JumpBase() : p(200) { for (unsigned src_len = 1; src_len < 200; ++src_len) { map& cpd = p[src_len]; diff --git a/gi/pf/base_measures.h b/gi/pf/base_measures.h index df17aa62..7ce7e2e6 100644 --- a/gi/pf/base_measures.h +++ b/gi/pf/base_measures.h @@ -97,6 +97,37 @@ struct PhraseJointBase { const prob_t kUNIFORM_TARGET; }; +struct PhraseJointBase_BiDir { + explicit PhraseJointBase_BiDir(const Model1& m1, + const Model1& im1, + const double m1mixture, + const unsigned vocab_e_size, + const unsigned vocab_f_size) : + model1(m1), + invmodel1(im1), + kM1MIXTURE(m1mixture), + kUNIFORM_MIXTURE(1.0 - m1mixture), + kUNIFORM_SOURCE(1.0 / vocab_f_size), + kUNIFORM_TARGET(1.0 / vocab_e_size) { + assert(m1mixture >= 0.0 && m1mixture <= 1.0); + assert(vocab_e_size > 0); + } + + // return p0 of rule.e_ | rule.f_ + prob_t operator()(const TRule& rule) const { + return p0(rule.f_, rule.e_, 0, 0); + } + + prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; + + const Model1& model1; + const Model1& invmodel1; + const prob_t kM1MIXTURE; // Model 1 mixture component + const prob_t kUNIFORM_MIXTURE; // uniform mixture component + const prob_t kUNIFORM_SOURCE; + const prob_t kUNIFORM_TARGET; +}; + // base distribution for jump size multinomials // basically p(0) = 0 and then, p(1) is max, and then // you drop as you move to the max jump distance diff --git a/gi/pf/dpnaive.cc b/gi/pf/dpnaive.cc index c926487b..db1c43c7 100644 --- a/gi/pf/dpnaive.cc +++ b/gi/pf/dpnaive.cc @@ -31,6 +31,7 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { ("max_src_phrase",po::value()->default_value(4),"Maximum length of source language phrases") ("max_trg_phrase",po::value()->default_value(4),"Maximum length of target language phrases") ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") + ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in base distribution)") ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") ("random_seed,S",po::value(), "Random seed"); po::options_description clo("Command line options"); @@ -58,7 +59,7 @@ shared_ptr prng; template struct ModelAndData { - explicit ModelAndData(MonotonicParallelSegementationModel& m, const Base& b, const vector >& ce, const vector >& cf, const set& ve, const set& vf) : + explicit ModelAndData(MonotonicParallelSegementationModel& m, const Base& b, const vector >& ce, const vector >& cf, const set& ve, const set& vf) : model(m), rng(&*prng), p0(b), @@ -139,7 +140,7 @@ struct ModelAndData { void Sample(); - MonotonicParallelSegementationModel& model; + MonotonicParallelSegementationModel& model; MT19937* rng; const Base& p0; prob_t baseprob; // cached value of generating the table table labels from p0 @@ -267,6 +268,10 @@ int main(int argc, char** argv) { cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; return 1; } + if (!conf.count("inverse_model1")) { + cerr << argv[0] << "Please use --inverse_model1 to specify inverse model 1 parameters\n"; + return 1; + } if (conf.count("random_seed")) prng.reset(new MT19937(conf["random_seed"].as())); else @@ -283,10 +288,12 @@ int main(int argc, char** argv) { assert(corpusf.size() == corpuse.size()); Model1 m1(conf["model1"].as()); - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel m(lp0); + Model1 invm1(conf["inverse_model1"].as()); +// PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); + PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); + MonotonicParallelSegementationModel m(alp0); - ModelAndData posterior(m, lp0, corpuse, corpusf, vocabe, vocabf); + ModelAndData posterior(m, alp0, corpuse, corpusf, vocabe, vocabf); posterior.Sample(); return 0; diff --git a/gi/pf/monotonic_pseg.h b/gi/pf/monotonic_pseg.h index 7e6af3fc..301aa6d8 100644 --- a/gi/pf/monotonic_pseg.h +++ b/gi/pf/monotonic_pseg.h @@ -8,8 +8,9 @@ #include "trule.h" #include "base_measures.h" +template struct MonotonicParallelSegementationModel { - explicit MonotonicParallelSegementationModel(PhraseJointBase& rcp0) : + explicit MonotonicParallelSegementationModel(BaseMeasure& rcp0) : rp0(rcp0), base(prob_t::One()), rules(1,1), stop(1.0) {} void DecrementRule(const TRule& rule) { @@ -78,7 +79,7 @@ struct MonotonicParallelSegementationModel { return prob_t(stop.prob(false, 0.5)); } - const PhraseJointBase& rp0; + const BaseMeasure& rp0; prob_t base; CCRP_NoTable rules; CCRP_NoTable stop; diff --git a/gi/pf/pfnaive.cc b/gi/pf/pfnaive.cc index 33dc08c3..d967958c 100644 --- a/gi/pf/pfnaive.cc +++ b/gi/pf/pfnaive.cc @@ -181,7 +181,17 @@ int main(int argc, char** argv) { Model1 invm1(conf["inverse_model1"].as()); PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel m(lp0); + PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); + MonotonicParallelSegementationModel m(alp0); + TRule xx("[X] ||| ms. kimura ||| MS. KIMURA ||| X=0"); + cerr << xx << endl << lp0(xx) << " " << alp0(xx) << endl; + TRule xx12("[X] ||| . ||| PHARMACY . ||| X=0"); + TRule xx21("[X] ||| pharmacy . ||| . ||| X=0"); +// TRule xx22("[X] ||| . ||| . ||| X=0"); + TRule xx22("[X] ||| . ||| THE . ||| X=0"); + cerr << xx12 << "\t" << lp0(xx12) << " " << alp0(xx12) << endl; + cerr << xx21 << "\t" << lp0(xx21) << " " << alp0(xx21) << endl; + cerr << xx22 << "\t" << lp0(xx22) << " " << alp0(xx22) << endl; cerr << "Initializing reachability limits...\n"; vector ps(corpusf.size()); -- cgit v1.2.3 From 297bbaab2b722e8fa4f83ab433dae8b4e2126d94 Mon Sep 17 00:00:00 2001 From: Guest_account Guest_account prguest11 Date: Thu, 20 Oct 2011 22:08:35 +0100 Subject: crazy bugs in parallelize --- vest/parallelize.pl | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/vest/parallelize.pl b/vest/parallelize.pl index b4783f91..869f430b 100755 --- a/vest/parallelize.pl +++ b/vest/parallelize.pl @@ -240,12 +240,11 @@ my $node_count = 0; my $script = ""; # fork == one thread runs the sentserver, while the # other spawns the sentclient commands. -if (my $pid = fork) { +my $pid = fork; +if ($pid == 0) { # child sleep 8; # give other thread time to start sentserver - $script = - qq{wait -$cdcmd$sentclient $host:$port:$key $cmd -}; + $script = "$cdcmd$sentclient $host:$port:$key $cmd"; + if ($verbose){ print STDERR "Client script:\n====\n"; print STDERR $script; @@ -270,13 +269,18 @@ $cdcmd$sentclient $host:$port:$key $cmd } } } - waitpid($pid, 0); - cleanup(); + print STDERR "CHILD PROCESSES SPAWNED ... WAITING\n"; + for my $p (@pids) { + waitpid($p, 0); + } } else { # my $todo = "$sentserver -k $key $multiflag $port "; my $todo = "$sentserver -k $key $multiflag $port $stay_alive_flag "; if ($verbose){ print STDERR "Running: $todo\n"; } check_call($todo); + print STDERR "Call to $sentserver returned.\n"; + cleanup(); + exit(0); } sub numof_live_jobs { @@ -343,16 +347,18 @@ sub launch_job_fork { push @errors,$errorfile; push @outs,$outfile; } - if (my $pid = fork) { + my $pid = fork; + if ($pid == 0) { my ($fh, $scr_name) = get_temp_script(); print $fh $script; close $fh; my $todo = "/bin/bash -xeo pipefail $scr_name 1> $outfile 2> $errorfile"; print STDERR "EXEC: $todo\n"; my $out = check_output("$todo"); - print STDERR "RES: $out\n"; unlink $scr_name or warn "Failed to remove $scr_name"; exit 0; + } else { + push @pids, $pid; } } -- cgit v1.2.3 From b2171f53c6c597ac4326f63250269aa13df84718 Mon Sep 17 00:00:00 2001 From: Guest_account Guest_account prguest11 Date: Fri, 21 Oct 2011 15:24:32 +0100 Subject: more particle filter stuff --- gi/pf/pf.h | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ gi/pf/pfdist.cc | 18 ++---------- gi/pf/pfnaive.cc | 24 ++++++---------- 3 files changed, 96 insertions(+), 30 deletions(-) create mode 100644 gi/pf/pf.h diff --git a/gi/pf/pf.h b/gi/pf/pf.h new file mode 100644 index 00000000..ede7cda8 --- /dev/null +++ b/gi/pf/pf.h @@ -0,0 +1,84 @@ +#ifndef _PF_H_ +#define _PF_H_ + +#include +#include +#include "sampler.h" +#include "prob.h" + +template +struct ParticleRenormalizer { + void operator()(std::vector* pv) const { + if (pv->empty()) return; + prob_t z = prob_t::Zero(); + for (unsigned i = 0; i < pv->size(); ++i) + z += (*pv)[i].weight; + assert(z > prob_t::Zero()); + for (unsigned i = 0; i < pv->size(); ++i) + (*pv)[i].weight /= z; + } +}; + +template +struct MultinomialResampleFilter { + explicit MultinomialResampleFilter(MT19937* rng) : rng_(rng) {} + + void operator()(std::vector* pv) { + if (pv->empty()) return; + std::vector& ps = *pv; + SampleSet ss; + for (int i = 0; i < ps.size(); ++i) + ss.add(ps[i].weight); + std::vector nps; nps.reserve(ps.size()); + const prob_t uniform_weight(1.0 / ps.size()); + for (int i = 0; i < ps.size(); ++i) { + nps.push_back(ps[rng_->SelectSample(ss)]); + nps[i].weight = uniform_weight; + } + nps.swap(ps); + } + + private: + MT19937* rng_; +}; + +template +struct SystematicResampleFilter { + explicit SystematicResampleFilter(MT19937* rng) : rng_(rng), renorm_() {} + + void operator()(std::vector* pv) { + if (pv->empty()) return; + renorm_(pv); + std::vector& ps = *pv; + std::vector nps; nps.reserve(ps.size()); + double lower = 0, upper = 0; + const double skip = 1.0 / ps.size(); + double u_j = rng_->next() * skip; + //std::cerr << "u_0: " << u_j << std::endl; + int j = 0; + for (unsigned i = 0; i < ps.size(); ++i) { + upper += ps[i].weight.as_float(); + //std::cerr << "lower: " << lower << " upper: " << upper << std::endl; + // how many children does ps[i] have? + while (u_j < lower) { u_j += skip; ++j; } + while (u_j >= lower && u_j <= upper) { + assert(j < ps.size()); + nps.push_back(ps[i]); + u_j += skip; + //std::cerr << " add u_j=" << u_j << std::endl; + ++j; + } + lower = upper; + } + //std::cerr << ps.size() << " " << nps.size() << "\n"; + assert(ps.size() == nps.size()); + //exit(1); + ps.swap(nps); + } + + private: + MT19937* rng_; + ParticleRenormalizer renorm_; +}; + +#endif diff --git a/gi/pf/pfdist.cc b/gi/pf/pfdist.cc index 81abd61b..aae5f798 100644 --- a/gi/pf/pfdist.cc +++ b/gi/pf/pfdist.cc @@ -6,6 +6,7 @@ #include #include +#include "pf.h" #include "base_measures.h" #include "reachability.h" #include "viterbi.h" @@ -413,20 +414,6 @@ ostream& operator<<(ostream& o, const Particle& p) { return o; } -void FilterCrapParticlesAndReweight(vector* pps) { - vector& ps = *pps; - SampleSet ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - vector nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[prng->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); -} - int main(int argc, char** argv) { po::variables_map conf; InitCommandLine(argc, argv, &conf); @@ -466,6 +453,7 @@ int main(int argc, char** argv) { MyJointModel m(lp0); #endif + MultinomialResampleFilter filter(&rng); cerr << "Initializing reachability limits...\n"; vector ps(corpusf.size()); vector reaches; reaches.reserve(corpusf.size()); @@ -500,7 +488,7 @@ int main(int argc, char** argv) { // all particles have now been extended a bit, we will reweight them now if (lps[0].trg_cov > 0) - FilterCrapParticlesAndReweight(&lps); + filter(&lps); // loop over all particles and extend them bool done_nothing = true; diff --git a/gi/pf/pfnaive.cc b/gi/pf/pfnaive.cc index d967958c..728ec00d 100644 --- a/gi/pf/pfnaive.cc +++ b/gi/pf/pfnaive.cc @@ -6,6 +6,7 @@ #include #include +#include "pf.h" #include "base_measures.h" #include "monotonic_pseg.h" #include "reachability.h" @@ -135,20 +136,6 @@ ostream& operator<<(ostream& o, const Particle& p) { return o; } -void FilterCrapParticlesAndReweight(vector* pps) { - vector& ps = *pps; - SampleSet ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - vector nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[prng->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); -} - int main(int argc, char** argv) { po::variables_map conf; InitCommandLine(argc, argv, &conf); @@ -204,6 +191,8 @@ int main(int argc, char** argv) { cerr << "Sampling...\n"; vector tmp_p(10000); // work space SampleSet pfss; + SystematicResampleFilter filter(&rng); + // MultinomialResampleFilter filter(&rng); for (int SS=0; SS < samples; ++SS) { for (int ci = 0; ci < corpusf.size(); ++ci) { vector& src = corpusf[ci]; @@ -223,7 +212,7 @@ int main(int argc, char** argv) { // all particles have now been extended a bit, we will reweight them now if (lps[0].trg_cov > 0) - FilterCrapParticlesAndReweight(&lps); + filter(&lps); // loop over all particles and extend them bool done_nothing = true; @@ -273,6 +262,11 @@ int main(int argc, char** argv) { } } // loop over particles (pi = 0 .. particles) if (done_nothing) all_complete = true; + prob_t wv = prob_t::Zero(); + for (int pp = 0; pp < lps.size(); ++pp) + wv += lps[pp].weight; + for (int pp = 0; pp < lps.size(); ++pp) + lps[pp].weight /= wv; } pfss.clear(); for (int i = 0; i < lps.size(); ++i) -- cgit v1.2.3 From 62f98f92a3ccf9a614eafe115dbfd04c79c55ea0 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 24 Oct 2011 18:32:13 +0100 Subject: noun-segmenting weights for german word segmenter --- compound-split/cdec-de.ini | 3 +- compound-split/de/charlm.rev.5gm.de.klm | Bin 17376711 -> 0 bytes compound-split/de/dev.nouns | 595 +++++++++++++++++++++++++ compound-split/de/weights.noun-only-1best-only | 20 + 4 files changed, 617 insertions(+), 1 deletion(-) delete mode 100644 compound-split/de/charlm.rev.5gm.de.klm create mode 100644 compound-split/de/dev.nouns create mode 100644 compound-split/de/weights.noun-only-1best-only diff --git a/compound-split/cdec-de.ini b/compound-split/cdec-de.ini index 2bfe63d2..85424e91 100644 --- a/compound-split/cdec-de.ini +++ b/compound-split/cdec-de.ini @@ -1,5 +1,6 @@ formalism=csplit intersection_strategy=full weights=de/weights.trained +#weights=de/weights.noun-only-1best-only feature_function=CSplit_BasicFeatures de/large_dict.de.gz de/badlist.de.gz -feature_function=CSplit_ReverseCharLM de/charlm.rev.5gm.de.klm +feature_function=CSplit_ReverseCharLM de/charlm.rev.5gm.de.lm.gz diff --git a/compound-split/de/charlm.rev.5gm.de.klm b/compound-split/de/charlm.rev.5gm.de.klm deleted file mode 100644 index 28d09b54..00000000 Binary files a/compound-split/de/charlm.rev.5gm.de.klm and /dev/null differ diff --git a/compound-split/de/dev.nouns b/compound-split/de/dev.nouns new file mode 100644 index 00000000..04f597f2 --- /dev/null +++ b/compound-split/de/dev.nouns @@ -0,0 +1,595 @@ +untersuchungsausschuss ||| # untersuchung ausschuss +hochzeiten ||| # hochzeiten +damalige ||| # damalige +siegfried ||| # siegfried +jaschinski ||| # jaschinski +verdächtigenkreis ||| # verdächtige kreis +staatsanwaltschaft ||| # staat anwaltschaft +landesbank ||| # land bank +feldherrn ||| # feld herrn +handelsblatt ||| # handel blatt +alexander ||| # alexander +großen ||| # großen +wachstumsplänen ||| # wachstum plänen +konsolidierung ||| # konsolidierung +branche ||| # branche +institut ||| # institut +augenhöhe ||| ((('#',0,1),),(('auge',0,1),('augen',0,1),),(('höhe',0,1),),) +wiederherstellung ||| # wiederherstellung +grundstück ||| ((('#',0,1),),(('grund',0,1),('grundstück',0,2),),(('stück',0,1),),) +ruhestand ||| # ruhestand +widerstand ||| # widerstand +versammlungsgesetz ||| ((('#',0,1),),(('versammlung',0,1),('versammlungs',0,1),),(('gesetz',0,1),),) +verstöße ||| # verstöße +sachbeschädigung ||| # sach beschädigung +vereinigungen ||| # vereinigungen +glashaus ||| # glas haus +auenlandschaft ||| # auen landschaft +fundament ||| # fundament +vierhock ||| # vierhock +idylle ||| # idylle +johannes ||| # johannes +sportjacke ||| # sport jacke +mittelmeerinseln ||| # mittelmeer inseln +stiefelförmige ||| # stiefel förmige +wiederaufnahme ||| # wiederaufnahme +tonbandaufnahme ||| # tonband aufnahme +korrekturzucker ||| # korrektur zucker +meeresspiegel ||| # meer spiegel +gesamtausgabe ||| # gesamt ausgabe +losfertigung ||| # los fertigung +reichstag ||| # reichstag +monntag ||| # monntag +mittwoch ||| # mittwoch +freitag ||| # freitag +sonntag ||| # sonntag +nachbarschaftsbeziehungen ||| # nachbarschaft beziehungen +saftbar ||| # saft bar +produktionsprozesses ||| # produktion prozesses +gesamtzusammenhang ||| # gesamt zusammenhang +volkswirtschaftslehre ||| # volkswirtschaft lehre +losverfahren ||| # los verfahren +schifffahrt ||| # schiff fahrt +dienstag ||| # dienstag +donnerstag ||| # donnerstag +samstag ||| # samstag +sonnabend ||| # sonnabend +mitglied ||| # mitglied +abblendlicht ||| # abblend licht +abbrucharbeiten ||| # abbruch arbeiten +abergläubischen ||| # abergläubischen +abfallbewirtschaftungshierarchie ||| # abfall bewirtschaftung hierarchie +abfallbewirtschaftungsplänen ||| # abfall bewirtschaftung plänen +abfertigungsgebühren ||| ((('#',0,1),),(('abfertigung',0,1),('abfertigungs',0,1),),(('gebühren',0,1),),) +beitragsunabhängig ||| ((('#',0,1),),(('beitrag',0,1),('beitrags',0,1),),(('unabhängig',0,1),),) +einreichungsfrist ||| ((('#',0,1),),(('einreichung',0,1),),(('frist',0,1),),) +europaabgeordnete ||| # europa abgeordnete +früherkennungskampagnen ||| ((('#',0,1),),(('früh',0,1),),(('erkennung',0,1),('erkennungs',0,1),),(('kampagnen',0,1),),) +hilfsagenturen ||| ((('#',0,1),),(('hilf',0,1),('hilfs',0,1),),(('agenturen',0,1),),) +hilfsinstitutionen ||| ((('#',0,1),),(('hilf',0,1),('hilfs',0,1),),(('institutionen',0,1),),) +kaisergranat ||| # kaiser granat +kalifornien ||| # kalifornien +kampfflugzeug ||| # kampf flugzeug +luftangriffen ||| # luft angriffen +luftsicherheit ||| # luft sicherheit +mithäftlingen ||| # mithäftlingen +wartungsarbeiten ||| # wartung arbeiten +wasseraufbereitungsanlagen ||| # wasser aufbereitung anlagen +wasserdienstleistungen ||| # wasser dienstleistungen +weihnachtsgeschenk ||| # weihnacht geschenk +weltarbeitsorganisation ||| # welt arbeit organisation +zeitdruck ||| # zeit druck +bundesagentur ||| # bund agentur +fahrstuhlschächten ||| # fahrstuhl schächten +elektroindustrie ||| # elektro industrie +sonderpreis ||| # sonder preis +modellrechnung ||| # modell rechnung +treibhauseffekt ||| # treibhaus effekt +europatournee ||| # europa tournee +wirtschaftskrise ||| # wirtschaft krise +mittwochabend ||| # mittwoch abend +abschiedsgala ||| # abschied gala +betriebstag ||| # betrieb tag +flughafen ||| # flughafen +truthahn ||| # truthahn +gehirnregion ||| # gehirn region +feststellung ||| # feststellung +thanksgiving ||| # thanks giving +börsenhändler ||| # börse händler +risikoforschung ||| # risiko forschung +finanzkrise ||| # finanz krise +erkenntnis ||| # erkenntnis +vergangenheit ||| # vergangenheit +festtagsbratenesser ||| # festtag braten esser +ungemütlichkeit ||| # ungemütlichkeit +privatanlegern ||| # privat anlegern +finanzmakler ||| # finanz makler +immobilienfonds ||| # immobilien fonds +wertzuwachs ||| # wert zuwachs +marmelade ||| # marmelade +roboterzofe ||| # roboter zofe +heinrich ||| # heinrich +zeitdehnung ||| # zeit dehnung +zeitlosigkeit ||| # zeitlosigkeit +auseinandersetzung ||| # auseinandersetzung +immobilienfondskrise ||| # immobilien fond krise +stackelberg ||| # stackelberg +bodenseedampfschifffahrtsgesellschaftskapitänsmütze ||| # bodensee dampf schiff fahrt gesellschaft kapitän mütze +hamburg ||| # hamburg +justiz ||| # justiz +bischof ||| # bischof +richard ||| # richard +williamson ||| # williamson +standes ||| # standes +soutane ||| # soutane +millionen ||| # millionen +gaskammern ||| # gas kammern +holocaustleugner ||| # holocaust leugner +großansicht ||| # groß ansicht +gegenfrage ||| # gegen frage +journalisten ||| # journalisten +fernsehsender ||| # fernseh sender +revisionisten ||| # revisionisten +dreihunderttausend ||| # drei hundert tausend +konzentrationslagern ||| # konzentration lagern +schornsteinhöhen ||| # schornstein höhen +touristen ||| # touristen +auschwitz ||| # auschwitz +birkenau ||| # birkenau +antisemitismus ||| # antisemitismus +interviewer ||| # interviewer +bischof ||| # bischof +williamson ||| # williamson +wahrheit ||| # wahrheit +schlecht ||| # schlecht +interessiert ||| # interessiert +entrückte ||| # entrückte +deutscher ||| # deutscher +blamiert ||| # blamiert +katholische ||| # katholische +ausgabe ||| # ausgabe +skandal ||| # skandal +interview ||| # interview +berichtet ||| # berichtet +bayerischen ||| # bayerischen +zaitzhofen ||| # zaitzhofen +fernsehen ||| # fernsehen +regensburger ||| # regensburger +ruckdäschel ||| # ruck däschel +artikel ||| # artikel +leitete ||| # leitete +angaben ||| # angaben +ermittlungsverfahren ||| # ermittlung verfahren +verdacht ||| # verdacht +volksverhetzung ||| # volk verhetzung +rechtsanwalt ||| # recht anwalt +einlassung ||| # einlassung +behörde ||| # behörde +gespräch ||| # gespräch +veröffentlichung ||| # veröffentlichung +deutschland ||| # deutschland +exkommunikation ||| # exkommunikation +hintergrund ||| # hintergrund +strafgesetzbuchs ||| # straf gesetz buchs +leugnung ||| # leugnung +strafverfolgung ||| # straf verfolgung +hintertür ||| # hintertür +entziehen ||| # entziehen +inzwischen ||| # inzwischen +einsicht ||| # einsicht +ermittlungsakten ||| # ermittlung akten +ruckdäschel ||| # ruckdäschel +überlegen ||| # überlegen +schwedischen ||| # schwedischen +journalisten ||| # journalisten +hintergründen ||| # hintergründen +absprachen ||| # absprachen +vorfeld ||| # vorfeld +oberstaatsanwalt ||| # ober staat anwalt +mendelssohn ||| # mendelssohn +bartholdy ||| # bartholdy +familie ||| # familie +philosophen ||| # philosophen +abraham ||| # abraham +bankkaufmannslehre ||| # bank kaufmann lehre +kompagnon ||| # kompagnon +mutter ||| # mutter +salomon ||| # salomon +fabrikantenfamilie ||| # fabrikanten familie +schwester ||| # schwester +hensel ||| # hensel +geschwister ||| # geschwister +rebecca ||| # rebecca +mathematiker ||| # mathematiker +dirichlet ||| # dirichlet +kinder ||| # kinder +pfarrer ||| # pfarrer +gemeinde ||| # gemeinde +berliner ||| # berliner +jerusalems ||| # jerusalems +haustaufe ||| # haus taufe +gelegenheit ||| # gelegenheit +taufnamen ||| # tauf namen +ludwig ||| # ludwig +familiennamen ||| # familie namen +christliche ||| # christliche +vorbesitzers ||| # vorbesitzers +gartens ||| # gartens +abraham ||| # abraham +mendelssohn ||| # mendelssohn +konvertierten ||| # konvertierten +christentum ||| # christentum +kindheit ||| # kindheit +besetzung ||| # besetzung +familie ||| # familie +großmutter ||| # groß mutter +großvater ||| # groß vater +musikunterricht ||| # musik unterricht +schülerin ||| # schülerin +philipp ||| # philipp +kirnbergers ||| # kirnbergers +großtante ||| # groß tante +tochter ||| # tochter +finanzmanns ||| # finanz manns +daniel ||| # daniel +preußischer ||| # preußischer +friedrich ||| # friedrich +wilhelm ||| # wilhelm +naturalisationspatent ||| # naturalisation patent +tradition ||| # tradition +gönnerin ||| # gönnerin +lehrerin ||| # lehrerin +aufenthalts ||| # aufenthalts +rückkehr ||| # rückkehr +unterricht ||| # unterricht +komposition ||| # komposition +ausbildung ||| # ausbildung +schriftstellers ||| # schriftstellers +neunjähriger ||| # neun jähriger +klavierpart ||| # klavier part +klaviertrio ||| # klavier trio +altsänger ||| # alt sänger +akademie ||| # akademie +kirchenmusik ||| # kirche musik +schnelligkeit ||| # schnelligkeit +klaviersonaten ||| # klavier sonaten +klaviertrio ||| # klavier trio +orgelstücke ||| # orgel stücke +streichersinfonien ||| # streicher sinfonien +motetten ||| # motetten +singspiele ||| # sing spiele +soldatenliebschaft ||| # soldaten liebschaft +singspiels ||| # sing spiels +komödianten ||| # komödianten +freundschaft ||| # freundschaft +barack ||| # barack +getappt ||| # getappt +anstehenden ||| # anstehenden +herkulesaufgaben ||| # herkules aufgaben +westen ||| # westen +regierungsgrundsätzen ||| # regierung grundsätzen +genügen ||| # genügen +problem ||| # problem +krankheit ||| # krankheit +washington ||| # washington +symptome ||| # symptome +verquickung ||| # verquickung +öffentlichen ||| # öffentlichen +interessen ||| # interessen +lobbyistenumtriebe ||| # lobbyisten umtriebe +steuerzahlungsphobie ||| # steuer zahlung phobie +krankheit ||| # krankheit +probleme ||| # probleme +problembewältiger ||| # problem bewältiger +verzichten ||| # verzichten +ausnahmen ||| # ausnahmen +nummer ||| # nummer +verteidigungsministerium ||| # verteidigung ministerium +fachmann ||| # fachmann +rüstungslobbyist ||| # rüstung lobbyist +geithner ||| # geithner +finanzminister ||| # finanz minister +fiskus ||| # fiskus +daschle ||| # daschle +obamas ||| # obamas +wunschkandidat ||| # wunsch kandidat +gesundheitsministerium ||| # gesundheit ministerium +begleichung ||| # begleichung +feststand ||| # fest stand +regierung ||| # regierung +proteststurm ||| # protest sturm +handtuch ||| # hand tuch +konzentrationslager ||| # konzentration lager +entwicklungspolitik ||| # entwicklung politik +zeitungsbericht ||| # zeitung bericht +managergehälter ||| # manager gehälter +unternehmen ||| # unternehmen +staatshilfen ||| # staat hilfen +präsidenten ||| # präsidenten +gehaltsobergrenze ||| # gehalt obergrenze +sonnensystems ||| # sonnen systems +planet ||| # planet +exoplanet ||| # exo planet +durchbruch ||| # durch bruch +ulrich ||| # ulrich +schnabel ||| # schnabel +kulturgeschichte ||| # kultur geschichte +milchwirtschaft ||| # milch wirtschaft +menschen ||| # menschen +geschäftsführung ||| # geschäft führung +amtsgericht ||| # amts gericht +göppingen ||| # göppingen +antrag ||| # antrag +eröffnung ||| # eröffnung +insolvenzverfahrens ||| # insolvenz verfahrens +dieter ||| # dieter +informationsangebote ||| # information angebote +vereinfacht ||| # vereinfacht +vokabelglossar ||| # vokabel glossar +gewaltige ||| # gewaltige +haushohe ||| # haushohe +segler ||| # segler +oktober ||| # oktober +gibraltar ||| # gibraltar +bronzekanonen ||| # bronze kanonen +tonnen ||| # tonnen +goldmünzen ||| # gold münzen +kaufleute ||| # kauf leute +schätzungen ||| # schätzungen +vergangenen ||| # vergangenen +millionen ||| # millionen +schiffe ||| # schiffe +meeresgrund ||| # meer grund +wasserfahrzeuge ||| # wasser fahrzeuge +kriegsschiffe ||| # krieg schiffe +passagierdampfer ||| # passagier dampfer +handelsflotten ||| # handel flotten +piratenboote ||| # pirate boote +tanker ||| # tanker +großbritannien ||| # groß britannien +ältesten ||| # ältesten +seefahrernationen ||| # see fahrer nationen +auffassung ||| # auffassung +marinehistoriker ||| # marine historiker +schiffe ||| # schiffe +ortung ||| # ortung +victory ||| # victory +bergungsunternehmen ||| # bergung unternehmen +meeresfunde ||| # meer funde +geschichte ||| # geschichte +entdeckern ||| # entdeckern +kriegsboot ||| # krieg boot +jahrhundert ||| # jahrhundert +tausend ||| # tausend +schiffen ||| # schiffen +entdeckung ||| # entdeckung +tauchjagd ||| # tauch jagd +überreste ||| # überreste +epochen ||| # epochen +länder ||| # länder +streiten ||| # streiten +schiffsfriedhöfe ||| # schiff friedhöfe +tauchjagd ||| # tauch jagd +schatzjäger ||| # schatz jäger +moderne ||| # moderne +technik ||| # technik +jahrhundertealte ||| # jahrhunderte alte +wracks ||| # wracks +geheimnisse ||| # geheimnisse +besitzrechte ||| # besitz rechte +wahrscheinlichkeit ||| # wahrscheinlichkeit +minimaltemperatur ||| # minimal temperatur +winterhalbjahr ||| # winter halb jahr +celsius ||| # celsius +sommerhalbjahr ||| # sommer halb jahr +maximaltemperatur ||| # maximal temperatur +kreuzungsversuche ||| # kreuzung versuche +erbsenpflanzen ||| # erbse pflanzen +klostergarten ||| # kloster garten +befruchtungen ||| # befruchtungen +zehntausende ||| # zehn tausende +pflanzenhybriden ||| # pflanze hybriden +fleißaufgabe ||| ((('#',0,1),),(('fleiß',0,1),('fleißaufgabe',0,2),),(('aufgabe',0,1),),) +hartnäckigkeit ||| # hartnäckigkeit +naturforschers ||| # natur forschers +innenminister ||| # innen minister +falschinformationen ||| # falsch informationen +analysen ||| # analysen +hintergründe ||| # hintergründe +menschen ||| # menschen +containerschiff ||| # container schiff +unfall ||| # unfall +umweltdesaster ||| # umwelt desaster +australien ||| # australien +containerschiff ||| # container schiff +unfall ||| # unfall +australien ||| # australien +schweröl ||| # schweröl +freitag ||| # freitag +flüssigkeit ||| # flüssigkeit +strände ||| # strände +kilometern ||| # kilometern +kapitän ||| # kapitän +pacific ||| # pacific +adventurer ||| # adventurer +hongkong ||| # hongkong +verlust ||| # verlust +tonnen ||| # tonnen +regierungschef ||| ((('#',0,1),),(('regierung',0,1),('regierungs',0,1),),(('chef',0,1),),) +queensland ||| # queensland +samstag ||| # samstag +kapitän ||| # kapitän +vortag ||| # vortag +vertuschung ||| # vertuschung +vorgeworfen ||| # vorgeworfen +mittwoch ||| # mittwoch +containerschiff ||| # container schiff +naturreservate ||| # natur reservate +touristenstrände ||| # touristen strände +ostküste ||| # ost küste +nördlich ||| # nördlich +brisbane ||| # brisbane +bestechungsversuch ||| ((('#',0,1),),(('bestechungs',0,1),('bestechung',0,1),),(('versuch',0,1),),) +campingkocher ||| # camping kocher +integration ||| # integration +funktechnik ||| # funk technik +mobiltelefone ||| # mobil telefone +aktenkoffers ||| ((('#',0,1),),(('akten',0,1),('aktenkoffers',0,2),),(('koffers',0,1),),) +vorstufe ||| # vorstufe +kohlendioxids ||| ((('#',0,1),),(('kohle',0,1),('kohlen',0,1),),(('dioxids',0,1),),) +computermodelle ||| # computer modelle +schützen ||| # schützen +climategate ||| ((('#',0,1),),(('climate',0,1),('climategate',0,2),),(('gate',0,1),),) +skandal ||| # skandal +forschungsinstitut ||| ((('#',0,1),),(('forschung',0,1),('forschungs',0,1),),(('institut',0,1),),) +climate ||| # climate +research ||| # research +folgenschwerste ||| ((('#',0,1),),(('folgenschwerste',0,2),('folgen',0,1),('folge',0,1),),(('schwerste',0,1),),) +wissenschaftsdebatte ||| ((('#',0,1),),(('wissenschaft',0,1),('wissenschafts',0,1),),(('debatte',0,1),),) +computerhacker ||| # computer hacker +instituts ||| # instituts +forscher ||| # forscher +prognose ||| # prognose +erwärmung ||| # erwärmung +wissenschaftler ||| # wissenschaftler +datenlieferanten ||| # daten lieferanten +weltklimarats ||| # welt klima rats +fraktionen ||| # fraktionen +debatte ||| # debatte +schule ||| # schule +repräsentiert ||| # repräsentiert +mehrheit ||| # mehrheit +wissenschaftler ||| # wissenschaftler +stelle ||| # stelle +forscher ||| # forscher +klimamodelle ||| # klimamodelle +computersimulationen ||| # computer simulationen +hauptverantwortlichen ||| # haupt verantwortlichen +erwärmung ||| # erwärmung +menschen ||| # menschen +spatenstich ||| # spaten stich +kirchenneubau ||| # kirche neu bau +ostdeutschlands ||| # ost deutschlands +rumänien ||| # rumänien +berlin ||| # berlin +regierung ||| # regierung +ankündigung ||| # ankündigung +deutschlands ||| # deutschlands +frankreichs ||| # frankreichs +rumäniens ||| # rumäniens +bulgariens ||| # bulgariens +schengen ||| # schengen +bukarest ||| # bukarest +informationen ||| # informationen +verletzung ||| # verletzung +vertrags ||| # vertrags +lissabon ||| # lissabon +rumänischer ||| # rumänischer +zollbeamter ||| # zoll beamter +grenze ||| # grenze +rumänien ||| # rumänien +republik ||| # republik +moldau ||| # moldau +dezember ||| # dezember +regierung ||| # regierung +bukarest ||| # bukarest +kontrollen ||| # kontrollen +grenzen ||| # grenzen +rumänien ||| # rumänien +bulgarien ||| # bulgarien +länder ||| # länder +fortschritte ||| # fortschritte +korruption ||| # korruption +kriminalität ||| # kriminalität +informationen ||| # informationen +zeitung ||| # zeitung +vertragsverletzung ||| # vertrag verletzung +deutschland ||| # deutschland +frankreich ||| # frankreich +haltung ||| # haltung +rumäniens ||| # rumäniens +außenministerium ||| ((('#',0,1),),(('außen',0,1),),(('ministerium',0,1),),) +präzedenzfall ||| ((('#',0,1),),(('präzedenzfall',0,2),('präzedenz',0,1),),(('fall',0,1),),) +staatspräsident ||| ((('#',0,1),),(('staatspräsident',0,2),('staats',0,1),('staat',0,1),),(('präsident',0,1),),) +georgi ||| # georgi +parwanow ||| # parwanow +verständnis ||| # verständnis +bulgarien ||| # bulgarien +verstehen ||| # verstehen +erklärung ||| # erklärung +verzögerung ||| # verzögerung +mittwoch ||| # mittwoch +haltung ||| # haltung +hintergrund ||| # hintergrund +streits ||| # streits +regierung ||| # regierung +ministerpräsident ||| ((('#',0,1),),(('minister',0,1),),(('präsident',0,1),),) +grenzkontrollen ||| ((('#',0,1),),(('grenz',0,1),),(('kontrollen',0,1),),) +weltkrieg ||| ((('#',0,1),),(('welt',0,1),('weltkrieg',0,2),),(('krieg',0,1),),) +abwehr ||| # abwehr +admirals ||| # admirals +canaris ||| # canaris +sprengsätze ||| # spreng sätze +apfelsinenkisten ||| # apfelsine kisten +hafenarbeiter ||| # hafen arbeiter +schiffe ||| # schiffe +zeiten ||| # zeiten +militärdiktatur ||| ((('#',0,1),),(('militär',0,1),),(('diktatur',0,1),),) +widerstandsgruppe ||| ((('#',0,1),),(('widerstand',0,1),('widerstands',0,1),),(('gruppe',0,1),),) +pfirsiche ||| # pfirsiche +aprikosen ||| # aprikosen +vergiftet ||| # vergiftet +kuklina ||| # kuklina +trägerin ||| # trägerin +alternativen ||| # alternativen +nobelpreis ||| # nobel preis +kämpft ||| # kämpft +rechte ||| # rechte +soldaten ||| # soldaten +russlands ||| # russlands +online ||| # online +sprach ||| # sprach +menschenrechte ||| # menschen rechte +heimat ||| # heimat +kaufrausch ||| ((('#',0,1),),(('kauf',0,1),),(('rausch',0,1),),) +kommerzialisierung ||| # kommerzialisierung +weihnachten ||| # weihnachten +wikileaks ||| # wikileaks +verfassungsgericht ||| ((('#',0,1),),(('verfassung',0,1),('verfassungs',0,1),),(('gericht',0,1),),) +berlusconis ||| # berlusconis +immunität ||| # immunität +zinspolitik ||| # zins politik +inflation ||| # inflation +eurozone ||| ((('#',0,1),),(('euro',0,1),('eurozone',0,2),),(('zone',0,1),),) +kontrolle ||| # kontrolle +futtermittelindustrie ||| # futter mittel industrie +deutschland ||| # deutschland +regierung ||| # regierung +vertrauen ||| # vertrauen +westerwelle ||| # westerwelle +abzugsdatum ||| ((('#',0,1),),(('abzugs',0,1),('abzug',0,1),),(('datum',0,1),),) +ghettoblaster ||| # ghetto blaster +alltag ||| # alltag +ikonen ||| # ikonen +jugendkultur ||| # jugend kultur +hochrechnungen ||| # hochrechnungen +mckinsey ||| # mckinsey +company ||| # company +kaufkraft ||| # kauf kraft +aufstrebenden ||| # aufstrebenden +mittelschicht ||| # mittel schicht +jahrzehnts ||| # jahrzehnts +billionen ||| # billionen +dollar ||| # dollar +erbrauchsniveau ||| ((('#',0,1),),(('erbrauch',0,1),('erbrauchs',0,1),),(('niveau',0,1),),) +staaten ||| # staaten +schwellenländer ||| ((('#',0,1),),(('schwellen',0,1),('schwelle',0,1),),(('länder',0,1),),) +brasilien ||| # brasilien +russland ||| # russland +indien ||| # indien +frühstück ||| # frühstück +fortschritt ||| # fortschritt +frühstückstisch ||| # frühstück tisch +familie ||| # familie +jahren ||| # jahren +tageszeitung ||| # tages zeitung diff --git a/compound-split/de/weights.noun-only-1best-only b/compound-split/de/weights.noun-only-1best-only new file mode 100644 index 00000000..d5f09b82 --- /dev/null +++ b/compound-split/de/weights.noun-only-1best-only @@ -0,0 +1,20 @@ +# Objective = 138.848 (eval count=264) +WordCount 11.333477067970925 +LettersSq -0.029215039089113905 +LettersSqrt 2.9632214600626146 +InDict -13.179505941917153 +InDictSubWord 1.3841593332953199 +Short 0.69941278454918432 +Long -0.25571511002085112 +OOV 24.512983009660331 +OOVSubWord -1.5433992715303351 +ShortRange -1.1723773936213888 +HighFreq -3.7705003072208156 +MedFreq -0.051578574665604303 +Freq 0.21221436730925425 +FreqLen1 -1.5685463970117162 +FreqLen2 -0.78986836749506351 +Bad -29.51141124634686 +RevCharLM 0.62803557957086842 +FugS 0.1285546128533254 +FugN 0.47201502660323558 -- cgit v1.2.3 From 6de8f58cd13813bf33af4903bf386439683c0fd6 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 31 Oct 2011 14:03:22 -0400 Subject: lbfgs + time-series minibatch optimization --- training/lbfgs_test.cc | 7 +- training/mpi_flex_optimize.cc | 145 +++++++++++++++++++++++++++--------------- training/optimize.cc | 3 +- word-aligner/aligner.pl | 8 ++- word-aligner/ortho-norm/mg.pl | 13 ++++ word-aligner/ortho-norm/rw.pl | 13 ++++ word-aligner/stemmers/mg.pl | 39 ++++++++++++ word-aligner/stemmers/rw.pl | 38 +++++++++++ 8 files changed, 208 insertions(+), 58 deletions(-) create mode 100755 word-aligner/ortho-norm/mg.pl create mode 100755 word-aligner/ortho-norm/rw.pl create mode 100755 word-aligner/stemmers/mg.pl create mode 100755 word-aligner/stemmers/rw.pl diff --git a/training/lbfgs_test.cc b/training/lbfgs_test.cc index fc21e98d..c94682e9 100644 --- a/training/lbfgs_test.cc +++ b/training/lbfgs_test.cc @@ -28,11 +28,14 @@ double TestOptimizer() { g[2] = 2 * x[2] + 6; obj = 4 * x[0]*x[0] + x[0] * x[1] + x[1]*x[1] + x[2]*x[2] + 6 * x[2] + 5; opt.run(x, obj, g); - + if (!opt.requests_f_and_g()) { + if (converged(x,g)) break; + opt.run(x, obj, g); + } cerr << x[0] << " " << x[1] << " " << x[2] << endl; cerr << " obj=" << obj << "\td/dx1=" << g[0] << " d/dx2=" << g[1] << " d/dx3=" << g[2] << endl; cerr << opt << endl; - } while (!converged(x, g)); + } while (true); return obj; } diff --git a/training/mpi_flex_optimize.cc b/training/mpi_flex_optimize.cc index 87c5f331..00746532 100644 --- a/training/mpi_flex_optimize.cc +++ b/training/mpi_flex_optimize.cc @@ -39,15 +39,12 @@ bool InitCommandLine(int argc, char** argv, po::variables_map* conf) { ("weights,w",po::value(),"Initial feature weights") ("training_data,d",po::value(),"Training data") ("minibatch_size_per_proc,s", po::value()->default_value(6), "Number of training instances evaluated per processor in each minibatch") - ("optimization_method,m", po::value()->default_value("lbfgs"), "Optimization method (options: lbfgs, sgd, rprop)") - ("minibatch_iterations,i", po::value()->default_value(10), "Number of optimization iterations per minibatch (1 = standard SGD)") + ("minibatch_iterations,i", po::value()->default_value(10), "Number of optimization iterations per minibatch") ("iterations,I", po::value()->default_value(50), "Number of passes through the training data before termination") + ("regularization_strength,C", po::value()->default_value(0.2), "Regularization strength") + ("time_series_strength,T", po::value()->default_value(0.0), "Time series regularization strength") ("random_seed,S", po::value(), "Random seed (if not specified, /dev/random will be used)") - ("lbfgs_memory_buffers,M", po::value()->default_value(10), "Number of memory buffers for LBFGS history") - ("eta_0,e", po::value()->default_value(0.1), "Initial learning rate for SGD") - ("L1,1","Use L1 regularization") - ("L2,2","Use L2 regularization") - ("regularization_strength,C", po::value()->default_value(1.0), "Regularization strength (C)"); + ("lbfgs_memory_buffers,M", po::value()->default_value(10), "Number of memory buffers for LBFGS history"); po::options_description clo("Command line options"); clo.add_options() ("config", po::value(), "Configuration file") @@ -64,7 +61,7 @@ bool InitCommandLine(int argc, char** argv, po::variables_map* conf) { po::notify(*conf); if (conf->count("help") || !conf->count("training_data") || !conf->count("cdec_config")) { - cerr << "General-purpose minibatch online optimizer (MPI support " + cerr << "LBFGS minibatch online optimizer (MPI support " #if HAVE_MPI << "enabled" #else @@ -166,6 +163,38 @@ void AddGrad(const SparseVector x, double s, SparseVector* acc) acc->add_value(it->first, it->second.as_float() * s); } +double PNorm(const vector& v, const double p) { + double acc = 0; + for (int i = 0; i < v.size(); ++i) + acc += pow(v[i], p); + return pow(acc, 1.0 / p); +} + +void VV(ostream&os, const vector& v) { + for (int i = 1; i < v.size(); ++i) + if (v[i]) os << FD::Convert(i) << "=" << v[i] << " "; +} + +double ApplyRegularizationTerms(const double C, + const double T, + const vector& weights, + const vector& prev_weights, + vector* g) { + assert(weights.size() == g->size()); + double reg = 0; + for (size_t i = 0; i < weights.size(); ++i) { + const double prev_w_i = (i < prev_weights.size() ? prev_weights[i] : 0.0); + const double& w_i = weights[i]; + double& g_i = (*g)[i]; + reg += C * w_i * w_i; + g_i += 2 * C * w_i; + + reg += T * (w_i - prev_w_i) * (w_i - prev_w_i); + g_i += 2 * T * (w_i - prev_w_i); + } + return reg; +} + int main(int argc, char** argv) { #ifdef HAVE_MPI mpi::environment env(argc, argv); @@ -176,7 +205,7 @@ int main(int argc, char** argv) { const int size = 1; const int rank = 0; #endif - if (size > 1) SetSilent(true); // turn off verbose decoder output + if (size > 0) SetSilent(true); // turn off verbose decoder output register_feature_functions(); MT19937* rng = NULL; @@ -186,56 +215,60 @@ int main(int argc, char** argv) { boost::shared_ptr o; const unsigned lbfgs_memory_buffers = conf["lbfgs_memory_buffers"].as(); - - istringstream ins; - ReadConfig(conf["cdec_config"].as(), &ins); - Decoder decoder(&ins); - - // load initial weights - vector init_weights; - if (conf.count("weights")) - Weights::InitFromFile(conf["weights"].as(), &init_weights); + const unsigned size_per_proc = conf["minibatch_size_per_proc"].as(); + const unsigned minibatch_iterations = conf["minibatch_iterations"].as(); + const double regularization_strength = conf["regularization_strength"].as(); + const double time_series_strength = conf["time_series_strength"].as(); + const bool use_time_series_reg = time_series_strength > 0.0; + const unsigned max_iteration = conf["iterations"].as(); vector corpus; vector ids; ReadTrainingCorpus(conf["training_data"].as(), rank, size, &corpus, &ids); assert(corpus.size() > 0); - const unsigned size_per_proc = conf["minibatch_size_per_proc"].as(); if (size_per_proc > corpus.size()) { - cerr << "Minibatch size must be smaller than corpus size!\n"; + cerr << "Minibatch size (per processor) must be smaller or equal to the local corpus size!\n"; return 1; } - size_t total_corpus_size = 0; -#ifdef HAVE_MPI - reduce(world, corpus.size(), total_corpus_size, std::plus(), 0); -#else - total_corpus_size = corpus.size(); -#endif + // initialize decoder (loads hash functions if necessary) + istringstream ins; + ReadConfig(conf["cdec_config"].as(), &ins); + Decoder decoder(&ins); + + // load initial weights + vector prev_weights; + if (conf.count("weights")) + Weights::InitFromFile(conf["weights"].as(), &prev_weights); if (conf.count("random_seed")) rng = new MT19937(conf["random_seed"].as()); else rng = new MT19937; - const unsigned minibatch_iterations = conf["minibatch_iterations"].as(); + size_t total_corpus_size = 0; +#ifdef HAVE_MPI + reduce(world, corpus.size(), total_corpus_size, std::plus(), 0); +#else + total_corpus_size = corpus.size(); +#endif - if (rank == 0) { + if (rank == 0) cerr << "Total corpus size: " << total_corpus_size << endl; - const unsigned batch_size = size_per_proc * size; - } - SparseVector x; - Weights::InitSparseVector(init_weights, &x); CopyHGsObserver observer; int write_weights_every_ith = 100; // TODO configure int titer = -1; - vector& lambdas = decoder.CurrentWeightVector(); - lambdas.swap(init_weights); - init_weights.clear(); + vector& cur_weights = decoder.CurrentWeightVector(); + if (use_time_series_reg) { + cur_weights = prev_weights; + } else { + cur_weights.swap(prev_weights); + prev_weights.clear(); + } int iter = -1; bool converged = false; @@ -243,26 +276,20 @@ int main(int argc, char** argv) { #ifdef HAVE_MPI mpi::timer timer; #endif - x.init_vector(&lambdas); ++iter; ++titer; -#if 0 if (rank == 0) { converged = (iter == max_iteration); - Weights::SanityCheck(lambdas); - Weights::ShowLargestFeatures(lambdas); string fname = "weights.cur.gz"; if (iter % write_weights_every_ith == 0) { - ostringstream o; o << "weights.epoch_" << (ai+1) << '.' << iter << ".gz"; + ostringstream o; o << "weights.epoch_" << iter << ".gz"; fname = o.str(); } - if (converged && ((ai+1)==agenda.size())) { fname = "weights.final.gz"; } + if (converged) { fname = "weights.final.gz"; } ostringstream vv; - vv << "total iter=" << titer << " (of current config iter=" << iter << ") minibatch=" << size_per_proc << " sentences/proc x " << size << " procs. num_feats=" << x.size() << '/' << FD::NumFeats() << " passes_thru_data=" << (titer * size_per_proc / static_cast(corpus.size())) << " eta=" << lr->eta(titer); + vv << "total iter=" << titer << " (of current config iter=" << iter << ") minibatch=" << size_per_proc << " sentences/proc x " << size << " procs. num_feats=" << FD::NumFeats() << " passes_thru_data=" << (titer * size_per_proc / static_cast(corpus.size())); const string svv = vv.str(); - cerr << svv << endl; - Weights::WriteToFile(fname, lambdas, true, &svv); + Weights::WriteToFile(fname, cur_weights, true, &svv); } -#endif vector hgs(size_per_proc); vector gold_hgs(size_per_proc); @@ -287,8 +314,8 @@ int main(int argc, char** argv) { Hypergraph& hg_gold = gold_hgs[i]; if (hg.edges_.size() < 2) continue; - hg.Reweight(lambdas); - hg_gold.Reweight(lambdas); + hg.Reweight(cur_weights); + hg_gold.Reweight(cur_weights); SparseVector model_exp, gold_exp; const prob_t z = InsideOutside gg(FD::NumFeats()); - if (gg.size() != lambdas.size()) { lambdas.resize(gg.size()); } + if (gg.size() != cur_weights.size()) { cur_weights.resize(gg.size()); } for (SparseVector::const_iterator it = g.begin(); it != g.end(); ++it) if (it->first) { gg[it->first] = it->second; } - cerr << "OBJ: " << obj << endl; - o->Optimize(obj, gg, &lambdas); + g.clear(); + double r = ApplyRegularizationTerms(regularization_strength, + time_series_strength * (iter == 0 ? 0.0 : 1.0), + cur_weights, + prev_weights, + &gg); + obj += r; + if (mi == 0 || mi == (minibatch_iterations - 1)) { + if (!mi) cerr << iter << ' '; else cerr << ' '; + cerr << "OBJ=" << obj << " (REG=" << r << ")" << " |g|=" << PNorm(gg, 2) << " |w|=" << PNorm(cur_weights, 2); + if (mi > 0) cerr << endl << flush; else cerr << ' '; + } else { cerr << '.' << flush; } + // cerr << "w = "; VV(cerr, cur_weights); cerr << endl; + // cerr << "g = "; VV(cerr, gg); cerr << endl; + o->Optimize(obj, gg, &cur_weights); } #ifdef HAVE_MPI - broadcast(world, x, 0); + // broadcast(world, x, 0); broadcast(world, converged, 0); world.barrier(); if (rank == 0) { cerr << " ELAPSED TIME THIS ITERATION=" << timer.elapsed() << endl; } #endif } + prev_weights = cur_weights; } return 0; } diff --git a/training/optimize.cc b/training/optimize.cc index f0740d5c..41ac90d8 100644 --- a/training/optimize.cc +++ b/training/optimize.cc @@ -96,6 +96,7 @@ void LBFGSOptimizer::OptimizeImpl(const double& obj, const vector& g, vector* x) { opt_.run(&(*x)[0], obj, &g[0]); - cerr << opt_ << endl; + if (!opt_.requests_f_and_g()) opt_.run(&(*x)[0], obj, &g[0]); + // cerr << opt_ << endl; } diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl index 3a385a88..c5078645 100755 --- a/word-aligner/aligner.pl +++ b/word-aligner/aligner.pl @@ -27,11 +27,13 @@ die "Expected format corpus.l1-l2 where l1 & l2 are two-letter abbreviations\nfo my $f_lang = $1; my $e_lang = $2; +print STDERR " Using mkcls in: $mkcls\n\n"; print STDERR "Source language: $f_lang\n"; print STDERR "Target language: $e_lang\n"; -print STDERR " Using mkcls in: $mkcls\n\n"; -die "Don't have an orthographic normalizer for $f_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl"; -die "Don't have an orthographic normalizer for $e_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl"; +die "Don't have an stemmer for $f_lang: please create $SCRIPT_DIR/stemmers/$f_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$f_lang.pl"; +die "Don't have an stemmer for $e_lang: please create $SCRIPT_DIR/stemmers/$e_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$e_lang.pl"; +die "Don't have an orthographic normalizer for $f_lang: please create $SCRIPT_DIR/ortho-norm/$f_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl"; +die "Don't have an orthographic normalizer for $e_lang: please create $SCRIPT_DIR/ortho-norm/$e_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl"; my @directions = qw(f-e); diff --git a/word-aligner/ortho-norm/mg.pl b/word-aligner/ortho-norm/mg.pl new file mode 100755 index 00000000..4cb0e8e7 --- /dev/null +++ b/word-aligner/ortho-norm/mg.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -w +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +while() { + $_ = lc $_; + s/([a-z])'( |$)/$1$2/g; + print; +} + diff --git a/word-aligner/ortho-norm/rw.pl b/word-aligner/ortho-norm/rw.pl new file mode 100755 index 00000000..4cb0e8e7 --- /dev/null +++ b/word-aligner/ortho-norm/rw.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -w +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +while() { + $_ = lc $_; + s/([a-z])'( |$)/$1$2/g; + print; +} + diff --git a/word-aligner/stemmers/mg.pl b/word-aligner/stemmers/mg.pl new file mode 100755 index 00000000..2f79a94e --- /dev/null +++ b/word-aligner/stemmers/mg.pl @@ -0,0 +1,39 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while() { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 5; + if ($w =~ /(ndz|ndr|nts|ntr)/) { $el++; } + if ($w =~ /^(mp|mb|nd)/) { $el++; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + diff --git a/word-aligner/stemmers/rw.pl b/word-aligner/stemmers/rw.pl new file mode 100755 index 00000000..6d873b40 --- /dev/null +++ b/word-aligner/stemmers/rw.pl @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while() { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 5; + if ($w =~ /(ny|jy|nk|nt|sh|cy)/) { $el++; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + -- cgit v1.2.3