From e26434979adc33bd949566ba7bf02dff64e80a3e Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 2 Oct 2012 00:19:43 -0400 Subject: cdec cleanup, remove bayesian stuff, parsing stuff --- utils/unigram_pyp_lm.cc | 214 ------------------------------------------------ 1 file changed, 214 deletions(-) delete mode 100644 utils/unigram_pyp_lm.cc (limited to 'utils/unigram_pyp_lm.cc') diff --git a/utils/unigram_pyp_lm.cc b/utils/unigram_pyp_lm.cc deleted file mode 100644 index 30b9fde1..00000000 --- a/utils/unigram_pyp_lm.cc +++ /dev/null @@ -1,214 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "corpus_tools.h" -#include "m.h" -#include "tdict.h" -#include "sampler.h" -#include "ccrp.h" -#include "gamma_poisson.h" - -// A not very memory-efficient implementation of an 1-gram LM based on PYPs -// as described in Y.-W. Teh. (2006) A Hierarchical Bayesian Language Model -// based on Pitman-Yor Processes. In Proc. ACL. - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,n",po::value()->default_value(50),"Number of samples") - ("train,i",po::value(),"Training data file") - ("test,T",po::value(),"Test data file") - ("discount_prior_a,a",po::value()->default_value(1.0), "discount ~ Beta(a,b): a=this") - ("discount_prior_b,b",po::value()->default_value(1.0), "discount ~ Beta(a,b): b=this") - ("strength_prior_s,s",po::value()->default_value(1.0), "strength ~ Gamma(s,r): s=this") - ("strength_prior_r,r",po::value()->default_value(1.0), "strength ~ Gamma(s,r): r=this") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("train") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct Histogram { - void increment(unsigned bin, unsigned delta = 1u) { - data[bin] += delta; - } - void decrement(unsigned bin, unsigned delta = 1u) { - data[bin] -= delta; - } - void move(unsigned from_bin, unsigned to_bin, unsigned delta = 1u) { - decrement(from_bin, delta); - increment(to_bin, delta); - } - map data; - // SparseVector data; -}; - -// Lord Rothschild. 1986. THE DISTRIBUTION OF ENGLISH DICTIONARY WORD LENGTHS. -// Journal of Statistical Planning and Inference 14 (1986) 311-322 -struct PoissonLengthUniformCharWordModel { - explicit PoissonLengthUniformCharWordModel(unsigned vocab_size) : plen(5,5), uc(-log(50)), llh() {} - void increment(WordID w, MT19937*) { - llh += log(prob(w)); // this isn't quite right - plen.increment(TD::Convert(w).size() - 1); - } - void decrement(WordID w, MT19937*) { - plen.decrement(TD::Convert(w).size() - 1); - llh -= log(prob(w)); // this isn't quite right - } - double prob(WordID w) const { - size_t len = TD::Convert(w).size(); - return plen.prob(len - 1) * exp(uc * len); - } - double log_likelihood() const { return llh; } - void resample_hyperparameters(MT19937*) {} - GammaPoisson plen; - const double uc; - double llh; -}; - -// uniform base distribution (0-gram model) -struct UniformWordModel { - explicit UniformWordModel(unsigned vocab_size) : p0(1.0 / vocab_size), draws() {} - void increment(WordID, MT19937*) { ++draws; } - void decrement(WordID, MT19937*) { --draws; assert(draws >= 0); } - double prob(WordID) const { return p0; } // all words have equal prob - double log_likelihood() const { return draws * log(p0); } - void resample_hyperparameters(MT19937*) {} - const double p0; - int draws; -}; - -// represents an Unigram LM -template -struct UnigramLM { - UnigramLM(unsigned vs, double da, double db, double ss, double sr) : - base(vs), - crp(da, db, ss, sr, 0.8, 1.0) {} - void increment(WordID w, MT19937* rng) { - const double backoff = base.prob(w); - if (crp.increment(w, backoff, rng)) - base.increment(w, rng); - } - void decrement(WordID w, MT19937* rng) { - if (crp.decrement(w, rng)) - base.decrement(w, rng); - } - double prob(WordID w) const { - const double backoff = base.prob(w); - return crp.prob(w, backoff); - } - - double log_likelihood() const { - double llh = base.log_likelihood(); - llh += crp.log_crp_prob(); - return llh; - } - - void resample_hyperparameters(MT19937* rng) { - crp.resample_hyperparameters(rng); - base.resample_hyperparameters(rng); - } - - double discount_a, discount_b, strength_s, strength_r; - double d, strength; - BaseGenerator base; - CCRP crp; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - const unsigned samples = conf["samples"].as(); - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector > corpuse; - set vocabe; - const WordID kEOS = TD::Convert(""); - cerr << "Reading corpus...\n"; - CorpusTools::ReadFromFile(conf["train"].as(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - vector > test; - if (conf.count("test")) - CorpusTools::ReadFromFile(conf["test"].as(), &test); - else - test = corpuse; -#if 1 - UnigramLM lm(vocabe.size(), -#else - UnigramLM lm(vocabe.size(), -#endif - conf["discount_prior_a"].as(), - conf["discount_prior_b"].as(), - conf["strength_prior_s"].as(), - conf["strength_prior_r"].as()); - for (unsigned SS=0; SS < samples; ++SS) { - for (unsigned ci = 0; ci < corpuse.size(); ++ci) { - const vector& s = corpuse[ci]; - for (unsigned i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - if (SS > 0) lm.decrement(w, &rng); - lm.increment(w, &rng); - } - if (SS > 0) lm.decrement(kEOS, &rng); - lm.increment(kEOS, &rng); - } - cerr << "LLH=" << lm.log_likelihood() << "\t tables=" << lm.crp.num_tables() << " " << endl; - if (SS % 10 == 9) lm.resample_hyperparameters(&rng); - } - double llh = 0; - unsigned cnt = 0; - unsigned oovs = 0; - for (unsigned ci = 0; ci < test.size(); ++ci) { - const vector& s = test[ci]; - for (unsigned i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - double lp = log(lm.prob(w)) / log(2); - if (i < s.size() && vocabe.count(w) == 0) { - cerr << "**OOV "; - ++oovs; - //lp = 0; - } - cerr << "p(" << TD::Convert(w) << ") = " << lp << endl; - llh -= lp; - cnt++; - } - } - cerr << " Log_10 prob: " << (-llh * log(2) / log(10)) << endl; - cerr << " Count: " << cnt << endl; - cerr << " OOVs: " << oovs << endl; - cerr << "Cross-entropy: " << (llh / cnt) << endl; - cerr << " Perplexity: " << pow(2, llh / cnt) << endl; - return 0; -} - -- cgit v1.2.3