diff options
| -rw-r--r-- | Makefile.am | 2 | ||||
| -rw-r--r-- | configure.ac | 2 | ||||
| -rw-r--r-- | phrasinator/Makefile.am | 6 | ||||
| -rw-r--r-- | phrasinator/ccrp.h | 294 | ||||
| -rw-r--r-- | phrasinator/train_plm.cc | 5 | ||||
| -rw-r--r-- | utils/slice_sampler.h | 191 |
6 files changed, 498 insertions, 2 deletions
diff --git a/Makefile.am b/Makefile.am index 08d1a5b7..a808c211 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ # warning - the subdirectories in the following list should # be kept in topologically sorted order. Also, DO NOT introduce # cyclic dependencies between these directories! -SUBDIRS = utils mteval klm/util klm/lm decoder training vest extools gi/pyp-topics/src gi/clda/src gi/posterior-regularisation/prjava +SUBDIRS = utils mteval klm/util klm/lm decoder phrasinator training vest extools gi/pyp-topics/src gi/clda/src gi/posterior-regularisation/prjava AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 3fb4ef47..c84d671f 100644 --- a/configure.ac +++ b/configure.ac @@ -114,4 +114,4 @@ then AM_CONDITIONAL([GLC], true) fi -AC_OUTPUT(Makefile utils/Makefile mteval/Makefile extools/Makefile decoder/Makefile training/Makefile vest/Makefile klm/util/Makefile klm/lm/Makefile gi/pyp-topics/src/Makefile gi/clda/src/Makefile) +AC_OUTPUT(Makefile utils/Makefile mteval/Makefile extools/Makefile decoder/Makefile phrasinator/Makefile training/Makefile vest/Makefile klm/util/Makefile klm/lm/Makefile gi/pyp-topics/src/Makefile gi/clda/src/Makefile) diff --git a/phrasinator/Makefile.am b/phrasinator/Makefile.am new file mode 100644 index 00000000..c9b2a513 --- /dev/null +++ b/phrasinator/Makefile.am @@ -0,0 +1,6 @@ +bin_PROGRAMS = train_plm + +train_plm_SOURCES = train_plm.cc +train_plm_LDADD = $(top_srcdir)/utils/libutils.a -lz + +AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval diff --git a/phrasinator/ccrp.h b/phrasinator/ccrp.h new file mode 100644 index 00000000..9acf12ab --- /dev/null +++ b/phrasinator/ccrp.h @@ -0,0 +1,294 @@ +#ifndef _CCRP_H_ +#define _CCRP_H_ + +#include <numeric> +#include <cassert> +#include <cmath> +#include <list> +#include <iostream> +#include <vector> +#include <tr1/unordered_map> +#include <boost/functional/hash.hpp> +#include "sampler.h" +#include "slice_sampler.h" + +// Chinese restaurant process (Pitman-Yor parameters) with table tracking. + +template <typename Dish, typename DishHash = boost::hash<Dish> > +class CCRP { + public: + CCRP(double disc, double conc) : + num_tables_(), + num_customers_(), + discount_(disc), + concentration_(conc), + discount_prior_alpha_(std::numeric_limits<double>::quiet_NaN()), + discount_prior_beta_(std::numeric_limits<double>::quiet_NaN()), + concentration_prior_shape_(std::numeric_limits<double>::quiet_NaN()), + concentration_prior_rate_(std::numeric_limits<double>::quiet_NaN()) {} + + CCRP(double d_alpha, double d_beta, double c_shape, double c_rate, double d = 0.1, double c = 10.0) : + num_tables_(), + num_customers_(), + discount_(d), + concentration_(c), + discount_prior_alpha_(d_alpha), + discount_prior_beta_(d_beta), + concentration_prior_shape_(c_shape), + concentration_prior_rate_(c_rate) {} + + double discount() const { return discount_; } + double concentration() const { return concentration_; } + + bool has_discount_prior() const { + return !std::isnan(discount_prior_alpha_); + } + + bool has_concentration_prior() const { + return !std::isnan(concentration_prior_shape_); + } + + void clear() { + num_tables_ = 0; + num_customers_ = 0; + dish_locs_.clear(); + } + + unsigned num_tables() const { + return num_tables_; + } + + unsigned num_tables(const Dish& dish) const { + const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); + if (it == dish_locs_.end()) return 0; + return it->second.table_counts_.size(); + } + + unsigned num_customers() const { + return num_customers_; + } + + unsigned num_customers(const Dish& dish) const { + const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); + if (it == dish_locs_.end()) return 0; + return it->total_dish_count_; + } + + // returns +1 or 0 indicating whether a new table was opened + int increment(const Dish& dish, const double& p0, MT19937* rng) { + DishLocations& loc = dish_locs_[dish]; + bool share_table = false; + if (loc.total_dish_count_) { + const double p_empty = (concentration_ + num_tables_ * discount_) * p0; + const double p_share = (loc.total_dish_count_ - loc.table_counts_.size() * discount_); + share_table = rng->SelectSample(p_empty, p_share); + } + if (share_table) { + double r = rng->next() * (loc.total_dish_count_ - loc.table_counts_.size() * discount_); + for (typename std::list<unsigned>::iterator ti = loc.table_counts_.begin(); + ti != loc.table_counts_.end(); ++ti) { + r -= (*ti - discount_); + if (r <= 0.0) { + ++(*ti); + break; + } + } + if (r > 0.0) { + std::cerr << "Serious error: r=" << r << std::endl; + Print(&std::cerr); + assert(r <= 0.0); + } + } else { + loc.table_counts_.push_back(1u); + ++num_tables_; + } + ++loc.total_dish_count_; + ++num_customers_; + return (share_table ? 0 : 1); + } + + // returns -1 or 0, indicating whether a table was closed + int decrement(const Dish& dish, MT19937* rng) { + DishLocations& loc = dish_locs_[dish]; + assert(loc.total_dish_count_); + if (loc.total_dish_count_ == 1) { + dish_locs_.erase(dish); + --num_tables_; + --num_customers_; + return -1; + } else { + int delta = 0; + // sample customer to remove UNIFORMLY. that is, do NOT use the discount + // here. if you do, it will introduce (unwanted) bias! + double r = rng->next() * loc.total_dish_count_; + --loc.total_dish_count_; + for (typename std::list<unsigned>::iterator ti = loc.table_counts_.begin(); + ti != loc.table_counts_.end(); ++ti) { + r -= *ti; + if (r <= 0.0) { + if ((--(*ti)) == 0) { + --num_tables_; + delta = -1; + loc.table_counts_.erase(ti); + } + break; + } + } + if (r > 0.0) { + std::cerr << "Serious error: r=" << r << std::endl; + Print(&std::cerr); + assert(r <= 0.0); + } + --num_customers_; + return delta; + } + } + + double prob(const Dish& dish, const double& p0) const { + const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); + const double r = num_tables_ * discount_ + concentration_; + if (it == dish_locs_.end()) { + return r * p0 / (num_customers_ + concentration_); + } else { + return (it->second.total_dish_count_ - discount_ * it->second.table_counts_.size() + r * p0) / + (num_customers_ + concentration_); + } + } + + double log_crp_prob() const { + return log_crp_prob(discount_, concentration_); + } + + static double log_beta_density(const double& x, const double& alpha, const double& beta) { + assert(x > 0.0); + assert(x < 1.0); + assert(alpha > 0.0); + assert(beta > 0.0); + const double lp = (alpha-1)*log(x)+(beta-1)*log(1-x)+lgamma(alpha+beta)-lgamma(alpha)-lgamma(beta); + return lp; + } + + static double log_gamma_density(const double& x, const double& shape, const double& rate) { + assert(x >= 0.0); + assert(shape > 0.0); + assert(rate > 0.0); + const double lp = (shape-1)*log(x) - shape*log(rate) - x/rate - lgamma(shape); + return lp; + } + + // taken from http://en.wikipedia.org/wiki/Chinese_restaurant_process + // does not include P_0's + double log_crp_prob(const double& discount, const double& concentration) const { + double lp = 0.0; + if (has_discount_prior()) + lp = log_beta_density(discount, discount_prior_alpha_, discount_prior_beta_); + if (has_concentration_prior()) + lp += log_gamma_density(concentration, concentration_prior_shape_, concentration_prior_rate_); + assert(lp <= 0.0); + if (num_customers_) { + if (discount > 0.0) { + const double r = lgamma(1.0 - discount); + lp += lgamma(concentration) - lgamma(concentration + num_customers_) + + num_tables_ * log(discount) + lgamma(concentration / discount + num_tables_) + - lgamma(concentration / discount); + assert(std::isfinite(lp)); + for (typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.begin(); + it != dish_locs_.end(); ++it) { + const DishLocations& cur = it->second; + for (std::list<unsigned>::const_iterator ti = cur.table_counts_.begin(); ti != cur.table_counts_.end(); ++ti) { + lp += lgamma(*ti - discount) - r; + } + } + } else { + assert(!"not implemented yet"); + } + } + assert(std::isfinite(lp)); + return lp; + } + + void resample_hyperparameters(MT19937* rng, const unsigned nloop = 5, const unsigned niterations = 10) { + assert(has_discount_prior() || has_concentration_prior()); + DiscountResampler dr(*this); + ConcentrationResampler cr(*this); + for (int iter = 0; iter < nloop; ++iter) { + if (has_concentration_prior()) { + concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, + std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); + } + if (has_discount_prior()) { + discount_ = slice_sampler1d(dr, discount_, *rng, std::numeric_limits<double>::min(), + 1.0, 0.0, niterations, 100*niterations); + } + } + concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, + std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); + } + + struct DiscountResampler { + DiscountResampler(const CCRP& crp) : crp_(crp) {} + const CCRP& crp_; + double operator()(const double& proposed_discount) const { + return crp_.log_crp_prob(proposed_discount, crp_.concentration_); + } + }; + + struct ConcentrationResampler { + ConcentrationResampler(const CCRP& crp) : crp_(crp) {} + const CCRP& crp_; + double operator()(const double& proposed_concentration) const { + return crp_.log_crp_prob(crp_.discount_, proposed_concentration); + } + }; + + struct DishLocations { + DishLocations() : total_dish_count_() {} + unsigned total_dish_count_; // customers at all tables with this dish + std::list<unsigned> table_counts_; // list<> gives O(1) deletion and insertion, which we want + // .size() is the number of tables for this dish + }; + + void Print(std::ostream* out) const { + std::cerr << "PYP(d=" << discount_ << ",c=" << concentration_ << ") customers=" << num_customers_ << std::endl; + for (typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.begin(); + it != dish_locs_.end(); ++it) { + (*out) << it->first << " (" << it->second.total_dish_count_ << " on " << it->second.table_counts_.size() << " tables): "; + for (typename std::list<unsigned>::const_iterator i = it->second.table_counts_.begin(); + i != it->second.table_counts_.end(); ++i) { + (*out) << " " << *i; + } + (*out) << std::endl; + } + } + + typedef typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator const_iterator; + const_iterator begin() const { + return dish_locs_.begin(); + } + const_iterator end() const { + return dish_locs_.end(); + } + + unsigned num_tables_; + unsigned num_customers_; + std::tr1::unordered_map<Dish, DishLocations, DishHash> dish_locs_; + + double discount_; + double concentration_; + + // optional beta prior on discount_ (NaN if no prior) + double discount_prior_alpha_; + double discount_prior_beta_; + + // optional gamma prior on concentration_ (NaN if no prior) + double concentration_prior_shape_; + double concentration_prior_rate_; +}; + +template <typename T,typename H> +std::ostream& operator<<(std::ostream& o, const CCRP<T,H>& c) { + c.Print(&o); + return o; +} + +#endif diff --git a/phrasinator/train_plm.cc b/phrasinator/train_plm.cc new file mode 100644 index 00000000..bb41ad26 --- /dev/null +++ b/phrasinator/train_plm.cc @@ -0,0 +1,5 @@ +#include <iostream> + +int main(int argc, char** argv) { + return 0; +} diff --git a/utils/slice_sampler.h b/utils/slice_sampler.h new file mode 100644 index 00000000..aa48a169 --- /dev/null +++ b/utils/slice_sampler.h @@ -0,0 +1,191 @@ +//! slice-sampler.h is an MCMC slice sampler +//! +//! Mark Johnson, 1st August 2008 + +#ifndef SLICE_SAMPLER_H +#define SLICE_SAMPLER_H + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <iostream> +#include <limits> + +//! slice_sampler_rfc_type{} returns the value of a user-specified +//! function if the argument is within range, or - infinity otherwise +// +template <typename F, typename Fn, typename U> +struct slice_sampler_rfc_type { + F min_x, max_x; + const Fn& f; + U max_nfeval, nfeval; + slice_sampler_rfc_type(F min_x, F max_x, const Fn& f, U max_nfeval) + : min_x(min_x), max_x(max_x), f(f), max_nfeval(max_nfeval), nfeval(0) { } + + F operator() (F x) { + if (min_x < x && x < max_x) { + assert(++nfeval <= max_nfeval); + F fx = f(x); + assert(std::isfinite(fx)); + return fx; + } + return -std::numeric_limits<F>::infinity(); + } +}; // slice_sampler_rfc_type{} + +//! slice_sampler1d() implements the univariate "range doubling" slice sampler +//! described in Neal (2003) "Slice Sampling", The Annals of Statistics 31(3), 705-767. +// +template <typename F, typename LogF, typename Uniform01> +F slice_sampler1d(const LogF& logF0, //!< log of function to sample + F x, //!< starting point + Uniform01& u01, //!< uniform [0,1) random number generator + F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support + F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support + F w = 0.0, //!< guess at initial width + unsigned nsamples=1, //!< number of samples to draw + unsigned max_nfeval=200) //!< max number of function evaluations +{ + typedef unsigned U; + slice_sampler_rfc_type<F,LogF,U> logF(min_x, max_x, logF0, max_nfeval); + + assert(std::isfinite(x)); + + if (w <= 0.0) { // set w to a default width + if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) + w = (max_x - min_x)/4; + else + w = std::max(((x < 0.0) ? -x : x)/4, (F) 0.1); + } + assert(std::isfinite(w)); + + F logFx = logF(x); + for (U sample = 0; sample < nsamples; ++sample) { + F logY = logFx + log(u01()+1e-100); //! slice logFx at this value + assert(std::isfinite(logY)); + + F xl = x - w*u01(); //! lower bound on slice interval + F logFxl = logF(xl); + F xr = xl + w; //! upper bound on slice interval + F logFxr = logF(xr); + + while (logY < logFxl || logY < logFxr) // doubling procedure + if (u01() < 0.5) + logFxl = logF(xl -= xr - xl); + else + logFxr = logF(xr += xr - xl); + + F xl1 = xl; + F xr1 = xr; + while (true) { // shrinking procedure + F x1 = xl1 + u01()*(xr1 - xl1); + if (logY < logF(x1)) { + F xl2 = xl; // acceptance procedure + F xr2 = xr; + bool d = false; + while (xr2 - xl2 > 1.1*w) { + F xm = (xl2 + xr2)/2; + if ((x < xm && x1 >= xm) || (x >= xm && x1 < xm)) + d = true; + if (x1 < xm) + xr2 = xm; + else + xl2 = xm; + if (d && logY >= logF(xl2) && logY >= logF(xr2)) + goto unacceptable; + } + x = x1; + goto acceptable; + } + goto acceptable; + unacceptable: + if (x1 < x) // rest of shrinking procedure + xl1 = x1; + else + xr1 = x1; + } + acceptable: + w = (4*w + (xr1 - xl1))/5; // update width estimate + } + return x; +} + +/* +//! slice_sampler1d() implements a 1-d MCMC slice sampler. +//! It should be correct for unimodal distributions, but +//! not for multimodal ones. +// +template <typename F, typename LogP, typename Uniform01> +F slice_sampler1d(const LogP& logP, //!< log of distribution to sample + F x, //!< initial sample + Uniform01& u01, //!< uniform random number generator + F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support + F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support + F w = 0.0, //!< guess at initial width + unsigned nsamples=1, //!< number of samples to draw + unsigned max_nfeval=200) //!< max number of function evaluations +{ + typedef unsigned U; + assert(std::isfinite(x)); + if (w <= 0.0) { + if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) + w = (max_x - min_x)/4; + else + w = std::max(((x < 0.0) ? -x : x)/4, 0.1); + } + // TRACE4(x, min_x, max_x, w); + F logPx = logP(x); + assert(std::isfinite(logPx)); + U nfeval = 1; + for (U sample = 0; sample < nsamples; ++sample) { + F x0 = x; + F logU = logPx + log(u01()+1e-100); + assert(std::isfinite(logU)); + F r = u01(); + F xl = std::max(min_x, x - r*w); + F xr = std::min(max_x, x + (1-r)*w); + // TRACE3(x, logPx, logU); + while (xl > min_x && logP(xl) > logU) { + xl -= w; + w *= 2; + ++nfeval; + if (nfeval >= max_nfeval) + std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << std::endl; + assert(nfeval < max_nfeval); + } + xl = std::max(xl, min_x); + while (xr < max_x && logP(xr) > logU) { + xr += w; + w *= 2; + ++nfeval; + if (nfeval >= max_nfeval) + std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xr = " << xr << std::endl; + assert(nfeval < max_nfeval); + } + xr = std::min(xr, max_x); + while (true) { + r = u01(); + x = r*xl + (1-r)*xr; + assert(std::isfinite(x)); + logPx = logP(x); + // TRACE4(logPx, x, xl, xr); + assert(std::isfinite(logPx)); + ++nfeval; + if (nfeval >= max_nfeval) + std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << ", xr = " << xr << ", x = " << x << std::endl; + assert(nfeval < max_nfeval); + if (logPx > logU) + break; + else if (x > x0) + xr = x; + else + xl = x; + } + // w = (4*w + (xr-xl))/5; // gradually adjust w + } + // TRACE2(logPx, x); + return x; +} // slice_sampler1d() +*/ + +#endif // SLICE_SAMPLER_H |